Box Backup 0.09 with a few tweeks

390 files changed, 58836 insertions, 0 deletions

diff --git a/BUGS.txt b/BUGS.txt
new file mode 100644
index 00000000..d2ef4cd3
--- /dev/null
+++ b/BUGS.txt
@@ -0,0 +1,10 @@
+================================================================================================================
+Bugs
+================================================================================================================
+
+* need a test to check that small files aren't tracked
+* things like object ids don't have proper typedefs
+* if a file changes while it's being streamed to the server, bad things will happen (exception in bbackupd, or corrupt file on server)
+* if bbackupd gets an error then a signal, it may not wait it's full 100 seconds before retrying. And then won't stop the cycle...
+* bbackupquery restore, if not root, then won't do file ownership properly, but won't alert the user to this fact
+* empty (real) directories in the store aren't deleted when they're empty (and will never be used again) -- uses up disc space unnecessarily
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 00000000..fc3f9f3e
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,37 @@
+ 
+Copyright (c) 2003, 2005
+     Ben Summers.  All rights reserved.
+ 
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. All use of this software and associated advertising materials must 
+   display the following acknowledgement:
+       This product includes software developed by Ben Summers.
+4. The names of the Authors may not be used to endorse or promote
+   products derived from this software without specific prior written
+   permission.
+
+[Where legally impermissible the Authors do not disclaim liability for 
+direct physical injury or death caused solely by defects in the software 
+unless it is modified by a third party.]
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+ 
+ 
+ 
\ No newline at end of file
diff --git a/bin/bbackupctl/bbackupctl.cpp b/bin/bbackupctl/bbackupctl.cpp
new file mode 100755
index 00000000..0dc4f98d
--- /dev/null
+++ b/bin/bbackupctl/bbackupctl.cpp
@@ -0,0 +1,217 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    bbackupctl.cpp
+//		Purpose: bbackupd daemon control program
+//		Created: 18/2/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+#include "MainHelper.h"
+#include "BoxPortsAndFiles.h"
+#include "BackupDaemonConfigVerify.h"
+#include "Socket.h"
+#include "SocketStream.h"
+#include "IOStreamGetLine.h"
+
+#include "MemLeakFindOn.h"
+
+void PrintUsageAndExit()
+{
+	printf("Usage: bbackupctl [-q] [-c config_file] <command>\n"
+	"Commands are:\n"
+	"  sync -- start a syncronisation run now\n"
+	"  force-sync -- force the start of a syncronisation run, even if SyncAllowScript says no\n"
+	"  reload -- reload daemon configuration\n"
+	"  terminate -- terminate daemon now\n"
+	"  wait-for-sync -- wait until the next sync starts, then exit\n"
+	);
+	exit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+	int returnCode = 0;
+
+	MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbackupctl.memleaks", "bbackupctl")
+
+	MAINHELPER_START
+
+	// Filename for configuraiton file?
+	const char *configFilename = BOX_FILE_BBACKUPD_DEFAULT_CONFIG;
+	
+	// Quiet?
+	bool quiet = false;
+	
+	// See if there's another entry on the command line
+	int c;
+	while((c = getopt(argc, (char * const *)argv, "qc:l:")) != -1)
+	{
+		switch(c)
+		{
+		case 'q':
+			// Quiet mode
+			quiet = true;
+			break;
+		
+		case 'c':
+			// store argument
+			configFilename = optarg;
+			break;
+		
+		case '?':
+		default:
+			PrintUsageAndExit();
+		}
+	}
+	// Adjust arguments
+	argc -= optind;
+	argv += optind;
+	
+	// Check there's a command
+	if(argc != 1)
+	{
+		PrintUsageAndExit();
+	}
+
+	// Read in the configuration file
+	if(!quiet) printf("Using configuration file %s\n", configFilename);
+	std::string errs;
+	std::auto_ptr<Configuration> config(Configuration::LoadAndVerify(configFilename, &BackupDaemonConfigVerify, errs));
+	if(config.get() == 0 || !errs.empty())
+	{
+		printf("Invalid configuration file:\n%s", errs.c_str());
+		return 1;
+	}
+	// Easier coding
+	const Configuration &conf(*config);
+
+	// Check there's a socket defined in the config file
+	if(!conf.KeyExists("CommandSocket"))
+	{
+		printf("Daemon isn't using a control socket, could not execute command.\nAdd a CommandSocket declaration to the bbackupd.conf file.\n");
+		return 1;
+	}
+	
+	// Connect to socket
+	SocketStream connection;
+	try
+	{
+		connection.Open(Socket::TypeUNIX, conf.GetKeyValue("CommandSocket").c_str());
+	}
+	catch(...)
+	{
+		printf("Failed to connect to daemon control socket.\n"	\
+			"Possible causes:\n"								\
+			"  * Daemon not running\n"							\
+			"  * Daemon busy syncing with store server\n"		\
+			"  * Another bbackupctl process is communicating with the daemon\n"	\
+			"  * Daemon is waiting to recover from an error\n"
+		);
+		return 1;
+	}
+	
+	// For receiving data
+	IOStreamGetLine getLine(connection);
+	
+	// Wait for the configuration summary
+	std::string configSummary;
+	if(!getLine.GetLine(configSummary))
+	{
+		printf("Failed to receive configuration summary from daemon\n");
+		return 1;
+	}
+
+	// Was the connection rejected by the server?
+	if(getLine.IsEOF())
+	{
+		printf("Server rejected the connection. Are you running bbackupctl as the same user as the daemon?\n");
+		return 1;
+	}
+
+	// Decode it
+	int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait;
+	if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", &autoBackup,
+			&updateStoreInterval, &minimumFileAge, &maxUploadWait) != 4)
+	{
+		printf("Config summary didn't decode\n");
+		return 1;
+	}
+	// Print summary?
+	if(!quiet)
+	{
+		printf("Daemon configuration summary:\n"	\
+			   "  AutomaticBackup = %s\n"			\
+			   "  UpdateStoreInterval = %d seconds\n"	\
+			   "  MinimumFileAge = %d seconds\n"	\
+			   "  MaxUploadWait = %d seconds\n",
+			   autoBackup?"true":"false", updateStoreInterval, minimumFileAge, maxUploadWait);
+	}
+
+	// Is the command the "wait for sync to start" command?
+	bool areWaitingForSync = false;
+	if(::strcmp(argv[0], "wait-for-sync") == 0)
+	{
+		// Check that it's not in non-automatic mode, because then it'll never start
+		if(!autoBackup)
+		{
+			printf("ERROR: Daemon is not in automatic mode -- sync will never start!\n");
+			return 1;
+		}
+	
+		// Yes... set the flag so we know what we're waiting for a sync to start
+		areWaitingForSync = true;
+	}
+	else
+	{
+		// No? Just send the command given plus a quit command.
+		std::string cmd(argv[0]);
+		cmd += "\nquit\n";
+		connection.Write(cmd.c_str(), cmd.size());		
+	}
+	
+	// Read the response
+	std::string line;
+	while(!getLine.IsEOF() && getLine.GetLine(line))
+	{
+		if(areWaitingForSync)
+		{
+			// Need to wait for the state change...
+			if(line == "start-sync")
+			{
+				// Send a quit command to finish nicely
+				connection.Write("quit\n", 5);
+				
+				// And we're done
+				break;
+			}		
+		}
+		else
+		{
+			// Is this an OK or error line?
+			if(line == "ok")
+			{
+				if(!quiet)
+				{
+					printf("Succeeded.\n");
+				}
+				break;
+			}
+			else if(line == "error")
+			{
+				printf("ERROR. (Check command spelling)\n");
+				returnCode = 1;
+				break;
+			}
+		}
+	}
+
+	MAINHELPER_END
+	
+	return returnCode;
+}
diff --git a/bin/bbackupd/BackupClientContext.cpp b/bin/bbackupd/BackupClientContext.cpp
new file mode 100755
index 00000000..08a203c1
--- /dev/null
+++ b/bin/bbackupd/BackupClientContext.cpp
@@ -0,0 +1,453 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientContext.cpp
+//		Purpose: Keep track of context
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <syslog.h>
+
+#include "BoxPortsAndFiles.h"
+#include "BoxTime.h"
+#include "BackupClientContext.h"
+#include "SocketStreamTLS.h"
+#include "Socket.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreException.h"
+#include "BackupDaemon.h"
+#include "autogen_BackupProtocolClient.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool)
+//		Purpose: Constructor
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+BackupClientContext::BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLSContext, const std::string &rHostname,
+			int32_t AccountNumber, bool ExtendedLogging)
+	: mrDaemon(rDaemon),
+	  mrTLSContext(rTLSContext),
+	  mHostname(rHostname),
+	  mAccountNumber(AccountNumber),
+	  mpSocket(0),
+	  mpConnection(0),
+	  mExtendedLogging(ExtendedLogging),
+	  mClientStoreMarker(ClientStoreMarker_NotKnown),
+	  mpDeleteList(0),
+	  mpCurrentIDMap(0),
+	  mpNewIDMap(0),
+	  mStorageLimitExceeded(false),
+	  mpExcludeFiles(0),
+	  mpExcludeDirs(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientContext::~BackupClientContext()
+//		Purpose: Destructor
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+BackupClientContext::~BackupClientContext()
+{
+	CloseAnyOpenConnection();
+	
+	// Delete delete list
+	if(mpDeleteList != 0)
+	{
+		delete mpDeleteList;
+		mpDeleteList = 0;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientContext::GetConnection()
+//		Purpose: Returns the connection, making the connection and logging into
+//				 the backup store if necessary.
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+BackupProtocolClient &BackupClientContext::GetConnection()
+{
+	// Already got it? Just return it.
+	if(mpConnection != 0)
+	{
+		return *mpConnection;
+	}
+	
+	// Get a socket connection
+	if(mpSocket == 0)
+	{
+		mpSocket = new SocketStreamTLS;
+		ASSERT(mpSocket != 0);	// will have exceptioned if this was a problem
+	}
+	
+	try
+	{
+		// Defensive.
+		if(mpConnection != 0)
+		{
+			delete mpConnection;
+			mpConnection = 0;
+		}
+		
+		// Log intention
+		::syslog(LOG_INFO, "Opening connection to server %s...", mHostname.c_str());
+
+		// Connect!
+		mpSocket->Open(mrTLSContext, Socket::TypeINET, mHostname.c_str(), BOX_PORT_BBSTORED);
+		
+		// And create a procotol object
+		mpConnection = new BackupProtocolClient(*mpSocket);
+		
+		// Set logging option
+		mpConnection->SetLogToSysLog(mExtendedLogging);
+		
+		// Handshake
+		mpConnection->Handshake();
+		
+		// Check the version of the server
+		{
+			std::auto_ptr<BackupProtocolClientVersion> serverVersion(mpConnection->QueryVersion(BACKUP_STORE_SERVER_VERSION));
+			if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION)
+			{
+				THROW_EXCEPTION(BackupStoreException, WrongServerVersion)
+			}
+		}
+
+		// Login -- if this fails, the Protocol will exception
+		std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(mpConnection->QueryLogin(mAccountNumber, 0 /* read/write */));
+		
+		// Check that the client store marker is the one we expect
+		if(mClientStoreMarker != ClientStoreMarker_NotKnown)
+		{
+			if(loginConf->GetClientStoreMarker() != mClientStoreMarker)
+			{
+				// Not good... finish the connection, abort, etc, ignoring errors
+				try
+				{
+					mpConnection->QueryFinished();
+					mpSocket->Shutdown();
+					mpSocket->Close();
+				}
+				catch(...)
+				{
+					// IGNORE
+				}
+				
+				// Then throw an exception about this
+				THROW_EXCEPTION(BackupStoreException, ClientMarkerNotAsExpected)
+			}
+		}
+		
+		// Log success
+		::syslog(LOG_INFO, "Connection made, login successful");
+
+		// Check to see if there is any space available on the server
+		int64_t softLimit = loginConf->GetBlocksSoftLimit();
+		int64_t hardLimit = loginConf->GetBlocksHardLimit();
+		// Threshold for uploading new stuff
+		int64_t stopUploadThreshold = softLimit + ((hardLimit - softLimit) / 3);
+		if(loginConf->GetBlocksUsed() > stopUploadThreshold)
+		{
+			// no -- flag so only things like deletions happen
+			mStorageLimitExceeded = true;
+			// Log
+			::syslog(LOG_INFO, "Exceeded storage limits on server -- not uploading changes to files");
+		}
+	}
+	catch(...)
+	{
+		// Clean up.
+		delete mpConnection;
+		mpConnection = 0;
+		delete mpSocket;
+		mpSocket = 0;
+		throw;
+	}
+	
+	return *mpConnection;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientContext::CloseAnyOpenConnection()
+//		Purpose: Closes a connection, if it's open
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+void BackupClientContext::CloseAnyOpenConnection()
+{
+	if(mpConnection)
+	{
+		try
+		{
+			// Need to set a client store marker?
+			if(mClientStoreMarker == ClientStoreMarker_NotKnown)
+			{
+				// Yes, choose one, the current time will do
+				int64_t marker = GetCurrentBoxTime();
+				
+				// Set it on the store
+				mpConnection->QuerySetClientStoreMarker(marker);
+				
+				// Record it so that it can be picked up later.
+				mClientStoreMarker = marker;
+			}
+		
+			// Quit nicely
+			mpConnection->QueryFinished();
+		}
+		catch(...)
+		{
+			// Ignore errors here
+		}
+		
+		// Delete it anyway.
+		delete mpConnection;
+		mpConnection = 0;
+	}
+	
+	if(mpSocket)
+	{
+		try
+		{
+			// Be nice about closing the socket
+			mpSocket->Shutdown();
+			mpSocket->Close();
+		}
+		catch(...)
+		{
+			// Ignore errors
+		}
+		
+		// Delete object
+		delete mpSocket;
+		mpSocket = 0;
+	}
+
+	// Delete any pending list
+	if(mpDeleteList != 0)
+	{
+		delete mpDeleteList;
+		mpDeleteList = 0;
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientContext::GetTimeout()
+//		Purpose: Gets the current timeout time.
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+int BackupClientContext::GetTimeout() const
+{
+	if(mpConnection)
+	{
+		return mpConnection->GetTimeout();
+	}
+	
+	return (15*60*1000);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientContext::GetDeleteList()
+//		Purpose: Returns the delete list, creating one if necessary
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+BackupClientDeleteList &BackupClientContext::GetDeleteList()
+{
+	// Already created?
+	if(mpDeleteList == 0)
+	{
+		mpDeleteList = new BackupClientDeleteList;
+	}
+
+	// Return reference to object	
+	return *mpDeleteList;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    
+//		Purpose: 
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientContext::PerformDeletions()
+{
+	// Got a list?
+	if(mpDeleteList == 0)
+	{
+		// Nothing to do
+		return;
+	}
+	
+	// Delegate to the delete list object
+	mpDeleteList->PerformDeletions(*this);
+	
+	// Delete the object
+	delete mpDeleteList;
+	mpDeleteList = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientContext::GetCurrentIDMap() const
+//		Purpose: Return a (const) reference to the current ID map
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+const BackupClientInodeToIDMap &BackupClientContext::GetCurrentIDMap() const
+{
+	ASSERT(mpCurrentIDMap != 0);
+	if(mpCurrentIDMap == 0)
+	{
+		THROW_EXCEPTION(CommonException, Internal)
+	}
+	return *mpCurrentIDMap;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientContext::GetNewIDMap() const
+//		Purpose: Return a reference to the new ID map
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+BackupClientInodeToIDMap &BackupClientContext::GetNewIDMap() const
+{
+	ASSERT(mpNewIDMap != 0);
+	if(mpNewIDMap == 0)
+	{
+		THROW_EXCEPTION(CommonException, Internal)
+	}
+	return *mpNewIDMap;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientContext::FindFilename(int64_t, int64_t, std::string &, bool &) const
+//		Purpose: Attempts to find the pathname of an object with a given ID on the server.
+//				 Returns true if it can be found, in which case rPathOut is the local filename,
+//				 and rIsDirectoryOut == true if the local object is a directory.
+//		Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut,
+	bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer, box_time_t *pAttributesHashOnServer, BackupStoreFilenameClear *pLeafname)
+{
+	// Make a connection to the server
+	BackupProtocolClient &connection(GetConnection());
+
+	// Request filenames from the server, in a "safe" manner to ignore errors properly
+	{
+		BackupProtocolClientGetObjectName send(ObjectID, ContainingDirectory);
+		connection.Send(send);
+	}
+	std::auto_ptr<BackupProtocolObjectCl> preply(connection.Receive());
+
+	// Is it of the right type?
+	if(preply->GetType() != BackupProtocolClientObjectName::TypeID)
+	{
+		// Was an error or something
+		return false;
+	}
+
+	// Cast to expected type.
+	BackupProtocolClientObjectName *names = (BackupProtocolClientObjectName *)(preply.get());
+
+	// Anything found?
+	int32_t numElements = names->GetNumNameElements();
+	if(numElements <= 0)
+	{
+		// No.
+		return false;
+	}
+	
+	// Get the stream containing all the names
+	std::auto_ptr<IOStream> nameStream(connection.ReceiveStream());
+
+	// Path
+	std::string path;
+
+	// Remember this is in reverse order!
+	for(int l = 0; l < numElements; ++l)
+	{
+		BackupStoreFilenameClear elementName;
+		elementName.ReadFromStream(*nameStream, GetTimeout());
+
+		// Store leafname for caller?
+		if(l == 0 && pLeafname)
+		{
+			*pLeafname = elementName;
+		}
+
+		// Is it part of the filename in the location?
+		if(l < (numElements - 1))
+		{
+			// Part of filename within
+			path = (path.empty())?(elementName.GetClearFilename()):(elementName.GetClearFilename() + DIRECTORY_SEPARATOR_ASCHAR + path);
+		}
+		else
+		{
+			// Location name -- look up in daemon's records
+			std::string locPath;
+			if(!mrDaemon.FindLocationPathName(elementName.GetClearFilename(), locPath))
+			{
+				// Didn't find the location... so can't give the local filename
+				return false;
+			}
+
+			// Add in location path
+			path = (path.empty())?(locPath):(locPath + DIRECTORY_SEPARATOR_ASCHAR + path);
+		}
+	}
+
+	// Is it a directory?
+	rIsDirectoryOut = ((names->GetFlags() & BackupProtocolClientListDirectory::Flags_Dir) == BackupProtocolClientListDirectory::Flags_Dir);
+	
+	// Is it the current version?
+	rIsCurrentVersionOut = ((names->GetFlags() & (BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted)) == 0);
+
+	// And other information which may be required
+	if(pModTimeOnServer) *pModTimeOnServer = names->GetModificationTime();
+	if(pAttributesHashOnServer) *pAttributesHashOnServer = names->GetAttributesHash();
+
+	// Tell caller about the pathname
+	rPathOut = path;
+
+	// Found
+	return true;
+}
+
+
diff --git a/bin/bbackupd/BackupClientContext.h b/bin/bbackupd/BackupClientContext.h
new file mode 100755
index 00000000..3933dbed
--- /dev/null
+++ b/bin/bbackupd/BackupClientContext.h
@@ -0,0 +1,156 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientContext.h
+//		Purpose: Keep track of context
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTCONTEXT__H
+#define BACKUPCLIENTCONTEXT__H
+
+#include "BoxTime.h"
+#include "BackupClientDeleteList.h"
+#include "ExcludeList.h"
+
+class TLSContext;
+class BackupProtocolClient;
+class SocketStreamTLS;
+class BackupClientInodeToIDMap;
+class BackupDaemon;
+class BackupStoreFilenameClear;
+
+#include <string>
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupClientContext
+//		Purpose: 
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+class BackupClientContext
+{
+public:
+	BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLSContext, const std::string &rHostname,
+		int32_t AccountNumber, bool ExtendedLogging);
+	~BackupClientContext();
+private:
+	BackupClientContext(const BackupClientContext &);
+public:
+
+	BackupProtocolClient &GetConnection();
+	
+	void CloseAnyOpenConnection();
+	
+	int GetTimeout() const;
+	
+	BackupClientDeleteList &GetDeleteList();
+	void PerformDeletions();
+
+	enum
+	{
+		ClientStoreMarker_NotKnown = 0
+	};
+
+	void SetClientStoreMarker(int64_t ClientStoreMarker) {mClientStoreMarker = ClientStoreMarker;}
+	int64_t GetClientStoreMarker() const {return mClientStoreMarker;}
+	
+	bool StorageLimitExceeded() {return mStorageLimitExceeded;}
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    BackupClientContext::SetIDMaps(const BackupClientInodeToIDMap *, BackupClientInodeToIDMap *)
+	//		Purpose: Store pointers to the Current and New ID maps
+	//		Created: 11/11/03
+	//
+	// --------------------------------------------------------------------------
+	void SetIDMaps(const BackupClientInodeToIDMap *pCurrent, BackupClientInodeToIDMap *pNew)
+	{
+		ASSERT(pCurrent != 0);
+		ASSERT(pNew != 0);
+		mpCurrentIDMap = pCurrent;
+		mpNewIDMap = pNew;
+	}
+	const BackupClientInodeToIDMap &GetCurrentIDMap() const;
+	BackupClientInodeToIDMap &GetNewIDMap() const;
+	
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    BackupClientContext::SetExcludeLists(ExcludeList *, ExcludeList *)
+	//		Purpose: Sets the exclude lists for the operation. Can be 0.
+	//		Created: 28/1/04
+	//
+	// --------------------------------------------------------------------------
+	void SetExcludeLists(ExcludeList *pExcludeFiles, ExcludeList *pExcludeDirs)
+	{
+		mpExcludeFiles = pExcludeFiles;
+		mpExcludeDirs = pExcludeDirs;
+	}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    BackupClientContext::ExcludeFile(const std::string &)
+	//		Purpose: Returns true is this file should be excluded from the backup
+	//		Created: 28/1/04
+	//
+	// --------------------------------------------------------------------------
+	inline bool ExcludeFile(const std::string &rFullFilename)
+	{
+		if(mpExcludeFiles != 0)
+		{
+			return mpExcludeFiles->IsExcluded(rFullFilename);
+		}
+		// If no list, don't exclude anything
+		return false;
+	}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    BackupClientContext::ExcludeDir(const std::string &)
+	//		Purpose: Returns true is this directory should be excluded from the backup
+	//		Created: 28/1/04
+	//
+	// --------------------------------------------------------------------------
+	inline bool ExcludeDir(const std::string &rFullDirName)
+	{
+		if(mpExcludeDirs != 0)
+		{
+			return mpExcludeDirs->IsExcluded(rFullDirName);
+		}
+		// If no list, don't exclude anything
+		return false;
+	}
+
+	// Utility functions -- may do a lot of work
+	bool FindFilename(int64_t ObjectID, int64_t ContainingDirectory, std::string &rPathOut, bool &rIsDirectoryOut,
+		bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer = 0, box_time_t *pAttributesHashOnServer = 0,
+		BackupStoreFilenameClear *pLeafname = 0); // not const as may connect to server
+
+private:
+	BackupDaemon &mrDaemon;
+	TLSContext &mrTLSContext;
+	std::string mHostname;
+	int32_t mAccountNumber;
+	SocketStreamTLS *mpSocket;
+	BackupProtocolClient *mpConnection;
+	bool mExtendedLogging;
+	int64_t mClientStoreMarker;
+	BackupClientDeleteList *mpDeleteList;
+	const BackupClientInodeToIDMap *mpCurrentIDMap;
+	BackupClientInodeToIDMap *mpNewIDMap;
+	bool mStorageLimitExceeded;
+	ExcludeList *mpExcludeFiles;
+	ExcludeList *mpExcludeDirs;
+};
+
+
+#endif // BACKUPCLIENTCONTEXT__H
+
diff --git a/bin/bbackupd/BackupClientDeleteList.cpp b/bin/bbackupd/BackupClientDeleteList.cpp
new file mode 100755
index 00000000..f6d8e0dc
--- /dev/null
+++ b/bin/bbackupd/BackupClientDeleteList.cpp
@@ -0,0 +1,195 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientDeleteList.cpp
+//		Purpose: List of pending deletes for backup
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <algorithm>
+
+#include "BackupClientDeleteList.h"
+#include "BackupClientContext.h"
+#include "autogen_BackupProtocolClient.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDeleteList::BackupClientDeleteList()
+//		Purpose: Constructor
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+BackupClientDeleteList::BackupClientDeleteList()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDeleteList::~BackupClientDeleteList()
+//		Purpose: Destructor
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+BackupClientDeleteList::~BackupClientDeleteList()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDeleteList::AddDirectoryDelete(int64_t)
+//		Purpose: Add a directory to the list of directories to be deleted.
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID)
+{
+	// Only add the delete to the list if it's not in the "no delete" set
+	if(mDirectoryNoDeleteList.find(ObjectID) == mDirectoryNoDeleteList.end())
+	{
+		// Not in the list, so should delete it
+		mDirectoryList.push_back(ObjectID);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDeleteList::AddFileDelete(int64_t, BackupStoreFilenameClear &)
+//		Purpose: 
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, const BackupStoreFilename &rFilename)
+{
+	// Try to find it in the no delete list
+	std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator delEntry(mFileNoDeleteList.begin());
+	while(delEntry != mFileNoDeleteList.end())
+	{
+		if((delEntry)->first == DirectoryID && (delEntry)->second == rFilename)
+		{
+			// Found!
+			break;
+		}
+		++delEntry;
+	}
+	
+	// Only add it to the delete list if it wasn't in the no delete list
+	if(delEntry == mFileNoDeleteList.end())
+	{
+		mFileList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename));
+	}
+}
+
+
+	
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext)
+//		Purpose: Perform all the pending deletes
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext)
+{
+	// Anything to do?
+	if(mDirectoryList.empty() && mFileList.empty())
+	{
+		// Nothing!
+		return;
+	}
+	
+	// Get a connection
+	BackupProtocolClient &connection(rContext.GetConnection());
+	
+	// Do the deletes
+	for(std::vector<int64_t>::iterator i(mDirectoryList.begin()); i != mDirectoryList.end(); ++i)
+	{
+		connection.QueryDeleteDirectory(*i);
+	}
+	
+	// Clear the directory list
+	mDirectoryList.clear();
+	
+	// Delete the files
+	for(std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator i(mFileList.begin()); i != mFileList.end(); ++i)
+	{
+		connection.QueryDeleteFile(i->first, i->second);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDeleteList::StopDirectoryDeletion(int64_t)
+//		Purpose: Stop a directory being deleted
+//		Created: 19/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID)
+{
+	// First of all, is it in the delete vector?
+	std::vector<int64_t>::iterator delEntry(std::find(mDirectoryList.begin(), mDirectoryList.end(), ObjectID));
+	if(delEntry != mDirectoryList.end())
+	{
+		// erase this entry
+		mDirectoryList.erase(delEntry);
+	}
+	else
+	{
+		// Haven't been asked to delete it yet, put it in the no delete list
+		mDirectoryNoDeleteList.insert(ObjectID);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDeleteList::StopFileDeletion(int64_t, const BackupStoreFilename &)
+//		Purpose: Stop a file from being deleted
+//		Created: 19/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, const BackupStoreFilename &rFilename)
+{
+	// Find this in the delete list
+	std::vector<std::pair<int64_t, BackupStoreFilename> >::iterator delEntry(mFileList.begin());
+	while(delEntry != mFileList.end())
+	{
+		if((delEntry)->first == DirectoryID && (delEntry)->second == rFilename)
+		{
+			// Found!
+			break;
+		}
+		++delEntry;
+	}
+	
+	if(delEntry != mFileList.end())
+	{
+		// erase this entry
+		mFileList.erase(delEntry);
+	}
+	else
+	{
+		// Haven't been asked to delete it yet, put it in the no delete list
+		mFileNoDeleteList.push_back(std::pair<int64_t, BackupStoreFilename>(DirectoryID, rFilename));
+	}
+
+}
+
+
+
+
+
diff --git a/bin/bbackupd/BackupClientDeleteList.h b/bin/bbackupd/BackupClientDeleteList.h
new file mode 100755
index 00000000..5940cf50
--- /dev/null
+++ b/bin/bbackupd/BackupClientDeleteList.h
@@ -0,0 +1,51 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientDeleteList.h
+//		Purpose: List of pending deletes for backup
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTDELETELIST__H
+#define BACKUPCLIENTDELETELIST__H
+
+#include "BackupStoreFilename.h"
+
+class BackupClientContext;
+
+#include <vector>
+#include <utility>
+#include <set>
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupClientDeleteList
+//		Purpose: List of pending deletes for backup
+//		Created: 10/11/03
+//
+// --------------------------------------------------------------------------
+class BackupClientDeleteList
+{
+public:
+	BackupClientDeleteList();
+	~BackupClientDeleteList();
+	
+	void AddDirectoryDelete(int64_t ObjectID);
+	void AddFileDelete(int64_t DirectoryID, const BackupStoreFilename &rFilename);
+
+	void StopDirectoryDeletion(int64_t ObjectID);
+	void StopFileDeletion(int64_t DirectoryID, const BackupStoreFilename &rFilename);
+	
+	void PerformDeletions(BackupClientContext &rContext);
+	
+private:
+	std::vector<int64_t> mDirectoryList;
+	std::set<int64_t> mDirectoryNoDeleteList;	// note: things only get in this list if they're not present in mDirectoryList when they are 'added'
+	std::vector<std::pair<int64_t, BackupStoreFilename> > mFileList;
+	std::vector<std::pair<int64_t, BackupStoreFilename> > mFileNoDeleteList;
+};
+
+#endif // BACKUPCLIENTDELETELIST__H
+
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp
new file mode 100755
index 00000000..eb4a8343
--- /dev/null
+++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp
@@ -0,0 +1,1203 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientDirectoryRecord.cpp
+//		Purpose: Implementation of record about directory for backup client
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <dirent.h>
+#include <errno.h>
+#include <string.h>
+
+#include "BackupClientDirectoryRecord.h"
+#include "autogen_BackupProtocolClient.h"
+#include "BackupClientContext.h"
+#include "IOStream.h"
+#include "MemBlockStream.h"
+#include "CommonException.h"
+#include "CollectInBufferStream.h"
+#include "BackupStoreFile.h"
+#include "BackupClientInodeToIDMap.h"
+#include "FileModificationTime.h"
+#include "BackupDaemon.h"
+#include "BackupStoreException.h"
+
+#ifdef PLATFORM_LINUX
+	#include "LinuxWorkaround.h"
+#endif
+
+#include "MemLeakFindOn.h"
+
+typedef std::map<std::string, BackupStoreDirectory::Entry *> DecryptedEntriesMap_t;
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::BackupClientDirectoryRecord()
+//		Purpose: Constructor
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName)
+	: mObjectID(ObjectID),
+	  mSubDirName(rSubDirName),
+	  mInitialSyncDone(false),
+	  mSyncDone(false),
+	  mpPendingEntries(0)
+{
+	::memset(mStateChecksum, 0, sizeof(mStateChecksum));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::~BackupClientDirectoryRecord()
+//		Purpose: Destructor
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+BackupClientDirectoryRecord::~BackupClientDirectoryRecord()
+{
+	// Make deletion recursive
+	DeleteSubDirectories();
+	
+	// Delete maps
+	if(mpPendingEntries != 0)
+	{
+		delete mpPendingEntries;
+		mpPendingEntries = 0;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::DeleteSubDirectories();
+//		Purpose: Delete all sub directory entries
+//		Created: 2003/10/09
+//
+// --------------------------------------------------------------------------
+void BackupClientDirectoryRecord::DeleteSubDirectories()
+{
+	// Delete all pointers
+	for(std::map<std::string, BackupClientDirectoryRecord *>::iterator i = mSubDirectories.begin();
+		i != mSubDirectories.end(); ++i)
+	{
+		delete i->second;
+	}
+	
+	// Empty list
+	mSubDirectories.clear();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &, int64_t, const std::string &, bool)
+//		Purpose: Syncronise, recusively, a local directory with the server.
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &rParams, int64_t ContainingDirectoryID,
+	const std::string &rLocalPath, bool ThisDirHasJustBeenCreated)
+{
+	// Signal received by daemon?
+	if(rParams.mrDaemon.StopRun())
+	{
+		// Yes. Stop now.
+		THROW_EXCEPTION(BackupStoreException, SignalReceived)
+	}
+
+	// Start by making some flag changes, marking this sync as not done,
+	// and on the immediate sub directories.
+	mSyncDone = false;
+	for(std::map<std::string, BackupClientDirectoryRecord *>::iterator i = mSubDirectories.begin();
+		i != mSubDirectories.end(); ++i)
+	{
+		i->second->mSyncDone = false;
+	}
+
+	// Work out the time in the future after which the file should be uploaded regardless.
+	// This is a simple way to avoid having too many problems with file servers when they have
+	// clients with badly out of sync clocks.
+	rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + rParams.mMaxFileTimeInFuture;
+	
+	// Build the current state checksum to compare against while getting info from dirs
+	// Note checksum is used locally only, so byte order isn't considered.
+	MD5Digest currentStateChecksum;
+	
+	// Stat the directory, to get attribute info
+	{
+		struct stat st;
+		if(::stat(rLocalPath.c_str(), &st) != 0)
+		{
+			// The directory has probably been deleted, so just ignore this error.
+			// In a future scan, this deletion will be noticed, deleted from server, and this object deleted.
+			TRACE1("Stat failed for '%s' (directory)\n", rLocalPath.c_str());
+			return;
+		}
+		// Store inode number in map so directories are tracked in case they're renamed
+		{
+			BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap());
+			idMap.AddToMap(st.st_ino, mObjectID, ContainingDirectoryID);
+		}
+		// Add attributes to checksum
+		currentStateChecksum.Add(&st.st_mode, sizeof(st.st_mode));
+		currentStateChecksum.Add(&st.st_uid, sizeof(st.st_uid));
+		currentStateChecksum.Add(&st.st_gid, sizeof(st.st_gid));
+		// Inode to be paranoid about things moving around
+		currentStateChecksum.Add(&st.st_ino, sizeof(st.st_ino));
+#ifndef PLATFORM_stat_NO_st_flags
+		currentStateChecksum.Add(&st.st_flags, sizeof(st.st_flags));
+#endif // n PLATFORM_stat_NO_st_flags
+	}
+	
+	// Read directory entries, building arrays of names
+	// First, need to read the contents of the directory.
+	std::vector<std::string> dirs;
+	std::vector<std::string> files;
+	bool downloadDirectoryRecordBecauseOfFutureFiles = false;
+	// BLOCK
+	{		
+		// read the contents...
+		DIR *dirHandle = 0;
+		try
+		{
+			dirHandle = ::opendir(rLocalPath.c_str());
+			if(dirHandle == 0)
+			{
+				// Report the error (logs and eventual email to administrator)
+				SetErrorWhenReadingFilesystemObject(rParams, rLocalPath.c_str());
+				// Ignore this directory for now.
+				return;
+			}
+			
+			// Basic structure for checksum info
+			struct {
+				box_time_t mModificationTime;
+				box_time_t mAttributeModificationTime;
+				int64_t mSize;
+				// And then the name follows
+			} checksum_info;
+			// Be paranoid about structure packing
+			::memset(&checksum_info, 0, sizeof(checksum_info));
+	
+			struct dirent *en = 0;
+			struct stat st;
+			std::string filename;
+			while((en = ::readdir(dirHandle)) != 0)
+			{
+				// Don't need to use LinuxWorkaround_FinishDirentStruct(en, rLocalPath.c_str());
+				// on Linux, as a stat is performed to get all this info
+
+				if(en->d_name[0] == '.' && 
+					(en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0')))
+				{
+					// ignore, it's . or ..
+					continue;
+				}
+
+				// Stat file to get info
+				filename = rLocalPath + DIRECTORY_SEPARATOR + en->d_name;
+				if(::lstat(filename.c_str(), &st) != 0)
+				{
+					TRACE1("Stat failed for '%s' (contents)\n", filename.c_str());
+					THROW_EXCEPTION(CommonException, OSFileError)
+				}
+
+				int type = st.st_mode & S_IFMT;
+				if(type == S_IFREG || type == S_IFLNK)
+				{
+					// File or symbolic link
+
+					// Exclude it?
+					if(rParams.mrContext.ExcludeFile(filename))
+					{
+						// Next item!
+						continue;
+					}
+
+					// Store on list
+					files.push_back(std::string(en->d_name));
+				}
+				else if(type == S_IFDIR)
+				{
+					// Directory
+
+					// Exclude it?
+					if(rParams.mrContext.ExcludeDir(filename))
+					{
+						// Next item!
+						continue;
+					}
+
+					// Store on list
+					dirs.push_back(std::string(en->d_name));
+				}
+				else
+				{
+					continue;
+				}
+				
+				// Here if the object is something to back up (file, symlink or dir, not excluded)
+				// So make the information for adding to the checksum
+				checksum_info.mModificationTime = FileModificationTime(st);
+				checksum_info.mAttributeModificationTime = FileAttrModificationTime(st);
+				checksum_info.mSize = st.st_size;
+				currentStateChecksum.Add(&checksum_info, sizeof(checksum_info));
+				currentStateChecksum.Add(en->d_name, strlen(en->d_name));
+				
+				// If the file has been modified madly into the future, download the 
+				// directory record anyway to ensure that it doesn't get uploaded
+				// every single time the disc is scanned.
+				if(checksum_info.mModificationTime > rParams.mUploadAfterThisTimeInTheFuture)
+				{
+					downloadDirectoryRecordBecauseOfFutureFiles = true;
+					// Log that this has happened
+					if(!rParams.mHaveLoggedWarningAboutFutureFileTimes)
+					{
+						::syslog(LOG_ERR, "Some files have modification times excessively in the future. Check clock syncronisation.\n");
+						::syslog(LOG_ERR, "Example file (only one shown) : %s\n", filename.c_str());
+						rParams.mHaveLoggedWarningAboutFutureFileTimes = true;
+					}
+				}
+			}
+	
+			if(::closedir(dirHandle) != 0)
+			{
+				THROW_EXCEPTION(CommonException, OSFileError)
+			}
+			dirHandle = 0;
+		}
+		catch(...)
+		{
+			if(dirHandle != 0)
+			{
+				::closedir(dirHandle);
+			}
+			throw;
+		}
+	}
+
+	// Finish off the checksum, and compare with the one currently stored
+	bool checksumDifferent = true;
+	currentStateChecksum.Finish();
+	if(mInitialSyncDone && currentStateChecksum.DigestMatches(mStateChecksum))
+	{
+		// The checksum is the same, and there was one to compare with
+		checksumDifferent = false;
+	}
+
+	// Pointer to potentially downloaded store directory info
+	BackupStoreDirectory *pdirOnStore = 0;
+	
+	try
+	{
+		// Want to get the directory listing?
+		if(ThisDirHasJustBeenCreated)
+		{
+			// Avoid sending another command to the server when we know it's empty
+			pdirOnStore = new BackupStoreDirectory(mObjectID, ContainingDirectoryID);
+		}
+		else
+		{
+			// Consider asking the store for it
+			if(!mInitialSyncDone || checksumDifferent || downloadDirectoryRecordBecauseOfFutureFiles)
+			{
+				pdirOnStore = FetchDirectoryListing(rParams);
+			}
+		}
+				
+		// Make sure the attributes are up to date -- if there's space on the server
+		// and this directory has not just been created (because it's attributes will be correct in this case)
+		// and the checksum is different, implying they *MIGHT* be different.
+		if((!ThisDirHasJustBeenCreated) && checksumDifferent && (!rParams.mrContext.StorageLimitExceeded()))
+		{
+			UpdateAttributes(rParams, pdirOnStore, rLocalPath);
+		}
+		
+		// Create the list of pointers to directory entries
+		std::vector<BackupStoreDirectory::Entry *> entriesLeftOver;
+		if(pdirOnStore)
+		{
+			entriesLeftOver.resize(pdirOnStore->GetNumberOfEntries(), 0);
+			BackupStoreDirectory::Iterator i(*pdirOnStore);
+			// Copy in pointers to all the entries
+			for(unsigned int l = 0; l < pdirOnStore->GetNumberOfEntries(); ++l)
+			{
+				entriesLeftOver[l] = i.Next();
+			}
+		}
+		
+		// Do the directory reading
+		bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, pdirOnStore, entriesLeftOver, files, dirs);
+		
+		// LAST THING! (think exception safety)
+		// Store the new checksum -- don't fetch things unnecessarily in the future
+		// But... only if 1) the storage limit isn't exceeded -- make sure things are done again if
+		// the directory is modified later
+		// and 2) All the objects within the directory were stored successfully.
+		if(!rParams.mrContext.StorageLimitExceeded() && updateCompleteSuccess)
+		{
+			currentStateChecksum.CopyDigestTo(mStateChecksum);
+		}
+	}
+	catch(...)
+	{
+		// Bad things have happened -- clean up
+		if(pdirOnStore != 0)
+		{
+			delete pdirOnStore;
+			pdirOnStore = 0;
+		}
+		
+		// Set things so that we get a full go at stuff later
+		::memset(mStateChecksum, 0, sizeof(mStateChecksum));
+		
+		throw;
+	}
+	
+	// Clean up directory on store
+	if(pdirOnStore != 0)
+	{
+		delete pdirOnStore;
+		pdirOnStore = 0;
+	}
+	
+	// Flag things as having happened.
+	mInitialSyncDone = true;
+	mSyncDone = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::FetchDirectoryListing(BackupClientDirectoryRecord::SyncParams &)
+//		Purpose: Fetch the directory listing of this directory from the store.
+//		Created: 2003/10/09
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory *BackupClientDirectoryRecord::FetchDirectoryListing(BackupClientDirectoryRecord::SyncParams &rParams)
+{
+	BackupStoreDirectory *pdir = 0;
+	
+	try
+	{
+		// Get connection to store
+		BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+
+		// Query the directory
+		std::auto_ptr<BackupProtocolClientSuccess> dirreply(connection.QueryListDirectory(
+				mObjectID,
+				BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,	// both files and directories
+				BackupProtocolClientListDirectory::Flags_Deleted | BackupProtocolClientListDirectory::Flags_OldVersion, // exclude old/deleted stuff
+				true /* want attributes */));
+
+		// Retrieve the directory from the stream following
+		pdir = new BackupStoreDirectory;
+		ASSERT(pdir != 0);
+		std::auto_ptr<IOStream> dirstream(connection.ReceiveStream());
+		pdir->ReadFromStream(*dirstream, connection.GetTimeout());
+	}
+	catch(...)
+	{
+		delete pdir;
+		pdir = 0;
+		throw;
+	}
+	
+	return pdir;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord::SyncParams &, const std::string &)
+//		Purpose: Sets the attributes of the directory on the store, if necessary
+//		Created: 2003/10/09
+//
+// --------------------------------------------------------------------------
+void BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord::SyncParams &rParams, BackupStoreDirectory *pDirOnStore, const std::string &rLocalPath)
+{
+	// Get attributes for the directory
+	BackupClientFileAttributes attr;
+	box_time_t attrModTime = 0;
+	attr.ReadAttributes(rLocalPath.c_str(), true /* directories have zero mod times */,
+		0 /* no modification time */, &attrModTime);
+
+	// Assume attributes need updating, unless proved otherwise
+	bool updateAttr = true;
+
+	// Got a listing to compare with?
+	ASSERT(pDirOnStore == 0 || (pDirOnStore != 0 && pDirOnStore->HasAttributes()));
+	if(pDirOnStore != 0 && pDirOnStore->HasAttributes())
+	{
+		const StreamableMemBlock &storeAttrEnc(pDirOnStore->GetAttributes());
+		// Explict decryption
+		BackupClientFileAttributes storeAttr(storeAttrEnc);
+		// Compare the attributes
+		if(attr.Compare(storeAttr, true, true /* ignore both modification times */))
+		{
+			// No update necessary
+			updateAttr = false;
+		}
+	}
+
+	// Update them?
+	if(updateAttr)
+	{
+		// Get connection to store
+		BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+
+		// Exception thrown if this doesn't work
+		MemBlockStream attrStream(attr);
+		connection.QueryChangeDirAttributes(mObjectID, attrModTime, attrStream);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &, const std::string &, BackupStoreDirectory *, std::vector<BackupStoreDirectory::Entry *> &)
+//		Purpose: Update the items stored on the server. The rFiles vector will be erased after it's used to save space.
+//				 Returns true if all items were updated successfully. (If not, the failures will have been logged).
+//		Created: 2003/10/09
+//
+// --------------------------------------------------------------------------
+bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &rParams,
+	const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore,
+	std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver,
+	std::vector<std::string> &rFiles, const std::vector<std::string> &rDirs)
+{
+	bool allUpdatedSuccessfully = true;
+
+	// Decrypt all the directory entries.
+	// It would be nice to be able to just compare the encrypted versions, however this doesn't work
+	// in practise because there can be multiple encodings of the same filename using different 
+	// methods (although each method will result in the same string for the same filename.) This
+	// happens when the server fixes a broken store, and gives plain text generated filenames.
+	// So if we didn't do things like this, then you wouldn't be able to recover from bad things
+	// happening with the server.
+	DecryptedEntriesMap_t decryptedEntries;
+	if(pDirOnStore != 0)
+	{
+		BackupStoreDirectory::Iterator i(*pDirOnStore);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next()) != 0)
+		{
+			decryptedEntries[BackupStoreFilenameClear(en->GetName()).GetClearFilename()] = en;
+		}
+	}
+
+	// Do files
+	for(std::vector<std::string>::const_iterator f = rFiles.begin();
+		f != rFiles.end(); ++f)
+	{
+		// Filename of this file
+		std::string filename(rLocalPath + DIRECTORY_SEPARATOR + *f);
+
+		// Get relevant info about file
+		box_time_t modTime = 0;
+		uint64_t attributesHash = 0;
+		int64_t fileSize = 0;
+		ino_t inodeNum = 0;
+		bool hasMultipleHardLinks = true;
+		// BLOCK
+		{
+			// Stat the file
+			struct stat st;
+			if(::lstat(filename.c_str(), &st) != 0)
+			{
+				THROW_EXCEPTION(CommonException, OSFileError)
+			}
+			
+			// Extract required data
+			modTime = FileModificationTime(st);
+			fileSize = st.st_size;
+			inodeNum = st.st_ino;
+			hasMultipleHardLinks = (st.st_nlink > 1);
+			attributesHash = BackupClientFileAttributes::GenerateAttributeHash(st, *f);
+		}
+
+		// See if it's in the listing (if we have one)
+		BackupStoreFilenameClear storeFilename(*f);
+		BackupStoreDirectory::Entry *en = 0;
+		int64_t latestObjectID = 0;
+		if(pDirOnStore != 0)
+		{
+			DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*f));
+			if(i != decryptedEntries.end())
+			{
+				en = i->second;
+				latestObjectID = en->GetObjectID();
+			}
+		}
+
+		// Check that the entry which might have been found is in fact a file
+		if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == 0))
+		{
+			// Directory exists in the place of this file -- sort it out
+			RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, en->GetObjectID(), *f);
+			en = 0;
+		}
+		
+		// Check for renaming?
+		if(pDirOnStore != 0 && en == 0)
+		{
+			// We now know...
+			// 1) File has just been added
+			// 2) It's not in the store
+			
+			// Do we know about the inode number?
+			const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap());
+			int64_t renameObjectID = 0, renameInDirectory = 0;
+			if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory))
+			{
+				// Look up on the server to get the name, to build the local filename
+				std::string localPotentialOldName;
+				bool isDir = false;
+				bool isCurrentVersion = false;
+				box_time_t srvModTime = 0, srvAttributesHash = 0;
+				BackupStoreFilenameClear oldLeafname;
+				if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname))
+				{	
+					// Only interested if it's a file and the latest version
+					if(!isDir && isCurrentVersion)
+					{
+						// Check that the object we found in the ID map doesn't exist on disc
+						struct stat st;
+						if(::stat(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT)
+						{
+							// Doesn't exist locally, but does exist on the server.
+							// Therefore we can safely rename it to this new file.
+
+							// Get the connection to the server 
+							BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+
+							// Only do this step if there is room on the server.
+							// This step will be repeated later when there is space available
+							if(!rParams.mrContext.StorageLimitExceeded())
+							{
+								// Rename the existing files (ie include old versions) on the server
+								connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */,
+									BackupProtocolClientMoveObject::Flags_MoveAllWithSameName | BackupProtocolClientMoveObject::Flags_AllowMoveOverDeletedObject,
+									storeFilename);
+									
+								// Stop the attempt to delete the file in the original location
+								BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList());
+								rdelList.StopFileDeletion(renameInDirectory, oldLeafname);
+								
+								// Create new entry in the directory for it
+								// -- will be near enough what's actually on the server for the rest to work.
+								en = pDirOnStore->AddEntry(storeFilename, srvModTime, renameObjectID, 0 /* size in blocks unknown, but not needed */,
+									BackupStoreDirectory::Entry::Flags_File, srvAttributesHash);
+							
+								// Store the object ID for the inode lookup map later
+								latestObjectID = renameObjectID;
+							}
+						}
+					}
+				}
+			}
+		}
+		
+		// Is it in the mPendingEntries list?
+		box_time_t pendingFirstSeenTime = 0;		// ie not seen
+		if(mpPendingEntries != 0)
+		{
+			std::map<std::string, box_time_t>::const_iterator i(mpPendingEntries->find(*f));
+			if(i != mpPendingEntries->end())
+			{
+				// found it -- set flag
+				pendingFirstSeenTime = i->second;
+			}
+		}
+		
+		// If pDirOnStore == 0, then this must have been after an initial sync:
+		ASSERT(pDirOnStore != 0 || mInitialSyncDone);
+		// So, if pDirOnStore == 0, then we know that everything before syncPeriodStart
+		// is either on the server, or in the toupload list. If the directory had changed,
+		// we'd have got a directory listing.
+		//
+		// At this point, if (pDirOnStore == 0 && en == 0), we can assume it's on the server with a
+		// mod time < syncPeriodStart, or didn't exist before that time.
+		//
+		// But if en != 0, then we need to compare modification times to avoid uploading it again.
+
+		// Need to update?
+		//
+		// Condition for upload:
+		//    modifiction time within sync period
+		//    if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait
+		//	  and if we know about it from a directory listing, that it hasn't got the same upload time as on the store
+		if(
+			(
+				// Check the file modified within the acceptable time period we're checking
+				// If the file isn't on the server, the acceptable time starts at zero.
+				// Check pDirOnStore and en, because if we didn't download a directory listing,
+				// pDirOnStore will be zero, but we know it's on the server.
+				( ((pDirOnStore != 0 && en == 0) || (modTime >= rParams.mSyncPeriodStart)) && modTime < rParams.mSyncPeriodEnd)
+
+				// However, just in case things are continually modified, we check the first seen time.
+				// The two compares of syncPeriodEnd and pendingFirstSeenTime are because the values are unsigned.
+				|| (pendingFirstSeenTime != 0 &&
+					(rParams.mSyncPeriodEnd > pendingFirstSeenTime)
+						&& ((rParams.mSyncPeriodEnd - pendingFirstSeenTime) > rParams.mMaxUploadWait))
+
+				// Then make sure that if files are added with a time less than the sync period start
+				// (which can easily happen on file server), it gets uploaded. The directory contents checksum
+				// will pick up the fact it has been added, so the store listing will be available when this happens.
+				|| ((modTime <= rParams.mSyncPeriodStart) && (en != 0) && (en->GetModificationTime() != modTime))
+				
+				// And just to catch really badly off clocks in the future for file server clients,
+				// just upload the file if it's madly in the future.
+				|| (modTime > rParams.mUploadAfterThisTimeInTheFuture)
+			)			
+			// But even then, only upload it if the mod time locally is different to that on the server.
+			&& (en == 0 || en->GetModificationTime() != modTime))
+		{
+			// Make sure we're connected -- must connect here so we know whether
+			// the storage limit has been exceeded, and hence whether or not
+			// to actually upload the file.
+			rParams.mrContext.GetConnection();
+
+			// Only do this step if there is room on the server.
+			// This step will be repeated later when there is space available
+			if(!rParams.mrContext.StorageLimitExceeded())
+			{
+				// Upload the file to the server, recording the object ID it returns
+				bool noPreviousVersionOnServer = ((pDirOnStore != 0) && (en == 0));
+				
+				// Surround this in a try/catch block, to catch errrors, but still continue
+				bool uploadSuccess = false;
+				try
+				{
+					latestObjectID = UploadFile(rParams, filename, storeFilename, fileSize, modTime, attributesHash, noPreviousVersionOnServer);
+					uploadSuccess = true;
+				}
+				catch(ConnectionException &e)
+				{
+					// Connection errors should just be passed on to the main handler, retries
+					// would probably just cause more problems.
+					throw;
+				}
+				catch(BoxException &e)
+				{
+					// an error occured -- make return code false, to show error in directory
+					allUpdatedSuccessfully = false;
+					// Log it.
+					SetErrorWhenReadingFilesystemObject(rParams, filename.c_str());
+					// Log error.
+					::syslog(LOG_ERR, "Error code when uploading was (%d/%d), %s", e.GetType(), e.GetSubType(), e.what());
+				}
+
+				// Update structures if the file was uploaded successfully.
+				if(uploadSuccess)
+				{
+					// delete from pending entries
+					if(pendingFirstSeenTime != 0 && mpPendingEntries != 0)
+					{
+						mpPendingEntries->erase(*f);
+					}
+				}
+			}
+		}
+		else if(en != 0 && en->GetAttributesHash() != attributesHash)
+		{
+			// Attributes have probably changed, upload them again.
+			// If the attributes have changed enough, the directory hash will have changed too,
+			// and so the dir will have been downloaded, and the entry will be available.
+
+			// Get connection
+			BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+
+			// Only do this step if there is room on the server.
+			// This step will be repeated later when there is space available
+			if(!rParams.mrContext.StorageLimitExceeded())
+			{
+				// Update store
+				BackupClientFileAttributes attr;
+				attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */);
+				MemBlockStream attrStream(attr);
+				connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream);
+			}
+		}
+
+		if(modTime >= rParams.mSyncPeriodEnd)
+		{
+			// Allocate?
+			if(mpPendingEntries == 0)
+			{
+				mpPendingEntries = new std::map<std::string, box_time_t>;
+			}
+			// Adding to mPendingEntries list 
+			if(pendingFirstSeenTime == 0)
+			{
+				// Haven't seen this before -- add to list!
+				(*mpPendingEntries)[*f] = modTime;
+			}
+		}
+		
+		// Zero pointer in rEntriesLeftOver, if we have a pointer to zero
+		if(en != 0)
+		{
+			for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l)
+			{
+				if(rEntriesLeftOver[l] == en)
+				{
+					rEntriesLeftOver[l] = 0;
+					break;
+				}
+			}
+		}
+		
+		// Does this file need an entry in the ID map?
+		if(fileSize >= rParams.mFileTrackingSizeThreshold)
+		{
+			// Get the map
+			BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap());
+		
+			// Need to get an ID from somewhere...
+			if(latestObjectID != 0)
+			{
+				// Use this one
+				idMap.AddToMap(inodeNum, latestObjectID, mObjectID /* containing directory */);
+			}
+			else
+			{
+				// Don't know it -- haven't sent anything to the store, and didn't get a listing.
+				// Look it up in the current map, and if it's there, use that.
+				const BackupClientInodeToIDMap &currentIDMap(rParams.mrContext.GetCurrentIDMap());
+				int64_t objid = 0, dirid = 0;
+				if(currentIDMap.Lookup(inodeNum, objid, dirid))
+				{
+					// Found
+					ASSERT(dirid == mObjectID);
+					// NOTE: If the above assert fails, an inode number has been reused by the OS,
+					// or there is a problem somewhere. If this happened on a short test run, look
+					// into it. However, in a long running process this may happen occasionally and
+					// not indiciate anything wrong.
+					// Run the release version for real life use, where this check is not made.
+					idMap.AddToMap(inodeNum, objid, mObjectID /* containing directory */);				
+				}
+			}
+		}
+	}
+
+	// Erase contents of files to save space when recursing
+	rFiles.clear();
+
+	// Delete the pending entries, if the map is entry
+	if(mpPendingEntries != 0 && mpPendingEntries->size() == 0)
+	{
+		TRACE1("Deleting mpPendingEntries from dir ID %lld\n", mObjectID);
+		delete mpPendingEntries;
+		mpPendingEntries = 0;
+	}
+	
+	// Do directories
+	for(std::vector<std::string>::const_iterator d = rDirs.begin();
+		d != rDirs.end(); ++d)
+	{
+		// Get the local filename
+		std::string dirname(rLocalPath + DIRECTORY_SEPARATOR + *d);		
+	
+		// See if it's in the listing (if we have one)
+		BackupStoreFilenameClear storeFilename(*d);
+		BackupStoreDirectory::Entry *en = 0;
+		if(pDirOnStore != 0)
+		{
+			DecryptedEntriesMap_t::iterator i(decryptedEntries.find(*d));
+			if(i != decryptedEntries.end())
+			{
+				en = i->second;
+			}
+		}
+		
+		// Check that the entry which might have been found is in fact a directory
+		if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == 0))
+		{
+			// Entry exists, but is not a directory. Bad. Get rid of it.
+			BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+			connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename);
+			
+			// Nothing found
+			en = 0;
+		}
+
+		// Flag for having created directory, so can optimise the recusive call not to
+		// read it again, because we know it's empty.
+		bool haveJustCreatedDirOnServer = false;
+
+		// Next, see if it's in the list of sub directories
+		BackupClientDirectoryRecord *psubDirRecord = 0;
+		std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(*d));
+		if(e != mSubDirectories.end())
+		{
+			// In the list, just use this pointer
+			psubDirRecord = e->second;
+		}
+		else if(!rParams.mrContext.StorageLimitExceeded())	// know we've got a connection if we get this far, as dir will have been modified.
+		{
+			// Note: only think about adding directory records if there's space left on the server.
+			// If there isn't, this step will be repeated when there is some available.
+		
+			// Need to create the record. But do we need to create the directory on the server?
+			int64_t subDirObjectID = 0;
+			if(en != 0)
+			{
+				// No. Exists on the server, and we know about it from the listing.
+				subDirObjectID = en->GetObjectID();
+			}
+			else
+			{
+				// Yes, creation required!
+				// It is known that the it doesn't exist:
+				//   if pDirOnStore == 0, then the directory has had an initial sync, and hasn't been modified.
+				//	 so it has definately been created already.
+				//   if en == 0 but pDirOnStore != 0, well... obviously it doesn't exist.
+
+				// Get attributes
+				box_time_t attrModTime = 0;
+				ino_t inodeNum = 0;
+				BackupClientFileAttributes attr;
+				attr.ReadAttributes(dirname.c_str(), true /* directories have zero mod times */,
+					0 /* not interested in mod time */, &attrModTime, 0 /* not file size */,
+					&inodeNum);
+
+				// Check to see if the directory been renamed
+				// First, do we have a record in the ID map?
+				int64_t renameObjectID = 0, renameInDirectory = 0;
+				bool renameDir = false;
+				const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap());
+				if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory))
+				{
+					// Look up on the server to get the name, to build the local filename
+					std::string localPotentialOldName;
+					bool isDir = false;
+					bool isCurrentVersion = false;
+					if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion))
+					{	
+						// Only interested if it's a directory
+						if(isDir && isCurrentVersion)
+						{
+							// Check that the object doesn't exist already
+							struct stat st;
+							if(::stat(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT)
+							{
+								// Doesn't exist locally, but does exist on the server.
+								// Therefore we can safely rename it.
+								renameDir = true;
+							}
+						}
+					}
+				}
+
+				// Get connection
+				BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+				
+				// Don't do a check for storage limit exceeded here, because if we get to this
+				// stage, a connection will have been opened, and the status known, so the check 
+				// in the else if(...) above will be correct.
+
+				// Build attribute stream for sending
+				MemBlockStream attrStream(attr);
+
+				if(renameDir)
+				{
+					// Rename the existing directory on the server
+					connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */,
+						BackupProtocolClientMoveObject::Flags_MoveAllWithSameName | BackupProtocolClientMoveObject::Flags_AllowMoveOverDeletedObject,
+						storeFilename);
+						
+					// Put the latest attributes on it
+					connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream);
+
+					// Stop it being deleted later
+					BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList());
+					rdelList.StopDirectoryDeletion(renameObjectID);
+
+					// This is the ID for the renamed directory
+					subDirObjectID = renameObjectID;
+				}
+				else
+				{
+					// Create a new directory
+					std::auto_ptr<BackupProtocolClientSuccess> dirCreate(connection.QueryCreateDirectory(
+						mObjectID, attrModTime, storeFilename, attrStream));
+					subDirObjectID = dirCreate->GetObjectID(); 
+					
+					// Flag as having done this for optimisation later
+					haveJustCreatedDirOnServer = true;
+				}
+			}
+			
+			// New an object for this
+			psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d);
+			
+			// Store in list
+			try
+			{
+				mSubDirectories[*d] = psubDirRecord;
+			}
+			catch(...)
+			{
+				delete psubDirRecord;
+				psubDirRecord = 0;
+				throw;
+			}
+		}
+		
+		ASSERT(psubDirRecord != 0 || rParams.mrContext.StorageLimitExceeded());
+		
+		if(psubDirRecord)
+		{
+			// Sync this sub directory too
+			psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, haveJustCreatedDirOnServer);
+		}
+
+		// Zero pointer in rEntriesLeftOver, if we have a pointer to zero
+		if(en != 0)
+		{
+			for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l)
+			{
+				if(rEntriesLeftOver[l] == en)
+				{
+					rEntriesLeftOver[l] = 0;
+					break;
+				}
+			}
+		}
+	}
+	
+	// Delete everything which is on the store, but not on disc
+	for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l)
+	{
+		if(rEntriesLeftOver[l] != 0)
+		{
+			BackupStoreDirectory::Entry *en = rEntriesLeftOver[l];
+		
+			// These entries can't be deleted immediately, as it would prevent
+			// renaming and moving of objects working properly. So we add them
+			// to a list, which is actually deleted at the very end of the session.
+			// If there's an error during the process, it doesn't matter if things
+			// aren't actually deleted, as the whole state will be reset anyway.
+			BackupClientDeleteList &rdel(rParams.mrContext.GetDeleteList());
+			
+			// Delete this entry -- file or directory?
+			if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0)
+			{
+				// Set a pending deletion for the file
+				rdel.AddFileDelete(mObjectID, en->GetName());				
+			}
+			else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0)
+			{
+				// Set as a pending deletion for the directory
+				rdel.AddDirectoryDelete(en->GetObjectID());
+				
+				// If there's a directory record for it in the sub directory map, delete it now
+				BackupStoreFilenameClear dirname(en->GetName());
+				std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(dirname.GetClearFilename()));
+				if(e != mSubDirectories.end())
+				{
+					// Carefully delete the entry from the map
+					BackupClientDirectoryRecord *rec = e->second;
+					mSubDirectories.erase(e);
+					delete rec;
+					TRACE2("Deleted directory record for %s/%s\n", rLocalPath.c_str(), dirname.GetClearFilename().c_str());
+				}				
+			}
+		}
+	}
+
+	// Return success flag (will be false if some files failed)
+	return allUpdatedSuccessfully;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &, BackupStoreDirectory *, int64_t, const std::string &)
+//		Purpose: Called to resolve difficulties when a directory is found on the
+//				 store where a file is to be uploaded.
+//		Created: 9/7/04
+//
+// --------------------------------------------------------------------------
+void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename)
+{
+	// First, delete the directory
+	BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+	connection.QueryDeleteDirectory(ObjectID);
+
+	// Then, delete any directory record
+	std::map<std::string, BackupClientDirectoryRecord *>::iterator e(mSubDirectories.find(rFilename));
+	if(e != mSubDirectories.end())
+	{
+		// A record exists for this, remove it
+		BackupClientDirectoryRecord *psubDirRecord = e->second;
+		mSubDirectories.erase(e);
+
+		// And delete the object
+		delete psubDirRecord;
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &, const std::string &, const BackupStoreFilename &, int64_t, box_time_t, box_time_t, bool)
+//		Purpose: Private. Upload a file to the server -- may send a patch instead of the whole thing
+//		Created: 20/1/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename,
+			int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer)
+{
+	// Get the connection
+	BackupProtocolClient &connection(rParams.mrContext.GetConnection());
+
+	// Info
+	int64_t objID = 0;
+	bool doNormalUpload = true;
+	
+	// Use a try block to catch store full errors
+	try
+	{
+		// Might an old version be on the server, and is the file size over the diffing threshold?
+		if(!NoPreviousVersionOnServer && FileSize >= rParams.mDiffingUploadSizeThreshold)
+		{
+			// YES -- try to do diff, if possible
+			// First, query the server to see if there's an old version available
+			std::auto_ptr<BackupProtocolClientSuccess> getBlockIndex(connection.QueryGetBlockIndexByName(mObjectID, rStoreFilename));
+			int64_t diffFromID = getBlockIndex->GetObjectID();
+			
+			if(diffFromID != 0)
+			{
+				// Found an old version -- get the index
+				std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream());
+			
+				// Diff the file
+				bool isCompletelyDifferent = false;
+				std::auto_ptr<IOStream> patchStream(BackupStoreFile::EncodeFileDiff(rFilename.c_str(),
+						mObjectID,	/* containing directory */
+						rStoreFilename, diffFromID, *blockIndexStream,
+						connection.GetTimeout(), 0 /* not interested in the modification time */, &isCompletelyDifferent));
+	
+				// Upload the patch to the store
+				std::auto_ptr<BackupProtocolClientSuccess> stored(connection.QueryStoreFile(mObjectID, ModificationTime,
+						AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream));
+				
+				// Don't attempt to upload it again!
+				doNormalUpload = false;
+			} 
+		}
+	
+		if(doNormalUpload)
+		{
+			// below threshold or nothing to diff from, so upload whole
+			
+			// Prepare to upload, getting a stream which will encode the file as we go along
+			std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(rFilename.c_str(), mObjectID, rStoreFilename));
+		
+			// Send to store
+			std::auto_ptr<BackupProtocolClientSuccess> stored(connection.QueryStoreFile(mObjectID, ModificationTime,
+					AttributesHash, 0 /* no diff from file ID */, rStoreFilename, *upload));
+	
+			// Get object ID from the result		
+			objID = stored->GetObjectID();
+		}
+	}
+	catch(BoxException &e)
+	{
+		if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply)
+		{
+			// Check and see what error the protocol has -- as it might be an error...
+			int type, subtype;
+			if(connection.GetLastError(type, subtype)
+				&& type == BackupProtocolClientError::ErrorType
+				&& subtype == BackupProtocolClientError::Err_StorageLimitExceeded)
+			{
+				// The hard limit was exceeded on the server, notify!
+				rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull);
+			}
+		}
+	
+		// Send the error on it's way
+		throw;
+	}
+
+	// Return the new object ID of this file
+	return objID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(SyncParams &, const char *)
+//		Purpose: Sets the error state when there were problems reading an object
+//				 from the filesystem.
+//		Created: 29/3/04
+//
+// --------------------------------------------------------------------------
+void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClientDirectoryRecord::SyncParams &rParams, const char *Filename)
+{
+	// Zero hash, so it gets synced properly next time round.
+	::memset(mStateChecksum, 0, sizeof(mStateChecksum));
+
+	// Log the error
+	::syslog(LOG_ERR, "Backup object failed, error when reading %s", Filename);
+
+	// Mark that an error occured in the parameters object
+	rParams.mReadErrorsOnFilesystemObjects = true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::SyncParams::SyncParams(BackupClientContext &)
+//		Purpose: Constructor
+//		Created: 8/3/04
+//
+// --------------------------------------------------------------------------
+BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, BackupClientContext &rContext)
+	: mSyncPeriodStart(0),
+	  mSyncPeriodEnd(0),
+	  mMaxUploadWait(0),
+	  mMaxFileTimeInFuture(99999999999999999LL),
+	  mFileTrackingSizeThreshold(16*1024),
+	  mDiffingUploadSizeThreshold(16*1024),
+	  mrDaemon(rDaemon),
+	  mrContext(rContext),
+	  mReadErrorsOnFilesystemObjects(false),
+	  mUploadAfterThisTimeInTheFuture(99999999999999999LL),
+	  mHaveLoggedWarningAboutFutureFileTimes(false)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientDirectoryRecord::SyncParams::~SyncParams()
+//		Purpose: Destructor
+//		Created: 8/3/04
+//
+// --------------------------------------------------------------------------
+BackupClientDirectoryRecord::SyncParams::~SyncParams()
+{
+}
+
+
+
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h
new file mode 100755
index 00000000..99354bc8
--- /dev/null
+++ b/bin/bbackupd/BackupClientDirectoryRecord.h
@@ -0,0 +1,115 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientDirectoryRecord.h
+//		Purpose: Implementation of record about directory for backup client
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTDIRECTORYRECORD__H
+#define BACKUPCLIENTDIRECTORYRECORD__H
+
+#include <string>
+#include <map>
+
+#include "BoxTime.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreDirectory.h"
+#include "MD5Digest.h"
+
+class BackupClientContext;
+class BackupDaemon;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupClientDirectoryRecord
+//		Purpose: Implementation of record about directory for backup client
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+class BackupClientDirectoryRecord
+{
+public:
+	BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName);
+	~BackupClientDirectoryRecord();
+private:
+	BackupClientDirectoryRecord(const BackupClientDirectoryRecord &);
+public:
+
+	enum
+	{
+		UnknownDirectoryID = 0
+	};
+
+	// --------------------------------------------------------------------------
+	//
+	// Class
+	//		Name:    BackupClientDirectoryRecord::SyncParams
+	//		Purpose: Holds parameters etc for directory syncing. Not passed as
+	//				 const, some parameters may be modified during sync.
+	//		Created: 8/3/04
+	//
+	// --------------------------------------------------------------------------
+	class SyncParams
+	{
+	public:
+		SyncParams(BackupDaemon &rDaemon, BackupClientContext &rContext);
+		~SyncParams();
+	private:
+		// No copying
+		SyncParams(const SyncParams&);
+		SyncParams &operator=(const SyncParams&);
+	public:
+
+		// Data members are public, as accessors are not justified here
+		box_time_t mSyncPeriodStart;
+		box_time_t mSyncPeriodEnd;
+		box_time_t mMaxUploadWait;
+		box_time_t mMaxFileTimeInFuture;
+		int32_t mFileTrackingSizeThreshold;
+		int32_t mDiffingUploadSizeThreshold;
+		BackupDaemon &mrDaemon;
+		BackupClientContext &mrContext;
+		bool mReadErrorsOnFilesystemObjects;
+		
+		// Member variables modified by syncing process
+		box_time_t mUploadAfterThisTimeInTheFuture;
+		bool mHaveLoggedWarningAboutFutureFileTimes;
+	};
+
+	void SyncDirectory(SyncParams &rParams, int64_t ContainingDirectoryID, const std::string &rLocalPath,
+		bool ThisDirHasJustBeenCreated = false);
+
+private:
+	void DeleteSubDirectories();
+	BackupStoreDirectory *FetchDirectoryListing(SyncParams &rParams);
+	void UpdateAttributes(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, const std::string &rLocalPath);
+	bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore,
+		std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver,
+		std::vector<std::string> &rFiles, const std::vector<std::string> &rDirs);
+	int64_t UploadFile(SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename,
+		int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer);
+	void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, const char *Filename);
+	void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename);
+
+private:
+	int64_t 		mObjectID;
+	std::string 	mSubDirName;
+	bool 			mInitialSyncDone;
+	bool 			mSyncDone;
+
+	// Checksum of directory contents and attributes, used to detect changes
+	uint8_t mStateChecksum[MD5Digest::DigestLength];
+
+	std::map<std::string, box_time_t>						*mpPendingEntries;
+	std::map<std::string, BackupClientDirectoryRecord *>	mSubDirectories;
+	// mpPendingEntries is a pointer rather than simple a member
+	// variables, because most of the time it'll be empty. This would waste a lot
+	// of memory because of STL allocation policies.
+};
+
+#endif // BACKUPCLIENTDIRECTORYRECORD__H
+
+
diff --git a/bin/bbackupd/BackupClientInodeToIDMap.cpp b/bin/bbackupd/BackupClientInodeToIDMap.cpp
new file mode 100755
index 00000000..23e91eba
--- /dev/null
+++ b/bin/bbackupd/BackupClientInodeToIDMap.cpp
@@ -0,0 +1,279 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientInodeToIDMap.cpp
+//		Purpose: Map of inode numbers to file IDs on the store
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#ifndef PLATFORM_BERKELEY_DB_NOT_SUPPORTED
+	// Include db headers and other OS files if they're needed for the disc implementation
+	#include <sys/types.h>
+	#include <fcntl.h>
+	#include <limits.h>
+	#ifdef PLATFORM_LINUX
+		#include "../../local/_linux_db.h"
+	#else
+		#include <db.h>
+	#endif
+	#include <sys/stat.h>
+#endif
+
+#define BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION
+#include "BackupClientInodeToIDMap.h"
+
+#include "BackupStoreException.h"
+
+
+#include "MemLeakFindOn.h"
+
+// What type of Berkeley DB shall we use?
+#define TABLE_DATABASE_TYPE DB_HASH
+
+typedef struct
+{
+	int64_t mObjectID;
+	int64_t mInDirectory;
+} IDBRecord;
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientInodeToIDMap::BackupClientInodeToIDMap()
+//		Purpose: Constructor
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+BackupClientInodeToIDMap::BackupClientInodeToIDMap()
+#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+	: mReadOnly(true),
+	  mEmpty(false),
+	  dbp(0)
+#endif
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientInodeToIDMap::~BackupClientInodeToIDMap()
+//		Purpose: Destructor
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+BackupClientInodeToIDMap::~BackupClientInodeToIDMap()
+{
+#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+	if(dbp != 0)
+	{
+		dbp->close(dbp);
+	}
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientInodeToIDMap::Open(const char *, bool, bool)
+//		Purpose: Open the database map, creating a file on disc to store everything
+//		Created: 20/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientInodeToIDMap::Open(const char *Filename, bool ReadOnly, bool CreateNew)
+{
+#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+	// Correct arguments?
+	ASSERT(!(CreateNew && ReadOnly));
+	
+	// Correct usage?
+	ASSERT(dbp == 0);
+	ASSERT(!mEmpty);
+	
+	// Open the database file
+	dbp = dbopen(Filename, (CreateNew?O_CREAT:0) | (ReadOnly?O_RDONLY:O_RDWR), S_IRUSR | S_IWUSR | S_IRGRP, TABLE_DATABASE_TYPE, NULL);
+	if(dbp == NULL)
+	{
+		THROW_EXCEPTION(BackupStoreException, BerkelyDBFailure);
+	}
+	
+	// Read only flag
+	mReadOnly = ReadOnly;
+#endif
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientInodeToIDMap::OpenEmpty()
+//		Purpose: 'Open' this map. Not associated with a disc file. Useful for when a map
+//				 is required, but is against an empty file on disc which shouldn't be created.
+//				 Implies read only.
+//		Created: 20/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientInodeToIDMap::OpenEmpty()
+{
+#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+	ASSERT(dbp == 0);
+	mEmpty = true;
+	mReadOnly = true;
+#endif
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientInodeToIDMap::Close()
+//		Purpose: Close the database file
+//		Created: 20/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientInodeToIDMap::Close()
+{
+#ifndef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+	if(dbp != 0)
+	{
+		if(dbp->close(dbp) != 0)
+		{
+			THROW_EXCEPTION(BackupStoreException, BerkelyDBFailure);
+		}
+		dbp = 0;
+	}
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientInodeToIDMap::AddToMap(InodeRefType, int64_t, int64_t)
+//		Purpose: Adds an entry to the map. Overwrites any existing entry.
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, int64_t InDirectory)
+{
+#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+	mMap[InodeRef] = std::pair<int64_t, int64_t>(ObjectID, InDirectory);
+#else
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, InodeMapIsReadOnly);
+	}
+
+	if(dbp == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen);
+	}
+
+	// Setup structures
+	IDBRecord rec;
+	rec.mObjectID = ObjectID;
+	rec.mInDirectory = InDirectory;
+	
+	DBT key;
+	key.data = &InodeRef;
+	key.size = sizeof(InodeRef);
+	
+	DBT data;
+	data.data = &rec;
+	data.size = sizeof(rec);
+	
+	// Add to map (or replace existing entry)
+	if(dbp->put(dbp, &key, &data, 0) != 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, BerkelyDBFailure);
+	}
+#endif
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientInodeToIDMap::Lookup(InodeRefType, int64_t &, int64_t &) const
+//		Purpose: Looks up an inode in the map, returning true if it exists, and the object
+//				 ids of it and the directory it's in the reference arguments.
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const
+{
+#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+	std::map<InodeRefType, std::pair<int64_t, int64_t> >::const_iterator i(mMap.find(InodeRef));
+	
+	// Found?
+	if(i == mMap.end())
+	{
+		return false;
+	}
+
+	// Yes. Return the details
+	rObjectIDOut = i->second.first;
+	rInDirectoryOut = i->second.second;
+	return true;
+#else
+	if(mEmpty)
+	{
+		// Map is empty
+		return false;
+	}
+
+	if(dbp == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, InodeMapNotOpen);
+	}
+
+	DBT key;
+	key.data = &InodeRef;
+	key.size = sizeof(InodeRef);
+	
+	DBT data;
+	data.data = 0;
+	data.size = 0;
+
+	switch(dbp->get(dbp, &key, &data, 0))
+	{
+	case 1:	// key not in file
+		return false;
+	
+	case -1:	// error
+	default:	// not specified in docs
+		THROW_EXCEPTION(BackupStoreException, BerkelyDBFailure);
+		return false;
+	
+	case 0:		// success, found it
+		break;		
+	}
+
+	// Check for sensible return
+	if(key.data == 0 || data.size != sizeof(IDBRecord))
+	{
+		// Assert in debug version
+		ASSERT(key.data == 0 || data.size != sizeof(IDBRecord));
+		
+		// Invalid entries mean it wasn't found
+		return false;
+	}
+	
+	// Data alignment isn't guarentted to be on a suitable bounday
+	IDBRecord rec;
+	::memcpy(&rec, data.data, sizeof(rec));
+	
+	// Return data
+	rObjectIDOut = rec.mObjectID;
+	rInDirectoryOut = rec.mInDirectory;
+
+	// Don't have to worry about freeing the returned data
+
+	// Found
+	return true;
+#endif
+}
+
+
diff --git a/bin/bbackupd/BackupClientInodeToIDMap.h b/bin/bbackupd/BackupClientInodeToIDMap.h
new file mode 100755
index 00000000..1ea7755d
--- /dev/null
+++ b/bin/bbackupd/BackupClientInodeToIDMap.h
@@ -0,0 +1,67 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientInodeToIDMap.h
+//		Purpose: Map of inode numbers to file IDs on the store
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTINODETOIDMAP_H
+#define BACKUPCLIENTINODETOIDMAP__H
+
+#include <sys/types.h>
+
+#include <map>
+#include <utility>
+
+// Use in memory implementation if there isn't access to the Berkely DB on this platform
+#ifdef PLATFORM_BERKELEY_DB_NOT_SUPPORTED
+	#define BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+#endif
+
+typedef ino_t InodeRefType;
+
+// avoid having to include the DB files when not necessary
+#ifndef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION
+	class DB;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupClientInodeToIDMap
+//		Purpose: Map of inode numbers to file IDs on the store
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+class BackupClientInodeToIDMap
+{
+public:
+	BackupClientInodeToIDMap();
+	~BackupClientInodeToIDMap();
+private:
+	BackupClientInodeToIDMap(const BackupClientInodeToIDMap &rToCopy);	// not allowed
+public:
+
+	void Open(const char *Filename, bool ReadOnly, bool CreateNew);
+	void OpenEmpty();
+
+	void AddToMap(InodeRefType InodeRef, int64_t ObjectID, int64_t InDirectory);
+	bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const;
+
+	void Close();
+
+private:
+#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+	std::map<InodeRefType, std::pair<int64_t, int64_t> > mMap;
+#else
+	bool mReadOnly;
+	bool mEmpty;
+	DB *dbp;	// C style interface, use notation from documentation
+#endif
+};
+
+#endif // BACKUPCLIENTINODETOIDMAP__H
+
+
diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp
new file mode 100755
index 00000000..7aa21a87
--- /dev/null
+++ b/bin/bbackupd/BackupDaemon.cpp
@@ -0,0 +1,1624 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupDaemon.cpp
+//		Purpose: Backup daemon
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <unistd.h>
+#include <syslog.h>
+#include <sys/param.h>
+#include <sys/mount.h>
+#include <signal.h>
+#ifdef PLATFORM_USES_MTAB_FILE_FOR_MOUNTS
+	#include <mntent.h>
+#endif
+#include <sys/wait.h>
+
+#include "Configuration.h"
+#include "IOStream.h"
+#include "MemBlockStream.h"
+#include "CommonException.h"
+
+#include "SSLLib.h"
+#include "TLSContext.h"
+
+#include "BackupDaemon.h"
+#include "BackupDaemonConfigVerify.h"
+#include "BackupClientContext.h"
+#include "BackupClientDirectoryRecord.h"
+#include "BackupStoreDirectory.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupClientInodeToIDMap.h"
+#include "autogen_BackupProtocolClient.h"
+#include "BackupClientCryptoKeys.h"
+#include "BannerText.h"
+#include "BackupStoreFile.h"
+#include "Random.h"
+#include "ExcludeList.h"
+#include "BackupClientMakeExcludeList.h"
+#include "IOStreamGetLine.h"
+#include "Utils.h"
+#include "FileStream.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "LocalProcessStream.h"
+#include "IOStreamGetLine.h"
+#include "Conversion.h"
+
+#include "MemLeakFindOn.h"
+
+#define 	MAX_SLEEP_TIME	((unsigned int)1024)
+
+// Make the actual sync period have a little bit of extra time, up to a 64th of the main sync period.
+// This prevents repetative cycles of load on the server
+#define		SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY	6
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::BackupDaemon()
+//		Purpose: constructor
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+BackupDaemon::BackupDaemon()
+	: mState(BackupDaemon::State_Initialising),
+	  mpCommandSocketInfo(0),
+	  mDeleteUnusedRootDirEntriesAfter(0)
+{
+	// Only ever one instance of a daemon
+	SSLLib::Initialise();
+	
+	// Initialise notifcation sent status
+	for(int l = 0; l <= NotifyEvent__MAX; ++l)
+	{
+		mNotificationsSent[l] = false;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::~BackupDaemon()
+//		Purpose: Destructor
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+BackupDaemon::~BackupDaemon()
+{
+	DeleteAllLocations();
+	DeleteAllIDMaps();
+	
+	if(mpCommandSocketInfo != 0)
+	{
+		delete mpCommandSocketInfo;
+		mpCommandSocketInfo = 0;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::DaemonName()
+//		Purpose: Get name of daemon
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+const char *BackupDaemon::DaemonName() const
+{
+	return "bbackupd";
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::DaemonBanner()
+//		Purpose: Daemon banner
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+const char *BackupDaemon::DaemonBanner() const
+{
+#ifndef NDEBUG
+	// Don't display banner in debug builds
+	return 0;
+#else
+	return BANNER_TEXT("Backup Client");
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::GetConfigVerify()
+//		Purpose: Get configuration specification
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+const ConfigurationVerify *BackupDaemon::GetConfigVerify() const
+{
+	// Defined elsewhere
+	return &BackupDaemonConfigVerify;
+}
+
+#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::SetupInInitialProcess()
+//		Purpose: Platforms with non-checkable credientals on local sockets only.
+//				 Prints a warning if the command socket is used.
+//		Created: 25/2/04
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::SetupInInitialProcess()
+{
+	// Print a warning on this platform if the CommandSocket is used.
+	if(GetConfiguration().KeyExists("CommandSocket"))
+	{
+		printf(
+				"============================================================================================\n" \
+				"SECURITY WARNING: This platform cannot check the credentials of connections to the\n" \
+				"command socket. This is a potential DoS security problem.\n" \
+				"Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl is not used.\n" \
+				"============================================================================================\n"
+			);
+	}
+}
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::DeleteAllLocations()
+//		Purpose: Deletes all records stored
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::DeleteAllLocations()
+{
+	// Run through, and delete everything
+	for(std::vector<Location *>::iterator i = mLocations.begin();
+		i != mLocations.end(); ++i)
+	{
+		delete *i;
+	}
+
+	// Clear the contents of the map, so it is empty
+	mLocations.clear();
+	
+	// And delete everything from the assoicated mount vector
+	mIDMapMounts.clear();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::Run()
+//		Purpose: Run function for daemon
+//		Created: 18/2/04
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::Run()
+{
+	// Ignore SIGPIPE (so that if a command connection is broken, the daemon doesn't terminate)
+	::signal(SIGPIPE, SIG_IGN);
+
+	// Create a command socket?
+	const Configuration &conf(GetConfiguration());
+	if(conf.KeyExists("CommandSocket"))
+	{
+		// Yes, create a local UNIX socket
+		mpCommandSocketInfo = new CommandSocketInfo;
+		const char *socketName = conf.GetKeyValue("CommandSocket").c_str();
+		::unlink(socketName);
+		mpCommandSocketInfo->mListeningSocket.Listen(Socket::TypeUNIX, socketName);
+	}
+
+	// Handle things nicely on exceptions
+	try
+	{
+		Run2();
+	}
+	catch(...)
+	{
+		if(mpCommandSocketInfo != 0)
+		{
+			delete mpCommandSocketInfo;
+			mpCommandSocketInfo = 0;
+		}
+
+		throw;
+	}
+	
+	// Clean up
+	if(mpCommandSocketInfo != 0)
+	{
+		delete mpCommandSocketInfo;
+		mpCommandSocketInfo = 0;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::Run2()
+//		Purpose: Run function for daemon (second stage)
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::Run2()
+{
+	// Read in the certificates creating a TLS context
+	TLSContext tlsContext;
+	const Configuration &conf(GetConfiguration());
+	std::string certFile(conf.GetKeyValue("CertificateFile"));
+	std::string keyFile(conf.GetKeyValue("PrivateKeyFile"));
+	std::string caFile(conf.GetKeyValue("TrustedCAsFile"));
+	tlsContext.Initialise(false /* as client */, certFile.c_str(), keyFile.c_str(), caFile.c_str());
+	
+	// Set up the keys for various things
+	BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str());
+
+	// Set maximum diffing time?
+	if(conf.KeyExists("MaximumDiffingTime"))
+	{
+		BackupStoreFile::SetMaximumDiffingTime(conf.GetKeyValueInt("MaximumDiffingTime"));
+	}
+
+	// Setup various timings
+	
+	// How often to connect to the store (approximate)
+	box_time_t updateStoreInterval = SecondsToBoxTime((uint32_t)conf.GetKeyValueInt("UpdateStoreInterval"));
+
+	// But are we connecting automatically?
+	bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup");
+	
+	// The minimum age a file needs to be before it will be considered for uploading
+	box_time_t minimumFileAge = SecondsToBoxTime((uint32_t)conf.GetKeyValueInt("MinimumFileAge"));
+
+	// The maximum time we'll wait to upload a file, regardless of how often it's modified
+	box_time_t maxUploadWait = SecondsToBoxTime((uint32_t)conf.GetKeyValueInt("MaxUploadWait"));
+	// Adjust by subtracting the minimum file age, so is relative to sync period end in comparisons
+	maxUploadWait = (maxUploadWait > minimumFileAge)?(maxUploadWait - minimumFileAge):(0);
+
+	// When the next sync should take place -- which is ASAP
+	box_time_t nextSyncTime = 0;
+
+	// When the last sync started (only updated if the store was not full when the sync ended)
+	box_time_t lastSyncTime = 0;
+	
+	// --------------------------------------------------------------------------------------------
+
+	// And what's the current client store marker?
+	int64_t clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;		// haven't contacted the store yet
+
+	// Set state
+	SetState(State_Idle);
+
+	// Loop around doing backups
+	do
+	{
+		// Flags used below
+		bool storageLimitExceeded = false;
+		bool doSync = false;
+		bool doSyncForcedByCommand = false;
+
+		// Is a delay necessary?
+		{
+			box_time_t currentTime;
+			do
+			{
+				// Need to check the stop run thing here too, so this loop isn't run if we should be stopping
+				if(StopRun()) break;
+				
+				currentTime = GetCurrentBoxTime();
+	
+				// Pause a while, but no more than MAX_SLEEP_TIME seconds (use the conditional because times are unsigned)
+				box_time_t requiredDelay = (nextSyncTime < currentTime)?(0):(nextSyncTime - currentTime);
+				// If there isn't automatic backup happening, set a long delay. And limit delays at the same time.
+				if(!automaticBackup || requiredDelay > SecondsToBoxTime((uint32_t)MAX_SLEEP_TIME)) requiredDelay = SecondsToBoxTime((uint32_t)MAX_SLEEP_TIME);
+
+				// Only do the delay if there is a delay required
+				if(requiredDelay > 0)
+				{
+					// Sleep somehow. There are choices on how this should be done, depending on the state of the control connection
+					if(mpCommandSocketInfo != 0)
+					{
+						// A command socket exists, so sleep by handling connections with it
+						WaitOnCommandSocket(requiredDelay, doSync, doSyncForcedByCommand);
+					}
+					else
+					{
+						// No command socket or connection, just do a normal sleep
+						int sleepSeconds = BoxTimeToSeconds(requiredDelay);
+						::sleep((sleepSeconds <= 0)?1:sleepSeconds);
+					}
+				}
+				
+			} while((!automaticBackup || (currentTime < nextSyncTime)) && !doSync && !StopRun());
+		}
+
+		// Time of sync start, and if it's time for another sync (and we're doing automatic syncs), set the flag
+		box_time_t currentSyncStartTime = GetCurrentBoxTime();
+		if(automaticBackup && currentSyncStartTime >= nextSyncTime)
+		{
+			doSync = true;
+		}
+		
+		// Use a script to see if sync is allowed now?
+		if(!doSyncForcedByCommand && doSync && !StopRun())
+		{
+			int d = UseScriptToSeeIfSyncAllowed();
+			if(d > 0)
+			{
+				// Script has asked for a delay
+				nextSyncTime = GetCurrentBoxTime() + SecondsToBoxTime((uint32_t)d);
+				doSync = false;
+			}
+		}
+
+		// Ready to sync? (but only if we're not supposed to be stopping)
+		if(doSync && !StopRun())
+		{
+			// Touch a file to record times in filesystem
+			TouchFileInWorkingDir("last_sync_start");
+		
+			// Tell anything connected to the command socket
+			SendSyncStartOrFinish(true /* start */);
+			
+			// Reset statistics on uploads
+			BackupStoreFile::ResetStats();
+			
+			// Calculate the sync period of files to examine
+			box_time_t syncPeriodStart = lastSyncTime;
+			box_time_t syncPeriodEnd = currentSyncStartTime - minimumFileAge;
+			// Check logic
+			ASSERT(syncPeriodEnd > syncPeriodStart);
+			// Paranoid check on sync times
+			if(syncPeriodStart >= syncPeriodEnd) continue;
+			
+			// Adjust syncPeriodEnd to emulate snapshot behaviour properly
+			box_time_t syncPeriodEndExtended = syncPeriodEnd;
+			// Using zero min file age?
+			if(minimumFileAge == 0)
+			{
+				// Add a year on to the end of the end time, to make sure we sync
+				// files which are modified after the scan run started.
+				// Of course, they may be eligable to be synced again the next time round,
+				// but this should be OK, because the changes only upload should upload no data.
+				syncPeriodEndExtended += SecondsToBoxTime((uint32_t)(356*24*3600));
+			}
+			
+			// Do sync
+			bool errorOccurred = false;
+			int errorCode = 0, errorSubCode = 0;
+			try
+			{
+				// Set state and log start
+				SetState(State_Connected);
+				::syslog(LOG_INFO, "Beginning scan of local files");
+
+				// Then create a client context object (don't just connect, as this may be unnecessary)
+				BackupClientContext clientContext(*this, tlsContext, conf.GetKeyValue("StoreHostname"),
+					conf.GetKeyValueInt("AccountNumber"), conf.GetKeyValueBool("ExtendedLogging"));
+					
+				// Set up the sync parameters
+				BackupClientDirectoryRecord::SyncParams params(*this, clientContext);
+				params.mSyncPeriodStart = syncPeriodStart;
+				params.mSyncPeriodEnd = syncPeriodEndExtended; // use potentially extended end time
+				params.mMaxUploadWait = maxUploadWait;
+				params.mFileTrackingSizeThreshold = conf.GetKeyValueInt("FileTrackingSizeThreshold");
+				params.mDiffingUploadSizeThreshold = conf.GetKeyValueInt("DiffingUploadSizeThreshold");
+				params.mMaxFileTimeInFuture = SecondsToBoxTime((uint32_t)conf.GetKeyValueInt("MaxFileTimeInFuture"));
+				
+				// Set store marker
+				clientContext.SetClientStoreMarker(clientStoreMarker);
+				
+				// Set up the locations, if necessary -- need to do it here so we have a (potential) connection to use
+				if(mLocations.empty())
+				{
+					const Configuration &locations(conf.GetSubConfiguration("BackupLocations"));
+					
+					// Make sure all the directory records are set up
+					SetupLocations(clientContext, locations);
+				}
+				
+				// Get some ID maps going
+				SetupIDMapsForSync();
+
+				// Delete any unused directories?
+				DeleteUnusedRootDirEntries(clientContext);
+								
+				// Go through the records, syncing them
+				for(std::vector<Location *>::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i)
+				{
+					// Set current and new ID map pointers in the context
+					clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], mNewIDMaps[(*i)->mIDMapIndex]);
+				
+					// Set exclude lists (context doesn't take ownership)
+					clientContext.SetExcludeLists((*i)->mpExcludeFiles, (*i)->mpExcludeDirs);
+
+					// Sync the directory
+					(*i)->mpDirectoryRecord->SyncDirectory(params, BackupProtocolClientListDirectory::RootDirectory, (*i)->mPath);
+
+					// Unset exclude lists (just in case)
+					clientContext.SetExcludeLists(0, 0);
+				}
+				
+				// Errors reading any files?
+				if(params.mReadErrorsOnFilesystemObjects)
+				{
+					// Notify administrator
+					NotifySysadmin(NotifyEvent_ReadError);
+				}
+				else
+				{
+					// Unset the read error flag, so the error is
+					// reported again in the future
+					mNotificationsSent[NotifyEvent_ReadError] = false;
+				}
+				
+				// Perform any deletions required -- these are delayed until the end
+				// to allow renaming to happen neatly.
+				clientContext.PerformDeletions();
+
+				// Close any open connection
+				clientContext.CloseAnyOpenConnection();
+				
+				// Get the new store marker
+				clientStoreMarker = clientContext.GetClientStoreMarker();
+				
+				// Check the storage limit
+				if(clientContext.StorageLimitExceeded())
+				{
+					// Tell the sysadmin about this
+					NotifySysadmin(NotifyEvent_StoreFull);
+				}
+				else
+				{
+					// The start time of the next run is the end time of this run
+					// This is only done if the storage limit wasn't exceeded (as things won't have been done properly if it was)
+					lastSyncTime = syncPeriodEnd;
+					// unflag the storage full notify flag so that next time the store is full, and alert will be sent
+					mNotificationsSent[NotifyEvent_StoreFull] = false;
+				}
+				
+				// Calculate when the next sync run should be
+				nextSyncTime = currentSyncStartTime + updateStoreInterval + Random::RandomInt(updateStoreInterval >> SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY);
+			
+				// Commit the ID Maps
+				CommitIDMapsAfterSync();
+
+				// Log
+				::syslog(LOG_INFO, "Finished scan of local files");
+			}
+			catch(BoxException &e)
+			{
+				errorOccurred = true;
+				errorCode = e.GetType();
+				errorSubCode = e.GetSubType();
+			}
+			catch(...)
+			{
+				// TODO: better handling of exceptions here... need to be very careful
+				errorOccurred = true;
+			}
+			
+			if(errorOccurred)
+			{
+				// Is it a berkely db failure?
+				bool isBerkelyDbFailure = (errorCode == BackupStoreException::ExceptionType
+					&& errorSubCode == BackupStoreException::BerkelyDBFailure);
+				if(isBerkelyDbFailure)
+				{
+					// Delete corrupt files
+					DeleteCorruptBerkelyDbFiles();
+				}
+
+				// Clear state data
+				syncPeriodStart = 0;	// go back to beginning of time
+				clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown;	// no store marker, so download everything
+				DeleteAllLocations();
+				DeleteAllIDMaps();
+
+				// Handle restart?
+				if(StopRun())
+				{
+					::syslog(LOG_INFO, "Exception (%d/%d) due to signal", errorCode, errorSubCode);
+					return;
+				}
+
+				// If the Berkely db files get corrupted, delete them and try again immediately
+				if(isBerkelyDbFailure)
+				{
+					::syslog(LOG_ERR, "Berkely db inode map files corrupted, deleting and restarting scan. Renamed files and directories will not be tracked until after this scan.\n");
+					::sleep(1);
+				}
+				else
+				{
+					// Not restart/terminate, pause and retry
+					SetState(State_Error);
+					::syslog(LOG_ERR, "Exception caught (%d/%d), reset state and waiting to retry...", errorCode, errorSubCode);
+					::sleep(100);
+				}
+			}
+
+			// Log the stats
+			::syslog(LOG_INFO, "File statistics: total file size uploaded %lld, bytes already on server %lld, encoded size %lld",
+				BackupStoreFile::msStats.mBytesInEncodedFiles, BackupStoreFile::msStats.mBytesAlreadyOnServer,
+				BackupStoreFile::msStats.mTotalFileStreamSize);
+			BackupStoreFile::ResetStats();
+
+			// Tell anything connected to the command socket
+			SendSyncStartOrFinish(false /* finish */);
+
+			// Touch a file to record times in filesystem
+			TouchFileInWorkingDir("last_sync_finish");
+		}
+		
+		// Set state
+		SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle);
+
+	} while(!StopRun());
+	
+	// Make sure we have a clean start next time round (if restart)
+	DeleteAllLocations();
+	DeleteAllIDMaps();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::UseScriptToSeeIfSyncAllowed()
+//		Purpose: Private. Use a script to see if the sync should be allowed (if configured)
+//				 Returns -1 if it's allowed, time in seconds to wait otherwise.
+//		Created: 21/6/04
+//
+// --------------------------------------------------------------------------
+int BackupDaemon::UseScriptToSeeIfSyncAllowed()
+{
+	const Configuration &conf(GetConfiguration());
+
+	// Got a script to run?
+	if(!conf.KeyExists("SyncAllowScript"))
+	{
+		// No. Do sync.
+		return -1;
+	}
+
+	// If there's no result, try again in five minutes
+	int waitInSeconds = (60*5);
+
+	// Run it?
+	pid_t pid = 0;
+	try
+	{
+		std::auto_ptr<IOStream> pscript(LocalProcessStream(conf.GetKeyValue("SyncAllowScript").c_str(), pid));
+
+		// Read in the result
+		IOStreamGetLine getLine(*pscript);
+		std::string line;
+		if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough
+		{
+			// Got a string, intepret
+			if(line == "now")
+			{
+				// Script says do it now. Obey.
+				waitInSeconds = -1;
+			}
+			else
+			{
+				// How many seconds to wait?
+				waitInSeconds = BoxConvert::Convert<int32_t, const std::string&>(line);
+				::syslog(LOG_INFO, "Delaying sync by %d seconds (SyncAllowScript '%s')", waitInSeconds, conf.GetKeyValue("SyncAllowScript").c_str());
+			}
+		}
+		
+		// Wait and then cleanup child process
+		int status = 0;
+		::waitpid(pid, &status, 0);
+	}
+	catch(...)
+	{
+		// Ignore any exceptions
+		// Log that something bad happened
+		::syslog(LOG_ERR, "Error running SyncAllowScript '%s'", conf.GetKeyValue("SyncAllowScript").c_str());
+		// Clean up though
+		if(pid != 0)
+		{
+			int status = 0;
+			::waitpid(pid, &status, 0);
+		}
+	}
+
+	return waitInSeconds;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &)
+//		Purpose: Waits on a the command socket for a time of UP TO the required time
+//				 but may be much less, and handles a command if necessary.
+//		Created: 18/2/04
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut)
+{
+	ASSERT(mpCommandSocketInfo != 0);
+	if(mpCommandSocketInfo == 0) {::sleep(1); return;} // failure case isn't too bad
+	
+	TRACE1("Wait on command socket, delay = %lld\n", RequiredDelay);
+	
+	try
+	{
+		// Timeout value for connections and things
+		int timeout = ((int)BoxTimeToMilliSeconds(RequiredDelay)) + 1;
+		// Handle bad boundary cases
+		if(timeout <= 0) timeout = 1;
+		if(timeout == INFTIM) timeout = 100000;
+
+		// Wait for socket connection, or handle a command?
+		if(mpCommandSocketInfo->mpConnectedSocket.get() == 0)
+		{
+			// No connection, listen for a new one
+			mpCommandSocketInfo->mpConnectedSocket.reset(mpCommandSocketInfo->mListeningSocket.Accept(timeout).release());
+			
+			if(mpCommandSocketInfo->mpConnectedSocket.get() == 0)
+			{
+				// If a connection didn't arrive, there was a timeout, which means we've
+				// waited long enough and it's time to go.
+				return;
+			}
+			else
+			{
+#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
+				bool uidOK = true;
+				::syslog(LOG_ERR, "On this platform, no security check can be made on the credientials of peers connecting to the command socket. (bbackupctl)");
+#else
+				// Security check -- does the process connecting to this socket have
+				// the same UID as this process?
+				bool uidOK = false;
+				// BLOCK
+				{
+					uid_t remoteEUID = 0xffff;
+					gid_t remoteEGID = 0xffff;
+					if(mpCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID))
+					{
+						// Credentials are available -- check UID
+						if(remoteEUID == ::getuid())
+						{
+							// Acceptable
+							uidOK = true;
+						}
+					}
+				}
+#endif
+				
+				// Is this an acceptible connection?
+				if(!uidOK)
+				{
+					// Dump the connection
+					::syslog(LOG_ERR, "Incoming command connection from peer had different user ID than this process, or security check could not be completed.");
+					mpCommandSocketInfo->mpConnectedSocket.reset();
+					return;
+				}
+				else
+				{
+					// Log
+					::syslog(LOG_INFO, "Connection from command socket");
+					
+					// Send a header line summarising the configuration and current state
+					const Configuration &conf(GetConfiguration());
+					char summary[256];
+					int summarySize = sprintf(summary, "bbackupd: %d %d %d %d\nstate %d\n",
+						conf.GetKeyValueBool("AutomaticBackup"),
+						conf.GetKeyValueInt("UpdateStoreInterval"),
+						conf.GetKeyValueInt("MinimumFileAge"),
+						conf.GetKeyValueInt("MaxUploadWait"),
+						mState);
+					mpCommandSocketInfo->mpConnectedSocket->Write(summary, summarySize);
+					
+					// Set the timeout to something very small, so we don't wait too long on waiting
+					// for any incoming data
+					timeout = 10; // milliseconds
+				}
+			}
+		}
+
+		// So there must be a connection now.
+		ASSERT(mpCommandSocketInfo->mpConnectedSocket.get() != 0);
+		
+		// Is there a getline object ready?
+		if(mpCommandSocketInfo->mpGetLine == 0)
+		{
+			// Create a new one
+			mpCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mpCommandSocketInfo->mpConnectedSocket.get()));
+		}
+		
+		// Ping the remote side, to provide errors which will mean the socket gets closed
+		mpCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5);
+		
+		// Wait for a command or something on the socket
+		std::string command;
+		while(mpCommandSocketInfo->mpGetLine != 0 && !mpCommandSocketInfo->mpGetLine->IsEOF()
+			&& mpCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout))
+		{
+			TRACE1("Receiving command '%s' over command socket\n", command.c_str());
+			
+			bool sendOK = false;
+			bool sendResponse = true;
+		
+			// Command to process!
+			if(command == "quit" || command == "")
+			{
+				// Close the socket.
+				CloseCommandConnection();
+				sendResponse = false;
+			}
+			else if(command == "sync")
+			{
+				// Sync now!
+				DoSyncFlagOut = true;
+				SyncIsForcedOut = false;
+				sendOK = true;
+			}
+			else if(command == "force-sync")
+			{
+				// Sync now (forced -- overrides any SyncAllowScript)
+				DoSyncFlagOut = true;
+				SyncIsForcedOut = true;
+				sendOK = true;
+			}
+			else if(command == "reload")
+			{
+				// Reload the configuration
+				SetReloadConfigWanted();
+				sendOK = true;
+			}
+			else if(command == "terminate")
+			{
+				// Terminate the daemon cleanly
+				SetTerminateWanted();
+				sendOK = true;
+			}
+			
+			// Send a response back?
+			if(sendResponse)
+			{
+				mpCommandSocketInfo->mpConnectedSocket->Write(sendOK?"ok\n":"error\n", sendOK?3:6);
+			}
+			
+			// Set timeout to something very small, so this just checks for data which is waiting
+			timeout = 1;
+		}
+		
+		// Close on EOF?
+		if(mpCommandSocketInfo->mpGetLine != 0 && mpCommandSocketInfo->mpGetLine->IsEOF())
+		{
+			CloseCommandConnection();
+		}
+	}
+	catch(...)
+	{
+		// If an error occurs, and there is a connection active, just close that
+		// connection and continue. Otherwise, let the error propagate.
+		if(mpCommandSocketInfo->mpConnectedSocket.get() == 0)
+		{
+			throw;
+		}
+		else
+		{
+			// Close socket and ignore error
+			CloseCommandConnection();
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::CloseCommandConnection()
+//		Purpose: Close the command connection, ignoring any errors
+//		Created: 18/2/04
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::CloseCommandConnection()
+{
+	try
+	{
+		TRACE0("Closing command connection\n");
+	
+		if(mpCommandSocketInfo->mpGetLine)
+		{
+			delete mpCommandSocketInfo->mpGetLine;
+			mpCommandSocketInfo->mpGetLine = 0;
+		}
+		mpCommandSocketInfo->mpConnectedSocket.reset();
+	}
+	catch(...)
+	{
+		// Ignore any errors
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupDaemon.cpp
+//		Purpose: Send a start or finish sync message to the command socket, if it's connected.
+//				 
+//		Created: 18/2/04
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::SendSyncStartOrFinish(bool SendStart)
+{
+
+	// The bbackupctl program can't rely on a state change, because it may never
+	// change if the server doesn't need to be contacted.
+	
+	if(mpCommandSocketInfo != 0 && mpCommandSocketInfo->mpConnectedSocket.get() != 0)
+	{
+		try
+		{
+			mpCommandSocketInfo->mpConnectedSocket->Write(SendStart?"start-sync\n":"finish-sync\n", SendStart?11:12);
+		}
+		catch(...)
+		{
+			CloseCommandConnection();
+		}
+	}
+}
+
+
+
+
+#ifdef PLATFORM_USES_MTAB_FILE_FOR_MOUNTS
+	// string comparison ordering for when mount points are handled
+	// by code, rather than the OS.
+	typedef struct
+	{
+		bool operator()(const std::string &s1, const std::string &s2)
+		{
+			if(s1.size() == s2.size())
+			{
+				// Equal size, sort according to natural sort order
+				return s1 < s2;
+			}
+			else
+			{
+				// Make sure longer strings go first
+				return s1.size() > s2.size();
+			}
+		}
+	} mntLenCompare;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::SetupLocations(BackupClientContext &, const Configuration &)
+//		Purpose: Makes sure that the list of directories records is correctly set up
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf)
+{
+	if(!mLocations.empty())
+	{
+		// Looks correctly set up
+		return;
+	}
+
+	// Make sure that if a directory is reinstated, then it doesn't get deleted	
+	mDeleteUnusedRootDirEntriesAfter = 0;
+	mUnusedRootDirEntries.clear();
+
+	// Just a check to make sure it's right.
+	DeleteAllLocations();
+	
+	// Going to need a copy of the root directory. Get a connection, and fetch it.
+	BackupProtocolClient &connection(rClientContext.GetConnection());
+	
+	// Ask server for a list of everything in the root directory, which is a directory itself
+	std::auto_ptr<BackupProtocolClientSuccess> dirreply(connection.QueryListDirectory(
+			BackupProtocolClientListDirectory::RootDirectory,
+			BackupProtocolClientListDirectory::Flags_Dir,	// only directories
+			BackupProtocolClientListDirectory::Flags_Deleted | BackupProtocolClientListDirectory::Flags_OldVersion, // exclude old/deleted stuff
+			false /* no attributes */));
+
+	// Retrieve the directory from the stream following
+	BackupStoreDirectory dir;
+	std::auto_ptr<IOStream> dirstream(connection.ReceiveStream());
+	dir.ReadFromStream(*dirstream, connection.GetTimeout());
+	
+	// Map of mount names to ID map index
+	std::map<std::string, int> mounts;
+	int numIDMaps = 0;
+
+#ifdef PLATFORM_USES_MTAB_FILE_FOR_MOUNTS
+	// Linux can't tell you where a directory is mounted. So we have to
+	// read the mount entries from /etc/mtab! Bizarre that the OS itself
+	// can't tell you, but there you go.
+	std::set<std::string, mntLenCompare> mountPoints;
+	// BLOCK
+	FILE *mountPointsFile = 0;
+	try
+	{
+		// Open mounts file
+		mountPointsFile = ::setmntent("/etc/mtab", "r");
+		if(mountPointsFile == 0)
+		{
+			THROW_EXCEPTION(CommonException, OSFileError);
+		}
+		
+		// Read all the entries, and put them in the set
+		struct mntent *entry = 0;
+		while((entry = ::getmntent(mountPointsFile)) != 0)
+		{
+			TRACE1("Found mount point at %s\n", entry->mnt_dir);
+			mountPoints.insert(std::string(entry->mnt_dir));
+		}
+		
+		// Close mounts file
+		::endmntent(mountPointsFile);
+	}
+	catch(...)
+	{
+		if(mountPointsFile != 0)
+		{
+			::endmntent(mountPointsFile);
+		}
+		throw;
+	}
+	// Check sorting and that things are as we expect
+	ASSERT(mountPoints.size() > 0);
+#ifndef NDEBUG
+	{
+		std::set<std::string, mntLenCompare>::const_reverse_iterator i(mountPoints.rbegin());
+		ASSERT(*i == "/");
+	}
+#endif // n NDEBUG
+#endif // PLATFORM_USES_MTAB_FILE_FOR_MOUNTS
+
+	// Then... go through each of the entries in the configuration,
+	// making sure there's a directory created for it.
+	for(std::list<std::pair<std::string, Configuration> >::const_iterator i = rLocationsConf.mSubConfigurations.begin();
+		i != rLocationsConf.mSubConfigurations.end(); ++i)
+	{
+TRACE0("new location\n");
+		// Create a record for it
+		Location *ploc = new Location;
+		try
+		{
+			// Setup names in the location record
+			ploc->mName = i->first;
+			ploc->mPath = i->second.GetKeyValue("Path");
+			
+			// Read the exclude lists from the Configuration
+			ploc->mpExcludeFiles = BackupClientMakeExcludeList_Files(i->second);
+			ploc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(i->second);
+			
+			// Do a fsstat on the pathname to find out which mount it's on
+			{
+#ifdef PLATFORM_USES_MTAB_FILE_FOR_MOUNTS
+				// Warn in logs if the directory isn't absolute
+				if(ploc->mPath[0] != '/')
+				{
+					::syslog(LOG_ERR, "Location path '%s' isn't absolute", ploc->mPath.c_str());
+				}
+				// Go through the mount points found, and find a suitable one
+				std::string mountName("/");
+				{
+					std::set<std::string, mntLenCompare>::const_iterator i(mountPoints.begin());
+					TRACE1("%d potential mount points\n", mountPoints.size());
+					for(; i != mountPoints.end(); ++i)
+					{
+						// Compare first n characters with the filename
+						// If it matches, the file belongs in that mount point
+						// (sorting order ensures this)
+						TRACE1("checking against mount point %s\n", i->c_str());
+						if(::strncmp(i->c_str(), ploc->mPath.c_str(), i->size()) == 0)
+						{
+							// Match
+							mountName = *i;
+							break;
+						}
+					}
+					TRACE2("mount point chosen for %s is %s\n", ploc->mPath.c_str(), mountName.c_str());
+				}
+#else
+				// BSD style statfs -- includes mount point, which is nice.
+				struct statfs s;
+				if(::statfs(ploc->mPath.c_str(), &s) != 0)
+				{
+					THROW_EXCEPTION(CommonException, OSFileError)
+				}
+				
+				// Where the filesystem is mounted
+				std::string mountName(s.f_mntonname);
+#endif
+				
+				// Got it?
+				std::map<std::string, int>::iterator f(mounts.find(mountName));
+				if(f != mounts.end())
+				{
+					// Yes -- store the index
+					ploc->mIDMapIndex = f->second;
+				}
+				else
+				{
+					// No -- new index
+					ploc->mIDMapIndex = numIDMaps;
+					mounts[mountName] = numIDMaps;
+					
+					// Store the mount name
+					mIDMapMounts.push_back(mountName);
+					
+					// Increment number of maps
+					++numIDMaps;
+				}
+			}
+		
+			// Does this exist on the server?
+			BackupStoreDirectory::Iterator iter(dir);
+			BackupStoreFilenameClear dirname(ploc->mName);	// generate the filename
+			BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname);
+			int64_t oid = 0;
+			if(en != 0)
+			{
+				oid = en->GetObjectID();
+				
+				// Delete the entry from the directory, so we get a list of
+				// unused root directories at the end of this.
+				dir.DeleteEntry(oid);
+			}
+			else
+			{
+				// Doesn't exist, so it has to be created on the server. Let's go!
+				// First, get the directory's attributes and modification time
+				box_time_t attrModTime = 0;
+				BackupClientFileAttributes attr;
+				attr.ReadAttributes(ploc->mPath.c_str(), true /* directories have zero mod times */,
+					0 /* not interested in mod time */, &attrModTime /* get the attribute modification time */);
+				
+				// Execute create directory command
+				MemBlockStream attrStream(attr);
+				std::auto_ptr<BackupProtocolClientSuccess> dirCreate(connection.QueryCreateDirectory(
+					BackupProtocolClientListDirectory::RootDirectory,
+					attrModTime, dirname, attrStream));
+					
+				// Object ID for later creation
+				oid = dirCreate->GetObjectID();
+			}
+
+			// Create and store the directory object for the root of this location
+			ASSERT(oid != 0);
+			BackupClientDirectoryRecord *precord = new BackupClientDirectoryRecord(oid, i->first);
+			ploc->mpDirectoryRecord.reset(precord);
+			
+			// Push it back on the vector of locations
+			mLocations.push_back(ploc);
+		}
+		catch(...)
+		{
+			delete ploc;
+			ploc = 0;
+			throw;
+		}
+	}
+	
+	// Any entries in the root directory which need deleting?
+	if(dir.GetNumberOfEntries() > 0)
+	{
+		::syslog(LOG_INFO, "%d redundant locations in root directory found, will delete from store after %d seconds.",
+			dir.GetNumberOfEntries(), BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER);
+
+		// Store directories in list of things to delete
+		mUnusedRootDirEntries.clear();
+		BackupStoreDirectory::Iterator iter(dir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = iter.Next()) != 0)
+		{
+			// Add name to list
+			BackupStoreFilenameClear clear(en->GetName());
+			const std::string &name(clear.GetClearFilename());
+			mUnusedRootDirEntries.push_back(std::pair<int64_t,std::string>(en->GetObjectID(), name));
+			// Log this
+			::syslog(LOG_INFO, "Unused location in root: %s", name.c_str());
+		}
+		ASSERT(mUnusedRootDirEntries.size() > 0);
+		// Time to delete them
+		mDeleteUnusedRootDirEntriesAfter =
+			GetCurrentBoxTime() + SecondsToBoxTime((uint32_t)BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::SetupIDMapsForSync()
+//		Purpose: Sets up ID maps for the sync process -- make sure they're all there
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::SetupIDMapsForSync()
+{
+	// Need to do different things depending on whether it's an in memory implementation,
+	// or whether it's all stored on disc.
+	
+#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+
+	// Make sure we have some blank, empty ID maps
+	DeleteIDMapVector(mNewIDMaps);
+	FillIDMapVector(mNewIDMaps, true /* new maps */);
+
+	// Then make sure that the current maps have objects, even if they are empty
+	// (for the very first run)
+	if(mCurrentIDMaps.empty())
+	{
+		FillIDMapVector(mCurrentIDMaps, false /* current maps */);
+	}
+
+#else
+
+	// Make sure we have some blank, empty ID maps
+	DeleteIDMapVector(mNewIDMaps);
+	FillIDMapVector(mNewIDMaps, true /* new maps */);
+	DeleteIDMapVector(mCurrentIDMaps);
+	FillIDMapVector(mCurrentIDMaps, false /* new maps */);
+
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &)
+//		Purpose: Fills the vector with the right number of empty ID maps
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps)
+{
+	ASSERT(rVector.size() == 0);
+	rVector.reserve(mIDMapMounts.size());
+	
+	for(unsigned int l = 0; l < mIDMapMounts.size(); ++l)
+	{
+		// Create the object
+		BackupClientInodeToIDMap *pmap = new BackupClientInodeToIDMap();
+		try
+		{
+			// Get the base filename of this map
+			std::string filename;
+			MakeMapBaseName(l, filename);
+			
+			// If it's a new one, add a suffix
+			if(NewMaps)
+			{
+				filename += ".n";
+			}
+
+			// If it's not a new map, it may not exist in which case an empty map should be created
+			if(!NewMaps && !FileExists(filename.c_str()))
+			{
+				pmap->OpenEmpty();
+			}
+			else
+			{
+				// Open the map
+				pmap->Open(filename.c_str(), !NewMaps /* read only */, NewMaps /* create new */);
+			}
+			
+			// Store on vector
+			rVector.push_back(pmap);
+		}
+		catch(...)
+		{
+			delete pmap;
+			throw;
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::DeleteCorruptBerkelyDbFiles()
+//		Purpose: Delete the Berkely db files from disc after they have been corrupted.
+//		Created: 14/9/04
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::DeleteCorruptBerkelyDbFiles()
+{
+	for(unsigned int l = 0; l < mIDMapMounts.size(); ++l)
+	{
+		// Get the base filename of this map
+		std::string filename;
+		MakeMapBaseName(l, filename);
+		
+		// Delete the file
+		TRACE1("Deleting %s\n", filename.c_str());
+		::unlink(filename.c_str());
+		
+		// Add a suffix for the new map
+		filename += ".n";
+
+		// Delete that too
+		TRACE1("Deleting %s\n", filename.c_str());
+		::unlink(filename.c_str());
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MakeMapBaseName(unsigned int, std::string &)
+//		Purpose: Makes the base name for a inode map
+//		Created: 20/11/03
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const
+{
+	// Get the directory for the maps
+	const Configuration &config(GetConfiguration());
+	std::string dir(config.GetKeyValue("DataDirectory"));
+
+	// Make a leafname
+	std::string leaf(mIDMapMounts[MountNumber]);
+	for(unsigned int z = 0; z < leaf.size(); ++z)
+	{
+		if(leaf[z] == DIRECTORY_SEPARATOR_ASCHAR)
+		{
+			leaf[z] = '_';
+		}
+	}
+
+	// Build the final filename
+	rNameOut = dir + DIRECTORY_SEPARATOR "mnt" + leaf;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::CommitIDMapsAfterSync()
+//		Purpose: Commits the new ID maps, so the 'new' maps are now the 'current' maps.
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::CommitIDMapsAfterSync()
+{
+	// Need to do different things depending on whether it's an in memory implementation,
+	// or whether it's all stored on disc.
+	
+#ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION
+	// Remove the current ID maps
+	DeleteIDMapVector(mCurrentIDMaps);
+
+	// Copy the (pointers to) "new" maps over to be the new "current" maps
+	mCurrentIDMaps = mNewIDMaps;
+	
+	// Clear the new ID maps vector (not delete them!)
+	mNewIDMaps.clear();
+
+#else
+
+	// Get rid of the maps in memory (leaving them on disc of course)
+	DeleteIDMapVector(mCurrentIDMaps);
+	DeleteIDMapVector(mNewIDMaps);
+
+	// Then move the old maps into the new places
+	for(unsigned int l = 0; l < mIDMapMounts.size(); ++l)
+	{
+		std::string target;
+		MakeMapBaseName(l, target);
+		std::string newmap(target + ".n");
+		
+		// Try to rename
+		if(::rename(newmap.c_str(), target.c_str()) != 0)
+		{
+			THROW_EXCEPTION(CommonException, OSFileError)
+		}
+	}
+
+#endif
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &)
+//		Purpose: Deletes the contents of a vector of ID maps
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector)
+{
+	while(!rVector.empty())
+	{
+		// Pop off list
+		BackupClientInodeToIDMap *toDel = rVector.back();
+		rVector.pop_back();
+		
+		// Close and delete
+		toDel->Close();
+		delete toDel;
+	}
+	ASSERT(rVector.size() == 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::FindLocationPathName(const std::string &, std::string &) const
+//		Purpose: Tries to find the path of the root of a backup location. Returns true (and path in rPathOut)
+//				 if it can be found, false otherwise.
+//		Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+bool BackupDaemon::FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const
+{
+	// Search for the location
+	for(std::vector<Location *>::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i)
+	{
+		if((*i)->mName == rLocationName)
+		{
+			rPathOut = (*i)->mPath;
+			return true;
+		}
+	}
+	
+	// Didn't find it
+	return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::SetState(int)
+//		Purpose: Record current action of daemon, and update process title to reflect this
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::SetState(int State)
+{
+	// Two little checks
+	if(State == mState) return;
+	if(State < 0) return;
+
+	// Update
+	mState = State;
+	
+	// Set process title
+	const static char *stateText[] = {"idle", "connected", "error -- waiting for retry", "over limit on server -- not backing up"};
+	SetProcessTitle(stateText[State]);
+	
+	// If there's a command socket connected, then inform it -- disconnecting from the
+	// command socket if there's an error
+	if(mpCommandSocketInfo != 0 && mpCommandSocketInfo->mpConnectedSocket.get() != 0)
+	{
+		// Something connected to the command socket, tell it about the new state
+		char newState[64];
+		char newStateSize = sprintf(newState, "state %d\n", State);
+		try
+		{
+			mpCommandSocketInfo->mpConnectedSocket->Write(newState, newStateSize);
+		}
+		catch(...)
+		{
+			CloseCommandConnection();
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::TouchFileInWorkingDir(const char *)
+//		Purpose: Make sure a zero length file of the name exists in the working directory.
+//				 Use for marking times of events in the filesystem.
+//		Created: 21/2/04
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::TouchFileInWorkingDir(const char *Filename)
+{
+	// Filename
+	const Configuration &config(GetConfiguration());
+	std::string fn(config.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR);
+	fn += Filename;
+	
+	// Open and close it to update the timestamp
+	FileStream touch(fn.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::NotifySysadmin(int)
+//		Purpose: Run the script to tell the sysadmin about events which need attention.
+//		Created: 25/2/04
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::NotifySysadmin(int Event)
+{
+	static const char *sEventNames[] = {"store-full", "read-error", 0};
+
+	TRACE1("BackupDaemon::NotifySysadmin() called, event = %d\n", Event);
+
+	if(Event < 0 || Event > NotifyEvent__MAX)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadNotifySysadminEventCode);
+	}
+
+	// Don't send lots of repeated messages
+	if(mNotificationsSent[Event])
+	{
+		return;
+	}
+
+	// Is there a notifation script?
+	const Configuration &conf(GetConfiguration());
+	if(!conf.KeyExists("NotifyScript"))
+	{
+		// Log, and then return
+		::syslog(LOG_ERR, "Not notifying administrator about event %s -- set NotifyScript to do this in future", sEventNames[Event]);
+		return;
+	}
+
+	// Script to run
+	std::string script(conf.GetKeyValue("NotifyScript") + ' ' + sEventNames[Event]);
+	
+	// Log what we're about to do
+	::syslog(LOG_INFO, "About to notify administrator about event %s, running script '%s'", sEventNames[Event], script.c_str());
+	
+	// Then do it
+	if(::system(script.c_str()) != 0)
+	{
+		::syslog(LOG_ERR, "Notify script returned an error code. ('%s')", script.c_str());
+	}
+
+	// Flag that this is done so the administrator isn't constantly bombarded with lots of errors
+	mNotificationsSent[Event] = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &)
+//		Purpose: Deletes any unused entries in the root directory, if they're scheduled to be deleted.
+//		Created: 13/5/04
+//
+// --------------------------------------------------------------------------
+void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext)
+{
+	if(mUnusedRootDirEntries.empty() || mDeleteUnusedRootDirEntriesAfter == 0)
+	{
+		// Nothing to do.
+		return;
+	}
+	
+	// Check time
+	if(GetCurrentBoxTime() < mDeleteUnusedRootDirEntriesAfter)
+	{
+		// Too early to delete files
+		return;
+	}
+
+	// Entries to delete, and it's the right time to do so...
+	::syslog(LOG_INFO, "Deleting unused locations from store root...");
+	BackupProtocolClient &connection(rContext.GetConnection());
+	for(std::vector<std::pair<int64_t,std::string> >::iterator i(mUnusedRootDirEntries.begin()); i != mUnusedRootDirEntries.end(); ++i)
+	{
+		connection.QueryDeleteDirectory(i->first);
+		
+		// Log this
+		::syslog(LOG_INFO, "Deleted %s (ID %08llx) from store root", i->second.c_str(), i->first);
+	}
+
+	// Reset state
+	mDeleteUnusedRootDirEntriesAfter = 0;
+	mUnusedRootDirEntries.clear();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::Location::Location()
+//		Purpose: Constructor
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+BackupDaemon::Location::Location()
+	: mIDMapIndex(0),
+	  mpExcludeFiles(0),
+	  mpExcludeDirs(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::Location::~Location()
+//		Purpose: Destructor
+//		Created: 11/11/03
+//
+// --------------------------------------------------------------------------
+BackupDaemon::Location::~Location()
+{
+	// Clean up exclude locations
+	if(mpExcludeDirs != 0)
+	{
+		delete mpExcludeDirs;
+		mpExcludeDirs = 0;
+	}
+	if(mpExcludeFiles != 0)
+	{
+		delete mpExcludeFiles;
+		mpExcludeFiles = 0;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::CommandSocketInfo::CommandSocketInfo()
+//		Purpose: Constructor
+//		Created: 18/2/04
+//
+// --------------------------------------------------------------------------
+BackupDaemon::CommandSocketInfo::CommandSocketInfo()
+	: mpGetLine(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupDaemon::CommandSocketInfo::~CommandSocketInfo()
+//		Purpose: Destructor
+//		Created: 18/2/04
+//
+// --------------------------------------------------------------------------
+BackupDaemon::CommandSocketInfo::~CommandSocketInfo()
+{
+	if(mpGetLine)
+	{
+		delete mpGetLine;
+		mpGetLine = 0;
+	}
+}
+
diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h
new file mode 100755
index 00000000..ffaf5783
--- /dev/null
+++ b/bin/bbackupd/BackupDaemon.h
@@ -0,0 +1,166 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupDaemon.h
+//		Purpose: Backup daemon
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPDAEMON__H
+#define BACKUPDAEMON__H
+
+#include <vector>
+#include <string>
+#include <memory>
+
+#include "Daemon.h"
+#include "BoxTime.h"
+#include "Socket.h"
+#include "SocketListen.h"
+#include "SocketStream.h"
+
+class BackupClientDirectoryRecord;
+class BackupClientContext;
+class Configuration;
+class BackupClientInodeToIDMap;
+class ExcludeList;
+class IOStreamGetLine;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupDaemon
+//		Purpose: Backup daemon
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+class BackupDaemon : public Daemon
+{
+public:
+	BackupDaemon();
+	~BackupDaemon();
+private:
+	BackupDaemon(const BackupDaemon &);
+public:
+
+	void Run();
+	virtual const char *DaemonName() const;
+	virtual const char *DaemonBanner() const;
+	const ConfigurationVerify *GetConfigVerify() const;
+
+	bool FindLocationPathName(const std::string &rLocationName, std::string &rPathOut) const;
+
+	enum
+	{
+		// Add stuff to this, make sure the textual equivalents in SetState() are changed too.
+		State_Initialising = -1,
+		State_Idle = 0,
+		State_Connected = 1,
+		State_Error = 2,
+		State_StorageLimitExceeded = 3
+	};
+
+	int GetState() {return mState;}
+
+	// Allow other classes to call this too
+	enum
+	{
+		NotifyEvent_StoreFull = 0,
+		NotifyEvent_ReadError = 1,
+		NotifyEvent__MAX = 1
+		// When adding notifications, remember to add strings to NotifySysadmin()
+	};
+	void NotifySysadmin(int Event);
+
+private:
+	void Run2();
+
+	void DeleteAllLocations();
+	void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf);
+
+	void DeleteIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector);
+	void DeleteAllIDMaps()
+	{
+		DeleteIDMapVector(mCurrentIDMaps);
+		DeleteIDMapVector(mNewIDMaps);
+	}
+	void FillIDMapVector(std::vector<BackupClientInodeToIDMap *> &rVector, bool NewMaps);
+	
+	void SetupIDMapsForSync();
+	void CommitIDMapsAfterSync();
+	void DeleteCorruptBerkelyDbFiles();
+	
+	void MakeMapBaseName(unsigned int MountNumber, std::string &rNameOut) const;
+
+	void SetState(int State);
+	
+	void WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut);
+	void CloseCommandConnection();
+	void SendSyncStartOrFinish(bool SendStart);
+	
+	void TouchFileInWorkingDir(const char *Filename);
+
+	void DeleteUnusedRootDirEntries(BackupClientContext &rContext);
+
+#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
+	// For warning user about potential security hole
+	virtual void SetupInInitialProcess();
+#endif
+
+	int UseScriptToSeeIfSyncAllowed();
+
+private:
+	class Location
+	{
+	public:
+		Location();
+		~Location();
+	private:
+		Location(const Location &);	// copy not allowed
+		Location &operator=(const Location &);
+	public:
+		std::string mName;
+		std::string mPath;
+		std::auto_ptr<BackupClientDirectoryRecord> mpDirectoryRecord;
+		int mIDMapIndex;
+		ExcludeList *mpExcludeFiles;
+		ExcludeList *mpExcludeDirs;
+	};
+
+	int mState;		// what the daemon is currently doing
+
+	std::vector<Location *> mLocations;
+	
+	std::vector<std::string> mIDMapMounts;
+	std::vector<BackupClientInodeToIDMap *> mCurrentIDMaps;
+	std::vector<BackupClientInodeToIDMap *> mNewIDMaps;
+	
+	// For the command socket
+	class CommandSocketInfo
+	{
+	public:
+		CommandSocketInfo();
+		~CommandSocketInfo();
+	private:
+		CommandSocketInfo(const CommandSocketInfo &);	// no copying
+		CommandSocketInfo &operator=(const CommandSocketInfo &);
+	public:
+		SocketListen<SocketStream, 1 /* listen backlog */> mListeningSocket;
+		std::auto_ptr<SocketStream> mpConnectedSocket;
+		IOStreamGetLine *mpGetLine;
+	};
+	
+	// Using a socket?
+	CommandSocketInfo *mpCommandSocketInfo;
+	
+	// Stop notifications being repeated.
+	bool mNotificationsSent[NotifyEvent__MAX + 1];
+
+	// Unused entries in the root directory wait a while before being deleted
+	box_time_t mDeleteUnusedRootDirEntriesAfter;	// time to delete them
+	std::vector<std::pair<int64_t,std::string> > mUnusedRootDirEntries;
+};
+
+#endif // BACKUPDAEMON__H
+
diff --git a/bin/bbackupd/bbackupd-config b/bin/bbackupd/bbackupd-config
new file mode 100755
index 00000000..c5e52282
--- /dev/null
+++ b/bin/bbackupd/bbackupd-config
@@ -0,0 +1,525 @@
+#!/usr/bin/perl
+use strict;
+
+# should be running as root
+if($> != 0)
+{
+	printf "\nWARNING: this should be run as root\n\n"
+}
+
+sub error_print_usage
+{
+	print <<__E;
+
+Setup bbackupd config utility.
+
+Bad command line parameters.
+Usage:
+	bbackupd-config config-dir backup-mode account-num server-hostname working-dir backup-dir [more backup directories]
+
+config-dir usually /etc/box
+backup-mode is lazy or snapshot
+    lazy mode runs continously, uploading files over a specified age
+    snapshot mode uploads a snapshot of the filesystem when instructed explicitly
+account-num (hexdecimal) and server-hostname as supplied from the server administrator
+working-dir usually /var/bbackupd
+backup-dir, list of directories to back up
+
+__E
+	print "=========\nERROR:\n",$_[0],"\n\n" if $_[0] ne '';
+	exit(1);
+}
+
+# check and get command line parameters
+if($#ARGV < 4)
+{
+	error_print_usage();
+}
+
+# check for OPENSSL_CONF environment var being set
+if(exists $ENV{'OPENSSL_CONF'})
+{
+	print <<__E;
+
+---------------------------------------
+
+WARNING:
+    You have the OPENSSL_CONF environment variable set.
+    Use of non-standard openssl configs may cause problems.
+
+---------------------------------------
+
+__E
+}
+
+# default locations
+my $default_config_location = '/etc/box/bbackupd.conf';
+
+# command line parameters
+my ($config_dir,$backup_mode,$account_num,$server,$working_dir,@tobackup) = @ARGV;
+
+# check backup mode is valid
+if($backup_mode ne 'lazy' && $backup_mode ne 'snapshot')
+{
+	error_print_usage("ERROR: backup mode must be 'lazy' or 'snapshot'");
+}
+
+# check server exists
+{
+	my @r = gethostbyname($server);
+	if($#r < 0)
+	{
+		error_print_usage("Backup server specified as '$server', but it could not found.\n(A test DNS lookup failed -- check arguments)");
+	}
+}
+
+if($working_dir !~ m~\A/~)
+{
+	error_print_usage("Working directory $working_dir is not specified as an absolute path");
+}
+
+# ssl stuff
+my $private_key = "$config_dir/bbackupd/$account_num-key.pem";
+my $certificate_request = "$config_dir/bbackupd/$account_num-csr.pem";
+my $certificate = "$config_dir/bbackupd/$account_num-cert.pem";
+my $ca_root_cert = "$config_dir/bbackupd/serverCA.pem";
+
+# encryption keys
+my $enc_key_file = "$config_dir/bbackupd/$account_num-FileEncKeys.raw";
+
+# other files
+my $config_file = "$config_dir/bbackupd.conf";
+my $notify_script = "$config_dir/bbackupd/NotifySysadmin.sh";
+
+# check that the directories are allowable
+for(@tobackup)
+{
+	if($_ eq '/')
+	{
+		die "It is not recommended that you backup the root directory of your disc";
+	}
+	if($_ !~ m/\A\//)
+	{
+		die "Directory $_ is not specified as an absolute path";
+	}
+	if(!-d $_)
+	{
+		die "$_ is not a directory";
+	}
+}
+
+# summarise configuration
+
+print <<__E;
+
+Setup bbackupd config utility.
+
+Configuration:
+   Writing configuration file: $config_file
+   Account: $account_num
+   Server hostname: $server
+   Directories to back up:
+__E
+print '      ',$_,"\n" for(@tobackup);
+print <<__E;
+
+Note: If other file systems are mounted inside these directories, then problems may occur
+with files on the store server being renamed incorrectly. This will cause efficiency
+problems, but not affect the integrity of the backups.
+
+WARNING: Directories not checked against mountpoints. Check mounted filesystems manually.
+
+__E
+
+# create directories
+if(!-d $config_dir)
+{
+	printf "Creating $config_dir...\n";
+	mkdir $config_dir,0755 or die "Can't create $config_dir";
+}
+
+if(!-d "$config_dir/bbackupd")
+{
+	printf "Creating $config_dir/bbackupd\n";
+	mkdir "$config_dir/bbackupd",0700 or die "Can't create $config_dir/bbackupd";
+}
+
+if(!-d "$working_dir")
+{
+	printf "Creating $working_dir\n";
+	if(!mkdir($working_dir,0700))
+	{
+		die "Couldn't create $working_dir -- create this manually and try again\n";
+	}
+}
+
+# generate the private key for the server
+if(!-f $private_key)
+{
+	print "Generating private key...\n";
+	if(system("openssl genrsa -out $private_key 2048") != 0)
+	{
+		die "Couldn't generate private key."
+	}
+}
+
+# generate a certificate request
+if(!-f $certificate_request)
+{
+	die "Couldn't run openssl for CSR generation" unless
+		open(CSR,"|openssl req -new -key $private_key -sha1 -out $certificate_request");
+	print CSR <<__E;
+.
+.
+.
+.
+.
+BACKUP-$account_num
+.
+.
+.
+
+__E
+	close CSR;
+	print "\n\n";
+	die "Certificate request wasn't created.\n" unless -f $certificate_request
+}
+
+# generate the key material for the file
+if(!-f $enc_key_file)
+{
+	print "Generating keys for file backup\n";
+	if(system("openssl rand -out $enc_key_file 1024") != 0)
+	{
+		die "Couldn't generate file backup keys."
+	}
+}
+
+# write the notify when store full script
+print "Writing notify script $notify_script\n";
+open NOTIFY,">$notify_script" or die "Can't open for writing";
+
+my $hostname = `hostname`; chomp $hostname;
+my $current_username = `whoami`; chomp $current_username;
+my $sendmail = `whereis sendmail`; chomp $sendmail;
+$sendmail =~ s/\n.\Z//s;
+# for Linux style whereis
+$sendmail = $1 if $sendmail =~ /^sendmail:\s+([\S]+)/;
+# last ditch guess
+$sendmail = 'sendmail' if $sendmail !~ m/\S/;
+
+print NOTIFY <<__EOS;
+#!/bin/sh
+
+SUBJECT="BACKUP PROBLEM on host $hostname"
+SENDTO="$current_username"
+
+if [ \$1 = store-full ]
+then
+$sendmail \$SENDTO <<EOM
+Subject: \$SUBJECT (store full)
+To: \$SENDTO
+
+
+The store account for $hostname is full.
+
+=============================
+FILES ARE NOT BEING BACKED UP
+=============================
+
+Please adjust the limits on account $account_num on server $server.
+
+EOM
+elif [ \$1 = read-error ]
+then
+$sendmail \$SENDTO <<EOM
+Subject: \$SUBJECT (read errors)
+To: \$SENDTO
+
+
+Errors occured reading some files or directories for backup on $hostname.
+
+===================================
+THESE FILES ARE NOT BEING BACKED UP
+===================================
+
+Check the logs on $hostname for the files and directories which caused
+these errors, and take appropraite action.
+
+Other files are being backed up.
+
+EOM
+else
+$sendmail \$SENDTO <<EOM
+Subject: \$SUBJECT (unknown)
+To: \$SENDTO
+
+
+The backup daemon on $hostname reported an unknown error.
+
+==========================
+FILES MAY NOT BE BACKED UP
+==========================
+
+Please check the logs on $hostname.
+
+EOM
+fi
+__EOS
+
+close NOTIFY;
+chmod 0700,$notify_script or die "Can't chmod $notify_script";
+
+
+# write the configuration file
+print "Writing configuration file $config_file\n";
+open CONFIG,">$config_file" or die "Can't open config file for writing";
+print CONFIG <<__E;
+
+StoreHostname = $server
+AccountNumber = 0x$account_num
+KeysFile = $enc_key_file
+
+CertificateFile = $certificate
+PrivateKeyFile = $private_key
+TrustedCAsFile = $ca_root_cert
+
+DataDirectory = $working_dir
+
+
+# This script is run whenever bbackupd encounters a problem which requires
+# the system administrator to assist:
+# 1) The store is full, and no more data can be uploaded.
+# 2) Some files or directories were not readable.
+# The default script emails the system administrator.
+
+NotifyScript = $notify_script
+
+__E
+
+if($backup_mode eq 'lazy')
+{
+	# lazy mode configuration
+	print CONFIG <<__E;
+
+# A scan of the local discs will be made once an hour (approximately).
+# To avoid cycles of load on the server, this time is randomly adjusted by a small
+# percentage as the daemon runs.
+
+UpdateStoreInterval = 3600
+
+
+# A file must have been modified at least 6 hours ago before it will be uploaded.
+
+MinimumFileAge = 21600
+
+
+# If a file is modified repeated, it won't be uploaded immediately in case it's modified again.
+# However, it should be uploaded eventually. This is how long we should wait after first noticing
+# a change. (1 day)
+
+MaxUploadWait = 86400
+
+__E
+}
+else
+{
+	# snapshot configuration
+	print CONFIG <<__E;
+
+# This configuration file is written for snapshot mode.
+# You will need to run bbackupctl to instruct the daemon to upload files.
+
+AutomaticBackup = no
+UpdateStoreInterval = 0
+MinimumFileAge = 0
+MaxUploadWait = 0
+
+__E
+}
+
+print CONFIG <<__E;
+
+# Files above this size (in bytes) are tracked, and if they are renamed they will simply be
+# renamed on the server, rather than being uploaded again. (64k - 1)
+
+FileTrackingSizeThreshold = 65535
+
+
+# The daemon does "changes only" uploads for files above this size (in bytes).
+# Files less than it are uploaded whole without this extra processing.
+
+DiffingUploadSizeThreshold = 8192
+
+
+# The limit on how much time is spent diffing files. Most files shouldn't take very long,
+# but if you have really big files you can use this to limit the time spent diffing them.
+# * Reduce if you are having problems with processor usage.
+# * Increase if you have large files, and think the upload of changes is too large and want
+#   to spend more time searching for unchanged blocks.
+
+MaximumDiffingTime = 20
+
+
+# Uncomment this line to see exactly what the daemon is going when it's connected to the server.
+
+# ExtendedLogging = yes
+
+
+# Use this to temporarily stop bbackupd from syncronising or connecting to the store.
+# This specifies a program or script script which is run just before each sync, and ideally
+# the full path to the interpreter. It will be run as the same user bbackupd is running as,
+# usually root.
+# The script prints either "now" or a number to STDOUT (and a terminating newline, no quotes).
+# If the result was "now", then the sync will happen. If it's a number, then the script will
+# be asked again in that number of seconds.
+# For example, you could use this on a laptop to only backup when on a specific network.
+
+# SyncAllowScript = /path/to/intepreter/or/exe script-name parameters etc
+
+
+# Where the command socket is created in the filesystem.
+
+CommandSocket = /var/run/bbackupd.sock
+
+
+Server
+{
+	PidFile = /var/run/bbackupd.pid
+}
+
+# 
+# BackupLocations specifies which locations on disc should be backed up. Each
+# directory is in the format
+# 
+# 	name
+# 	{
+# 		Path = /path/of/directory
+# 		(optional exclude directives)
+# 	}
+# 
+# 'name' is derived from the Path by the config script, but should merely be
+# unique.
+# 
+# The exclude directives are of the form
+# 
+# 	[Exclude|AlwaysInclude][File|Dir][|sRegex] = regex or full pathname
+# 
+# (The regex suffix is shown as 'sRegex' to make File or Dir plural)
+#
+# For example:
+# 
+# 	ExcludeDir = /home/guest-user
+# 	ExcludeFilesRegex = *.(mp3|MP3)\$
+# 	AlwaysIncludeFile = /home/username/veryimportant.mp3
+# 
+# This excludes the directory /home/guest-user from the backup along with all mp3
+# files, except one MP3 file in particular.
+# 
+# In general, Exclude excludes a file or directory, unless the directory is
+# explicitly mentioned in a AlwaysInclude directive.
+# 
+# If a directive ends in Regex, then it is a regular expression rather than a
+# explicit full pathname. See
+# 
+# 	man 7 re_format
+# 
+# for the regex syntax on your platform.
+# 
+
+BackupLocations
+{
+__E
+
+# write the dirs to backup
+for my $d (@tobackup)
+{
+	$d =~ m/\A.(.+)\Z/;
+	my $n = $1;
+	$n =~ tr`/`-`;
+	
+	my $excludekeys = '';
+	if(substr($enc_key_file, 0, length($d)+1) eq $d.'/')
+	{
+		$excludekeys = "\t\tExcludeFile = $enc_key_file\n";
+		print <<__E;
+
+NOTE: Keys file has been explicitly excluded from the backup.
+
+__E
+	}
+	
+	print CONFIG <<__E
+	$n
+	{
+		Path = $d
+$excludekeys	}
+__E
+}
+
+print CONFIG "}\n\n";
+close CONFIG;
+
+# explain to the user what they need to do next
+my $daemon_args = ($config_file eq $default_config_location)?'':" $config_file";
+my $ctl_daemon_args = ($config_file eq $default_config_location)?'':" -c $config_file";
+
+print <<__E;
+
+===================================================================
+
+bbackupd basic configuration complete.
+
+What you need to do now...
+
+1) Make a backup of $enc_key_file
+   This should be a secure offsite backup.
+   Without it, you cannot restore backups. Everything else can
+   be replaced. But this cannot.
+   KEEP IT IN A SAFE PLACE, OTHERWISE YOUR BACKUPS ARE USELESS.
+
+2) Send $certificate_request
+   to the administrator of the backup server, and ask for it to
+   be signed.
+
+3) The administrator will send you two files. Install them as
+      $certificate
+      $ca_root_cert
+   after checking their authenticity.
+
+4) You may wish to read the configuration file
+      $config_file
+   and adjust as appropraite.
+   
+   There are some notes in it on excluding files you do not
+   wish to be backed up.
+
+5) Review the script
+      $notify_script
+   and check that it will email the right person when the store
+   becomes full. This is important -- when the store is full, no
+   more files will be backed up. You want to know about this.
+
+6) Start the backup daemon with the command
+      /usr/local/bin/bbackupd$daemon_args
+   in /etc/rc.local, or your local equivalent.
+   Note that bbackupd must run as root.
+__E
+if($backup_mode eq 'snapshot')
+{
+	print <<__E;
+
+7) Set up a cron job to run whenever you want a snapshot of the
+   file system to be taken. Run the command
+      /usr/local/bin/bbackupctl -q$ctl_daemon_args sync
+__E
+}
+print <<__E;
+
+===================================================================
+
+Remember to make a secure, offsite backup of your backup keys,
+as described in step 1 above. If you do not, you have no backups.
+
+__E
+
diff --git a/bin/bbackupd/bbackupd.cpp b/bin/bbackupd/bbackupd.cpp
new file mode 100755
index 00000000..ca843105
--- /dev/null
+++ b/bin/bbackupd/bbackupd.cpp
@@ -0,0 +1,26 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    bbackupd.cpp
+//		Purpose: main file for backup daemon
+//		Created: 2003/10/11
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupDaemon.h"
+#include "MainHelper.h"
+#include "BoxPortsAndFiles.h"
+
+#include "MemLeakFindOn.h"
+
+int main(int argc, const char *argv[])
+{
+	MAINHELPER_START
+
+	BackupDaemon daemon;
+	return daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv);
+	
+	MAINHELPER_END
+}
+
diff --git a/bin/bbackupobjdump/bbackupobjdump.cpp b/bin/bbackupobjdump/bbackupobjdump.cpp
new file mode 100644
index 00000000..74fff510
--- /dev/null
+++ b/bin/bbackupobjdump/bbackupobjdump.cpp
@@ -0,0 +1,82 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    bbackupobjdump.cpp
+//		Purpose: Dump contents of backup objects
+//		Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "MainHelper.h"
+#include "FileStream.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    int main(int, const char *[])
+//		Purpose: Main fn for bbackupobjdump
+//		Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+int main(int argc, const char *argv[])
+{
+	MAINHELPER_START
+
+	if(argc != 2)
+	{
+		::printf("Input file not specified.\nUsage: bbackupobjdump <input file>\n");
+		return 1;
+	}
+
+	// Open file
+	FileStream file(argv[1]);
+	
+	// Read magic number
+	uint32_t signature;
+	if(file.Read(&signature, sizeof(signature)) != sizeof(signature))
+	{
+		// Too short, can't read signature from it
+		return false;
+	}
+	// Seek back to beginning
+	file.Seek(0, IOStream::SeekType_Absolute);
+	
+	// Then... check depending on the type
+	switch(ntohl(signature))
+	{
+	case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+	case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+#endif
+		BackupStoreFile::DumpFile(stdout, false, file);
+		break;
+
+	case OBJECTMAGIC_DIR_MAGIC_VALUE:
+		{
+			BackupStoreDirectory dir;
+			dir.ReadFromStream(file, IOStream::TimeOutInfinite);
+			dir.Dump(stdout, false);
+			if(dir.CheckAndFix())
+			{
+				::printf("Directory didn't pass checking\n");
+			}
+		}
+		break;
+
+	default:
+		::printf("File does not appear to be a valid box backup object.\n");
+		break;
+	}
+
+	MAINHELPER_END
+}
+
diff --git a/bin/bbackupquery/BackupQueries.cpp b/bin/bbackupquery/BackupQueries.cpp
new file mode 100755
index 00000000..0d08f1eb
--- /dev/null
+++ b/bin/bbackupquery/BackupQueries.cpp
@@ -0,0 +1,1700 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupQueries.cpp
+//		Purpose: Perform various queries on the backup store server.
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+
+#include <set>
+
+#include "BackupQueries.h"
+#include "Utils.h"
+#include "Configuration.h"
+#include "autogen_BackupProtocolClient.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreDirectory.h"
+#include "IOStream.h"
+#include "BoxTimeToText.h"
+#include "FileStream.h"
+#include "BackupStoreFile.h"
+#include "TemporaryDirectory.h"
+#include "FileModificationTime.h"
+#include "BackupClientFileAttributes.h"
+#include "CommonException.h"
+#include "BackupClientRestore.h"
+#include "BackupStoreException.h"
+#include "ExcludeList.h"
+#include "BackupClientMakeExcludeList.h"
+
+#include "MemLeakFindOn.h"
+
+#define COMPARE_RETURN_SAME			1
+#define COMPARE_RETURN_DIFFERENT	2
+#define COMPARE_RETURN_ERROR		3
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::BackupQueries()
+//		Purpose: Constructor
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+BackupQueries::BackupQueries(BackupProtocolClient &rConnection, const Configuration &rConfiguration)
+	: mrConnection(rConnection),
+	  mrConfiguration(rConfiguration),
+	  mQuitNow(false),
+	  mRunningAsRoot(false),
+	  mWarnedAboutOwnerAttributes(false),
+	  mReturnCode(0)		// default return code
+{
+	mRunningAsRoot = (::geteuid() == 0);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::~BackupQueries()
+//		Purpose: Destructor
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+BackupQueries::~BackupQueries()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::DoCommand(const char *)
+//		Purpose: Perform a command
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+void BackupQueries::DoCommand(const char *Command)
+{
+	// is the command a shell command?
+	if(Command[0] == 's' && Command[1] == 'h' && Command[2] == ' ' && Command[3] != '\0')
+	{
+		// Yes, run shell command
+		::system(Command + 3);
+		return;
+	}
+
+	// split command into components
+	std::vector<std::string> cmdElements;
+	std::string options;
+	{
+		const char *c = Command;
+		bool inQuoted = false;
+		bool inOptions = false;
+		
+		std::string s;
+		while(*c != 0)
+		{
+			// Terminating char?
+			if(*c == ((inQuoted)?'"':' '))
+			{
+				if(!s.empty()) cmdElements.push_back(s);
+				s.resize(0);
+				inQuoted = false;
+				inOptions = false;
+			}
+			else
+			{
+				// No. Start of quoted parameter?
+				if(s.empty() && *c == '"')
+				{
+					inQuoted = true;
+				}
+				// Start of options?
+				else if(s.empty() && *c == '-')
+				{
+					inOptions = true;
+				}
+				else
+				{
+					if(inOptions)
+					{
+						// Option char
+						options += *c;
+					}
+					else
+					{
+						// Normal string char
+						s += *c;
+					}
+				}
+			}
+		
+			++c;
+		}
+		if(!s.empty()) cmdElements.push_back(s);
+	}
+	
+	// Check...
+	if(cmdElements.size() < 1)
+	{
+		// blank command
+		return;
+	}
+	
+	// Data about commands
+	static const char *commandNames[] = {"quit", "exit", "list",	 "pwd", "cd", "lcd",	"sh", "getobject", "get", "compare", "restore", "help", "usage", "undelete", 0};
+	static const char *validOptions[] = {"",	 "",	 "rodIFtsh", "",	   "od", "",	"",	  "",		   "i",   "alcqE",   "dri",     "",     "",      "",		 0};
+	#define COMMAND_Quit		0
+	#define COMMAND_Exit		1
+	#define COMMAND_List		2
+	#define COMMAND_pwd			3
+	#define COMMAND_cd			4
+	#define COMMAND_lcd			5
+	#define COMMAND_sh			6
+	#define COMMAND_GetObject	7
+	#define COMMAND_Get			8
+	#define COMMAND_Compare		9
+	#define COMMAND_Restore		10
+	#define COMMAND_Help		11
+	#define COMMAND_Usage		12
+	#define COMMAND_Undelete	13
+	static const char *alias[] = {"ls",			0};
+	static const int aliasIs[] = {COMMAND_List, 0};
+	
+	// Work out which command it is...
+	int cmd = 0;
+	while(commandNames[cmd] != 0 && ::strcmp(cmdElements[0].c_str(), commandNames[cmd]) != 0)
+	{
+		cmd++;
+	}
+	if(commandNames[cmd] == 0)
+	{
+		// Check for aliases
+		int a;
+		for(a = 0; alias[a] != 0; ++a)
+		{
+			if(::strcmp(cmdElements[0].c_str(), alias[a]) == 0)
+			{
+				// Found an alias
+				cmd = aliasIs[a];
+				break;
+			}
+		}
+	
+		// No such command
+		if(alias[a] == 0)
+		{
+			printf("Unrecognised command: %s\n", Command);
+			return;
+		}
+	}
+
+	// Arguments
+	std::vector<std::string> args(cmdElements.begin() + 1, cmdElements.end());
+
+	// Set up options
+	bool opts[256];
+	for(int o = 0; o < 256; ++o) opts[o] = false;
+	// BLOCK
+	{
+		// options
+		const char *c = options.c_str();
+		while(*c != 0)
+		{
+			// Valid option?
+			if(::strchr(validOptions[cmd], *c) == NULL)
+			{
+				printf("Invalid option '%c' for command %s\n", *c, commandNames[cmd]);
+				return;
+			}
+			opts[(int)*c] = true;
+			++c;
+		}
+	}
+
+	if(cmd != COMMAND_Quit && cmd != COMMAND_Exit)
+	{
+		// If not a quit command, set the return code to zero
+		SetReturnCode(0);
+	}
+
+	// Handle command
+	switch(cmd)
+	{
+	case COMMAND_Quit:
+	case COMMAND_Exit:
+		mQuitNow = true;
+		break;
+		
+	case COMMAND_List:
+		CommandList(args, opts);
+		break;
+		
+	case COMMAND_pwd:
+		{
+			// Simple implementation, so do it here
+			std::string dir(GetCurrentDirectoryName());
+			printf("%s (%08llx)\n", dir.c_str(), GetCurrentDirectoryID());
+		}
+		break;
+
+	case COMMAND_cd:
+		CommandChangeDir(args, opts);
+		break;
+		
+	case COMMAND_lcd:
+		CommandChangeLocalDir(args);
+		break;
+		
+	case COMMAND_sh:
+		printf("The command to run must be specified as an argument.\n");
+		break;
+		
+	case COMMAND_GetObject:
+		CommandGetObject(args, opts);
+		break;
+		
+	case COMMAND_Get:
+		CommandGet(args, opts);
+		break;
+		
+	case COMMAND_Compare:
+		CommandCompare(args, opts);
+		break;
+		
+	case COMMAND_Restore:
+		CommandRestore(args, opts);
+		break;
+		
+	case COMMAND_Usage:
+		CommandUsage();
+		break;
+		
+	case COMMAND_Help:
+		CommandHelp(args);
+		break;
+
+	case COMMAND_Undelete:
+		CommandUndelete(args, opts);
+		break;
+		
+	default:
+		break;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandList(const std::vector<std::string> &, const bool *)
+//		Purpose: List directories (optionally recursive)
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandList(const std::vector<std::string> &args, const bool *opts)
+{
+	#define LIST_OPTION_RECURSIVE		'r'
+	#define LIST_OPTION_ALLOWOLD		'o'
+	#define LIST_OPTION_ALLOWDELETED	'd'
+	#define LIST_OPTION_NOOBJECTID		'I'
+	#define LIST_OPTION_NOFLAGS			'F'
+	#define LIST_OPTION_TIMES			't'
+	#define LIST_OPTION_SIZEINBLOCKS	's'
+	#define LIST_OPTION_DISPLAY_HASH	'h'
+
+	// default to using the current directory
+	int64_t rootDir = GetCurrentDirectoryID();
+
+	// name of base directory
+	std::string listRoot;	// blank
+
+	// Got a directory in the arguments?
+	if(args.size() > 0)
+	{
+		// Attempt to find the directory
+		rootDir = FindDirectoryObjectID(args[0], opts[LIST_OPTION_ALLOWOLD], opts[LIST_OPTION_ALLOWDELETED]);
+		if(rootDir == 0)
+		{
+			printf("Directory %s not found on store\n", args[0].c_str());
+			return;
+		}
+	}
+	
+	// List it
+	List(rootDir, listRoot, opts, true /* first level to list */);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandList2(int64_t, const std::string &, const bool *)
+//		Purpose: Do the actual listing of directories and files
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool *opts, bool FirstLevel)
+{
+	// Generate exclude flags
+	int16_t excludeFlags = BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING;
+	if(!opts[LIST_OPTION_ALLOWOLD]) excludeFlags |= BackupProtocolClientListDirectory::Flags_OldVersion;
+	if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolClientListDirectory::Flags_Deleted;
+
+	// Do communication
+	mrConnection.QueryListDirectory(
+			DirID,
+			BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,	// both files and directories
+			excludeFlags,
+			true /* want attributes */);
+
+	// Retrieve the directory from the stream following
+	BackupStoreDirectory dir;
+	std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream());
+	dir.ReadFromStream(*dirstream, mrConnection.GetTimeout());
+
+	// Then... display everything
+	BackupStoreDirectory::Iterator i(dir);
+	BackupStoreDirectory::Entry *en = 0;
+	while((en = i.Next()) != 0)
+	{
+		// Display this entry
+		BackupStoreFilenameClear clear(en->GetName());
+		std::string line;
+		
+		// Object ID?
+		if(!opts[LIST_OPTION_NOOBJECTID])
+		{
+			// add object ID to line
+			char oid[32];
+			sprintf(oid, "%08llx ", en->GetObjectID());
+			line += oid;
+		}
+		
+		// Flags?
+		if(!opts[LIST_OPTION_NOFLAGS])
+		{
+			static const char *flags = BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES;
+			char displayflags[16];
+			// make sure f is big enough
+			ASSERT(sizeof(displayflags) >= sizeof(BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES) + 3);
+			// Insert flags
+			char *f = displayflags;
+			const char *t = flags;
+			int16_t en_flags = en->GetFlags();
+			while(*t != 0)
+			{
+				*f = ((en_flags&1) == 0)?'-':*t;
+				en_flags >>= 1;
+				f++;
+				t++;
+			}
+			// attributes flags
+			*(f++) = (en->HasAttributes())?'a':'-';
+			// terminate
+			*(f++) = ' ';
+			*(f++) = '\0';
+			line += displayflags;
+			if(en_flags != 0)
+			{
+				line += "[ERROR: Entry has additional flags set] ";
+			}
+		}
+		
+		if(opts[LIST_OPTION_TIMES])
+		{
+			// Show times...
+			line += BoxTimeToISO8601String(en->GetModificationTime());
+			line += ' ';
+		}
+		
+		if(opts[LIST_OPTION_DISPLAY_HASH])
+		{
+			char hash[64];
+			::sprintf(hash, "%016llx ", en->GetAttributesHash());
+			line += hash;
+		}
+		
+		if(opts[LIST_OPTION_SIZEINBLOCKS])
+		{
+			char num[32];
+			sprintf(num, "%05lld ", en->GetSizeInBlocks());
+			line += num;
+		}
+		
+		// add name
+		if(!FirstLevel)
+		{
+			line += rListRoot;
+			line += '/';
+		}
+		line += clear.GetClearFilename().c_str();
+		
+		if(!en->GetName().IsEncrypted())
+		{
+			line += "[FILENAME NOT ENCRYPTED]";
+		}
+		
+		// print line
+		printf("%s\n", line.c_str());
+		
+		// Directory?
+		if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0)
+		{
+			// Recurse?
+			if(opts[LIST_OPTION_RECURSIVE])
+			{
+				std::string subroot(rListRoot);
+				if(!FirstLevel) subroot += '/';
+				subroot += clear.GetClearFilename();
+				List(en->GetObjectID(), subroot, opts, false /* not the first level to list */);
+			}
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::FindDirectoryObjectID(const std::string &)
+//		Purpose: Find the object ID of a directory on the store, or return 0 for not found.
+//				 If pStack != 0, the object is set to the stack of directories.
+//				 Will start from the current directory stack.
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, bool AllowOldVersion,
+			bool AllowDeletedDirs, std::vector<std::pair<std::string, int64_t> > *pStack)
+{
+	// Split up string into elements
+	std::vector<std::string> dirElements;
+	SplitString(rDirName, DIRECTORY_SEPARATOR_ASCHAR, dirElements);
+
+	// Start from current stack, or root, whichever is required
+	std::vector<std::pair<std::string, int64_t> > stack;
+	int64_t dirID = BackupProtocolClientListDirectory::RootDirectory;
+	if(rDirName.size() > 0 && rDirName[0] == '/')
+	{
+		// Root, do nothing
+	}
+	else
+	{
+		// Copy existing stack
+		stack = mDirStack;
+		if(stack.size() > 0)
+		{
+			dirID = stack[stack.size() - 1].second;
+		}
+	}
+
+	// Generate exclude flags
+	int16_t excludeFlags = BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING;
+	if(!AllowOldVersion) excludeFlags |= BackupProtocolClientListDirectory::Flags_OldVersion;
+	if(!AllowDeletedDirs) excludeFlags |= BackupProtocolClientListDirectory::Flags_Deleted;
+
+	// Read directories
+	for(unsigned int e = 0; e < dirElements.size(); ++e)
+	{
+		if(dirElements[e].size() > 0)
+		{
+			if(dirElements[e] == ".")
+			{
+				// Ignore.
+			}
+			else if(dirElements[e] == "..")
+			{
+				// Up one!
+				if(stack.size() > 0)
+				{
+					// Remove top element
+					stack.pop_back();
+					
+					// New dir ID
+					dirID = (stack.size() > 0)?(stack[stack.size() - 1].second):BackupProtocolClientListDirectory::RootDirectory;
+				}
+				else
+				{	
+					// At root anyway
+					dirID = BackupProtocolClientListDirectory::RootDirectory;
+				}
+			}
+			else
+			{
+				// Not blank element. Read current directory.
+				std::auto_ptr<BackupProtocolClientSuccess> dirreply(mrConnection.QueryListDirectory(
+						dirID,
+						BackupProtocolClientListDirectory::Flags_Dir,	// just directories
+						excludeFlags,
+						true /* want attributes */));
+
+				// Retrieve the directory from the stream following
+				BackupStoreDirectory dir;
+				std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream());
+				dir.ReadFromStream(*dirstream, mrConnection.GetTimeout());
+
+				// Then... find the directory within it
+				BackupStoreDirectory::Iterator i(dir);
+				BackupStoreFilenameClear dirname(dirElements[e]);
+				BackupStoreDirectory::Entry *en = i.FindMatchingClearName(dirname);
+				if(en == 0)
+				{
+					// Not found
+					return 0;
+				}
+				
+				// Object ID for next round of searching
+				dirID = en->GetObjectID();
+
+				// Push onto stack
+				stack.push_back(std::pair<std::string, int64_t>(dirElements[e], dirID));
+			}
+		}
+	}
+	
+	// If required, copy the new stack to the caller
+	if(pStack)
+	{
+		*pStack = stack;
+	}
+
+	return dirID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::GetCurrentDirectoryID()
+//		Purpose: Returns the ID of the current directory
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+int64_t BackupQueries::GetCurrentDirectoryID()
+{
+	// Special case for root
+	if(mDirStack.size() == 0)
+	{
+		return BackupProtocolClientListDirectory::RootDirectory;
+	}
+	
+	// Otherwise, get from the last entry on the stack
+	return mDirStack[mDirStack.size() - 1].second;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::GetCurrentDirectoryName()
+//		Purpose: Gets the name of the current directory
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+std::string BackupQueries::GetCurrentDirectoryName()
+{
+	// Special case for root
+	if(mDirStack.size() == 0)
+	{
+		return std::string("/");
+	}
+
+	// Build path
+	std::string r;
+	for(unsigned int l = 0; l < mDirStack.size(); ++l)
+	{
+		r += "/";
+		r += mDirStack[l].first;
+	}
+	
+	return r;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandChangeDir(const std::vector<std::string> &)
+//		Purpose: Change directory command
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandChangeDir(const std::vector<std::string> &args, const bool *opts)
+{
+	if(args.size() != 1 || args[0].size() == 0)
+	{
+		printf("Incorrect usage.\ncd [-o] [-d] <directory>\n");
+		return;
+	}
+	
+	std::vector<std::pair<std::string, int64_t> > newStack;
+	int64_t id = FindDirectoryObjectID(args[0], opts['o'], opts['d'], &newStack);
+	
+	if(id == 0)
+	{
+		printf("Directory '%s' not found\n", args[0].c_str());
+		return;
+	}
+	
+	// Store new stack
+	mDirStack = newStack;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &)
+//		Purpose: Change local directory command
+//		Created: 2003/10/11
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &args)
+{
+	if(args.size() != 1 || args[0].size() == 0)
+	{
+		printf("Incorrect usage.\nlcd <local-directory>\n");
+		return;
+	}
+	
+	// Try changing directory
+	if(::chdir(args[0].c_str()) != 0)
+	{
+		printf((errno == ENOENT || errno == ENOTDIR)?"Directory '%s' does not exist\n":"Error changing dir to '%s'\n",
+			args[0].c_str());
+		return;
+	}
+	
+	// Report current dir
+	char wd[PATH_MAX];
+	if(::getcwd(wd, PATH_MAX) == 0)
+	{
+		printf("Error getting current directory\n");
+		return;
+	}
+	
+	printf("Local current directory is now '%s'\n", wd);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandGetObject(const std::vector<std::string> &, const bool *)
+//		Purpose: Gets an object without any translation.
+//		Created: 2003/10/11
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const bool *opts)
+{
+	// Check args
+	if(args.size() != 2)
+	{
+		printf("Incorrect usage.\ngetobject <object-id> <local-filename>\n");
+		return;
+	}
+	
+	int64_t id = ::strtoll(args[0].c_str(), 0, 16);
+	if(id == LLONG_MIN || id == LLONG_MAX || id == 0)
+	{
+		printf("Not a valid object ID (specified in hex)\n");
+		return;
+	}
+	
+	// Does file exist?
+	struct stat st;
+	if(::stat(args[1].c_str(), &st) == 0 || errno != ENOENT)
+	{
+		printf("The local file %s already exists\n", args[1].c_str());
+		return;
+	}
+	
+	// Open file
+	FileStream out(args[1].c_str(), O_WRONLY | O_CREAT | O_EXCL);
+	
+	// Request that object
+	try
+	{
+		// Request object
+		std::auto_ptr<BackupProtocolClientSuccess> getobj(mrConnection.QueryGetObject(id));
+		if(getobj->GetObjectID() != BackupProtocolClientGetObject::NoObject)
+		{
+			// Stream that object out to the file
+			std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream());
+			objectStream->CopyStreamTo(out);
+			
+			printf("Object ID %08llx fetched successfully.\n", id);
+		}
+		else
+		{
+			printf("Object does not exist on store.\n");
+			::unlink(args[1].c_str());
+		}
+	}
+	catch(...)
+	{
+		::unlink(args[1].c_str());
+		printf("Error occured fetching object.\n");
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandGet(const std::vector<std::string> &, const bool *)
+//		Purpose: Command to get a file from the store
+//		Created: 2003/10/12
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandGet(const std::vector<std::string> &args, const bool *opts)
+{
+	// At least one argument?
+	// Check args
+	if(args.size() < 1 || (opts['i'] && args.size() != 2) || args.size() > 2)
+	{
+		printf("Incorrect usage.\ngetobject <object-id> <local-filename>\n or get -i <object-id> <local-filename>\n");
+		return;
+	}
+
+	// Find object ID somehow
+	int64_t id;
+	std::string localName;
+	// BLOCK
+	{
+		// Need to look it up in the current directory
+		mrConnection.QueryListDirectory(
+				GetCurrentDirectoryID(),
+				BackupProtocolClientListDirectory::Flags_File,	// just files
+				(opts['i'])?(BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING):(BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted), // only current versions
+				false /* don't want attributes */);
+
+		// Retrieve the directory from the stream following
+		BackupStoreDirectory dir;
+		std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream());
+		dir.ReadFromStream(*dirstream, mrConnection.GetTimeout());
+
+		if(opts['i'])
+		{
+			// Specified as ID. 
+			id = ::strtoll(args[0].c_str(), 0, 16);
+			if(id == LLONG_MIN || id == LLONG_MAX || id == 0)
+			{
+				printf("Not a valid object ID (specified in hex)\n");
+				return;
+			}
+			
+			// Check that the item is actually in the directory
+			if(dir.FindEntryByID(id) == 0)
+			{
+				printf("ID '%08llx' not found in current directory on store.\n(You can only download objects by ID from the current directory.)\n", id);
+				return;
+			}
+			
+			// Must have a local name in the arguments (check at beginning of function ensures this)
+			localName = args[1];
+		}
+		else
+		{				
+			// Specified by name, find the object in the directory to get the ID
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreFilenameClear fn(args[0]);
+			BackupStoreDirectory::Entry *en = i.FindMatchingClearName(fn);
+			
+			if(en == 0)
+			{
+				printf("Filename '%s' not found in current directory on store.\n(Subdirectories in path not searched.)\n", args[0].c_str());
+				return;
+			}
+			
+			id = en->GetObjectID();
+			
+			// Local name is the last argument, which is either the looked up filename, or
+			// a filename specified by the user.
+			localName = args[args.size() - 1];
+		}
+	}
+	
+	// Does local file already exist? (don't want to overwrite)
+	struct stat st;
+	if(::stat(localName.c_str(), &st) == 0 || errno != ENOENT)
+	{
+		printf("The local file %s already exists, will not overwrite it.\n", localName.c_str());
+		return;
+	}
+	
+	// Request it from the store
+	try
+	{
+		// Request object
+		mrConnection.QueryGetFile(GetCurrentDirectoryID(), id);
+
+		// Stream containing encoded file
+		std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream());
+		
+		// Decode it
+		BackupStoreFile::DecodeFile(*objectStream, localName.c_str(), mrConnection.GetTimeout());
+
+		// Done.
+		printf("Object ID %08llx fetched sucessfully.\n", id);
+	}
+	catch(...)
+	{
+		::unlink(args[1].c_str());
+		printf("Error occured fetching file.\n");
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CompareParams::CompareParams()
+//		Purpose: Constructor
+//		Created: 29/1/04
+//
+// --------------------------------------------------------------------------
+BackupQueries::CompareParams::CompareParams()
+	: mQuickCompare(false),
+	  mIgnoreExcludes(false),
+	  mDifferences(0),
+	  mDifferencesExplainedByModTime(0),
+	  mExcludedDirs(0),
+	  mExcludedFiles(0),
+	  mpExcludeFiles(0),
+	  mpExcludeDirs(0),
+	  mLatestFileUploadTime(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CompareParams::~CompareParams()
+//		Purpose: Destructor
+//		Created: 29/1/04
+//
+// --------------------------------------------------------------------------
+BackupQueries::CompareParams::~CompareParams()
+{
+	DeleteExcludeLists();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CompareParams::DeleteExcludeLists()
+//		Purpose: Delete the include lists contained
+//		Created: 29/1/04
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CompareParams::DeleteExcludeLists()
+{
+	if(mpExcludeFiles != 0)
+	{
+		delete mpExcludeFiles;
+		mpExcludeFiles = 0;
+	}
+	if(mpExcludeDirs != 0)
+	{
+		delete mpExcludeDirs;
+		mpExcludeDirs = 0;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandCompare(const std::vector<std::string> &, const bool *)
+//		Purpose: Command to compare data on the store with local data
+//		Created: 2003/10/12
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandCompare(const std::vector<std::string> &args, const bool *opts)
+{
+	// Parameters, including count of differences
+	BackupQueries::CompareParams params;
+	params.mQuickCompare = opts['q'];
+	params.mIgnoreExcludes = opts['E'];
+	
+	// Try and work out the time before which all files should be on the server
+	{
+		std::string syncTimeFilename(mrConfiguration.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR);
+		syncTimeFilename += "last_sync_start";
+		// Stat it to get file time
+		struct stat st;
+		if(::stat(syncTimeFilename.c_str(), &st) == 0)
+		{
+			// Files modified after this time shouldn't be on the server, so report errors slightly differently
+			params.mLatestFileUploadTime = FileModificationTime(st)
+					- SecondsToBoxTime((uint32_t)mrConfiguration.GetKeyValueInt("MinimumFileAge"));
+		}
+		else
+		{
+			printf("Warning: couldn't determine the time of the last syncronisation -- checks not performed.\n");
+		}
+	}
+
+	// Quick compare?
+	if(params.mQuickCompare)
+	{
+		printf("WARNING: Quick compare used -- file attributes are not checked.\n");
+	}
+	
+	if(!opts['l'] && opts['a'] && args.size() == 0)
+	{
+		// Compare all locations
+		const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations"));
+		for(std::list<std::pair<std::string, Configuration> >::const_iterator i = locations.mSubConfigurations.begin();
+				i != locations.mSubConfigurations.end(); ++i)
+		{
+			CompareLocation(i->first, params);
+		}
+	}
+	else if(opts['l'] && !opts['a'] && args.size() == 1)
+	{
+		// Compare one location
+		CompareLocation(args[0], params);
+	}
+	else if(!opts['l'] && !opts['a'] && args.size() == 2)
+	{
+		// Compare directory to directory
+		
+		// Can't be bothered to do all the hard work to work out which location it's on, and hence which exclude list
+		if(!params.mIgnoreExcludes)
+		{
+			printf("Cannot use excludes on directory to directory comparison -- use -E flag to specify ignored excludes\n");
+			return;
+		}
+		else
+		{
+			// Do compare
+			Compare(args[0], args[1], params);
+		}
+	}
+	else
+	{
+		printf("Incorrect usage.\ncompare -a\n or compare -l <location-name>\n or compare <store-dir-name> <local-dir-name>\n");
+		return;
+	}
+	
+	printf("\n[ %d (of %d) differences probably due to file modifications after the last upload ]\nDifferences: %d (%d dirs excluded, %d files excluded)\n",
+		params.mDifferencesExplainedByModTime, params.mDifferences, params.mDifferences, params.mExcludedDirs, params.mExcludedFiles);
+	
+	// Set return code?
+	if(opts['c'])
+	{
+		SetReturnCode((params.mDifferences == 0)?COMPARE_RETURN_SAME:COMPARE_RETURN_DIFFERENT);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CompareLocation(const std::string &, BackupQueries::CompareParams &)
+//		Purpose: Compare a location
+//		Created: 2003/10/13
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CompareLocation(const std::string &rLocation, BackupQueries::CompareParams &rParams)
+{
+	// Find the location's sub configuration
+	const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations"));
+	if(!locations.SubConfigurationExists(rLocation.c_str()))
+	{
+		printf("Location %s does not exist.\n", rLocation.c_str());
+		return;
+	}
+	const Configuration &loc(locations.GetSubConfiguration(rLocation.c_str()));
+	
+	try
+	{
+		// Generate the exclude lists
+		if(!rParams.mIgnoreExcludes)
+		{
+			rParams.mpExcludeFiles = BackupClientMakeExcludeList_Files(loc);
+			rParams.mpExcludeDirs = BackupClientMakeExcludeList_Dirs(loc);
+		}
+				
+		// Then get it compared
+		Compare(std::string("/") + rLocation, loc.GetKeyValue("Path"), rParams);
+	}
+	catch(...)
+	{
+		// Clean up
+		rParams.DeleteExcludeLists();
+		throw;
+	}
+	
+	// Delete exclude lists
+	rParams.DeleteExcludeLists();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::Compare(const std::string &, const std::string &, BackupQueries::CompareParams &)
+//		Purpose: Compare a store directory against a local directory
+//		Created: 2003/10/13
+//
+// --------------------------------------------------------------------------
+void BackupQueries::Compare(const std::string &rStoreDir, const std::string &rLocalDir, BackupQueries::CompareParams &rParams)
+{
+	// Get the directory ID of the directory -- only use current data
+	int64_t dirID = FindDirectoryObjectID(rStoreDir);
+	
+	// Found?
+	if(dirID == 0)
+	{
+		printf("Local directory '%s' exists, but server directory '%s' does not exist\n", rLocalDir.c_str(), rStoreDir.c_str());		
+		rParams.mDifferences ++;
+		return;
+	}
+	
+	// Go!
+	Compare(dirID, rStoreDir, rLocalDir, rParams);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::Compare(int64_t, const std::string &, BackupQueries::CompareParams &)
+//		Purpose: Compare a store directory against a local directory
+//		Created: 2003/10/13
+//
+// --------------------------------------------------------------------------
+void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const std::string &rLocalDir, BackupQueries::CompareParams &rParams)
+{
+	// Get info on the local directory
+	struct stat st;
+	if(::lstat(rLocalDir.c_str(), &st) != 0)
+	{
+		// What kind of error?
+		if(errno == ENOTDIR)
+		{
+			printf("Local object '%s' is a file, server object '%s' is a directory\n", rLocalDir.c_str(), rStoreDir.c_str());
+			rParams.mDifferences ++;
+		}
+		else if(errno == ENOENT)
+		{
+			printf("Local directory '%s' does not exist (compared to server directory '%s')\n", rLocalDir.c_str(), rStoreDir.c_str());
+		}
+		else
+		{
+			printf("ERROR: stat on local dir '%s'\n", rLocalDir.c_str());
+		}
+		return;
+	}
+
+	// Get the directory listing from the store
+	mrConnection.QueryListDirectory(
+			DirID,
+			BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,	// get everything
+			BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted,	// except for old versions and deleted files
+			true /* want attributes */);
+
+	// Retrieve the directory from the stream following
+	BackupStoreDirectory dir;
+	std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream());
+	dir.ReadFromStream(*dirstream, mrConnection.GetTimeout());
+
+	// Test out the attributes
+	if(!dir.HasAttributes())
+	{
+		printf("Store directory '%s' doesn't have attributes.\n", rStoreDir.c_str());
+	}
+	else
+	{
+		// Fetch the attributes
+		const StreamableMemBlock &storeAttr(dir.GetAttributes());
+		BackupClientFileAttributes attr(storeAttr);
+
+		// Get attributes of local directory
+		BackupClientFileAttributes localAttr;
+		localAttr.ReadAttributes(rLocalDir.c_str(), true /* directories have zero mod times */);
+
+		if(!(attr.Compare(localAttr, true, true /* ignore modification times */)))
+		{
+			printf("Local directory '%s' has different attributes to store directory '%s'.\n",
+				rLocalDir.c_str(), rStoreDir.c_str());
+			rParams.mDifferences ++;
+		}
+	}
+
+	// Open the local directory
+	DIR *dirhandle = ::opendir(rLocalDir.c_str());
+	if(dirhandle == 0)
+	{
+		printf("ERROR: opendir on local dir '%s'\n", rLocalDir.c_str());
+		return;
+	}
+	try
+	{
+		// Read the files and directories into sets
+		std::set<std::string> localFiles;
+		std::set<std::string> localDirs;
+		struct dirent *localDirEn = 0;
+		while((localDirEn = readdir(dirhandle)) != 0)
+		{
+			// Not . and ..!
+			if(localDirEn->d_name[0] == '.' && 
+				(localDirEn->d_name[1] == '\0' || (localDirEn->d_name[1] == '.' && localDirEn->d_name[2] == '\0')))
+			{
+				// ignore, it's . or ..
+				continue;
+			}
+
+#ifdef PLATFORM_dirent_BROKEN_d_type
+			std::string fn(rLocalDir);
+			fn += '/';
+			fn += localDirEn->d_name;
+			struct stat st;
+			if(::lstat(fn.c_str(), &st) != 0)
+			{
+			    THROW_EXCEPTION(CommonException, OSFileError)
+			}
+			
+			// Entry -- file or dir?
+			if(S_ISREG(st.st_mode) || S_ISLNK(st.st_mode))
+			{	
+			    // File or symbolic link
+			    localFiles.insert(std::string(localDirEn->d_name));
+			}
+			else if(S_ISDIR(st.st_mode))
+			{
+			    // Directory
+			    localDirs.insert(std::string(localDirEn->d_name));
+			}			
+#else
+			// Entry -- file or dir?
+			if(localDirEn->d_type == DT_REG || localDirEn->d_type == DT_LNK)
+			{
+				// File or symbolic link
+				localFiles.insert(std::string(localDirEn->d_name));
+			}
+			else if(localDirEn->d_type == DT_DIR)
+			{
+				// Directory
+				localDirs.insert(std::string(localDirEn->d_name));
+			}
+#endif // PLATFORM_dirent_BROKEN_d_type
+		}
+		// Close directory
+		if(::closedir(dirhandle) != 0)
+		{
+			printf("ERROR: closedir on local dir '%s'\n", rLocalDir.c_str());
+		}
+		dirhandle = 0;
+	
+		// Do the same for the store directories
+		std::set<std::pair<std::string, BackupStoreDirectory::Entry *> > storeFiles;
+		std::set<std::pair<std::string, BackupStoreDirectory::Entry *> > storeDirs;
+		
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *storeDirEn = 0;
+		while((storeDirEn = i.Next()) != 0)
+		{
+			// Decrypt filename
+			BackupStoreFilenameClear name(storeDirEn->GetName());
+		
+			// What is it?
+			if((storeDirEn->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == BackupStoreDirectory::Entry::Flags_File)
+			{
+				// File
+				storeFiles.insert(std::pair<std::string, BackupStoreDirectory::Entry *>(name.GetClearFilename(), storeDirEn));
+			}
+			else
+			{
+				// Dir
+				storeDirs.insert(std::pair<std::string, BackupStoreDirectory::Entry *>(name.GetClearFilename(), storeDirEn));
+			}
+		}
+		
+		// Now compare files.
+		for(std::set<std::pair<std::string, BackupStoreDirectory::Entry *> >::const_iterator i = storeFiles.begin(); i != storeFiles.end(); ++i)
+		{
+			// Does the file exist locally?
+			std::set<std::string>::const_iterator local(localFiles.find(i->first));
+			if(local == localFiles.end())
+			{
+				// Not found -- report
+				printf("Local file '%s/%s' does not exist, but store file '%s/%s' does.\n",
+					rLocalDir.c_str(), i->first.c_str(), rStoreDir.c_str(), i->first.c_str());
+				rParams.mDifferences ++;
+			}
+			else
+			{
+				try
+				{
+					// make local name of file for comparison
+					std::string localName(rLocalDir + DIRECTORY_SEPARATOR + i->first);
+
+					// Files the same flag?
+					bool equal = true;
+					
+					// File modified after last sync flag
+					bool modifiedAfterLastSync = false;
+						
+					if(rParams.mQuickCompare)
+					{
+						// Compare file -- fetch it
+						mrConnection.QueryGetBlockIndexByID(i->second->GetObjectID());
+
+						// Stream containing block index
+						std::auto_ptr<IOStream> blockIndexStream(mrConnection.ReceiveStream());
+						
+						// Compare
+						equal = BackupStoreFile::CompareFileContentsAgainstBlockIndex(localName.c_str(), *blockIndexStream, mrConnection.GetTimeout());
+					}
+					else
+					{
+						// Compare file -- fetch it
+						mrConnection.QueryGetFile(DirID, i->second->GetObjectID());
+	
+						// Stream containing encoded file
+						std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream());
+	
+						// Decode it
+						std::auto_ptr<BackupStoreFile::DecodedStream> fileOnServerStream;
+						// Got additional attibutes?
+						if(i->second->HasAttributes())
+						{
+							// Use these attributes
+							const StreamableMemBlock &storeAttr(i->second->GetAttributes());
+							BackupClientFileAttributes attr(storeAttr);
+							fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout(), &attr).release());
+						}
+						else
+						{
+							// Use attributes stored in file
+							fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout()).release());
+						}
+						
+						// Should always be something in the auto_ptr, it's how the interface is defined. But be paranoid.
+						if(!fileOnServerStream.get())
+						{
+							THROW_EXCEPTION(BackupStoreException, Internal)
+						}
+						
+						// Compare attributes
+						BackupClientFileAttributes localAttr;
+						box_time_t fileModTime = 0;
+						localAttr.ReadAttributes(localName.c_str(), false /* don't zero mod times */, &fileModTime);					
+						modifiedAfterLastSync = (fileModTime > rParams.mLatestFileUploadTime);
+						if(!localAttr.Compare(fileOnServerStream->GetAttributes(),
+								true /* ignore attr mod time */,
+								fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */))
+						{
+							printf("Local file '%s/%s' has different attributes to store file '%s/%s'.\n",
+								rLocalDir.c_str(), i->first.c_str(), rStoreDir.c_str(), i->first.c_str());						
+							rParams.mDifferences ++;
+							if(modifiedAfterLastSync)
+							{
+								rParams.mDifferencesExplainedByModTime ++;
+								printf("(the file above was modified after the last sync time -- might be reason for difference)\n");
+							}
+							else if(i->second->HasAttributes())
+							{
+								printf("(the file above has had new attributes applied)\n");
+							}
+						}
+	
+						// Compare contents, if it's a regular file not a link
+						// Remember, we MUST read the entire stream from the server.
+						if(!fileOnServerStream->IsSymLink())
+						{
+							// Open the local file
+							FileStream l(localName.c_str());
+							
+							// Size
+							IOStream::pos_type fileSizeLocal = l.BytesLeftToRead();
+							IOStream::pos_type fileSizeServer = 0;
+							
+							// Test the contents
+							char buf1[2048];
+							char buf2[2048];
+							while(fileOnServerStream->StreamDataLeft() && l.StreamDataLeft())
+							{
+								int size = fileOnServerStream->Read(buf1, sizeof(buf1), mrConnection.GetTimeout());
+								fileSizeServer += size;
+								
+								if(l.Read(buf2, size) != size
+										|| ::memcmp(buf1, buf2, size) != 0)
+								{
+									equal = false;
+									break;
+								}
+							}
+	
+							// Check read all the data from the server and file -- can't be equal if local and remote aren't the same length
+							// Can't use StreamDataLeft() test on file, because if it's the same size, it won't know
+							// it's EOF yet.
+							if(fileOnServerStream->StreamDataLeft() || fileSizeServer != fileSizeLocal)
+							{
+								equal = false;
+							}
+
+							// Must always read the entire decoded string, if it's not a symlink
+							if(fileOnServerStream->StreamDataLeft())
+							{
+								// Absorb all the data remaining
+								char buffer[2048];
+								while(fileOnServerStream->StreamDataLeft())
+								{
+									fileOnServerStream->Read(buffer, sizeof(buffer), mrConnection.GetTimeout());
+								}
+							}
+						}
+					}
+
+					// Report if not equal.
+					if(!equal)
+					{
+						printf("Local file '%s/%s' has different contents to store file '%s/%s'.\n",
+							rLocalDir.c_str(), i->first.c_str(), rStoreDir.c_str(), i->first.c_str());
+						rParams.mDifferences ++;
+						if(modifiedAfterLastSync)
+						{
+							rParams.mDifferencesExplainedByModTime ++;
+							printf("(the file above was modified after the last sync time -- might be reason for difference)\n");
+						}
+						else if(i->second->HasAttributes())
+						{
+							printf("(the file above has had new attributes applied)\n");
+						}
+					}
+				}
+				catch(BoxException &e)
+				{
+					printf("ERROR: (%d/%d) during file fetch and comparsion for '%s/%s'\n",
+						e.GetType(),
+						e.GetSubType(),
+						rStoreDir.c_str(), i->first.c_str());
+				}
+				catch(...)
+				{
+					printf("ERROR: (unknown) during file fetch and comparsion for '%s/%s'\n", rStoreDir.c_str(), i->first.c_str());
+				}
+
+				// Remove from set so that we know it's been compared
+				localFiles.erase(local);
+			}
+		}
+		
+		// Report any files which exist on the locally, but not on the store
+		for(std::set<std::string>::const_iterator i = localFiles.begin(); i != localFiles.end(); ++i)
+		{
+			std::string localName(rLocalDir + DIRECTORY_SEPARATOR + *i);
+			// Should this be ignored (ie is excluded)?
+			if(rParams.mpExcludeFiles == 0 || !(rParams.mpExcludeFiles->IsExcluded(localName)))
+			{
+				printf("Local file '%s/%s' exists, but store file '%s/%s' does not exist.\n",
+					rLocalDir.c_str(), (*i).c_str(), rStoreDir.c_str(), (*i).c_str());
+				rParams.mDifferences ++;
+				
+				// Check the file modification time
+				{
+					struct stat st;
+					if(::stat(localName.c_str(), &st) == 0)
+					{
+						if(FileModificationTime(st) > rParams.mLatestFileUploadTime)
+						{
+							rParams.mDifferencesExplainedByModTime ++;
+							printf("(the file above was modified after the last sync time -- might be reason for difference)\n");
+						}
+					}
+				}
+			}
+			else
+			{
+				rParams.mExcludedFiles ++;
+			}
+		}		
+		
+		// Finished with the files, clear the sets to reduce memory usage slightly
+		localFiles.clear();
+		storeFiles.clear();
+		
+		// Now do the directories, recusively to check subdirectories
+		for(std::set<std::pair<std::string, BackupStoreDirectory::Entry *> >::const_iterator i = storeDirs.begin(); i != storeDirs.end(); ++i)
+		{
+			// Does the directory exist locally?
+			std::set<std::string>::const_iterator local(localDirs.find(i->first));
+			if(local == localDirs.end())
+			{
+				// Not found -- report
+				printf("Local directory '%s/%s' does not exist, but store directory '%s/%s' does.\n",
+					rLocalDir.c_str(), i->first.c_str(), rStoreDir.c_str(), i->first.c_str());
+				rParams.mDifferences ++;
+			}
+			else
+			{
+				// Compare directory
+				Compare(i->second->GetObjectID(), rStoreDir + "/" + i->first, rLocalDir + DIRECTORY_SEPARATOR + i->first, rParams);
+				
+				// Remove from set so that we know it's been compared
+				localDirs.erase(local);
+			}
+		}
+		
+		// Report any files which exist on the locally, but not on the store
+		for(std::set<std::string>::const_iterator i = localDirs.begin(); i != localDirs.end(); ++i)
+		{
+			std::string localName(rLocalDir + DIRECTORY_SEPARATOR + *i);
+			// Should this be ignored (ie is excluded)?
+			if(rParams.mpExcludeDirs == 0 || !(rParams.mpExcludeDirs->IsExcluded(localName)))
+			{
+				printf("Local directory '%s/%s' exists, but store directory '%s/%s' does not exist.\n",
+					rLocalDir.c_str(), (*i).c_str(), rStoreDir.c_str(), (*i).c_str());
+				rParams.mDifferences ++;
+			}
+			else
+			{
+				rParams.mExcludedDirs ++;
+			}
+		}		
+		
+	}
+	catch(...)
+	{
+		if(dirhandle != 0)
+		{
+			::closedir(dirhandle);
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandRestore(const std::vector<std::string> &, const bool *)
+//		Purpose: Restore a directory
+//		Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandRestore(const std::vector<std::string> &args, const bool *opts)
+{
+	// Check arguments
+	if(args.size() != 2)
+	{
+		printf("Incorrect usage.\nrestore [-d] [-r] [-i] <directory-name> <local-directory-name>\n");
+		return;
+	}
+
+	// Restoring deleted things?
+	bool restoreDeleted = opts['d'];
+
+	// Get directory ID
+	int64_t dirID = 0;
+	if(opts['i'])
+	{
+		// Specified as ID. 
+		dirID = ::strtoll(args[0].c_str(), 0, 16);
+		if(dirID == LLONG_MIN || dirID == LLONG_MAX || dirID == 0)
+		{
+			printf("Not a valid object ID (specified in hex)\n");
+			return;
+		}
+	}
+	else
+	{
+		// Look up directory ID
+		dirID = FindDirectoryObjectID(args[0], false /* no old versions */, restoreDeleted /* find deleted dirs */);
+	}
+	
+	// Allowable?
+	if(dirID == 0)
+	{
+		printf("Directory %s not found on server\n", args[0].c_str());
+		return;
+	}
+	if(dirID == BackupProtocolClientListDirectory::RootDirectory)
+	{
+		printf("Cannot restore the root directory -- restore locations individually.\n");
+		return;
+	}
+	
+	// Go and restore...
+	switch(BackupClientRestore(mrConnection, dirID, args[1].c_str(), true /* print progress dots */, restoreDeleted, 
+		false /* don't undelete after restore! */, opts['r'] /* resume? */))
+	{
+	case Restore_Complete:
+		printf("Restore complete\n");
+		break;
+	
+	case Restore_ResumePossible:
+		printf("Resume possible -- repeat command with -r flag to resume\n");
+		break;
+	
+	case Restore_TargetExists:
+		printf("The target directory exists. You cannot restore over an existing directory.\n");
+		break;
+		
+	default:
+		printf("ERROR: Unknown restore result.\n");
+		break;
+	}
+}
+
+
+
+// These are autogenerated by a script.
+extern char *help_commands[];
+extern char *help_text[];
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandHelp(const std::vector<std::string> &args)
+//		Purpose: Display help on commands
+//		Created: 15/2/04
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandHelp(const std::vector<std::string> &args)
+{
+	if(args.size() == 0)
+	{
+		// Display a list of all commands
+		printf("Available commands are:\n");
+		for(int c = 0; help_commands[c] != 0; ++c)
+		{
+			printf("    %s\n", help_commands[c]);
+		}
+		printf("Type \"help <command>\" for more information on a command.\n\n");
+	}
+	else
+	{
+		// Display help on a particular command
+		int c;
+		for(c = 0; help_commands[c] != 0; ++c)
+		{
+			if(::strcmp(help_commands[c], args[0].c_str()) == 0)
+			{
+				// Found the command, print help
+				printf("\n%s\n", help_text[c]);
+				break;
+			}
+		}
+		if(help_commands[c] == 0)
+		{
+			printf("No help found for command '%s'\n", args[0].c_str());
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandUsage()
+//		Purpose: Display storage space used on server
+//		Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandUsage()
+{
+	// Request full details from the server
+	std::auto_ptr<BackupProtocolClientAccountUsage> usage(mrConnection.QueryGetAccountUsage());
+
+	// Display each entry in turn
+	int64_t hardLimit = usage->GetBlocksHardLimit();
+	int32_t blockSize = usage->GetBlockSize();
+	CommandUsageDisplayEntry("Used", usage->GetBlocksUsed(), hardLimit, blockSize);
+	CommandUsageDisplayEntry("Old files", usage->GetBlocksInOldFiles(), hardLimit, blockSize);
+	CommandUsageDisplayEntry("Deleted files", usage->GetBlocksInDeletedFiles(), hardLimit, blockSize);
+	CommandUsageDisplayEntry("Directories", usage->GetBlocksInDirectories(), hardLimit, blockSize);
+	CommandUsageDisplayEntry("Soft limit", usage->GetBlocksSoftLimit(), hardLimit, blockSize);
+	CommandUsageDisplayEntry("Hard limit", hardLimit, hardLimit, blockSize);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandUsageDisplayEntry(const char *, int64_t, int64_t, int32_t)
+//		Purpose: Display an entry in the usage table
+//		Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandUsageDisplayEntry(const char *Name, int64_t Size, int64_t HardLimit, int32_t BlockSize)
+{
+	// Calculate size in Mb
+	double mb = (((double)Size) * ((double)BlockSize)) / ((double)(1024*1024));
+	int64_t percent = (Size * 100) / HardLimit;
+
+	// Bar graph
+	char bar[41];
+	unsigned int b = (int)((Size * (sizeof(bar)-1)) / HardLimit);
+	if(b > sizeof(bar)-1) {b = sizeof(bar)-1;}
+	for(unsigned int l = 0; l < b; l++)
+	{
+		bar[l] = '*';
+	}
+	bar[b] = '\0';
+
+	// Print the entryj
+	::printf("%14s %10.1fMb %3d%% %s\n", Name, mb, (int32_t)percent, bar);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupQueries::CommandUndelete(const std::vector<std::string> &, const bool *)
+//		Purpose: Undelete a directory
+//		Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+void BackupQueries::CommandUndelete(const std::vector<std::string> &args, const bool *opts)
+{
+	// Check arguments
+	if(args.size() != 1)
+	{
+		printf("Incorrect usage.\nundelete <directory-name>\n");
+		return;
+	}
+
+	// Get directory ID
+	int64_t dirID = FindDirectoryObjectID(args[0], false /* no old versions */, true /* find deleted dirs */);
+	
+	// Allowable?
+	if(dirID == 0)
+	{
+		printf("Directory %s not found on server\n", args[0].c_str());
+		return;
+	}
+	if(dirID == BackupProtocolClientListDirectory::RootDirectory)
+	{
+		printf("Cannot restore the root directory -- restore locations individually.\n");
+		return;
+	}
+
+	// Undelete
+	mrConnection.QueryUndeleteDirectory(dirID);
+}
+
+
+
+
+
diff --git a/bin/bbackupquery/BackupQueries.h b/bin/bbackupquery/BackupQueries.h
new file mode 100755
index 00000000..e84de6ab
--- /dev/null
+++ b/bin/bbackupquery/BackupQueries.h
@@ -0,0 +1,101 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupQueries.h
+//		Purpose: Perform various queries on the backup store server.
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPQUERIES__H
+#define BACKUPQUERIES__H
+
+#include <vector>
+#include <string>
+
+#include "BoxTime.h"
+
+class BackupProtocolClient;
+class Configuration;
+class ExcludeList;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupQueries
+//		Purpose: Perform various queries on the backup store server.
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+class BackupQueries
+{
+public:
+	BackupQueries(BackupProtocolClient &rConnection, const Configuration &rConfiguration);
+	~BackupQueries();
+private:
+	BackupQueries(const BackupQueries &);
+public:
+
+	void DoCommand(const char *Command);
+
+	// Ready to stop?
+	bool Stop() {return mQuitNow;}
+	
+	// Return code?
+	int GetReturnCode() {return mReturnCode;}
+
+private:
+	// Commands
+	void CommandList(const std::vector<std::string> &args, const bool *opts);
+	void CommandChangeDir(const std::vector<std::string> &args, const bool *opts);
+	void CommandChangeLocalDir(const std::vector<std::string> &args);
+	void CommandGetObject(const std::vector<std::string> &args, const bool *opts);
+	void CommandGet(const std::vector<std::string> &args, const bool *opts);
+	void CommandCompare(const std::vector<std::string> &args, const bool *opts);
+	void CommandRestore(const std::vector<std::string> &args, const bool *opts);
+	void CommandUndelete(const std::vector<std::string> &args, const bool *opts);
+	void CommandUsage();
+	void CommandUsageDisplayEntry(const char *Name, int64_t Size, int64_t HardLimit, int32_t BlockSize);
+	void CommandHelp(const std::vector<std::string> &args);
+
+	// Implementations
+	void List(int64_t DirID, const std::string &rListRoot, const bool *opts, bool FirstLevel);
+	class CompareParams
+	{
+	public:
+		CompareParams();
+		~CompareParams();
+		void DeleteExcludeLists();
+		bool mQuickCompare;
+		bool mIgnoreExcludes;
+		int mDifferences;
+		int mDifferencesExplainedByModTime;
+		int mExcludedDirs;
+		int mExcludedFiles;
+		const ExcludeList *mpExcludeFiles;
+		const ExcludeList *mpExcludeDirs;
+		box_time_t mLatestFileUploadTime;
+	};
+	void CompareLocation(const std::string &rLocation, CompareParams &rParams);
+	void Compare(const std::string &rStoreDir, const std::string &rLocalDir, CompareParams &rParams);
+	void Compare(int64_t DirID, const std::string &rStoreDir, const std::string &rLocalDir, CompareParams &rParams);
+
+	// Utility functions
+	int64_t FindDirectoryObjectID(const std::string &rDirName, bool AllowOldVersion = false,
+		bool AllowDeletedDirs = false, std::vector<std::pair<std::string, int64_t> > *pStack = 0);
+	int64_t GetCurrentDirectoryID();
+	std::string GetCurrentDirectoryName();
+	void SetReturnCode(int code) {mReturnCode = code;}
+
+private:
+	BackupProtocolClient &mrConnection;
+	const Configuration &mrConfiguration;
+	bool mQuitNow;
+	std::vector<std::pair<std::string, int64_t> > mDirStack;
+	bool mRunningAsRoot;
+	bool mWarnedAboutOwnerAttributes;
+	int mReturnCode;
+};
+
+#endif // BACKUPQUERIES__H
+
diff --git a/bin/bbackupquery/Makefile.extra b/bin/bbackupquery/Makefile.extra
new file mode 100755
index 00000000..633ec0fc
--- /dev/null
+++ b/bin/bbackupquery/Makefile.extra
@@ -0,0 +1,6 @@
+
+# AUTOGEN SEEDING
+autogen_Documentation.cpp:	makedocumentation.pl documentation.txt
+	perl makedocumentation.pl
+
+
diff --git a/bin/bbackupquery/bbackupquery.cpp b/bin/bbackupquery/bbackupquery.cpp
new file mode 100755
index 00000000..aea0faa8
--- /dev/null
+++ b/bin/bbackupquery/bbackupquery.cpp
@@ -0,0 +1,243 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    bbackupquery.cpp
+//		Purpose: Backup query utility
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/types.h>
+#ifndef PLATFORM_READLINE_NOT_SUPPORTED
+	#ifdef PLATFORM_LINUX
+		#include "../../local/_linux_readline.h"
+	#else
+		#include <readline/readline.h>
+		#include <readline/history.h>
+	#endif
+#endif
+
+#include "MainHelper.h"
+#include "BoxPortsAndFiles.h"
+#include "BackupDaemonConfigVerify.h"
+#include "SocketStreamTLS.h"
+#include "Socket.h"
+#include "TLSContext.h"
+#include "SSLLib.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreException.h"
+#include "autogen_BackupProtocolClient.h"
+#include "BackupQueries.h"
+#include "FdGetLine.h"
+#include "BackupClientCryptoKeys.h"
+#include "BannerText.h"
+
+#include "MemLeakFindOn.h"
+
+void PrintUsageAndExit()
+{
+	printf("Usage: bbackupquery [-q] [-c config_file] [-l log_file] [commands]\nAs many commands as you require.\n"	\
+	"If commands are multiple words, remember to enclose the command in quotes.\n"	\
+	"Remember to use quit command if you don't want to drop into interactive mode.\n");
+	exit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+	MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbackupquery.memleaks", "bbackupquery")
+
+	// Really don't want trace statements happening, even in debug mode
+	#ifndef NDEBUG
+		BoxDebugTraceOn = false;
+	#endif
+	
+	int returnCode = 0;
+
+	MAINHELPER_START
+	
+	FILE *logFile = 0;
+
+	// Filename for configuraiton file?
+	const char *configFilename = BOX_FILE_BBACKUPD_DEFAULT_CONFIG;
+	
+	// Flags
+	bool quiet = false;
+	bool readWrite = false;
+	
+	// See if there's another entry on the command line
+	int c;
+	while((c = getopt(argc, (char * const *)argv, "qwc:l:")) != -1)
+	{
+		switch(c)
+		{
+		case 'q':
+			// Quiet mode
+			quiet = true;
+			break;
+		
+		case 'w':
+			// Read/write mode
+			readWrite = true;
+			break;
+		
+		case 'c':
+			// store argument
+			configFilename = optarg;
+			break;
+		
+		case 'l':
+			// open log file
+			logFile = ::fopen(optarg, "w");
+			if(logFile == 0)
+			{
+				printf("Can't open log file '%s'\n", optarg);
+			}
+			break;
+		
+		case '?':
+		default:
+			PrintUsageAndExit();
+		}
+	}
+	// Adjust arguments
+	argc -= optind;
+	argv += optind;
+	
+	// Print banner?
+	if(!quiet)
+	{
+		const char *banner = BANNER_TEXT("Backup Query Tool");
+		printf(banner);
+	}
+
+	// Read in the configuration file
+	if(!quiet) printf("Using configuration file %s\n", configFilename);
+	std::string errs;
+	std::auto_ptr<Configuration> config(Configuration::LoadAndVerify(configFilename, &BackupDaemonConfigVerify, errs));
+	if(config.get() == 0 || !errs.empty())
+	{
+		printf("Invalid configuration file:\n%s", errs.c_str());
+		return 1;
+	}
+	// Easier coding
+	const Configuration &conf(*config);
+	
+	// Setup and connect
+	// 1. TLS context
+	SSLLib::Initialise();
+	// Read in the certificates creating a TLS context
+	TLSContext tlsContext;
+	std::string certFile(conf.GetKeyValue("CertificateFile"));
+	std::string keyFile(conf.GetKeyValue("PrivateKeyFile"));
+	std::string caFile(conf.GetKeyValue("TrustedCAsFile"));
+	tlsContext.Initialise(false /* as client */, certFile.c_str(), keyFile.c_str(), caFile.c_str());
+	
+	// Initialise keys
+	BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str());
+
+	// 2. Connect to server
+	if(!quiet) printf("Connecting to store...\n");
+	SocketStreamTLS socket;
+	socket.Open(tlsContext, Socket::TypeINET, conf.GetKeyValue("StoreHostname").c_str(), BOX_PORT_BBSTORED);
+	
+	// 3. Make a protocol, and handshake
+	if(!quiet) printf("Handshake with store...\n");
+	BackupProtocolClient connection(socket);
+	connection.Handshake();
+	
+	// logging?
+	if(logFile != 0)
+	{
+		connection.SetLogToFile(logFile);
+	}
+	
+	// 4. Log in to server
+	if(!quiet) printf("Login to store...\n");
+	// Check the version of the server
+	{
+		std::auto_ptr<BackupProtocolClientVersion> serverVersion(connection.QueryVersion(BACKUP_STORE_SERVER_VERSION));
+		if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION)
+		{
+			THROW_EXCEPTION(BackupStoreException, WrongServerVersion)
+		}
+	}
+	// Login -- if this fails, the Protocol will exception
+	connection.QueryLogin(conf.GetKeyValueInt("AccountNumber"),
+		(readWrite)?0:(BackupProtocolClientLogin::Flags_ReadOnly));
+
+	// 5. Tell user.
+	if(!quiet) printf("Login complete.\n\nType \"help\" for a list of commands.\n\n");
+	
+	// Set up a context for our work
+	BackupQueries context(connection, conf);
+	
+	// Start running commands... first from the command line
+	{
+		int c = 0;
+		while(c < argc && !context.Stop())
+		{
+			context.DoCommand(argv[c++]);
+		}
+	}
+	
+	// Get commands from input
+#ifndef PLATFORM_READLINE_NOT_SUPPORTED
+	using_history();
+	char *last_cmd = 0;
+	while(!context.Stop())
+	{
+		char *command = readline("query > ");
+		if(command == NULL)
+		{
+			// Ctrl-D pressed -- terminate now
+			break;
+		}
+		context.DoCommand(command);
+		if(last_cmd != 0 && ::strcmp(last_cmd, command) == 0)
+		{
+			free(command);
+		}
+		else
+		{
+			add_history(command);
+			last_cmd = command;
+		}
+	}
+#else
+	// Version for platforms which don't have readline by default
+	FdGetLine getLine(fileno(stdin));
+	while(!context.Stop())
+	{
+		printf("query > ");
+		fflush(stdout);
+		std::string command(getLine.GetLine());
+		context.DoCommand(command.c_str());
+	}
+#endif
+	
+	// Done... stop nicely
+	if(!quiet) printf("Logging off...\n");
+	connection.QueryFinished();
+	if(!quiet) printf("Session finished.\n");
+	
+	// Return code
+	returnCode = context.GetReturnCode();
+	
+	// Close log file?
+	if(logFile)
+	{
+		::fclose(logFile);
+	}
+	
+	// Let everything be cleaned up on exit.
+	
+	MAINHELPER_END
+	
+	exit(returnCode);	
+	return returnCode;
+}
+
diff --git a/bin/bbackupquery/documentation.txt b/bin/bbackupquery/documentation.txt
new file mode 100755
index 00000000..429caabe
--- /dev/null
+++ b/bin/bbackupquery/documentation.txt
@@ -0,0 +1,165 @@
+
+bbackupquery utility -- examine store, compare files, restore, etc.
+
+This file has markers for automatic help generation script -- '>' marks a start of a command/help topic,
+and '<' marks the end of a section.
+
+Command line:
+=============
+
+> bbackupquery [-q] [-c configfile] [commands ...]
+
+	-q -- quiet, no information prompts
+	-c -- specify another bbackupd configuation file
+
+The commands following the options are executed, then (if there was no quit
+command) an interactive mode is entered.
+
+If a command contains a space, enclose it in quotes. Example
+
+	bbackupquery "list testdir1" quit
+
+to list the contents of testdir1, and then exit without interactive mode.
+<
+
+Commands:
+=========
+
+All directory names relative to a "current" directory, or from root if they
+start with '/'. The initial directory is always the root directory.
+
+
+> list [options] [directory-name]
+
+	List contents of current directory, or specified directory.
+	
+	-r -- recursively list all files
+	-d -- list deleted files/directories
+	-o -- list old versions of files/directories
+	-I -- don't display object ID
+	-F -- don't display flags
+	-t -- show file modification time
+			(and attr mod time if has the object has attributes, ~ separated)
+	-s -- show file size in blocks used on server
+			(only very approximate indication of size locally)
+
+ls can be used as an alias.
+<
+
+> ls
+
+	Alias for 'list'. Type 'help list' for options.
+<
+
+> cd [options] <directory-name>
+
+	Change directory
+	
+	-d -- consider deleted directories for traversal
+	-o -- consider old versions of directories for traversal
+		(this option should never be useful in a correctly formed store)
+<
+
+> pwd
+
+	Print current directory, always root relative.
+<
+
+> lcd <local-directory-name>
+
+	Change local directory.
+	
+	Type "sh ls" to list the contents.
+<
+
+> sh <shell command>
+
+	All of the parameters after the "sh" are run as a shell command.
+	
+	For example, to list the contents of the location directory, type "sh ls"
+<
+
+> get <object-filename> [<local-filename>]
+get -i <object-id> <local-filename>
+
+	Gets a file from the store. Object is specified as the filename within
+	the current directory, and local filename is optional. Ignores old and
+	deleted files when searching the directory for the file to retrieve.
+	
+	To get an old or deleted file, use the -i option and select the object
+	as a hex object ID (first column in listing). The local filename must
+	be specified.
+<
+
+> compare -a
+compare -l <location-name>
+compare <store-dir-name> <local-dir-name>
+
+	Compares the (current) data on the store with the data on the disc.
+	All the data will be downloaded -- this is potentially a very long
+	operation.
+	
+	-a -- compare all locations
+	-l -- compare one backup location as specified in the configuration file.
+	-c -- set return code
+	-q -- quick compare. Only checks file contents against checksums,
+			doesn't do a full download
+	-E -- ignore exclusion settings
+	
+	Comparing with the root directory is an error, use -a option instead.
+
+	If -c is set, then the return code (if quit is the next command) will be
+		1	Comparison was exact
+		2	Differences were found
+		3	An error occured
+	This can be used for automated tests.
+<
+
+> restore [-d] [-r] [-i] <directory-name> <local-directory-name>
+
+	Restores a directory to the local disc. The local directory specified
+	must not exist (unless a previous restore is being restarted).
+
+	The root cannot be restored -- restore locations individually.
+
+	-d -- restore a deleted directory.
+	-r -- resume an interrupted restoration
+	-i -- directory name is actually an ID
+
+	If a restore operation is interrupted for any reason, it can be restarted
+	using the -r switch. Restore progress information is saved in a file at
+	regular intervals during the restore operation to allow restarts. 
+<
+
+> getobject <object-id> <local-filename>
+
+	Gets the object specified by the object id (in hex) and stores the raw
+	contents in the local file specified.
+
+	This is only useful for debugging as it does not decode files from the
+	stored format, which is encrypted and compressed.
+<
+
+> usage
+
+	Show space used on the server for this account.
+
+	Used: Total amount of space used on the server.
+	Old files: Space used by old files
+	Deleted files: Space used by deleted files
+	Directories: Space used by the directory structure.
+	
+	When Used exceeds the soft limit, the server will start to remove old and
+	deleted files until the usage drops below the soft limit.
+	
+	After a while, you would expect to see the usage stay at just below the
+	soft limit. You only need more space if the space used by old and deleted
+	files is near zero.
+<
+
+> quit
+
+	End session and exit.
+<
+
+
diff --git a/bin/bbackupquery/makedocumentation.pl b/bin/bbackupquery/makedocumentation.pl
new file mode 100755
index 00000000..a3632848
--- /dev/null
+++ b/bin/bbackupquery/makedocumentation.pl
@@ -0,0 +1,75 @@
+#!/usr/bin/perl
+use strict;
+
+print "Creating built-in documentation for bbackupquery...\n";
+
+open DOC,"documentation.txt" or die "Can't open documentation.txt file";
+my $section;
+my %help;
+my @in_order;
+
+while(<DOC>)
+{
+	if(m/\A>\s+(\w+)/)
+	{
+		$section = $1;
+		m/\A>\s+(.+)\Z/;
+		$help{$section} = $1."\n";
+		push @in_order,$section;
+	}
+	elsif(m/\A</)
+	{
+		$section = '';
+	}
+	elsif($section ne '')
+	{
+		$help{$section} .= $_;
+	}
+}
+
+close DOC;
+
+open OUT,">autogen_Documentation.cpp" or die "Can't open output file for writing";
+
+print OUT <<__E;
+//
+// Automatically generated file, do not edit.
+//
+
+#include "Box.h"
+
+#include "MemLeakFindOn.h"
+
+char *help_commands[] =
+{
+__E
+
+for(@in_order)
+{
+	print OUT qq:\t"$_",\n:;
+}
+
+print OUT <<__E;
+	0
+};
+
+char *help_text[] =
+{
+__E
+
+for(@in_order)
+{
+	my $t = $help{$_};
+	$t =~ s/\t/    /g;
+	$t =~ s/\n/\\n/g;
+	$t =~ s/"/\\"/g;
+	print OUT qq:\t"$t",\n:;	
+}
+
+print OUT <<__E;
+	0
+};
+
+__E
+
+close OUT;
diff --git a/bin/bbstoreaccounts/bbstoreaccounts.cpp b/bin/bbstoreaccounts/bbstoreaccounts.cpp
new file mode 100755
index 00000000..89edd0b2
--- /dev/null
+++ b/bin/bbstoreaccounts/bbstoreaccounts.cpp
@@ -0,0 +1,548 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    bbstoreaccounts
+//		Purpose: backup store administration tool
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <limits.h>
+#include <vector>
+#include <algorithm>
+
+#include "BoxPortsAndFiles.h"
+#include "BackupStoreConfigVerify.h"
+#include "RaidFileController.h"
+#include "BackupStoreAccounts.h"
+#include "BackupStoreAccountDatabase.h"
+#include "MainHelper.h"
+#include "BackupStoreInfo.h"
+#include "StoreStructure.h"
+#include "NamedLock.h"
+#include "UnixUser.h"
+#include "BackupStoreCheck.h"
+
+#include "MemLeakFindOn.h"
+
+// max size of soft limit as percent of hard limit
+#define MAX_SOFT_LIMIT_SIZE		97
+
+void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit)
+{
+	if(SoftLimit >= HardLimit)
+	{
+		printf("ERROR: Soft limit must be less than the hard limit.\n");
+		exit(1);
+	}
+	if(SoftLimit > ((HardLimit * MAX_SOFT_LIMIT_SIZE) / 100))
+	{
+		printf("ERROR: Soft limit must be no more than %d%% of the hard limit.\n", MAX_SOFT_LIMIT_SIZE);
+		exit(1);
+	}
+}
+
+int BlockSizeOfDiscSet(int DiscSet)
+{
+	// Get controller, check disc set number
+	RaidFileController &controller(RaidFileController::GetController());
+	if(DiscSet < 0 || DiscSet >= controller.GetNumDiscSets())
+	{
+		printf("Disc set %d does not exist\n", DiscSet);
+		exit(1);
+	}
+	
+	// Return block size
+	return controller.GetDiscSet(DiscSet).GetBlockSize();
+}
+
+const char *BlockSizeToString(int64_t Blocks, int DiscSet)
+{
+	// Not reentrant, nor can be used in the same function call twice, etc.
+	static char string[256];
+	
+	// Work out size in Mb.
+	double mb = (Blocks * BlockSizeOfDiscSet(DiscSet)) / (1024.0*1024.0);
+	
+	// Format string
+	sprintf(string, "%lld (%.2fMb)", Blocks, mb);
+	
+	return string;
+}
+
+int64_t SizeStringToBlocks(const char *string, int DiscSet)
+{
+	// Find block size
+	int blockSize = BlockSizeOfDiscSet(DiscSet);
+	
+	// Get number
+	char *endptr = (char*)string;
+	int64_t number = strtol(string, &endptr, 0);
+	if(endptr == string || number == LONG_MIN || number == LONG_MAX)
+	{
+		printf("%s is an invalid number\n", string);
+		exit(1);
+	}
+	
+	// Check units
+	switch(*endptr)
+	{
+	case 'M':
+	case 'm':
+		// Units: Mb
+		return (number * 1024*1024) / blockSize;
+		break;
+		
+	case 'G':
+	case 'g':
+		// Units: Gb
+		return (number * 1024*1024*1024) / blockSize;
+		break;
+		
+	case 'B':
+	case 'b':
+		// Units: Blocks
+		// Easy! Just return the number specified.
+		return number;
+		break;
+	
+	default:
+		printf("%s has an invalid units specifier\nUse B for blocks, M for Mb, G for Gb, eg 2Gb\n", string);
+		exit(1);
+		break;		
+	}
+}
+
+bool GetWriteLockOnAccount(NamedLock &rLock, const std::string rRootDir, int DiscSetNum)
+{
+	std::string writeLockFilename;
+	StoreStructure::MakeWriteLockFilename(rRootDir, DiscSetNum, writeLockFilename);
+
+	bool gotLock = false;
+	int triesLeft = 8;
+	do
+	{
+		gotLock = rLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */);
+		
+		if(!gotLock)
+		{
+			--triesLeft;
+			::sleep(1);
+		}
+	} while(!gotLock && triesLeft > 0);
+
+	if(!gotLock)
+	{
+		// Couldn't lock the account -- just stop now
+		printf("Couldn't lock the account -- did not change the limits\nTry again later.\n");
+		return 1;
+	}
+
+	return gotLock;
+}
+
+int SetLimit(Configuration &rConfig, const std::string &rUsername, int32_t ID, const char *SoftLimitStr, const char *HardLimitStr)
+{
+	// Become the user specified in the config file?
+	std::auto_ptr<UnixUser> user;
+	if(!rUsername.empty())
+	{
+		// Username specified, change...
+		user.reset(new UnixUser(rUsername.c_str()));
+		user->ChangeProcessUser(true /* temporary */);
+		// Change will be undone at the end of this function
+	}
+
+	// Load in the account database 
+	std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str()));
+	
+	// Already exists?
+	if(!db->EntryExists(ID))
+	{
+		printf("Account %x does not exist\n", ID);
+		return 1;
+	}
+	
+	// Load it in
+	BackupStoreAccounts acc(*db);
+	std::string rootDir;
+	int discSet;
+	acc.GetAccountRoot(ID, rootDir, discSet);
+	
+	// Attempt to lock
+	NamedLock writeLock;
+	if(!GetWriteLockOnAccount(writeLock, rootDir, discSet))
+	{
+		// Failed to get lock
+		return 1;
+	}
+
+	// Load the info
+	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, discSet, false /* Read/Write */));
+
+	// Change the limits
+	int64_t softlimit = SizeStringToBlocks(SoftLimitStr, discSet);
+	int64_t hardlimit = SizeStringToBlocks(HardLimitStr, discSet);
+	CheckSoftHardLimits(softlimit, hardlimit);
+	info->ChangeLimits(softlimit, hardlimit);
+	
+	// Save
+	info->Save();
+
+	printf("Limits on account 0x%08x changed to %lld soft, %lld hard\n", ID, softlimit, hardlimit);
+
+	return 0;
+}
+
+int AccountInfo(Configuration &rConfig, int32_t ID)
+{
+	// Load in the account database 
+	std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str()));
+	
+	// Exists?
+	if(!db->EntryExists(ID))
+	{
+		printf("Account %x does not exist\n", ID);
+		return 1;
+	}
+	
+	// Load it in
+	BackupStoreAccounts acc(*db);
+	std::string rootDir;
+	int discSet;
+	acc.GetAccountRoot(ID, rootDir, discSet);
+	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, discSet, true /* ReadOnly */));
+	
+	// Then print out lots of info
+	printf("                  Account ID: %08x\n", ID);
+	printf("              Last object ID: %lld\n", info->GetLastObjectIDUsed());
+	printf("                 Blocks used: %s\n", BlockSizeToString(info->GetBlocksUsed(), discSet));
+	printf("    Blocks used by old files: %s\n", BlockSizeToString(info->GetBlocksInOldFiles(), discSet));
+	printf("Blocks used by deleted files: %s\n", BlockSizeToString(info->GetBlocksInDeletedFiles(), discSet));
+	printf("  Blocks used by directories: %s\n", BlockSizeToString(info->GetBlocksInDirectories(), discSet));
+	printf("            Block soft limit: %s\n", BlockSizeToString(info->GetBlocksSoftLimit(), discSet));
+	printf("            Block hard limit: %s\n", BlockSizeToString(info->GetBlocksHardLimit(), discSet));
+	printf("         Client store marker: %lld\n", info->GetClientStoreMarker());
+	
+	return 0;
+}
+
+int DeleteAccount(Configuration &rConfig, const std::string &rUsername, int32_t ID, bool AskForConfirmation)
+{
+	// Check user really wants to do this
+	if(AskForConfirmation)
+	{
+		::printf("Really delete account %08x?\n(type 'yes' to confirm)\n", ID);
+		char response[256];
+		if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0)
+		{
+			printf("Deletion cancelled\n");
+			return 0;
+		}
+	}
+	
+	// Load in the account database 
+	std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str()));
+	
+	// Exists?
+	if(!db->EntryExists(ID))
+	{
+		printf("Account %x does not exist\n", ID);
+		return 1;
+	}
+	
+	// Get info from the database
+	BackupStoreAccounts acc(*db);
+	std::string rootDir;
+	int discSetNum;
+	acc.GetAccountRoot(ID, rootDir, discSetNum);
+	
+	// Obtain a write lock, as the daemon user
+	NamedLock writeLock;
+	{
+		// Bbecome the user specified in the config file
+		std::auto_ptr<UnixUser> user;
+		if(!rUsername.empty())
+		{
+			// Username specified, change...
+			user.reset(new UnixUser(rUsername.c_str()));
+			user->ChangeProcessUser(true /* temporary */);
+			// Change will be undone at the end of this function
+		}
+	
+		// Get a write lock
+		if(!GetWriteLockOnAccount(writeLock, rootDir, discSetNum))
+		{
+			// Failed to get lock
+			return 1;
+		}
+		
+		// Back to original user, but write is maintained
+	}
+
+	// Delete from account database
+	db->DeleteEntry(ID);
+	
+	// Write back to disc
+	db->Write();
+	
+	// Remove the store files...
+
+	// First, become the user specified in the config file
+	std::auto_ptr<UnixUser> user;
+	if(!rUsername.empty())
+	{
+		// Username specified, change...
+		user.reset(new UnixUser(rUsername.c_str()));
+		user->ChangeProcessUser(true /* temporary */);
+		// Change will be undone at the end of this function
+	}
+	
+	// Secondly, work out which directories need wiping
+	std::vector<std::string> toDelete;
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet discSet(rcontroller.GetDiscSet(discSetNum));
+	for(RaidFileDiscSet::const_iterator i(discSet.begin()); i != discSet.end(); ++i)
+	{
+		if(std::find(toDelete.begin(), toDelete.end(), *i) == toDelete.end())
+		{
+			toDelete.push_back((*i) + DIRECTORY_SEPARATOR + rootDir);
+		}
+	}
+	
+	// Thirdly, delete the directories...
+	for(std::vector<std::string>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
+	{
+		::printf("Deleting store directory %s...\n", (*d).c_str());
+		// Just use the rm command to delete the files
+		std::string cmd("rm -rf ");
+		cmd += *d;
+		// Run command
+		if(::system(cmd.c_str()) != 0)
+		{
+			::printf("ERROR: Deletion of %s failed.\n(when cleaning up, remember to delete all raid directories)\n", (*d).c_str());
+			return 1;
+		}
+	}
+	
+	// Success!
+	return 0;
+}
+
+int CheckAccount(Configuration &rConfig, const std::string &rUsername, int32_t ID, bool FixErrors, bool Quiet)
+{
+	// Load in the account database 
+	std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str()));
+	
+	// Exists?
+	if(!db->EntryExists(ID))
+	{
+		printf("Account %x does not exist\n", ID);
+		return 1;
+	}
+	
+	// Get info from the database
+	BackupStoreAccounts acc(*db);
+	std::string rootDir;
+	int discSetNum;
+	acc.GetAccountRoot(ID, rootDir, discSetNum);
+	
+	// Become the right user
+	std::auto_ptr<UnixUser> user;
+	if(!rUsername.empty())
+	{
+		// Username specified, change...
+		user.reset(new UnixUser(rUsername.c_str()));
+		user->ChangeProcessUser(true /* temporary */);
+		// Change will be undone at the end of this function
+	}
+
+	// Check it
+	BackupStoreCheck check(rootDir, discSetNum, ID, FixErrors, Quiet);
+	check.Check();
+	
+	return check.ErrorsFound()?1:0;
+}
+
+int CreateAccount(Configuration &rConfig, const std::string &rUsername, int32_t ID, int32_t DiscNumber, int32_t SoftLimit, int32_t HardLimit)
+{
+	// Load in the account database 
+	std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str()));
+	
+	// Already exists?
+	if(db->EntryExists(ID))
+	{
+		printf("Account %x already exists\n", ID);
+		return 1;
+	}
+	
+	// Create it.
+	BackupStoreAccounts acc(*db);
+	acc.Create(ID, DiscNumber, SoftLimit, HardLimit, rUsername);
+	
+	printf("Account %x created\n", ID);
+
+	return 0;
+}
+
+void PrintUsageAndExit()
+{
+	printf("Usage: bbstoreaccounts [-c config_file] action account_id [args]\nAccount ID is integer specified in hex\n");
+	exit(1);
+}
+
+int main(int argc, const char *argv[])
+{
+	MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbstoreaccounts.memleaks", "bbstoreaccounts")
+
+	MAINHELPER_START
+
+	// Filename for configuraiton file?
+	const char *configFilename = BOX_FILE_BBSTORED_DEFAULT_CONFIG;
+	
+	// See if there's another entry on the command line
+	int c;
+	while((c = getopt(argc, (char * const *)argv, "c:")) != -1)
+	{
+		switch(c)
+		{
+		case 'c':
+			// store argument
+			configFilename = optarg;
+			break;
+		
+		case '?':
+		default:
+			PrintUsageAndExit();
+		}
+	}
+	// Adjust arguments
+	argc -= optind;
+	argv += optind;
+
+	// Read in the configuration file
+	std::string errs;
+	std::auto_ptr<Configuration> config(Configuration::LoadAndVerify(configFilename, &BackupConfigFileVerify, errs));
+	if(config.get() == 0 || !errs.empty())
+	{
+		printf("Invalid configuration file:\n%s", errs.c_str());
+	}
+	
+	// Get the user under which the daemon runs
+	std::string username;
+	{
+		const Configuration &rserverConfig(config->GetSubConfiguration("Server"));
+		if(rserverConfig.KeyExists("User"))
+		{
+			username = rserverConfig.GetKeyValue("User");
+		}
+	}
+	
+	// Initialise the raid file controller
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	rcontroller.Initialise(config->GetKeyValue("RaidFileConf").c_str());
+
+	// Then... check we have two arguments
+	if(argc < 2)
+	{
+		PrintUsageAndExit();
+	}
+	
+	// Get the id
+	int32_t id;
+	if(::sscanf(argv[1], "%x", &id) != 1)
+	{
+		PrintUsageAndExit();
+	}
+	
+	// Now do the command.
+	if(::strcmp(argv[0], "create") == 0)
+	{
+		// which disc?
+		int32_t discnum;
+		int32_t softlimit;
+		int32_t hardlimit;
+		if(argc < 5
+			|| ::sscanf(argv[2], "%d", &discnum) != 1)
+		{
+			printf("create requires raid file disc number, soft and hard limits\n");
+			return 1;
+		}
+		
+		// Decode limits
+		softlimit = SizeStringToBlocks(argv[3], discnum);
+		hardlimit = SizeStringToBlocks(argv[4], discnum);
+		CheckSoftHardLimits(softlimit, hardlimit);
+	
+		// Create the account...
+		return CreateAccount(*config, username, id, discnum, softlimit, hardlimit);
+	}
+	else if(::strcmp(argv[0], "info") == 0)
+	{
+		// Print information on this account
+		return AccountInfo(*config, id);
+	}
+	else if(::strcmp(argv[0], "setlimit") == 0)
+	{
+		// Change the limits on this account
+		if(argc < 4)
+		{
+			printf("setlimit requires soft and hard limits\n");
+			return 1;
+		}
+		
+		return SetLimit(*config, username, id, argv[2], argv[3]);
+	}
+	else if(::strcmp(argv[0], "delete") == 0)
+	{
+		// Delete an account
+		bool askForConfirmation = true;
+		if(argc >= 3 && (::strcmp(argv[2], "yes") == 0))
+		{
+			askForConfirmation = false;
+		}
+		return DeleteAccount(*config, username, id, askForConfirmation);
+	}
+	else if(::strcmp(argv[0], "check") == 0)
+	{
+		bool fixErrors = false;
+		bool quiet = false;
+		
+		// Look at other options
+		for(int o = 2; o < argc; ++o)
+		{
+			if(::strcmp(argv[o], "fix") == 0)
+			{
+				fixErrors = true;
+			}
+			else if(::strcmp(argv[o], "quiet") == 0)
+			{
+				quiet = true;
+			}
+			else
+			{
+				::printf("Unknown option %s.\n", argv[o]);
+				return 2;
+			}
+		}
+	
+		// Check the account
+		return CheckAccount(*config, username, id, fixErrors, quiet);
+	}
+	else
+	{
+		printf("Unknown command '%s'\n", argv[0]);
+		return 1;
+	}
+
+	return 0;
+	
+	MAINHELPER_END
+}
+
+
diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp
new file mode 100755
index 00000000..2eb19089
--- /dev/null
+++ b/bin/bbstored/BBStoreDHousekeeping.cpp
@@ -0,0 +1,175 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BBStoreDHousekeeping.cpp
+//		Purpose: Implementation of housekeeping functions for bbstored
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <syslog.h>
+
+#include "BackupStoreDaemon.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "HousekeepStoreAccount.h"
+#include "BoxTime.h"
+#include "Configuration.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::HousekeepingProcess()
+//		Purpose: Do housekeeping
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::HousekeepingProcess()
+{
+	// Get the time between housekeeping runs
+	const Configuration &rconfig(GetConfiguration());
+	int64_t housekeepingInterval = SecondsToBoxTime((uint32_t)rconfig.GetKeyValueInt("TimeBetweenHousekeeping"));
+	
+	int64_t lastHousekeepingRun = 0;
+
+	while(!StopRun())
+	{
+		// Time now
+		int64_t timeNow = GetCurrentBoxTime();
+		// Do housekeeping if the time interval has elapsed since the last check
+		if((timeNow - lastHousekeepingRun) >= housekeepingInterval)
+		{
+			// Store the time
+			lastHousekeepingRun = timeNow;
+			::syslog(LOG_INFO, "Starting housekeeping");
+
+			// Get the list of accounts
+			std::vector<int32_t> accounts;
+			if(mpAccountDatabase)
+			{
+				mpAccountDatabase->GetAllAccountIDs(accounts);
+			}
+			
+			SetProcessTitle("housekeeping, active");
+			
+			// Check them all
+			for(std::vector<int32_t>::const_iterator i = accounts.begin(); i != accounts.end(); ++i)
+			{
+				try
+				{
+					if(mpAccounts)
+					{
+						// Get the account root
+						std::string rootDir;
+						int discSet = 0;
+						mpAccounts->GetAccountRoot(*i, rootDir, discSet);
+						
+						// Do housekeeping on this account
+						HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this);
+						housekeeping.DoHousekeeping();
+					}
+				}
+				catch(BoxException &e)
+				{
+					::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account",
+						*i, e.what(), e.GetType(), e.GetSubType());
+				}
+				catch(std::exception &e)
+				{
+					::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account",
+						*i, e.what());
+				}
+				catch(...)
+				{
+					::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account",
+						*i);
+				}
+				
+				// Check to see if there's any message pending
+				CheckForInterProcessMsg(0 /* no account */);
+		
+				// Stop early?
+				if(StopRun())
+				{
+					break;
+				}
+			}
+			
+			::syslog(LOG_INFO, "Finished housekeeping");
+		}
+
+		// Placed here for accuracy, if StopRun() is true, for example.
+		SetProcessTitle("housekeeping, idle");
+		
+		// Calculate how long should wait before doing the next housekeeping run
+		timeNow = GetCurrentBoxTime();
+		int64_t secondsToGo = BoxTimeToSeconds((lastHousekeepingRun + housekeepingInterval) - timeNow);
+		if(secondsToGo < 1) secondsToGo = 1;
+		if(secondsToGo > 60) secondsToGo = 60;
+		int32_t millisecondsToGo = ((int)secondsToGo) * 1000;
+	
+		// Check to see if there's any message pending
+		CheckForInterProcessMsg(0 /* no account */, millisecondsToGo);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::CheckForInterProcessMsg(int, int)
+//		Purpose: Process a message, returning true if the housekeeping process
+//				 should abort for the specified account.
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitTime)
+{
+	// First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate.
+	if(mInterProcessComms.IsEOF())
+	{
+		SetTerminateWanted();
+		return true;
+	}
+
+	// Get a line, and process the message
+	std::string line;
+	if(mInterProcessComms.GetLine(line, false /* no pre-processing */, MaximumWaitTime))
+	{
+		TRACE1("housekeeping received command '%s' over interprocess comms\n", line.c_str());
+	
+		int account = 0;
+	
+		if(line == "h")
+		{
+			// HUP signal received by main process
+			SetReloadConfigWanted();
+			return true;
+		}
+		else if(line == "t")
+		{
+			// Terminate signal received by main process
+			SetTerminateWanted();
+			return true;
+		}
+		else if(sscanf(line.c_str(), "r%x", &account) == 1)
+		{
+			// Main process is trying to lock an account -- are we processing it?
+			if(account == AccountNum)
+			{
+				// Yes! -- need to stop now so when it retries to get the lock, it will succeed
+				::syslog(LOG_INFO, "Housekeeping giving way to connection for account 0x%08x", AccountNum);
+				return true;
+			}
+		}
+	}
+	
+	return false;
+}
+
+
diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp
new file mode 100755
index 00000000..1a021a58
--- /dev/null
+++ b/bin/bbstored/BackupCommands.cpp
@@ -0,0 +1,861 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupCommands.cpp
+//		Purpose: Implement commands for the Backup store protocol
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <syslog.h>
+
+#include "autogen_BackupProtocolServer.h"
+#include "BackupConstants.h"
+#include "BackupContext.h"
+#include "CollectInBufferStream.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreException.h"
+#include "BackupStoreFile.h"
+#include "StreamableMemBlock.h"
+#include "BackupStoreConstants.h"
+#include "RaidFileController.h"
+#include "BackupStoreInfo.h"
+#include "RaidFileController.h"
+#include "FileStream.h"
+
+#include "MemLeakFindOn.h"
+
+#define CHECK_PHASE(phase)																						\
+	if(rContext.GetPhase() != BackupContext::phase)																\
+	{																											\
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(										\
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_NotInRightProtocolPhase));		\
+	}
+
+#define CHECK_WRITEABLE_SESSION																					\
+	if(rContext.SessionIsReadOnly())																			\
+	{																											\
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(										\
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_SessionReadOnly));				\
+	}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerVersion::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Return the current version, or an error if the requested version isn't allowed
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerVersion::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Version)
+
+	// Correct version?
+	if(mVersion != BACKUP_STORE_SERVER_VERSION)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_WrongVersion));
+	}
+
+	// Mark the next phase
+	rContext.SetPhase(BackupContext::Phase_Login);
+
+	// Return our version
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerVersion(BACKUP_STORE_SERVER_VERSION));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerLogin::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Return the current version, or an error if the requested version isn't allowed
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerLogin::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Login)
+
+	// Check given client ID against the ID in the certificate certificate
+	// and that the client actually has an account on this machine
+	if(mClientID != rContext.GetClientID() || !rContext.GetClientHasAccount())
+	{
+		::syslog(LOG_INFO, "Failed login: Client ID presented was %08X", mClientID);
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_BadLogin));
+	}
+
+	// If we need to write, check that nothing else has got a write lock
+	if((mFlags & Flags_ReadOnly) != Flags_ReadOnly)
+	{
+		// See if the context will get the lock
+		if(!rContext.AttemptToGetWriteLock())
+		{
+			::syslog(LOG_INFO, "Failed to get write lock (for Client ID %08X)", mClientID);
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_CannotLockStoreForWriting));			
+		}
+		
+		// Debug: check we got the lock
+		ASSERT(!rContext.SessionIsReadOnly());
+	}
+	
+	// Load the store info
+	rContext.LoadStoreInfo();
+
+	// Get the last client store marker
+	int64_t clientStoreMarker = rContext.GetClientStoreMarker();
+
+	// Mark the next phase
+	rContext.SetPhase(BackupContext::Phase_Commands);
+	
+	// Log login
+	::syslog(LOG_INFO, "Login: Client ID %08X, %s", mClientID, ((mFlags & Flags_ReadOnly) != Flags_ReadOnly)?"Read/Write":"Read-only");
+
+	// Get the usage info for reporting to the client
+	int64_t blocksUsed = 0, blocksSoftLimit = 0, blocksHardLimit = 0;
+	rContext.GetStoreDiscUsageInfo(blocksUsed, blocksSoftLimit, blocksHardLimit);
+
+	// Return success
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerLoginConfirmed(clientStoreMarker, blocksUsed, blocksSoftLimit, blocksHardLimit));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerFinished::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Marks end of conversation (Protocol framework handles this)
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	::syslog(LOG_INFO, "Session finished");
+
+	// Let the context know about it
+	rContext.ReceivedFinishCommand();
+
+	// can be called in any phase
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerFinished);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerListDirectory::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Command to list a directory
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerListDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Ask the context for a directory
+	const BackupStoreDirectory &rdir(rContext.GetDirectory(mObjectID));
+	
+	// Store the listing to a stream
+	std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream);
+	rdir.WriteToStream(*stream, mFlagsMustBeSet, mFlagsNotToBeSet, mSendAttributes,
+		false /* never send dependency info to the client */);
+	stream->SetForReading();
+	
+	// Get the protocol to send the stream
+	rProtocol.SendStreamAfterCommand(stream.release());
+
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerStoreFile::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Command to store a file on the server
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerStoreFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+	
+	// Check that the diff from file actually exists, if it's specified
+	if(mDiffFromFileID != 0)
+	{
+		if(!rContext.ObjectExists(mDiffFromFileID, BackupContext::ObjectExists_File))
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DiffFromFileDoesNotExist));	
+		}
+	}
+	
+	// A stream follows, which contains the file
+	std::auto_ptr<IOStream> dirstream(rProtocol.ReceiveStream());
+	
+	// Ask the context to store it
+	int64_t id = 0;
+	try
+	{
+		id = rContext.AddFile(*dirstream, mDirectoryObjectID, mModificationTime, mAttributesHash, mDiffFromFileID,
+			 mFilename, true /* mark files with same name as old versions */);
+	}
+	catch(BackupStoreException &e)
+	{
+		if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify)
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_FileDoesNotVerify));			
+		}
+		else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit)
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_StorageLimitExceeded));			
+		}
+		else
+		{
+			throw;
+		}
+	}
+	
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id));
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetObject::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Command to get an arbitary object from the server
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetObject::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Check the object exists
+	if(!rContext.ObjectExists(mObjectID))
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(NoObject));
+	}
+
+	// Open the object
+	std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID));
+
+	// Stream it to the peer
+	rProtocol.SendStreamAfterCommand(object.release());
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetFile::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Command to get an file object from the server -- may have to do a bit of 
+//				 work to get the object.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Check the objects exist
+	if(!rContext.ObjectExists(mObjectID)
+		|| !rContext.ObjectExists(mInDirectory))
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DoesNotExist));			
+	}
+
+	// Get the directory it's in
+	const BackupStoreDirectory &rdir(rContext.GetDirectory(mInDirectory));
+
+	// Find the object within the directory
+	BackupStoreDirectory::Entry *pfileEntry = rdir.FindEntryByID(mObjectID);
+	if(pfileEntry == 0)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DoesNotExistInDirectory));			
+	}
+
+	// The result
+	std::auto_ptr<IOStream> stream;
+
+	// Does this depend on anything?
+	if(pfileEntry->GetDependsNewer() != 0)
+	{
+		// File exists, but is a patch from a new version. Generate the older version.
+		std::vector<int64_t> patchChain;
+		int64_t id = mObjectID;
+		BackupStoreDirectory::Entry *en = 0;
+		do
+		{
+			patchChain.push_back(id);
+			en = rdir.FindEntryByID(id);
+			if(en == 0)
+			{
+				::syslog(LOG_ERR, "Object %llx in dir %llx for account %x references object %llx which does not exist in dir",
+					mObjectID, mInDirectory, rContext.GetClientID(), id);
+				return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+					BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_PatchConsistencyError));			
+			}
+			id = en->GetDependsNewer();
+		} while(en != 0 && id != 0);
+		
+		// OK! The last entry in the chain is the full file, the others are patches back from it.
+		// Open the last one, which is the current from file
+		std::auto_ptr<IOStream> from(rContext.OpenObject(patchChain[patchChain.size() - 1]));
+		
+		// Then, for each patch in the chain, do a combine
+		for(int p = ((int)patchChain.size()) - 2; p >= 0; --p)
+		{
+			// ID of patch
+			int64_t patchID = patchChain[p];
+			
+			// Open it a couple of times
+			std::auto_ptr<IOStream> diff(rContext.OpenObject(patchID));
+			std::auto_ptr<IOStream> diff2(rContext.OpenObject(patchID));
+			
+			// Choose a temporary filename for the result of the combination
+			std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp",
+				p + 16 /* rotate which disc it's on */));
+			
+			// Open the temporary file
+			std::auto_ptr<IOStream> combined;
+			try
+			{
+				{
+					// Write nastily to allow this to work with gcc 2.x
+					std::auto_ptr<IOStream> t(new FileStream(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL));
+					combined = t;
+				}
+				// Unlink immediately as it's a temporary file
+				if(::unlink(tempFn.c_str()) != 0)
+				{
+					THROW_EXCEPTION(CommonException, OSFileError);
+				}
+			}
+			catch(...)
+			{
+				// Make sure it goes
+				::unlink(tempFn.c_str());
+				throw;
+			}
+			
+			// Do the combining
+			BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined);
+			
+			// Move to the beginning of the combined file
+			combined->Seek(0, IOStream::SeekType_Absolute);
+			
+			// Then shuffle round for the next go
+			from = combined;
+		}
+		
+		// Now, from contains a nice file to send to the client. Reorder it
+		{
+			// Write nastily to allow this to work with gcc 2.x
+			std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */));
+			stream = t;
+		}
+		
+		// Release from file to avoid double deletion
+		from.release();
+	}
+	else
+	{
+		// Simple case: file already exists on disc ready to go
+	
+		// Open the object
+		std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID));
+		
+		// Verify it
+		if(!BackupStoreFile::VerifyEncodedFileFormat(*object))
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_FileDoesNotVerify));			
+		}
+		
+		// Reset stream -- seek to beginning
+		object->Seek(0, IOStream::SeekType_Absolute);
+		
+		// Reorder the stream/file into stream order
+		{
+			// Write nastily to allow this to work with gcc 2.x
+			std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(object.get(), true /* take ownership */));
+			stream = t;
+		}
+
+		// Object will be deleted when the stream is deleted, so can release the object auto_ptr here to
+		// avoid premature deletiong
+		object.release();
+	}
+
+	// Stream the reordered stream to the peer
+	rProtocol.SendStreamAfterCommand(stream.get());
+	
+	// Don't delete the stream here
+	stream.release();
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerCreateDirectory::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Create directory command
+//		Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerCreateDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+	
+	// Get the stream containing the attributes
+	std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
+	// Collect the attributes -- do this now so no matter what the outcome, 
+	// the data has been absorbed.
+	StreamableMemBlock attr;
+	attr.Set(*attrstream, rProtocol.GetTimeout());
+	
+	// Check to see if the hard limit has been exceeded
+	if(rContext.HardLimitExceeded())
+	{
+		// Won't allow creation if the limit has been exceeded
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_StorageLimitExceeded));			
+	}
+
+	bool alreadyExists = false;
+	int64_t id = rContext.AddDirectory(mContainingDirectoryID, mDirectoryName, attr, mAttributesModTime, alreadyExists);
+	
+	if(alreadyExists)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DirectoryAlreadyExists));			
+	}
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(id));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerChangeDirAttributes::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Change attributes on directory
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerChangeDirAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Get the stream containing the attributes
+	std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
+	// Collect the attributes -- do this now so no matter what the outcome, 
+	// the data has been absorbed.
+	StreamableMemBlock attr;
+	attr.Set(*attrstream, rProtocol.GetTimeout());
+
+	// Get the context to do it's magic
+	rContext.ChangeDirAttributes(mObjectID, attr, mAttributesModTime);
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerSetReplacementFileAttributes::DoCommand(Protocol &, BackupContext &)
+//		Purpose: Change attributes on directory
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerSetReplacementFileAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Get the stream containing the attributes
+	std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream());
+	// Collect the attributes -- do this now so no matter what the outcome, 
+	// the data has been absorbed.
+	StreamableMemBlock attr;
+	attr.Set(*attrstream, rProtocol.GetTimeout());
+
+	// Get the context to do it's magic
+	int64_t objectID = 0;
+	if(!rContext.ChangeFileAttributes(mFilename, mInDirectory, attr, mAttributesHash, objectID))
+	{
+		// Didn't exist
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DoesNotExist));			
+	}
+
+	// Tell the caller what the file was
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Delete a file
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Context handles this
+	int64_t objectID = 0;
+	rContext.DeleteFile(mFilename, mInDirectory, objectID);
+
+	// return the object ID or zero for not found
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Delete a directory
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Check it's not asking for the root directory to be deleted
+	if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_CannotDeleteRoot));			
+	}
+
+	// Context handles this
+	rContext.DeleteDirectory(mObjectID);
+
+	// return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Undelete a directory
+//		Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Check it's not asking for the root directory to be deleted
+	if(mObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+	{
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+			BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_CannotDeleteRoot));			
+	}
+
+	// Context handles this
+	rContext.DeleteDirectory(mObjectID, true /* undelete */);
+
+	// return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Command to set the client's store marker
+//		Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+
+	// Set the marker
+	rContext.SetClientStoreMarker(mClientStoreMarker);
+
+	// return store marker set
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mClientStoreMarker));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Command to move an object from one directory to another
+//		Created: 2003/11/12
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	CHECK_WRITEABLE_SESSION
+	
+	// Let context do this, but modify error reporting on exceptions...
+	try
+	{
+		rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory,
+			mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName,
+			(mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject);
+	}
+	catch(BackupStoreException &e)
+	{
+		if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory)
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DoesNotExist));			
+		}
+		else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory)
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError(
+				BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_TargetNameExists));			
+		}
+		else
+		{
+			throw;
+		}
+	}
+
+	// Return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Command to find the name of an object
+//		Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+	
+	// Create a stream for the list of filenames
+	std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream);
+
+	// Object and directory IDs
+	int64_t objectID = mObjectID;
+	int64_t dirID = mContainingDirectoryID;
+	
+	// Data to return in the reply
+	int32_t numNameElements = 0;
+	int16_t objectFlags = 0;
+	int64_t modTime = 0;
+	uint64_t attrModHash = 0;
+	bool haveModTimes = false;
+	
+	do
+	{
+		// Check the directory really exists
+		if(!rContext.ObjectExists(dirID, BackupContext::ObjectExists_Directory))
+		{
+			return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0));
+		}
+
+		// Load up the directory
+		const BackupStoreDirectory &rdir(rContext.GetDirectory(dirID));
+
+		// Find the element in this directory and store it's name
+		if(objectID != ObjectID_DirectoryOnly)
+		{
+			const BackupStoreDirectory::Entry *en = rdir.FindEntryByID(objectID);
+
+			// If this can't be found, then there is a problem... tell the caller it can't be found
+			if(en == 0)
+			{
+				// Abort!
+				return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0));
+			}
+			
+			// Store flags?
+			if(objectFlags == 0)
+			{
+				objectFlags = en->GetFlags();
+			}
+			
+			// Store modification times?
+			if(!haveModTimes)
+			{
+				modTime = en->GetModificationTime();
+				attrModHash = en->GetAttributesHash();
+				haveModTimes = true;
+			}
+			
+			// Store the name in the stream
+			en->GetName().WriteToStream(*stream);
+			
+			// Count of name elements
+			++numNameElements;
+		}
+		
+		// Setup for next time round
+		objectID = dirID;
+		dirID = rdir.GetContainerID();
+
+	} while(objectID != 0 && objectID != BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+	// Stream to send?
+	if(numNameElements > 0)
+	{
+		// Get the stream ready to go
+		stream->SetForReading();	
+		// Tell the protocol to send the stream
+		rProtocol.SendStreamAfterCommand(stream.release());
+	}
+
+	// Make reply
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerObjectName(numNameElements, modTime, attrModHash, objectFlags));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Get the block index from a file, by ID
+//		Created: 19/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Open the file
+	std::auto_ptr<IOStream> stream(rContext.OpenObject(mObjectID));
+	
+	// Move the file pointer to the block index
+	BackupStoreFile::MoveStreamPositionToBlockIndex(*stream);
+	
+	// Return the stream to the client
+	rProtocol.SendStreamAfterCommand(stream.release());
+
+	// Return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(mObjectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Get the block index from a file, by name within a directory
+//		Created: 19/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Get the directory
+	const BackupStoreDirectory &dir(rContext.GetDirectory(mInDirectory));
+	
+	// Find the latest object ID within it which has the same name
+	int64_t objectID = 0;
+	BackupStoreDirectory::Iterator i(dir);
+	BackupStoreDirectory::Entry *en = 0;
+	while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+	{
+		if(en->GetName() == mFilename)
+		{
+			// Store the ID, if it's a newer ID than the last one
+			if(en->GetObjectID() > objectID)
+			{
+				objectID = en->GetObjectID();
+			}
+		}
+	}
+	
+	// Found anything?
+	if(objectID == 0)
+	{
+		// No... return a zero object ID
+		return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(0));
+	}
+
+	// Open the file
+	std::auto_ptr<IOStream> stream(rContext.OpenObject(objectID));
+	
+	// Move the file pointer to the block index
+	BackupStoreFile::MoveStreamPositionToBlockIndex(*stream);
+	
+	// Return the stream to the client
+	rProtocol.SendStreamAfterCommand(stream.release());
+
+	// Return the object ID
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerSuccess(objectID));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &, BackupContext &)
+//		Purpose: Return the amount of disc space used
+//		Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext)
+{
+	CHECK_PHASE(Phase_Commands)
+
+	// Get store info from context
+	const BackupStoreInfo &rinfo(rContext.GetBackupStoreInfo());
+	
+	// Find block size
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(rinfo.GetDiscSetNumber()));
+	
+	// Return info
+	return std::auto_ptr<ProtocolObject>(new BackupProtocolServerAccountUsage(
+		rinfo.GetBlocksUsed(),
+		rinfo.GetBlocksInOldFiles(),
+		rinfo.GetBlocksInDeletedFiles(),
+		rinfo.GetBlocksInDirectories(),
+		rinfo.GetBlocksSoftLimit(),
+		rinfo.GetBlocksHardLimit(),
+		rdiscSet.GetBlockSize()
+	));
+}
+
diff --git a/bin/bbstored/BackupConstants.h b/bin/bbstored/BackupConstants.h
new file mode 100755
index 00000000..515b3bcd
--- /dev/null
+++ b/bin/bbstored/BackupConstants.h
@@ -0,0 +1,23 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupConstants.h
+//		Purpose: Constants for the backup server and client
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCONSTANTS__H
+#define BACKUPCONSTANTS__H
+
+#define BACKUP_STORE_DEFAULT_ACCOUNT_DATABASE_FILE	"/etc/box/backupstoreaccounts"
+
+// 15 minutes to timeout (milliseconds)
+#define	BACKUP_STORE_TIMEOUT			(15*60*1000)
+
+// Should the store daemon convert files to Raid immediately?
+#define	BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY	true
+
+#endif // BACKUPCONSTANTS__H
+
+
diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp
new file mode 100755
index 00000000..c796c13a
--- /dev/null
+++ b/bin/bbstored/BackupContext.cpp
@@ -0,0 +1,1650 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupContext.cpp
+//		Purpose: Context for backup store server
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "BackupContext.h"
+#include "RaidFileWrite.h"
+#include "RaidFileRead.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreException.h"
+#include "BackupStoreInfo.h"
+#include "BackupConstants.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreObjectMagic.h"
+#include "StoreStructure.h"
+#include "BackupStoreDaemon.h"
+#include "RaidFileController.h"
+#include "FileStream.h"
+
+#include "MemLeakFindOn.h"
+
+
+// Maximum number of directories to keep in the cache
+// When the cache is bigger than this, everything gets
+// deleted.
+#ifdef NDEBUG
+	#define	MAX_CACHE_SIZE	32
+#else
+	#define	MAX_CACHE_SIZE	2
+#endif
+
+// Allow the housekeeping process 4 seconds to release an account
+#define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT	4
+
+// Maximum amount of store info updates before it's actually saved to disc.
+#define STORE_INFO_SAVE_DELAY	96
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::BackupContext()
+//		Purpose: Constructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupContext::BackupContext(int32_t ClientID, BackupStoreDaemon &rDaemon)
+	: mClientID(ClientID),
+	  mrDaemon(rDaemon),
+	  mProtocolPhase(Phase_START),
+	  mClientHasAccount(false),
+	  mStoreDiscSet(-1),
+	  mReadOnly(true),
+	  mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::~BackupContext()
+//		Purpose: Destructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupContext::~BackupContext()
+{
+	// Delete the objects in the cache
+	for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i)
+	{
+		delete (i->second);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::CleanUp()
+//		Purpose: Clean up after a connection
+//		Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::CleanUp()
+{
+	// Make sure the store info is saved, if it has been loaded, isn't read only and has been modified
+	if(mpStoreInfo.get() && !(mpStoreInfo->IsReadOnly()) && mpStoreInfo->IsModified())
+	{
+		mpStoreInfo->Save();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::ReceivedFinishCommand()
+//		Purpose: Called when the finish command is received by the protocol
+//		Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::ReceivedFinishCommand()
+{
+	if(!mReadOnly && mpStoreInfo.get())
+	{
+		// Save the store info, not delayed
+		SaveStoreInfo(false);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::AttemptToGetWriteLock()
+//		Purpose: Attempt to get a write lock for the store, and if so, unset the read only flags
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+bool BackupContext::AttemptToGetWriteLock()
+{
+	// Make the filename of the write lock file
+	std::string writeLockFile;
+	StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile);
+
+	// Request the lock
+	bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */);
+	
+	if(!gotLock)
+	{
+		// The housekeeping process might have the thing open -- ask it to stop
+		char msg[256];
+		int msgLen = sprintf(msg, "r%x\n", mClientID);
+		// Send message
+		mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen);
+		
+		// Then try again a few times
+		int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT;
+		do
+		{
+			::sleep(1 /* second */);
+			--tries;
+			gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */);
+			
+		} while(!gotLock && tries > 0);
+	}
+	
+	if(gotLock)
+	{
+		// Got the lock, mark as not read only
+		mReadOnly = false;
+	}
+	
+	return gotLock;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::LoadStoreInfo()
+//		Purpose: Load the store info from disc
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::LoadStoreInfo()
+{
+	if(mpStoreInfo.get() != 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded)
+	}
+	
+	// Load it up!
+	std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly));
+	
+	// Check it
+	if(i->GetAccountID() != mClientID)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount)
+	}
+	
+	// Keep the pointer to it
+	mpStoreInfo = i;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::SaveStoreInfo(bool)
+//		Purpose: Potentially delayed saving of the store info
+//		Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::SaveStoreInfo(bool AllowDelay)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	// Can delay saving it a little while?
+	if(AllowDelay)
+	{
+		--mSaveStoreInfoDelay;
+		if(mSaveStoreInfoDelay > 0)
+		{
+			return;
+		}
+	}
+
+	// Want to save now	
+	mpStoreInfo->Save();
+
+	// Set count for next delay
+	mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::MakeObjectFilename(int64_t, std::string &, bool)
+//		Purpose: Create the filename of an object in the store, optionally creating the 
+//				 containing directory if it doesn't already exist.
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+void BackupContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists)
+{
+	// Delegate to utility function
+	StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::GetDirectoryInternal(int64_t)
+//		Purpose: Return a reference to a directory. Valid only until the 
+//				 next time a function which affects directories is called.
+//				 Mainly this funciton, and creation of files.
+//				 Private version of this, which returns non-const directories.
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory &BackupContext::GetDirectoryInternal(int64_t ObjectID)
+{
+	// Get the filename
+	std::string filename;
+	MakeObjectFilename(ObjectID, filename);
+	
+	// Already in cache?
+	std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID));
+	if(item != mDirectoryCache.end())
+	{
+		// Check the revision ID of the file -- does it need refreshing?
+		int64_t revID = 0;
+		if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID))
+		{
+			THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted)
+		}
+	
+		if(revID == item->second->GetRevisionID())
+		{
+			// Looks good... return the cached object
+			return *(item->second);
+		}
+		
+		// Delete this cached object
+		delete item->second;
+		mDirectoryCache.erase(item);
+	}
+	
+	// Need to load it up
+	
+	// First check to see if the cache is too big
+	if(mDirectoryCache.size() > MAX_CACHE_SIZE)
+	{
+		// Very simple. Just delete everything!
+		for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i)
+		{
+			delete (i->second);
+		}
+		mDirectoryCache.clear();
+	}
+
+	// Get a RaidFileRead to read it
+	int64_t revID = 0;
+	std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID));
+	ASSERT(revID != 0);
+	
+	// New directory object
+	std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory);
+	
+	// Read it from the stream, then set it's revision ID
+	dir->ReadFromStream(*objectFile, IOStream::TimeOutInfinite);
+	dir->SetRevisionID(revID);
+			
+	// Make sure the size of the directory is available for writing the dir back
+	int64_t dirSize = objectFile->GetDiscUsageInBlocks();
+	ASSERT(dirSize > 0);
+	dir->SetUserInfo1_SizeInBlocks(dirSize);
+
+	// Store in cache
+	BackupStoreDirectory *pdir = dir.release();
+	try
+	{	
+		mDirectoryCache[ObjectID] = pdir;
+	}
+	catch(...)
+	{
+		delete pdir;
+		throw;
+	}
+	
+	// Return it
+	return *pdir;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::AllocateObjectID()
+//		Purpose: Allocate a new object ID, tolerant of failures to save store info
+//		Created: 16/12/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupContext::AllocateObjectID()
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+
+	// Given that the store info may not be saved for STORE_INFO_SAVE_DELAY
+	// times after it has been updated, this is a reasonable number of times
+	// to try for finding an unused ID.
+	// (Sizes used in the store info are fixed by the housekeeping process)
+	int retryLimit = (STORE_INFO_SAVE_DELAY * 2);
+	
+	while(retryLimit > 0)
+	{
+		// Attempt to allocate an ID from the store
+		int64_t id = mpStoreInfo->AllocateObjectID();
+		
+		// Generate filename
+		std::string filename;
+		MakeObjectFilename(id, filename);
+		// Check it doesn't exist
+		if(!RaidFileRead::FileExists(mStoreDiscSet, filename))
+		{
+			// Success!
+			return id;
+		}
+		
+		// Decrement retry count, and try again
+		--retryLimit;
+		
+		// Mark that the store info should be saved as soon as possible
+		mSaveStoreInfoDelay = 0;
+		
+		TRACE1("When allocating object ID, found that %lld is already in use\n", id);
+	}
+	
+	THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
+//		Purpose: Add a file to the store, from a given stream, into a specified directory.
+//				 Returns object ID of the new file.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime,
+		int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
+		bool MarkFileWithSameNameAsOldVersions)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+	
+	// This is going to be a bit complex to make sure it copes OK
+	// with things going wrong.
+	// The only thing which isn't safe is incrementing the object ID
+	// and keeping the blocks used entirely accurate -- but these
+	// aren't big problems if they go horribly wrong. The sizes will
+	// be corrected the next time the account has a housekeeping run,
+	// and the object ID allocation code is tolerant of missed IDs.
+	// (the info is written lazily, so these are necessary)
+	
+	// Get the directory we want to modify
+	BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+	
+	// Allocate the next ID
+	int64_t id = AllocateObjectID();
+	
+	// Stream the file to disc
+	std::string fn;
+	MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
+	int64_t blocksUsed = 0;
+	RaidFileWrite *ppreviousVerStoreFile = 0;
+	bool reversedDiffIsCompletelyDifferent = false;
+	int64_t oldVersionNewBlocksUsed = 0;
+	try
+	{
+		RaidFileWrite storeFile(mStoreDiscSet, fn);
+		storeFile.Open(false /* no overwriting */);
+		int64_t spaceAdjustFromDiff = 0;	// size adjustment from use of patch in old file
+
+		// Diff or full file?
+		if(DiffFromFileID == 0)
+		{
+			// A full file, just store to disc
+			if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT))
+			{
+				THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut)
+			}
+		}
+		else
+		{
+			// Check that the diffed from ID actually exists in the directory
+			if(dir.FindEntryByID(DiffFromFileID) == 0)
+			{
+				THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory)
+			}
+		
+			// Diff file, needs to be recreated.
+			// Choose a temporary filename.
+			std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp",
+				1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */));
+			
+			try
+			{
+				// Open it twice
+				FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL);
+				FileStream diff2(tempFn.c_str(), O_RDONLY);
+				// Unlink it immediately, so it definately goes away
+				if(::unlink(tempFn.c_str()) != 0)
+				{
+					THROW_EXCEPTION(CommonException, OSFileError);
+				}
+				
+				// Stream the incoming diff to this temporary file
+				if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT))
+				{
+					THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut)
+				}
+				
+				// Verify the diff
+				diff.Seek(0, IOStream::SeekType_Absolute);
+				if(!BackupStoreFile::VerifyEncodedFileFormat(diff))
+				{
+					THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify)
+				}
+
+				// Seek to beginning of diff file
+				diff.Seek(0, IOStream::SeekType_Absolute);
+
+				// Filename of the old version
+				std::string oldVersionFilename;
+				MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */);
+				
+				// Reassemble that diff -- open previous file, and combine the patch and file
+				std::auto_ptr<RaidFileRead> from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename));
+				BackupStoreFile::CombineFile(diff, diff2, *from, storeFile);
+
+				// Then... reverse the patch back (open the from file again, and create a write file to overwrite it)
+				std::auto_ptr<RaidFileRead> from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename));
+				ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename);
+				ppreviousVerStoreFile->Open(true /* allow overwriting */);
+				from->Seek(0, IOStream::SeekType_Absolute);
+				diff.Seek(0, IOStream::SeekType_Absolute);
+				BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile,
+						DiffFromFileID, &reversedDiffIsCompletelyDifferent);
+				
+				// Store disc space used
+				oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks();
+				
+				// And make a space adjustment for the size calculation
+				spaceAdjustFromDiff = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed;
+
+				// Everything cleans up here...
+			}
+			catch(...)
+			{
+				// Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway
+				::unlink(tempFn.c_str());
+				throw;
+			}
+		}
+		
+		// Get the blocks used
+		blocksUsed = storeFile.GetDiscUsageInBlocks();
+		
+		// Exceeds the hard limit?
+		if((mpStoreInfo->GetBlocksUsed() + blocksUsed - spaceAdjustFromDiff) > mpStoreInfo->GetBlocksHardLimit())
+		{
+			THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit)
+			// The store file will be deleted automatically by the RaidFile object
+		}
+
+		// Commit the file
+		storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+	}
+	catch(...)
+	{
+		// Delete any previous version store file
+		if(ppreviousVerStoreFile != 0)
+		{
+			delete ppreviousVerStoreFile;
+			ppreviousVerStoreFile = 0;
+		}
+		
+		throw;
+	}
+
+	// Verify the file -- only necessary for non-diffed versions
+	// NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because
+	// in the non-diffed code path it's never allocated.
+	if(DiffFromFileID == 0)
+	{
+		std::auto_ptr<RaidFileRead> checkFile(RaidFileRead::Open(mStoreDiscSet, fn));
+		if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile))
+		{
+			// Error! Delete the file
+			RaidFileWrite del(mStoreDiscSet, fn);
+			del.Delete();
+			
+			// Exception
+			THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify)
+		}
+	}			
+	
+	// Modify the directory -- first make all files with the same name
+	// marked as an old version
+	int64_t blocksInOldFiles = 0;
+	try
+	{
+		if(MarkFileWithSameNameAsOldVersions)
+		{
+			BackupStoreDirectory::Iterator i(dir);
+
+			BackupStoreDirectory::Entry *e = 0;
+			while((e = i.Next()) != 0)
+			{
+				// First, check it's not an old version (cheaper comparison)
+				if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0)
+				{
+					// Compare name
+					if(e->GetName() == rFilename)
+					{
+						// Check that it's definately not an old version
+						ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0);
+						// Set old version flag
+						e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion);
+						// Can safely do this, because we know we won't be here if it's already 
+						// an old version
+						blocksInOldFiles += e->GetSizeInBlocks();
+					}
+				}
+			}
+		}
+		
+		// Then the new entry
+		BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename,
+				ModificationTime, id, blocksUsed, BackupStoreDirectory::Entry::Flags_File, AttributesHash);
+
+		// Adjust for the patch back stuff?
+		if(DiffFromFileID != 0)
+		{
+			// Get old version entry
+			BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID);
+			ASSERT(poldEntry != 0);
+		
+			// Adjust dependency info of file?
+			if(!reversedDiffIsCompletelyDifferent)
+			{
+				poldEntry->SetDependsNewer(id);
+				pnewEntry->SetDependsOlder(DiffFromFileID);
+			}
+			
+			// Adjust size of old entry
+			int64_t oldSize = poldEntry->GetSizeInBlocks();
+			poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed);
+			
+			// And adjust blocks used count, for later adjustment
+			blocksUsed += (oldVersionNewBlocksUsed - oldSize);
+			blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize);
+		}
+
+		// Write the directory back to disc
+		SaveDirectory(dir, InDirectory);
+
+		// Commit the old version's new patched version, now that the directory safely reflects
+		// the state of the files on disc.
+		if(ppreviousVerStoreFile != 0)
+		{
+			ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+			delete ppreviousVerStoreFile;
+			ppreviousVerStoreFile = 0;
+		}
+	}
+	catch(...)
+	{
+		// Back out on adding that file
+		RaidFileWrite del(mStoreDiscSet, fn);
+		del.Delete();
+		
+		// Remove this entry from the cache
+		RemoveDirectoryFromCache(InDirectory);
+		
+		// Delete any previous version store file
+		if(ppreviousVerStoreFile != 0)
+		{
+			delete ppreviousVerStoreFile;
+			ppreviousVerStoreFile = 0;
+		}
+		
+		// Don't worry about the incremented number in the store info
+		throw;
+	}
+	
+	// Check logic
+	ASSERT(ppreviousVerStoreFile == 0);
+	
+	// Modify the store info
+	mpStoreInfo->ChangeBlocksUsed(blocksUsed);
+	mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
+	
+	// Save the store info -- can cope if this exceptions because infomation
+	// will be rebuilt by housekeeping, and ID allocation can recover.
+	SaveStoreInfo();
+	
+	// Return the ID to the caller
+	return id;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
+//		Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+bool BackupContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut)
+{
+	// Essential checks!
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	// Find the directory the file is in (will exception if it fails)
+	BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+
+	// Setup flags
+	bool fileExisted = false;
+	bool madeChanges = false;
+	rObjectIDOut = 0;		// not found
+
+	// Count of deleted blocks
+	int64_t blocksDel = 0;
+
+	try
+	{
+		// Iterate through directory, only looking at files which haven't been deleted
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *e = 0;
+		while((e = i.Next(BackupStoreDirectory::Entry::Flags_File,
+			BackupStoreDirectory::Entry::Flags_Deleted)) != 0)
+		{
+			// Compare name
+			if(e->GetName() == rFilename)
+			{
+				// Check that it's definately not already deleted
+				ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0);
+				// Set deleted flag
+				e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				// Mark as made a change
+				madeChanges = true;
+				// Can safely do this, because we know we won't be here if it's already 
+				// an old version
+				blocksDel += e->GetSizeInBlocks();
+				// Is this the last version?
+				if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0)
+				{
+					// Yes. It's been found.
+					rObjectIDOut = e->GetObjectID();
+					fileExisted = true;
+				}
+			}
+		}
+		
+		// Save changes?
+		if(madeChanges)
+		{
+			// Save the directory back
+			SaveDirectory(dir, InDirectory);
+			
+			// Modify the store info, and write
+			mpStoreInfo->ChangeBlocksInDeletedFiles(blocksDel);
+			
+			// Maybe postponed save of store info
+			SaveStoreInfo();
+		}
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(InDirectory);
+		throw;
+	}
+		
+
+	return fileExisted;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::RemoveDirectoryFromCache(int64_t)
+//		Purpose: Remove directory from cache
+//		Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+void BackupContext::RemoveDirectoryFromCache(int64_t ObjectID)
+{
+	std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID));
+	if(item != mDirectoryCache.end())
+	{
+		// Delete this cached object
+		delete item->second;
+		// Erase the entry form the map
+		mDirectoryCache.erase(item);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::SaveDirectory(BackupStoreDirectory &, int64_t)
+//		Purpose: Save directory back to disc, update time in cache
+//		Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+void BackupContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(rDir.GetObjectID() != ObjectID)
+	{
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+
+	try
+	{
+		// Write to disc, adjust size in store info
+		std::string dirfn;
+		MakeObjectFilename(ObjectID, dirfn);
+		{
+			RaidFileWrite writeDir(mStoreDiscSet, dirfn);
+			writeDir.Open(true /* allow overwriting */);
+			rDir.WriteToStream(writeDir);
+
+			// get the disc usage (must do this before commiting it)
+			int64_t dirSize = writeDir.GetDiscUsageInBlocks();
+
+			// Commit directory
+			writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+			
+			// Make sure the size of the directory is available for writing the dir back
+			ASSERT(dirSize > 0);
+			int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks();
+			mpStoreInfo->ChangeBlocksUsed(sizeAdjustment);
+			mpStoreInfo->ChangeBlocksInDirectories(sizeAdjustment);
+			// Update size stored in directory
+			rDir.SetUserInfo1_SizeInBlocks(dirSize);
+		}
+		// Refresh revision ID in cache
+		{
+			int64_t revid = 0;
+			if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid))
+			{
+				THROW_EXCEPTION(BackupStoreException, Internal)
+			}
+			rDir.SetRevisionID(revid);
+		}
+	}
+	catch(...)
+	{
+		// Remove it from the cache if anything went wrong
+		RemoveDirectoryFromCache(ObjectID);
+		throw;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &)
+//		Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely.
+//		Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+	
+	// Flags as not already existing
+	rAlreadyExists = false;
+	
+	// Get the directory we want to modify
+	BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+
+	// Scan the directory for the name (only looking for directories which already exist)
+	{
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING,
+			BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)	// Ignore deleted and old directories
+		{
+			if(en->GetName() == rFilename)
+			{
+				// Already exists
+				rAlreadyExists = true;
+				return en->GetObjectID();
+			}
+		}
+	}
+
+	// Allocate the next ID
+	int64_t id = AllocateObjectID();
+
+	// Create a blank directory with the given attributes on disc
+	std::string fn;
+	MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
+	{
+		BackupStoreDirectory emptyDir(id, InDirectory);
+		// add the atttribues
+		emptyDir.SetAttributes(Attributes, AttributesModTime);
+		
+		// Write...
+		RaidFileWrite dirFile(mStoreDiscSet, fn);
+		dirFile.Open(false /* no overwriting */);
+		emptyDir.WriteToStream(dirFile);
+		// Get disc usage, before it's commited
+		int64_t dirSize = dirFile.GetDiscUsageInBlocks();
+		// Commit the file
+		dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);		
+
+		// Make sure the size of the directory is added to the usage counts in the info
+		ASSERT(dirSize > 0);
+		mpStoreInfo->ChangeBlocksUsed(dirSize);
+		mpStoreInfo->ChangeBlocksInDirectories(dirSize);
+		// Not added to cache, so don't set the size in the directory
+	}
+	
+	// Then add it into the directory
+	try
+	{
+		dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
+		SaveDirectory(dir, InDirectory);
+	}
+	catch(...)
+	{
+		// Back out on adding that directory
+		RaidFileWrite del(mStoreDiscSet, fn);
+		del.Delete();
+		
+		// Remove this entry from the cache
+		RemoveDirectoryFromCache(InDirectory);
+		
+		// Don't worry about the incremented number in the store info
+		throw;	
+	}
+	
+	// Save the store info (may be postponed)
+	SaveStoreInfo();
+
+	// tell caller what the ID was
+	return id;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &, bool)
+//		Purpose: Recusively deletes a directory (or undeletes if Undelete = true)
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+void BackupContext::DeleteDirectory(int64_t ObjectID, bool Undelete)
+{
+	// Essential checks!
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	// Containing directory
+	int64_t InDirectory = 0;
+	
+	// Count of blocks deleted
+	int64_t blocksDeleted = 0;
+
+	try
+	{
+		// Get the directory that's to be deleted
+		{
+			// In block, because dir may not be valid after the delete directory call
+			BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
+			
+			// Store the directory it's in for later
+			InDirectory = dir.GetContainerID();
+		
+			// Depth first delete of contents
+			DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete);
+		}
+		
+		// Remove the entry from the directory it's in
+		ASSERT(InDirectory != 0);
+		BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory));
+		
+		BackupStoreDirectory::Iterator i(parentDir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING),
+			Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0)	// Ignore deleted directories (or not deleted if Undelete)
+		{
+			if(en->GetObjectID() == ObjectID)
+			{
+				// This is the one to delete
+				if(Undelete)
+				{
+					en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				}
+				else
+				{
+					en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				}
+							
+				// Save it
+				SaveDirectory(parentDir, InDirectory);
+				
+				// Done
+				break;
+			}
+		}
+		
+		// Update blocks deleted count
+		mpStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted));
+		
+		// Save store info, may be postponed
+		SaveStoreInfo();
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(InDirectory);
+		throw;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::DeleteDirectoryRecurse(BackupStoreDirectory &, int64_t)
+//		Purpose: Private. Deletes a directory depth-first recusively.
+//		Created: 2003/10/21
+//
+// --------------------------------------------------------------------------
+void BackupContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete)
+{
+	try
+	{
+		// Does things carefully to avoid using a directory in the cache after recursive call
+		// because it may have been deleted.
+		
+		// Do sub directories
+		{
+			// Get the directory...
+			BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
+			
+			// Then scan it for directories
+			std::vector<int64_t> subDirs;
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+			if(Undelete)
+			{
+				while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted,	// deleted dirs
+					BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING)) != 0)
+				{
+					// Store the directory ID.
+					subDirs.push_back(en->GetObjectID());
+				}
+			}
+			else
+			{
+				while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir,	// dirs only
+					BackupStoreDirectory::Entry::Flags_Deleted)) != 0)		// but not deleted ones
+				{
+					// Store the directory ID.
+					subDirs.push_back(en->GetObjectID());
+				}
+			}
+			
+			// Done with the directory for now. Recurse to sub directories
+			for(std::vector<int64_t>::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i)
+			{
+				DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete);	
+			}
+		}
+		
+		// Then, delete the files. Will need to load the directory again because it might have
+		// been removed from the cache.
+		{
+			// Get the directory...
+			BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID));
+	
+			// Changes made?
+			bool changesMade = false;
+	
+			// Run through files		
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+
+			while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING),
+				Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0)	// Ignore deleted directories (or not deleted if Undelete)
+			{
+				// Add/remove the deleted flags
+				if(Undelete)
+				{
+					en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				}
+				else
+				{
+					en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted);
+				}
+							
+				// Keep count of the deleted blocks
+				if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0)
+				{
+					rBlocksDeletedOut += en->GetSizeInBlocks();
+				}
+				
+				// Did something
+				changesMade = true;
+			}
+			
+			// Save the directory
+			if(changesMade)
+			{
+				SaveDirectory(dir, ObjectID);
+			}
+		}
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(ObjectID);
+		throw;
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::ChangeDirAttributes(int64_t, const StreamableMemBlock &, int64_t)
+//		Purpose: Change the attributes of a directory
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+void BackupContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	try
+	{	
+		// Get the directory we want to modify
+		BackupStoreDirectory &dir(GetDirectoryInternal(Directory));
+	
+		// Set attributes
+		dir.SetAttributes(Attributes, AttributesModTime);
+		
+		// Save back
+		SaveDirectory(dir, Directory);
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(Directory);
+		throw;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::ChangeFileAttributes(int64_t, int64_t, const StreamableMemBlock &, int64_t)
+//		Purpose: Sets the attributes on a directory entry. Returns true if the object existed, false if it didn't.
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+bool BackupContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+	
+	try
+	{
+		// Get the directory we want to modify
+		BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory));
+	
+		// Find the file entry
+		BackupStoreDirectory::Entry *en = 0;
+		// Iterate through current versions of files, only
+		BackupStoreDirectory::Iterator i(dir);
+		while((en = i.Next(
+			BackupStoreDirectory::Entry::Flags_File,
+			BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)
+			) != 0)
+		{
+			if(en->GetName() == rFilename)
+			{
+				// Set attributes
+				en->SetAttributes(Attributes, AttributesHash);
+				
+				// Tell caller the object ID
+				rObjectIDOut = en->GetObjectID();
+				
+				// Done
+				break;
+			}
+		}
+		if(en == 0)
+		{
+			// Didn't find it
+			return false;
+		}
+	
+		// Save back
+		SaveDirectory(dir, InDirectory);
+	}
+	catch(...)
+	{
+		RemoveDirectoryFromCache(InDirectory);
+		throw;
+	}
+	
+	// Changed, everything OK
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::ObjectExists(int64_t)
+//		Purpose: Test to see if an object of this ID exists in the store
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+bool BackupContext::ObjectExists(int64_t ObjectID, int MustBe)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	
+	// Note that we need to allow object IDs a little bit greater than the last one in the store info,
+	// because the store info may not have got saved in an error condition. Max greater ID is
+	// STORE_INFO_SAVE_DELAY in this case, *2 to be safe.
+	if(ObjectID <= 0 || ObjectID > (mpStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2)))
+	{
+		// Obviously bad object ID
+		return false;
+	}
+	
+	// Test to see if it exists on the disc
+	std::string filename;
+	MakeObjectFilename(ObjectID, filename);
+	if(!RaidFileRead::FileExists(mStoreDiscSet, filename))
+	{
+		// RaidFile reports no file there
+		return false;
+	}
+	
+	// Do we need to be more specific?
+	if(MustBe != ObjectExists_Anything)
+	{
+		// Open the file
+		std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename));
+
+		// Read the first integer
+		u_int32_t magic;
+		if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */))
+		{
+			// Failed to get any bytes, must have failed
+			return false;
+		}
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		if(MustBe == ObjectExists_File && ntohl(magic) == OBJECTMAGIC_FILE_MAGIC_VALUE_V0)
+		{
+			// Old version detected
+			return true;
+		}
+#endif
+
+		// Right one?
+		u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE;
+	
+		// Check
+		if(ntohl(magic) != requiredMagic)
+		{
+			return false;
+		}
+		
+		// File is implicitly closed
+	}
+	
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::OpenObject(int64_t)
+//		Purpose: Opens an object
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupContext::OpenObject(int64_t ObjectID)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	
+	// Attempt to open the file
+	std::string fn;
+	MakeObjectFilename(ObjectID, fn);
+	return std::auto_ptr<IOStream>(RaidFileRead::Open(mStoreDiscSet, fn).release());
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::GetClientStoreMarker()
+//		Purpose: Retrieve the client store marker
+//		Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+int64_t BackupContext::GetClientStoreMarker()
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	
+	return mpStoreInfo->GetClientStoreMarker();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::GetStoreDiscUsageInfo(int64_t &, int64_t &, int64_t &)
+//		Purpose: Get disc usage info from store info
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+void BackupContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+
+	rBlocksUsed = mpStoreInfo->GetBlocksUsed();
+	rBlocksSoftLimit = mpStoreInfo->GetBlocksSoftLimit();
+	rBlocksHardLimit = mpStoreInfo->GetBlocksHardLimit();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::HardLimitExceeded()
+//		Purpose: Returns true if the hard limit has been exceeded
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+bool BackupContext::HardLimitExceeded()
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+
+	return mpStoreInfo->GetBlocksUsed() > mpStoreInfo->GetBlocksHardLimit();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::SetClientStoreMarker(int64_t)
+//		Purpose: Sets the client store marker, and commits it to disc
+//		Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+void BackupContext::SetClientStoreMarker(int64_t ClientStoreMarker)
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+	
+	mpStoreInfo->SetClientStoreMarker(ClientStoreMarker);
+	SaveStoreInfo(false /* don't delay saving this */);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::MoveObject(int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
+//		Purpose: Move an object (and all objects with the same name) from one directory to another
+//		Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+void BackupContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly)
+	}
+
+	// Should deleted files be excluded when checking for the existance of objects with the target name?
+	int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject)
+		?(BackupStoreDirectory::Entry::Flags_Deleted)
+		:(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
+	
+	// Special case if the directories are the same...
+	if(MoveFromDirectory == MoveToDirectory)
+	{
+		try
+		{
+			// Get the first directory
+			BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory));
+		
+			// Find the file entry
+			BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID);
+	
+			// Error if not found
+			if(en == 0)
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+			}
+			
+			// Check the new name doens't already exist (optionally ignoring deleted files)
+			{
+				BackupStoreDirectory::Iterator i(dir);
+				BackupStoreDirectory::Entry *c = 0;
+				while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0)
+				{
+					if(c->GetName() == rNewFilename)
+					{
+						THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory)
+					}
+				}
+			}
+			
+			// Need to get all the entries with the same name?
+			if(MoveAllWithSameName)
+			{
+				// Iterate through the directory, copying all with matching names
+				BackupStoreDirectory::Iterator i(dir);
+				BackupStoreDirectory::Entry *c = 0;
+				while((c = i.Next()) != 0)
+				{
+					if(c->GetName() == en->GetName())
+					{
+						// Rename this one
+						c->SetName(rNewFilename);
+					}
+				}
+			}
+			else
+			{
+				// Just copy this one
+				en->SetName(rNewFilename);
+			}
+			
+			// Save the directory back
+			SaveDirectory(dir, MoveFromDirectory);
+		}
+		catch(...)
+		{
+			RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same
+			throw;
+		}
+	
+		return;
+	}
+
+	// Got to be careful how this is written, as we can't guarentte that if we have two
+	// directories open, the first won't be deleted as the second is opened. (cache)
+
+	// List of entries to move
+	std::vector<BackupStoreDirectory::Entry *> moving;
+	
+	// list of directory IDs which need to have containing dir id changed
+	std::vector<int64_t> dirsToChangeContainingID;
+
+	try
+	{
+		// First of all, get copies of the entries to move to the to directory.
+		
+		{
+			// Get the first directory
+			BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory));
+		
+			// Find the file entry
+			BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID);
+	
+			// Error if not found
+			if(en == 0)
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+			}
+			
+			// Need to get all the entries with the same name?
+			if(MoveAllWithSameName)
+			{
+				// Iterate through the directory, copying all with matching names
+				BackupStoreDirectory::Iterator i(from);
+				BackupStoreDirectory::Entry *c = 0;
+				while((c = i.Next()) != 0)
+				{
+					if(c->GetName() == en->GetName())
+					{
+						// Copy
+						moving.push_back(new BackupStoreDirectory::Entry(*c));
+						
+						// Check for containing directory correction
+						if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID());
+					}
+				}
+				ASSERT(!moving.empty());
+			}
+			else
+			{
+				// Just copy this one
+				moving.push_back(new BackupStoreDirectory::Entry(*en));
+
+				// Check for containing directory correction
+				if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID());
+			}
+		}
+		
+		// Secondly, insert them into the to directory, and save it
+		
+		{
+			// To directory
+			BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory));
+	
+			// Check the new name doens't already exist
+			{
+				BackupStoreDirectory::Iterator i(to);
+				BackupStoreDirectory::Entry *c = 0;
+				while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0)
+				{
+					if(c->GetName() == rNewFilename)
+					{
+						THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory)
+					}
+				}
+			}
+			
+			// Copy the entries into it, changing the name as we go
+			for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
+			{
+				BackupStoreDirectory::Entry *en = (*i);
+				en->SetName(rNewFilename);
+				to.AddEntry(*en);	// adds copy
+			}
+	
+			// Save back
+			SaveDirectory(to, MoveToDirectory);
+		}
+
+		// Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory
+		try
+		{
+			// Get directory
+			BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory));
+		
+			// Delete each one
+			for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
+			{
+				from.DeleteEntry((*i)->GetObjectID());
+			}
+	
+			// Save back
+			SaveDirectory(from, MoveFromDirectory);		
+		}
+		catch(...)
+		{
+			// UNDO modification to To directory
+					
+			// Get directory
+			BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory));
+		
+			// Delete each one
+			for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i)
+			{
+				to.DeleteEntry((*i)->GetObjectID());
+			}
+	
+			// Save back
+			SaveDirectory(to, MoveToDirectory);
+
+			// Throw the error
+			throw;
+		}
+		
+		// Finally... for all the directories we moved, modify their containing directory ID
+		for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i)
+		{
+			// Load the directory
+			BackupStoreDirectory &change(GetDirectoryInternal(*i));
+			
+			// Modify containing dir ID
+			change.SetContainerID(MoveToDirectory);
+			
+			// Save it back
+			SaveDirectory(change, *i);
+		}
+	}
+	catch(...)
+	{
+		// Make sure directories aren't in the cache, as they may have been modified		
+		RemoveDirectoryFromCache(MoveToDirectory);
+		RemoveDirectoryFromCache(MoveFromDirectory);
+		for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i)
+		{
+			RemoveDirectoryFromCache(*i);			
+		}
+
+		while(!moving.empty())
+		{
+			delete moving.back();
+			moving.pop_back();
+		}
+		throw;
+	}	
+
+	// Clean up
+	while(!moving.empty())
+	{
+		delete moving.back();
+		moving.pop_back();
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupContext::GetBackupStoreInfo()
+//		Purpose: Return the backup store info object, exception if it isn't loaded
+//		Created: 19/4/04
+//
+// --------------------------------------------------------------------------
+const BackupStoreInfo &BackupContext::GetBackupStoreInfo() const
+{
+	if(mpStoreInfo.get() == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded)
+	}
+	
+	return *(mpStoreInfo.get());
+}
+
+
diff --git a/bin/bbstored/BackupContext.h b/bin/bbstored/BackupContext.h
new file mode 100755
index 00000000..18f2f25c
--- /dev/null
+++ b/bin/bbstored/BackupContext.h
@@ -0,0 +1,149 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupContext.h
+//		Purpose: Context for backup store server
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCONTEXT__H
+#define BACKUPCONTEXT__H
+
+#include <string>
+#include <map>
+#include <memory>
+
+#include "NamedLock.h"
+#include "Utils.h"
+
+class BackupStoreDirectory;
+class BackupStoreFilename;
+class BackupStoreDaemon;
+class BackupStoreInfo;
+class IOStream;
+class StreamableMemBlock;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupContext
+//		Purpose: Context for backup store server
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupContext
+{
+public:
+	BackupContext(int32_t ClientID, BackupStoreDaemon &rDaemon);
+	~BackupContext();
+private:
+	BackupContext(const BackupContext &rToCopy);
+public:
+
+	void ReceivedFinishCommand();
+	void CleanUp();
+
+	int32_t GetClientID() {return mClientID;}
+
+	enum
+	{
+		Phase_START		= 0,
+		Phase_Version	= 0,
+		Phase_Login		= 1,
+		Phase_Commands	= 2
+	};
+	
+	int GetPhase() const {return mProtocolPhase;}
+	void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;}
+	
+	// Read only locking
+	bool SessionIsReadOnly() {return mReadOnly;}
+	bool AttemptToGetWriteLock();
+
+	void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;}
+	bool GetClientHasAccount() const {return mClientHasAccount;}
+	const std::string &GetStoreRoot() const {return mStoreRoot;}
+	int GetStoreDiscSet() const {return mStoreDiscSet;}
+
+	// Store info
+	void LoadStoreInfo();
+	void SaveStoreInfo(bool AllowDelay = true);
+	const BackupStoreInfo &GetBackupStoreInfo() const;
+
+	// Client marker
+	int64_t GetClientStoreMarker();
+	void SetClientStoreMarker(int64_t ClientStoreMarker);
+	
+	// Usage information
+	void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit);
+	bool HardLimitExceeded();
+
+	// Reading directories
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    BackupContext::GetDirectory(int64_t)
+	//		Purpose: Return a reference to a directory. Valid only until the 
+	//				 next time a function which affects directories is called.
+	//				 Mainly this funciton, and creation of files.
+	//		Created: 2003/09/02
+	//
+	// --------------------------------------------------------------------------
+	const BackupStoreDirectory &GetDirectory(int64_t ObjectID)
+	{
+		// External callers aren't allowed to change it -- this function
+		// merely turns the the returned directory const.
+		return GetDirectoryInternal(ObjectID);
+	}
+	
+	// Manipulating files/directories
+	int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions);
+	int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists);
+	void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime);
+	bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut);
+	bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut);
+	void DeleteDirectory(int64_t ObjectID, bool Undelete = false);
+	void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject);
+
+	// Manipulating objects
+	enum
+	{
+		ObjectExists_Anything = 0,
+		ObjectExists_File = 1,
+		ObjectExists_Directory = 2
+	};
+	bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything);
+	std::auto_ptr<IOStream> OpenObject(int64_t ObjectID);
+	
+	// Info
+	int32_t GetClientID() const {return mClientID;}
+
+private:
+	void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false);
+	BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID);
+	void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID);
+	void RemoveDirectoryFromCache(int64_t ObjectID);
+	void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
+	int64_t AllocateObjectID();
+
+private:
+	int32_t mClientID;
+	BackupStoreDaemon &mrDaemon;
+	int mProtocolPhase;
+	bool mClientHasAccount;
+	std::string mStoreRoot;	// has final directory separator
+	int mStoreDiscSet;
+	bool mReadOnly;
+	NamedLock mWriteLock;
+	int mSaveStoreInfoDelay; // how many times to delay saving the store info
+	
+	// Store info
+	std::auto_ptr<BackupStoreInfo> mpStoreInfo;
+	
+	// Directory cache
+	std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;
+};
+
+#endif // BACKUPCONTEXT__H
+
diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp
new file mode 100755
index 00000000..0afdaa5d
--- /dev/null
+++ b/bin/bbstored/BackupStoreDaemon.cpp
@@ -0,0 +1,284 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreDaemon.cpp
+//		Purpose: Backup store daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <syslog.h>
+#include <signal.h>
+
+#include "BackupContext.h"
+#include "BackupStoreDaemon.h"
+#include "BackupStoreConfigVerify.h"
+#include "autogen_BackupProtocolServer.h"
+#include "RaidFileController.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "BannerText.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::BackupStoreDaemon()
+//		Purpose: Constructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreDaemon::BackupStoreDaemon()
+	: mpAccountDatabase(0),
+	  mpAccounts(0),
+	  mExtendedLogging(false),
+	  mHaveForkedHousekeeping(false),
+	  mIsHousekeepingProcess(false),
+	  mInterProcessComms(mInterProcessCommsSocket)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::~BackupStoreDaemon()
+//		Purpose: Destructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreDaemon::~BackupStoreDaemon()
+{
+	// Must delete this one before the database ...
+	if(mpAccounts != 0)
+	{
+		delete mpAccounts;
+		mpAccounts = 0;
+	}
+	// ... as the mpAccounts object has a reference to it
+	if(mpAccountDatabase != 0)
+	{
+		delete mpAccountDatabase;
+		mpAccountDatabase = 0;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::DaemonName()
+//		Purpose: Name of daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+const char *BackupStoreDaemon::DaemonName() const
+{
+	return "bbstored";
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::DaemonBanner()
+//		Purpose: Daemon banner
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+const char *BackupStoreDaemon::DaemonBanner() const
+{
+#ifndef NDEBUG
+	// Don't display banner in debug builds
+	return 0;
+#else
+	return BANNER_TEXT("Backup Store Server");
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::GetConfigVerify()
+//		Purpose: Configuration definition
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+const ConfigurationVerify *BackupStoreDaemon::GetConfigVerify() const
+{
+	return &BackupConfigFileVerify;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::SetupInInitialProcess()
+//		Purpose: Setup before we fork -- get raid file controller going
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::SetupInInitialProcess()
+{
+	const Configuration &config(GetConfiguration());
+	
+	// Initialise the raid files controller
+	RaidFileController &rcontroller = RaidFileController::GetController();
+	rcontroller.Initialise(config.GetKeyValue("RaidFileConf").c_str());
+	
+	// Load the account database
+	std::auto_ptr<BackupStoreAccountDatabase> pdb(BackupStoreAccountDatabase::Read(config.GetKeyValue("AccountDatabase").c_str()));
+	mpAccountDatabase = pdb.release();
+	
+	// Create a accounts object
+	mpAccounts = new BackupStoreAccounts(*mpAccountDatabase);
+	
+	// Ready to go!
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::Run()
+//		Purpose: Run shim for the store daemon -- read some config details
+//		Created: 2003/10/24
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::Run()
+{
+	// Get extended logging flag
+	mExtendedLogging = false;
+	const Configuration &config(GetConfiguration());
+	mExtendedLogging = config.GetKeyValueBool("ExtendedLogging");
+	
+	// Fork off housekeeping daemon -- must only do this the first time Run() is called
+	if(!mHaveForkedHousekeeping)
+	{
+		// Open a socket pair for communication
+		int sv[2] = {-1,-1};
+		if(::socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, sv) != 0)
+		{
+			THROW_EXCEPTION(ServerException, SocketPairFailed)
+		}
+		int whichSocket = 0;
+		
+		// Fork
+		switch(::fork())
+		{
+		case -1:
+			{
+				// Error
+				THROW_EXCEPTION(ServerException, ServerForkError)
+			}
+			break;
+		case 0:
+			{
+				// In child process
+				mIsHousekeepingProcess = true;
+				SetProcessTitle("housekeeping, idle");
+				whichSocket = 1;
+				// Change the log name
+				::openlog("bbstored/hk", LOG_PID, LOG_LOCAL6);
+				// Log that housekeeping started
+				::syslog(LOG_INFO, "Housekeeping process started");
+				// Ignore term and hup
+				// Parent will handle these and alert the child via the socket, don't want to randomly die
+				::signal(SIGHUP, SIG_IGN);
+				::signal(SIGTERM, SIG_IGN);
+			}
+			break;
+		default:
+			{
+				// Parent process
+				whichSocket = 0;
+			}
+			break;
+		}
+		
+		// Mark that this has been, so -HUP doesn't try and do this again
+		mHaveForkedHousekeeping = true;
+		
+		// Attach the comms thing to the right socket, and close the other one
+		mInterProcessCommsSocket.Attach(sv[whichSocket]);
+		
+		if(::close(sv[(whichSocket == 0)?1:0]) != 0)
+		{
+			THROW_EXCEPTION(ServerException, SocketCloseError)
+		}
+	}
+
+	if(mIsHousekeepingProcess)
+	{
+		// Housekeeping process -- do other stuff
+		HousekeepingProcess();
+	}
+	else
+	{
+		// In server process -- use the base class to do the magic
+		ServerTLS<BOX_PORT_BBSTORED>::Run();
+		
+		// Why did it stop? Tell the housekeeping process to do the same
+		if(IsReloadConfigWanted())
+		{
+			mInterProcessCommsSocket.Write("h\n", 2);
+		}
+		if(IsTerminateWanted())
+		{
+			mInterProcessCommsSocket.Write("t\n", 2);
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::Connection(SocketStreamTLS &)
+//		Purpose: Handles a connection
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::Connection(SocketStreamTLS &rStream)
+{
+	// Get the common name from the certificate
+	std::string clientCommonName(rStream.GetPeerCommonName());
+	
+	// Log the name
+	::syslog(LOG_INFO, "Certificate CN: %s\n", clientCommonName.c_str());
+	
+	// Check it
+	int32_t id;
+	if(::sscanf(clientCommonName.c_str(), "BACKUP-%x", &id) != 1)
+	{
+		// Bad! Disconnect immediately
+		return;
+	}
+
+	// Make ps listings clearer
+	SetProcessTitle("client %08x", id);
+
+	// Create a context, using this ID
+	BackupContext context(id, *this);
+	
+	// See if the client has an account?
+	if(mpAccounts && mpAccounts->AccountExists(id))
+	{
+		std::string root;
+		int discSet;
+		mpAccounts->GetAccountRoot(id, root, discSet);
+		context.SetClientHasAccount(root, discSet);
+	}
+
+	// Handle a connection with the backup protocol
+	BackupProtocolServer server(rStream);
+	server.SetLogToSysLog(mExtendedLogging);
+	server.SetTimeout(BACKUP_STORE_TIMEOUT);
+	server.DoServer(context);
+	context.CleanUp();
+}
+
diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h
new file mode 100755
index 00000000..6c4a45bc
--- /dev/null
+++ b/bin/bbstored/BackupStoreDaemon.h
@@ -0,0 +1,77 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreDaemon.h
+//		Purpose: Backup store daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREDAEMON__H
+#define BACKUPSTOREDAEMON__H
+
+#include "ServerTLS.h"
+#include "BoxPortsAndFiles.h"
+#include "BackupConstants.h"
+#include "IOStreamGetLine.h"
+
+class BackupStoreAccounts;
+class BackupStoreAccountDatabase;
+class HousekeepStoreAccount;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreDaemon
+//		Purpose: Backup store daemon implementation
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupStoreDaemon : public ServerTLS<BOX_PORT_BBSTORED>
+{
+	friend class HousekeepStoreAccount;
+
+public:
+	BackupStoreDaemon();
+	~BackupStoreDaemon();
+private:
+	BackupStoreDaemon(const BackupStoreDaemon &rToCopy);
+public:
+
+	// For BackupContext to comminicate with housekeeping process
+	void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen)
+	{
+		mInterProcessCommsSocket.Write(Msg, MsgLen);
+	}
+
+protected:
+	
+	virtual void SetupInInitialProcess();
+
+	virtual void Run();
+
+	void Connection(SocketStreamTLS &rStream);
+	
+	virtual const char *DaemonName() const;
+	virtual const char *DaemonBanner() const;
+
+	const ConfigurationVerify *GetConfigVerify() const;
+	
+	// Housekeeping functions
+	void HousekeepingProcess();
+	bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0);
+
+private:
+	BackupStoreAccountDatabase *mpAccountDatabase;
+	BackupStoreAccounts *mpAccounts;
+	bool mExtendedLogging;
+	bool mHaveForkedHousekeeping;
+	bool mIsHousekeepingProcess;
+	
+	SocketStream mInterProcessCommsSocket;
+	IOStreamGetLine mInterProcessComms;
+};
+
+
+#endif // BACKUPSTOREDAEMON__H
+
diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp
new file mode 100755
index 00000000..4aa1999e
--- /dev/null
+++ b/bin/bbstored/HousekeepStoreAccount.cpp
@@ -0,0 +1,844 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    HousekeepStoreAccount.cpp
+//		Purpose: 
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <map>
+#include <stdio.h>
+
+#include "HousekeepStoreAccount.h"
+#include "BackupStoreDaemon.h"
+#include "StoreStructure.h"
+#include "BackupStoreConstants.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreInfo.h"
+#include "NamedLock.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreFile.h"
+
+#include "MemLeakFindOn.h"
+
+// check every 32 directories scanned/files deleted
+#define POLL_INTERPROCESS_MSG_CHECK_FREQUENCY	32
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::HousekeepStoreAccount(int, const std::string &, int, BackupStoreDaemon &)
+//		Purpose: Constructor
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon)
+	: mAccountID(AccountID),
+	  mStoreRoot(rStoreRoot),
+	  mStoreDiscSet(StoreDiscSet),
+	  mrDaemon(rDaemon),
+	  mDeletionSizeTarget(0),
+  	  mPotentialDeletionsTotalSize(0),
+	  mMaxSizeInPotentialDeletions(0),
+	  mBlocksUsed(0),
+	  mBlocksInOldFiles(0),
+	  mBlocksInDeletedFiles(0),
+	  mBlocksInDirectories(0),
+	  mBlocksUsedDelta(0),
+	  mBlocksInOldFilesDelta(0),
+	  mBlocksInDeletedFilesDelta(0),
+	  mBlocksInDirectoriesDelta(0),
+	  mFilesDeleted(0),
+	  mEmptyDirectoriesDeleted(0),
+	  mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::~HousekeepStoreAccount()
+//		Purpose: Destructor
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+HousekeepStoreAccount::~HousekeepStoreAccount()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DoHousekeeping()
+//		Purpose: Perform the housekeeping
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::DoHousekeeping()
+{
+	// Attempt to lock the account
+	std::string writeLockFilename;
+	StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFilename);
+	NamedLock writeLock;
+	if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */))
+	{
+		// Couldn't lock the account -- just stop now
+		return;
+	}
+
+	// Load the store info to find necessary info for the housekeeping
+	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet, false /* Read/Write */));
+
+	// Calculate how much should be deleted
+	mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit();
+	if(mDeletionSizeTarget < 0)
+	{
+		mDeletionSizeTarget = 0;
+	}
+
+	// Scan the directory for potential things to delete
+	// This will also remove elegiable items marked with RemoveASAP
+	bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+	// If scan directory stopped for some reason, probably parent instructed to teminate, stop now.
+	if(!continueHousekeeping)
+	{
+		// If any files were marked "delete now", then update the size of the store.
+		if(mBlocksUsedDelta != 0 || mBlocksInOldFilesDelta != 0 || mBlocksInDeletedFilesDelta != 0)
+		{
+			info->ChangeBlocksUsed(mBlocksUsedDelta);
+			info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+			info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+			
+			// Save the store info back
+			info->Save();
+		}
+	
+		return;
+	}
+
+	// Log any difference in opinion between the values recorded in the store info, and
+	// the values just calculated for space usage.
+	// BLOCK
+	{
+		int64_t used = info->GetBlocksUsed();
+		int64_t usedOld = info->GetBlocksInOldFiles();
+		int64_t usedDeleted = info->GetBlocksInDeletedFiles();
+		int64_t usedDirectories = info->GetBlocksInDirectories();
+
+		// If the counts were wrong, taking into account RemoveASAP items deleted, log a message
+		if((used + mBlocksUsedDelta) != mBlocksUsed || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles
+			|| (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles || usedDirectories != mBlocksInDirectories)
+		{
+			// Log this
+			::syslog(LOG_ERR, "On housekeeping, sizes in store do not match calculated sizes, correcting");
+			::syslog(LOG_ERR, "different (store,calc): acc 0x%08x, used (%lld,%lld), old (%lld,%lld), deleted (%lld,%lld), dirs (%lld,%lld)",
+				mAccountID,
+				(used + mBlocksUsedDelta), mBlocksUsed, (usedOld + mBlocksInOldFilesDelta), mBlocksInOldFiles,
+				(usedDeleted + mBlocksInDeletedFilesDelta), mBlocksInDeletedFiles, usedDirectories, mBlocksInDirectories);
+		}
+		
+		// If the current values don't match, store them
+		if(used != mBlocksUsed || usedOld != mBlocksInOldFiles
+			|| usedDeleted != mBlocksInDeletedFiles || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta))
+		{	
+			// Set corrected values in store info
+			info->CorrectAllUsedValues(mBlocksUsed, mBlocksInOldFiles, mBlocksInDeletedFiles, mBlocksInDirectories + mBlocksInDirectoriesDelta);
+			info->Save();
+		}
+	}
+	
+	// Reset the delta counts for files, as they will include RemoveASAP flagged files deleted
+	// during the initial scan.
+	int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta;	// keep for reporting
+	mBlocksUsedDelta = 0;
+	mBlocksInOldFilesDelta = 0;
+	mBlocksInDeletedFilesDelta = 0;
+	
+	// Go and delete items from the accounts
+	bool deleteInterrupted = DeleteFiles();
+	
+	// If that wasn't interrupted, remove any empty directories which are also marked as deleted in their containing directory
+	if(!deleteInterrupted)
+	{
+		deleteInterrupted = DeleteEmptyDirectories();
+	}
+	
+	// Log deletion if anything was deleted
+	if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0)
+	{
+		::syslog(LOG_INFO, "Account 0x%08x, removed %lld blocks (%lld files, %lld dirs)%s", mAccountID, 0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta),
+			mFilesDeleted, mEmptyDirectoriesDeleted,
+			deleteInterrupted?" was interrupted":"");
+	}
+	
+	// Make sure the delta's won't cause problems if the counts are really wrong, and
+	// it wasn't fixed because the store was updated during the scan.
+	if(mBlocksUsedDelta 			< (0 - info->GetBlocksUsed())) 				mBlocksUsedDelta = 			(0 - info->GetBlocksUsed());
+	if(mBlocksInOldFilesDelta 		< (0 - info->GetBlocksInOldFiles())) 		mBlocksInOldFilesDelta = 	(0 - info->GetBlocksInOldFiles());
+	if(mBlocksInDeletedFilesDelta 	< (0 - info->GetBlocksInDeletedFiles())) 	mBlocksInDeletedFilesDelta =(0 - info->GetBlocksInDeletedFiles());
+	if(mBlocksInDirectoriesDelta 	< (0 - info->GetBlocksInDirectories()))		mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories());
+	
+	// Update the usage counts in the store
+	info->ChangeBlocksUsed(mBlocksUsedDelta);
+	info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta);
+	info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta);
+	info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta);
+	
+	// Save the store info back
+	info->Save();
+	
+	// Explicity release the lock (would happen automatically on going out of scope, included for code clarity)
+	writeLock.ReleaseLock();
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::MakeObjectFilename(int64_t, std::string &)
+//		Purpose: Generate and place the filename for a given object ID
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut)
+{
+	// Delegate to utility function
+	StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rFilenameOut, false /* don't bother ensuring the directory exists */);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::ScanDirectory(int64_t)
+//		Purpose: Private. Scan a directory for potenitally deleteable items, and
+//				 add them to the list. Returns true if the scan should continue.
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
+{
+	if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+	{
+		mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+		// Check for having to stop
+		if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is locked
+		{
+			// Need to abort now
+			return false;
+		}
+	}
+
+	// Get the filename
+	std::string objectFilename;
+	MakeObjectFilename(ObjectID, objectFilename);
+
+	// Open it.
+	std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename));
+	
+	// Add the size of the directory on disc to the size being calculated
+	int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+	mBlocksInDirectories += originalDirSizeInBlocks;
+	mBlocksUsed += originalDirSizeInBlocks;
+	
+	// Read the directory in
+	BackupStoreDirectory dir;
+	dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+	
+	// Is it empty?
+	if(dir.GetNumberOfEntries() == 0)
+	{
+		// Add it to the list of directories to potentially delete
+		mEmptyDirectories.push_back(dir.GetObjectID());
+	}
+	
+	// BLOCK
+	{
+		// Remove any files which are marked for removal as soon as they become old
+		// or deleted.
+		bool deletedSomething = false;
+		do
+		{
+			// Iterate through the directory
+			deletedSomething = false;
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+			while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+			{
+				int16_t enFlags = en->GetFlags();
+				if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0
+					&& (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+				{
+					// Delete this immediately.
+					DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks);
+					
+					// flag as having done something
+					deletedSomething = true;
+					
+					// Must start the loop from the beginning again, as iterator is now
+					// probably invalid.
+					break;
+				}
+			}
+		} while(deletedSomething);
+	}
+	
+	// BLOCK
+	{
+		// Add files to the list of potential deletions
+
+		// map to count the distance from the mark
+		std::map<std::pair<BackupStoreFilename, int32_t>, int32_t> markVersionAges;
+			// map of pair (filename, mark number) -> version age
+
+		// NOTE: use a reverse iterator to allow the distance from mark stuff to work
+		BackupStoreDirectory::ReverseIterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+		{
+			// Update recalculated usage sizes
+			int16_t enFlags = en->GetFlags();
+			int64_t enSizeInBlocks = en->GetSizeInBlocks();
+			mBlocksUsed += enSizeInBlocks;
+			if(enFlags & BackupStoreDirectory::Entry::Flags_OldVersion) mBlocksInOldFiles += enSizeInBlocks;
+			if(enFlags & BackupStoreDirectory::Entry::Flags_Deleted) mBlocksInDeletedFiles += enSizeInBlocks;
+					
+			// Work out ages of this version from the last mark
+			int32_t enVersionAge = 0;
+			std::map<std::pair<BackupStoreFilename, int32_t>, int32_t>::iterator enVersionAgeI(markVersionAges.find(std::pair<BackupStoreFilename, int32_t>(en->GetName(), en->GetMarkNumber())));
+			if(enVersionAgeI != markVersionAges.end())
+			{
+				enVersionAge = enVersionAgeI->second + 1;
+				enVersionAgeI->second = enVersionAge;
+			}
+			else
+			{
+				markVersionAges[std::pair<BackupStoreFilename, int32_t>(en->GetName(), en->GetMarkNumber())] = enVersionAge;
+			}
+			// enVersionAge is now the age of this version.
+			
+			// Potentially add it to the list if it's deleted, if it's an old version or deleted
+			if((enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0)
+			{
+				// Is deleted / old version.
+				DelEn d;
+				d.mObjectID = en->GetObjectID();
+				d.mInDirectory = ObjectID;
+				d.mSizeInBlocks = en->GetSizeInBlocks();
+				d.mMarkNumber = en->GetMarkNumber();
+				d.mVersionAgeWithinMark = enVersionAge;
+				
+				// Add it to the list
+				mPotentialDeletions.insert(d);
+				
+				// Update various counts
+				mPotentialDeletionsTotalSize += d.mSizeInBlocks;
+				if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks;
+				
+				// Too much in the list of potential deletions?
+				// (check against the deletion target + the max size in deletions, so that we never delete things
+				// and take the total size below the deletion size target)
+				if(mPotentialDeletionsTotalSize > (mDeletionSizeTarget + mMaxSizeInPotentialDeletions))
+				{
+					int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions);
+					bool recalcMaxSize = false;
+					
+					while(sizeToRemove > 0)
+					{
+						// Make iterator for the last element, while checking that there's something there in the first place.
+						std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.end());
+						if(i != mPotentialDeletions.begin())
+						{
+							// Nothing left in set
+							break;
+						}
+						// Make this into an iterator pointing to the last element in the set
+						--i;
+						
+						// Delete this one?
+						if(sizeToRemove > i->mSizeInBlocks)
+						{
+							sizeToRemove -= i->mSizeInBlocks;
+							if(i->mSizeInBlocks >= mMaxSizeInPotentialDeletions)
+							{
+								// Will need to recalculate the maximum size now, because we've just deleted that element
+								recalcMaxSize = true;
+							}
+							mPotentialDeletions.erase(i);
+						}
+						else
+						{
+							// Over the size to remove, so stop now
+							break;
+						}
+					}
+					
+					if(recalcMaxSize)
+					{
+						// Because an object which was the maximum size recorded was deleted from the set
+						// it's necessary to recalculate this maximum.
+						mMaxSizeInPotentialDeletions = 0;
+						std::set<DelEn, DelEnCompare>::const_iterator i(mPotentialDeletions.begin());
+						for(; i != mPotentialDeletions.end(); ++i)
+						{
+							if(i->mSizeInBlocks > mMaxSizeInPotentialDeletions)
+							{
+								mMaxSizeInPotentialDeletions = i->mSizeInBlocks;
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	{
+		// Recurse into subdirectories
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
+		{
+			// Next level
+			ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir);
+			
+			if(!ScanDirectory(en->GetObjectID()))
+			{
+				// Halting operation
+				return false;
+			}
+		}
+	}
+	
+	return true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &, const HousekeepStoreAccount::DelEnd &)
+//		Purpose: Comparison function for set
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &x, const HousekeepStoreAccount::DelEn &y)
+{
+	// STL spec says this:
+	// A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. 
+
+	// The sort order here is intended to preserve the entries of most value, that is, the newest objects
+	// which are on a mark boundary.
+	
+	// Reverse order age, so oldest goes first
+	if(x.mVersionAgeWithinMark > y.mVersionAgeWithinMark)
+	{
+		return true;
+	}
+	else if(x.mVersionAgeWithinMark < y.mVersionAgeWithinMark)
+	{
+		return false;
+	}
+
+	// but mark number in ascending order, so that the oldest marks are deleted first
+	if(x.mMarkNumber < y.mMarkNumber)
+	{
+		return true;
+	}
+	else if(x.mMarkNumber > y.mMarkNumber)
+	{
+		return false;
+	}
+
+	// Just compare object ID now to put the oldest objects first
+	return x.mObjectID < y.mObjectID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DeleteFiles()
+//		Purpose: Delete the files targetted for deletion, returning true if the operation was interrupted
+//		Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DeleteFiles()
+{
+	// Only delete files if the deletion target is greater than zero
+	// (otherwise we delete one file each time round, which gradually deletes the old versions)
+	if(mDeletionSizeTarget <= 0)
+	{
+		// Not interrupted
+		return false;
+	}
+
+	// Iterate through the set of potential deletions, until enough has been deleted.
+	// (there is likely to be more in the set than should be actually deleted).
+	for(std::set<DelEn, DelEnCompare>::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i)
+	{
+		if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+		{
+			mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+			// Check for having to stop
+			if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+			{
+				// Need to abort now
+				return true;
+			}
+		}
+
+		// Load up the directory it's in
+		// Get the filename
+		std::string dirFilename;
+		BackupStoreDirectory dir;
+		int64_t dirSizeInBlocksOrig = 0;
+		{
+			MakeObjectFilename(i->mInDirectory, dirFilename);
+			std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
+			dirSizeInBlocksOrig = dirStream->GetDiscUsageInBlocks();
+			dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+		}
+		
+		// Delete the file
+		DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig);
+		
+		// Stop if the deletion target has been matched or exceeded
+		// (checking here rather than at the beginning will tend to reduce the
+		// space to slightly less than the soft limit, which will allow the backup
+		// client to start uploading files again)
+		if((0 - mBlocksUsedDelta) >= mDeletionSizeTarget)
+		{
+			break;
+		}
+	}
+
+	return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DeleteFile(int64_t, int64_t, BackupStoreDirectory &, const std::string &, int64_t)
+//		Purpose: Delete a file. Takes the directory already loaded in and the filename,
+//				 for efficiency in both the usage senarios.
+//		Created: 15/7/04
+//
+// --------------------------------------------------------------------------
+void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks)
+{
+	// Find the entry inside the directory
+	bool wasDeleted = false;
+	bool wasOldVersion = false;
+	int64_t deletedFileSizeInBlocks = 0;
+	// A pointer to an object which requires commiting if the directory save goes OK
+	std::auto_ptr<RaidFileWrite> padjustedEntry;
+	// BLOCK
+	{
+		BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID);
+		if(pentry == 0)
+		{
+			::syslog(LOG_ERR, "acc 0x%08x, object %lld not found in dir %lld, logic error/corruption? Run bbstoreaccounts check <accid> fix", mAccountID, ObjectID, InDirectory);
+			return;
+		}
+		
+		// Record the flags it's got set
+		wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0);
+		wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0);
+		// Check this should be deleted
+		if(!wasDeleted && !wasOldVersion)
+		{
+			// Things changed size we were last around
+			return;
+		}
+		
+		// Record size
+		deletedFileSizeInBlocks = pentry->GetSizeInBlocks();
+		
+		// If the entry is involved in a chain of patches, it needs to be handled
+		// a bit more carefully.
+		if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0)
+		{
+			// This entry is a patch from a newer entry. Just need to update the info on that entry.
+			BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
+			if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID)
+			{
+				THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+			}
+			// Change the info in the newer entry so that this no longer points to this entry
+			pnewer->SetDependsOlder(0);
+		}
+		else if(pentry->GetDependsOlder() != 0)
+		{
+			BackupStoreDirectory::Entry *polder = rDirectory.FindEntryByID(pentry->GetDependsOlder());
+			if(pentry->GetDependsNewer() == 0)
+			{
+				// There exists an older version which depends on this one. Need to combine the two over that one.
+
+				// Adjust the other entry in the directory
+				if(polder == 0 || polder->GetDependsNewer() != ObjectID)
+				{
+					THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+				}
+				// Change the info in the older entry so that this no longer points to this entry
+				polder->SetDependsNewer(0);
+			}
+			else
+			{
+				// This entry is in the middle of a chain, and two patches need combining.
+				
+				// First, adjust the directory entries
+				BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer());
+				if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID
+					|| polder == 0 || polder->GetDependsNewer() != ObjectID)
+				{
+					THROW_EXCEPTION(BackupStoreException, PatchChainInfoBadInDirectory);
+				}
+				// Remove the middle entry from the linked list by simply using the values from this entry
+				pnewer->SetDependsOlder(pentry->GetDependsOlder());
+				polder->SetDependsNewer(pentry->GetDependsNewer());
+			}
+			
+			// COMMON CODE to both cases
+
+			// Generate the filename of the older version
+			std::string objFilenameOlder;
+			MakeObjectFilename(pentry->GetDependsOlder(), objFilenameOlder);
+			// Open it twice (it's the diff)
+			std::auto_ptr<RaidFileRead> pdiff(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
+			std::auto_ptr<RaidFileRead> pdiff2(RaidFileRead::Open(mStoreDiscSet, objFilenameOlder));
+			// Open this file
+			std::string objFilename;
+			MakeObjectFilename(ObjectID, objFilename);
+			std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename));
+			// And open a write file to overwrite the other directory entry
+			padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, objFilenameOlder));
+			padjustedEntry->Open(true /* allow overwriting */);
+
+			if(pentry->GetDependsNewer() == 0)
+			{
+				// There exists an older version which depends on this one. Need to combine the two over that one.
+				BackupStoreFile::CombineFile(*pdiff, *pdiff2, *pobjectBeingDeleted, *padjustedEntry);
+			}
+			else
+			{
+				// This entry is in the middle of a chain, and two patches need combining.
+				BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry);
+			}
+			// The file will be committed later when the directory is safely commited.
+			
+			// Work out the adjusted size
+			int64_t newSize = padjustedEntry->GetDiscUsageInBlocks();
+			int64_t sizeDelta = newSize - polder->GetSizeInBlocks();
+			mBlocksUsedDelta += sizeDelta;
+			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)
+			{
+				mBlocksInDeletedFilesDelta += sizeDelta;
+			}
+			if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0)
+			{
+				mBlocksInOldFilesDelta += sizeDelta;
+			}
+			polder->SetSizeInBlocks(newSize);
+		}
+		
+		// pentry no longer valid
+	}
+	
+	// Delete it from the directory
+	rDirectory.DeleteEntry(ObjectID);
+	
+	// Save directory back to disc
+	// BLOCK
+	int64_t dirRevisedSize = 0;
+	{
+		RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename);
+		writeDir.Open(true /* allow overwriting */);
+		rDirectory.WriteToStream(writeDir);
+
+		// get the disc usage (must do this before commiting it)
+		dirRevisedSize = writeDir.GetDiscUsageInBlocks();
+
+		// Commit directory
+		writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+		// adjust usage counts for this directory
+		if(dirRevisedSize > 0)
+		{
+			int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks;
+			mBlocksUsedDelta += adjust;
+			mBlocksInDirectoriesDelta += adjust;
+		}
+	}
+
+	// Commit any new adjusted entry
+	if(padjustedEntry.get() != 0)
+	{
+		padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+		padjustedEntry.reset();	// delete it now
+	}
+
+	// Delete from disc
+	{
+		std::string objFilename;
+		MakeObjectFilename(ObjectID, objFilename);
+		RaidFileWrite del(mStoreDiscSet, objFilename);
+		del.Delete();
+	}
+
+	// Adjust counts for the file
+	++mFilesDeleted;
+	mBlocksUsedDelta -= deletedFileSizeInBlocks;
+	if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks;
+	if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks;
+	
+	// Delete the directory?
+	// Do this if... dir has zero entries, and is marked as deleted in it's containing directory
+	if(rDirectory.GetNumberOfEntries() == 0)
+	{
+		// Candidate for deletion
+		mEmptyDirectories.push_back(InDirectory);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    HousekeepStoreAccount::DeleteEmptyDirectories()
+//		Purpose: Remove any empty directories which are also marked as deleted in their containing directory,
+//				 returning true if the opertaion was interrupted
+//		Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+bool HousekeepStoreAccount::DeleteEmptyDirectories()
+{
+	while(mEmptyDirectories.size() > 0)
+	{
+		std::vector<int64_t> toExamine;
+
+		// Go through list
+		for(std::vector<int64_t>::const_iterator i(mEmptyDirectories.begin()); i != mEmptyDirectories.end(); ++i)
+		{
+			if((--mCountUntilNextInterprocessMsgCheck) <= 0)
+			{
+				mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
+				// Check for having to stop
+				if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+				{
+					// Need to abort now
+					return true;
+				}
+			}
+
+			// Do not delete the root directory
+			if(*i == BACKUPSTORE_ROOT_DIRECTORY_ID)
+			{
+				continue;
+			}
+
+			// Load up the directory to potentially delete
+			std::string dirFilename;
+			BackupStoreDirectory dir;
+			int64_t dirSizeInBlocks = 0;
+			{
+				MakeObjectFilename(*i, dirFilename);
+				// Check it actually exists (just in case it gets added twice to the list)
+				if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename))
+				{
+					// doesn't exist, next!
+					continue;
+				}
+				// load
+				std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename));
+				dirSizeInBlocks = dirStream->GetDiscUsageInBlocks();
+				dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+			}
+
+			// Make sure this directory is actually empty
+			if(dir.GetNumberOfEntries() != 0)
+			{
+				// Not actually empty, try next one
+				continue;
+			}
+
+			// Candiate for deletion... open containing directory
+			std::string containingDirFilename;
+			BackupStoreDirectory containingDir;
+			int64_t containingDirSizeInBlocksOrig = 0;
+			{
+				MakeObjectFilename(dir.GetContainerID(), containingDirFilename);
+				std::auto_ptr<RaidFileRead> containingDirStream(RaidFileRead::Open(mStoreDiscSet, containingDirFilename));
+				containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks();
+				containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite);
+			}
+
+			// Find the entry
+			BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID());
+			if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0))
+			{
+				// Should be deleted
+				containingDir.DeleteEntry(dir.GetObjectID());
+
+				// Is the containing dir now a candidate for deletion?
+				if(containingDir.GetNumberOfEntries() == 0)
+				{
+					toExamine.push_back(containingDir.GetObjectID());
+				}
+
+				// Write revised parent directory
+				RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename);
+				writeDir.Open(true /* allow overwriting */);
+				containingDir.WriteToStream(writeDir);
+
+				// get the disc usage (must do this before commiting it)
+				int64_t dirSize = writeDir.GetDiscUsageInBlocks();
+
+				// Commit directory
+				writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY);
+
+				// adjust usage counts for this directory
+				if(dirSize > 0)
+				{
+					int64_t adjust = dirSize - containingDirSizeInBlocksOrig;
+					mBlocksUsedDelta += adjust;
+					mBlocksInDirectoriesDelta += adjust;
+				}
+
+				// Delete the directory itself
+				{
+					RaidFileWrite del(mStoreDiscSet, dirFilename);
+					del.Delete();
+				}
+
+				// And adjust usage counts for the directory that's just been deleted
+				mBlocksUsedDelta -= dirSizeInBlocks;
+				mBlocksInDirectoriesDelta -= dirSizeInBlocks;
+
+				// Update count
+				++mEmptyDirectoriesDeleted;
+			}
+		}
+
+		// Remove contents of empty directories
+		mEmptyDirectories.clear();
+		// Swap in new, so it's examined next time round
+		mEmptyDirectories.swap(toExamine);
+	}
+	
+	// Not interrupted
+	return false;
+}
+
+
+
+
diff --git a/bin/bbstored/HousekeepStoreAccount.h b/bin/bbstored/HousekeepStoreAccount.h
new file mode 100755
index 00000000..6c8f251d
--- /dev/null
+++ b/bin/bbstored/HousekeepStoreAccount.h
@@ -0,0 +1,97 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    HousekeepStoreAccount.h
+//		Purpose: Action class to perform housekeeping on a store account
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef HOUSEKEEPSTOREACCOUNT__H
+#define HOUSEKEEPSTOREACCOUNT__H
+
+#include <string>
+#include <set>
+#include <vector>
+
+class BackupStoreDaemon;
+class BackupStoreDirectory;
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    HousekeepStoreAccount
+//		Purpose: Action class to perform housekeeping on a store account
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+class HousekeepStoreAccount
+{
+public:
+	HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon);
+	~HousekeepStoreAccount();
+	
+	void DoHousekeeping();
+	
+	
+private:
+	// utility functions
+	void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut);
+
+	bool ScanDirectory(int64_t ObjectID);
+	bool DeleteFiles();
+	bool DeleteEmptyDirectories();
+	void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks);
+
+private:
+	typedef struct
+	{
+		int64_t mObjectID;
+		int64_t mInDirectory;
+		int64_t mSizeInBlocks;
+		int32_t mMarkNumber;
+		int32_t mVersionAgeWithinMark;	// 0 == current, 1 latest old version, etc
+	} DelEn;
+	
+	struct DelEnCompare
+	{
+		bool operator()(const DelEn &x, const DelEn &y);
+	};
+	
+	int mAccountID;
+	std::string mStoreRoot;
+	int mStoreDiscSet;
+	BackupStoreDaemon &mrDaemon;
+	
+	int64_t mDeletionSizeTarget;
+	
+	std::set<DelEn, DelEnCompare> mPotentialDeletions;
+	int64_t mPotentialDeletionsTotalSize;
+	int64_t mMaxSizeInPotentialDeletions;
+	
+	// List of directories which are empty, and might be good for deleting
+	std::vector<int64_t> mEmptyDirectories;
+	
+	// The re-calculated blocks used stats
+	int64_t mBlocksUsed;
+	int64_t mBlocksInOldFiles;
+	int64_t mBlocksInDeletedFiles;
+	int64_t mBlocksInDirectories;
+
+	// Deltas from deletion
+	int64_t mBlocksUsedDelta;
+	int64_t mBlocksInOldFilesDelta;
+	int64_t mBlocksInDeletedFilesDelta;
+	int64_t mBlocksInDirectoriesDelta;
+	
+	// Deletion count
+	int64_t mFilesDeleted;
+	int64_t mEmptyDirectoriesDeleted;
+	
+	// Poll frequency
+	int mCountUntilNextInterprocessMsgCheck;
+};
+
+#endif // HOUSEKEEPSTOREACCOUNT__H
+
diff --git a/bin/bbstored/Makefile.extra b/bin/bbstored/Makefile.extra
new file mode 100755
index 00000000..187d53ef
--- /dev/null
+++ b/bin/bbstored/Makefile.extra
@@ -0,0 +1,9 @@
+
+MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
+
+GEN_CMD_SRV = $(MAKEPROTOCOL) Server backupprotocol.txt
+
+# AUTOGEN SEEDING
+autogen_BackupProtocolServer.cpp autogen_BackupProtocolServer.h:	$(MAKEPROTOCOL) backupprotocol.txt
+	perl $(GEN_CMD_SRV)
+
diff --git a/bin/bbstored/backupprotocol.txt b/bin/bbstored/backupprotocol.txt
new file mode 100755
index 00000000..39cb1fb3
--- /dev/null
+++ b/bin/bbstored/backupprotocol.txt
@@ -0,0 +1,221 @@
+#
+# backup protocol definition
+#
+
+Name 				Backup
+IdentString			Box-Backup:v=B
+ServerContextClass	BackupContext	BackupContext.h
+
+ClientType		Filename	BackupStoreFilenameClear	BackupStoreFilenameClear.h
+ServerType		Filename	BackupStoreFilename			BackupStoreFilename.h
+
+ImplementLog	Server	syslog
+ImplementLog	Client	syslog
+ImplementLog	Client	file
+
+LogTypeToText	Client	Filename	\"%s\"	VAR.GetClearFilename().c_str()
+
+BEGIN_OBJECTS
+
+# -------------------------------------------------------------------------------------
+#  Session commands
+# -------------------------------------------------------------------------------------
+
+Error		0	IsError(Type,SubType)	Reply
+	int32		Type
+	int32		SubType
+	CONSTANT	ErrorType						1000
+	CONSTANT	Err_WrongVersion				1
+	CONSTANT	Err_NotInRightProtocolPhase		2
+	CONSTANT	Err_BadLogin					3
+	CONSTANT	Err_CannotLockStoreForWriting	4
+	CONSTANT	Err_SessionReadOnly				5
+	CONSTANT	Err_FileDoesNotVerify			6
+	CONSTANT	Err_DoesNotExist				7
+	CONSTANT	Err_DirectoryAlreadyExists		8
+	CONSTANT	Err_CannotDeleteRoot			9
+	CONSTANT	Err_TargetNameExists			10
+	CONSTANT	Err_StorageLimitExceeded		11
+	CONSTANT	Err_DiffFromFileDoesNotExist	12
+	CONSTANT	Err_DoesNotExistInDirectory		13
+	CONSTANT	Err_PatchConsistencyError		14
+
+Version		1	Command(Version)	Reply
+	int32	Version
+
+
+Login		2	Command(LoginConfirmed)
+	int32		ClientID
+	int32		Flags
+	CONSTANT	Flags_ReadOnly	1
+
+
+LoginConfirmed	3	Reply
+	int64		ClientStoreMarker
+	int64		BlocksUsed
+	int64		BlocksSoftLimit
+	int64		BlocksHardLimit
+
+
+Finished	4	Command(Finished)	Reply	EndsConversation
+
+
+# generic success object
+Success		5	Reply
+	int64		ObjectID
+
+
+SetClientStoreMarker	6	Command(Success)
+	int64		ClientStoreMarker
+
+
+# -------------------------------------------------------------------------------------
+#  Generic object commands
+# -------------------------------------------------------------------------------------
+
+GetObject	10	Command(Success)
+	int64		ObjectID
+	CONSTANT	NoObject	0
+	# reply has stream following, if ObjectID != NoObject
+
+
+MoveObject	11	Command(Success)
+	int64		ObjectID
+	int64		MoveFromDirectory
+	int64		MoveToDirectory
+	int32		Flags
+	Filename	NewFilename
+	
+	CONSTANT Flags_MoveAllWithSameName			1
+	CONSTANT Flags_AllowMoveOverDeletedObject	2
+
+# consider this an object command as, although it deals with directory entries,
+# it's not specific to either a file or a directory
+
+
+GetObjectName	12	Command(ObjectName)
+	int64		ObjectID
+	int64		ContainingDirectoryID
+	CONSTANT	ObjectID_DirectoryOnly	0
+
+	# set ObjectID to ObjectID_DirectoryOnly to only get info on the directory
+
+
+ObjectName		13	Reply
+	int32		NumNameElements
+	int64		ModificationTime
+	int64		AttributesHash
+	int16		Flags
+	# NumNameElements is zero if the object doesn't exist
+	CONSTANT	NumNameElements_ObjectDoesntExist	0
+	# a stream of Filename objects follows, if and only if NumNameElements > 0
+
+
+# -------------------------------------------------------------------------------------
+#  Directory commands
+# -------------------------------------------------------------------------------------
+
+CreateDirectory	20	Command(Success)	StreamWithCommand
+	int64		ContainingDirectoryID
+	int64		AttributesModTime
+	Filename	DirectoryName
+	# stream following containing attributes
+
+
+ListDirectory	21	Command(Success)
+	int64		ObjectID
+	int16		FlagsMustBeSet
+	int16		FlagsNotToBeSet
+	bool		SendAttributes
+	# make sure these flags are synced with those in BackupStoreDirectory
+	CONSTANT	Flags_INCLUDE_EVERYTHING 	-1
+	CONSTANT	Flags_EXCLUDE_NOTHING 		0
+	CONSTANT	Flags_EXCLUDE_EVERYTHING	15
+	CONSTANT	Flags_File			1
+	CONSTANT	Flags_Dir			2
+	CONSTANT	Flags_Deleted			4
+	CONSTANT	Flags_OldVersion		8
+	# make sure this is the same as in BackupStoreConstants.h
+	CONSTANT	RootDirectory			1
+
+	# reply has stream following Success object, containing a stored BackupStoreDirectory
+
+
+ChangeDirAttributes	22	Command(Success)	StreamWithCommand
+	int64		ObjectID
+	int64		AttributesModTime
+	# stream following containing attributes
+
+
+DeleteDirectory	23	Command(Success)
+	int64		ObjectID
+
+UndeleteDirectory	24	Command(Success)
+	int64		ObjectID
+	# may not have exactly the desired effect if files within in have been deleted before the directory was deleted.
+
+
+# -------------------------------------------------------------------------------------
+#  File commands
+# -------------------------------------------------------------------------------------
+
+StoreFile	30	Command(Success)	StreamWithCommand
+	int64		DirectoryObjectID
+	int64		ModificationTime
+	int64		AttributesHash
+	int64		DiffFromFileID		# 0 if the file is not a diff
+	Filename	Filename
+	# then send a stream containing the encoded file
+
+
+GetFile		31	Command(Success)
+	int64		InDirectory
+	int64		ObjectID
+	# error returned if not a file, or does not exist
+	# reply has stream following, containing an encoded file IN STREAM ORDER
+	# (use GetObject to get it in file order)
+
+
+SetReplacementFileAttributes	32	Command(Success)	StreamWithCommand
+	int64		InDirectory
+	int64		AttributesHash
+	Filename	Filename
+	# stream follows containing attributes
+
+
+DeleteFile	33	Command(Success)
+	int64		InDirectory
+	Filename	Filename
+	# will return 0 if the object couldn't be found in the specified directory
+
+
+GetBlockIndexByID	34	Command(Success)
+	int64		ObjectID
+
+	# stream of the block index follows the reply
+	# returns an error if the object didn't exist
+
+
+GetBlockIndexByName	35	Command(Success)
+	int64		InDirectory
+	Filename	Filename
+
+	# Success object contains the found ID -- or 0 if the entry wasn't found in the directory
+	# stream of the block index follows the reply if found ID != 0
+
+
+# -------------------------------------------------------------------------------------
+#  Information commands
+# -------------------------------------------------------------------------------------
+
+GetAccountUsage	40	Command(AccountUsage)
+	# no data members
+
+AccountUsage	41	Reply
+	int64	BlocksUsed
+	int64	BlocksInOldFiles
+	int64	BlocksInDeletedFiles
+	int64	BlocksInDirectories
+	int64	BlocksSoftLimit
+	int64	BlocksHardLimit
+	int32	BlockSize
diff --git a/bin/bbstored/bbstored-certs b/bin/bbstored/bbstored-certs
new file mode 100755
index 00000000..d1fa8dea
--- /dev/null
+++ b/bin/bbstored/bbstored-certs
@@ -0,0 +1,319 @@
+#!/usr/bin/perl
+use strict;
+
+# validity period for root certificates -- default is a very long time
+my $root_sign_period = '10000';
+
+# but less so for client certificates
+my $sign_period = '5000';
+
+# check and get command line parameters
+if($#ARGV < 1)
+{
+	print <<__E;
+
+bbstored certificates utility.
+
+Bad command line parameters.
+Usage:
+	bbstored-certs certs-dir command [arguments]
+
+certs-dir is the directory holding the root keys and certificates for the backup system
+command is the action to perform, taking parameters.
+
+Commands are
+
+	init
+		-- generate initial root certificates (certs-dir must not already exist)
+	sign certificate-name
+		-- sign a client certificate
+	sign-server certificate-name
+		-- sign a server certificate
+
+Signing requires confirmation that the certificate is correct and should be signed.
+
+__E
+	exit(1);
+}
+
+# check for OPENSSL_CONF environment var being set
+if(exists $ENV{'OPENSSL_CONF'})
+{
+	print <<__E;
+
+---------------------------------------
+
+WARNING:
+    You have the OPENSSL_CONF environment variable set.
+    Use of non-standard openssl configs may cause problems.
+
+---------------------------------------
+
+__E
+}
+
+# directory structure:
+#
+# roots/
+#	clientCA.pem -- root certificate for client (used on server)
+#	serverCA.pem -- root certificate for servers (used on clients)
+# keys/
+#   clientRootKey.pem -- root key for clients
+#   serverRootKey.pem -- root key for servers
+# servers/
+#   hostname.pem -- certificate for server 'hostname'
+# clients/
+#   account.pem -- certficiate for account 'account' (ID in hex)
+#
+
+
+# check parameters
+my ($cert_dir,$command,@args) = @ARGV;
+
+# check directory exists
+if($command ne 'init')
+{
+	if(!-d $cert_dir)
+	{
+		die "$cert_dir does not exist";
+	}
+}
+
+# run command
+if($command eq 'init') {&cmd_init;}
+elsif($command eq 'sign') {&cmd_sign;}
+elsif($command eq 'sign-server') {&cmd_sign_server;}
+else
+{
+	die "Unknown command $command"
+}
+
+sub cmd_init
+{
+	# create directories
+	unless(mkdir($cert_dir,0700)
+		&& mkdir($cert_dir.'/roots',0700)
+		&& mkdir($cert_dir.'/keys',0700)
+		&& mkdir($cert_dir.'/servers',0700)
+		&& mkdir($cert_dir.'/clients',0700))
+	{
+		die "Failed to create directory structure"
+	}
+
+	# create root keys and certrs
+	cmd_init_create_root('client');
+	cmd_init_create_root('server');
+}
+
+sub cmd_init_create_root
+{
+	my $entity = $_[0];
+
+	my $cert = "$cert_dir/roots/".$entity.'CA.pem';
+	my $serial = "$cert_dir/roots/".$entity.'CA.srl';
+	my $key = "$cert_dir/keys/".$entity.'RootKey.pem';
+	my $csr = "$cert_dir/keys/".$entity.'RootCSR.pem';
+
+	# generate key
+	if(system("openssl genrsa -out $key 2048") != 0)
+	{
+		die "Couldn't generate private key."
+	}
+	
+	# make CSR
+	die "Couldn't run openssl for CSR generation" unless
+		open(CSR,"|openssl req -new -key $key -sha1 -out $csr");
+	print CSR <<__E;
+.
+.
+.
+.
+.
+Backup system $entity root
+.
+.
+.
+
+__E
+	close CSR;
+	print "\n\n";
+	die "Certificate request wasn't created.\n" unless -f $csr;
+	
+	# sign it to make a self-signed root CA key
+	if(system("openssl x509 -req -in $csr -sha1 -extensions v3_ca -signkey $key -out $cert -days $root_sign_period") != 0)
+	{
+		die "Couldn't generate root certificate."
+	}
+	
+	# write the initial serial number
+	open SERIAL,">$serial" or die "Can't open $serial for writing";
+	print SERIAL "00\n";
+	close SERIAL;
+}
+
+sub cmd_sign
+{
+	my $csr = $args[0];
+	
+	if(!-f $csr)
+	{
+		die "$csr does not exist";
+	}
+	
+	# get the common name specified in this certificate
+	my $common_name = get_csr_common_name($csr);
+	
+	# look OK?
+	unless($common_name =~ m/\ABACKUP-([A-Fa-f0-9]+)\Z/)
+	{
+		die "The certificate presented does not appear to be a backup client certificate"
+	}
+	
+	my $acc = $1;
+	
+	# check against filename
+	if(!($csr =~ m/(\A|\/)([A-Fa-f0-9]+)-/) || $2 ne $acc)
+	{
+		die "Certificate request filename does not match name in certificate ($common_name)"
+	}
+		
+	print <<__E;
+
+This certificate is for backup account
+
+   $acc
+
+Ensure this matches the account number you are expecting. The filename is
+
+   $csr
+
+which should include this account number, and additionally, you should check
+that you received it from the right person.
+
+Signing the wrong certificate compromises the security of your backup system.
+
+Would you like to sign this certificate? (type 'yes' to confirm)
+__E
+
+	return unless get_confirmation();
+
+	# out certificate
+	my $out_cert = "$cert_dir/clients/$acc"."-cert.pem";
+
+	# sign it!
+	if(system("openssl x509 -req -in $csr -sha1 -extensions usr_crt -CA $cert_dir/roots/clientCA.pem -CAkey $cert_dir/keys/clientRootKey.pem -out $out_cert -days $sign_period") != 0)
+	{
+		die "Signing failed"
+	}
+	
+	# tell user what to do next
+	print <<__E;
+
+
+Certificate signed.
+
+Send the files
+
+   $out_cert
+   $cert_dir/roots/serverCA.pem
+
+to the client.
+
+__E
+}
+
+sub cmd_sign_server
+{
+	my $csr = $args[0];
+	
+	if(!-f $csr)
+	{
+		die "$csr does not exist";
+	}
+	
+	# get the common name specified in this certificate
+	my $common_name = get_csr_common_name($csr);
+	
+	# look OK?
+	if($common_name !~ m/\A[-a-zA-Z0-9.]+\Z/)
+	{
+		die "Invalid server name"
+	}
+	
+	print <<__E;
+
+This certificate is for backup server
+
+   $common_name
+
+Signing the wrong certificate compromises the security of your backup system.
+
+Would you like to sign this certificate? (type 'yes' to confirm)
+__E
+
+	return unless get_confirmation();
+
+	# out certificate
+	my $out_cert = "$cert_dir/servers/$common_name"."-cert.pem";
+
+	# sign it!
+	if(system("openssl x509 -req -in $csr -sha1 -extensions usr_crt -CA $cert_dir/roots/serverCA.pem -CAkey $cert_dir/keys/serverRootKey.pem -out $out_cert -days $sign_period") != 0)
+	{
+		die "Signing failed"
+	}
+	
+	# tell user what to do next
+	print <<__E;
+
+
+Certificate signed.
+
+Install the files
+
+   $out_cert
+   $cert_dir/roots/clientCA.pem
+
+on the server.
+
+__E
+}
+
+
+sub get_csr_common_name
+{
+	my $csr = $_[0];
+	
+	open CSRTEXT,"openssl req -text -in $csr |" or die "Can't open openssl for reading";
+	
+	my $subject;
+	while(<CSRTEXT>)
+	{
+		$subject = $1 if m/Subject:.+?CN=([-\.\w]+)/
+	}	
+	close CSRTEXT;
+
+	if($subject eq '')
+	{
+		die "No subject found in CSR $csr"
+	}
+	
+	return $subject
+}
+
+sub get_confirmation()
+{
+	my $line = <STDIN>;
+	chomp $line;
+	if(lc $line ne 'yes')
+	{
+		print "CANCELLED\n";
+		return 0;
+	}
+	
+	return 1;
+}
+
+
+
+
+
diff --git a/bin/bbstored/bbstored-config b/bin/bbstored/bbstored-config
new file mode 100755
index 00000000..0032f4c6
--- /dev/null
+++ b/bin/bbstored/bbstored-config
@@ -0,0 +1,242 @@
+#!/usr/bin/perl
+use strict;
+
+# should be running as root
+if($> != 0)
+{
+	printf "\nWARNING: this should be run as root\n\n"
+}
+
+# check and get command line parameters
+if($#ARGV < 2)
+{
+	print <<__E;
+
+Setup bbstored config utility.
+
+Bad command line parameters.
+Usage:
+	bbstored-config config-dir server-hostname username [raidfile-config]
+
+config-dir usually /etc/box
+server-hostname is the hostname used by clients to connect to this server
+username is the user to run the server under
+raidfile-config is optional. Use if you have a non-standard raidfile.conf file.
+
+__E
+	exit(1);
+}
+
+# check for OPENSSL_CONF environment var being set
+if(exists $ENV{'OPENSSL_CONF'})
+{
+	print <<__E;
+
+---------------------------------------
+
+WARNING:
+    You have the OPENSSL_CONF environment variable set.
+    Use of non-standard openssl configs may cause problems.
+
+---------------------------------------
+
+__E
+}
+
+# default locations
+my $default_config_location = '/etc/box/bbstored.conf';
+
+# command line parameters
+my ($config_dir,$server,$username,$raidfile_config) = @ARGV;
+
+$raidfile_config = $config_dir . '/raidfile.conf' unless $raidfile_config ne '';
+
+# check server exists, but don't bother checking that it's actually this machine.
+{
+	my @r = gethostbyname($server);
+	if($#r < 0)
+	{
+		die "Server '$server' not found. (check server name, test DNS lookup failed.)"
+	}
+}
+
+# check this exists
+if(!-f $raidfile_config)
+{
+	print "The RaidFile configuration file $raidfile_config doesn't exist.\nYou may need to create it with raidfile-config.\nWon't configure bbstored without it.\n";
+	exit(1);
+}
+
+# check that the user exists
+die "You shouldn't run bbstored as root" if $username eq 'root';
+my $user_uid = 0;
+(undef,undef,$user_uid) = getpwnam($username);
+if($user_uid == 0)
+{
+	die "User $username doesn't exist\n";
+}
+
+# check that directories are writeable
+open RAIDCONF,$raidfile_config or die "Can't open $raidfile_config";
+{
+	my %done = ();
+	while(<RAIDCONF>)
+	{
+		next unless m/Dir\d\s*=\s*(.+)/;
+		my $d = $1;
+		$d = $d.'/backup' if -e $d.'/backup';
+		print "Checking permissions on $d\n";
+		my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($d);
+		my $req_perms = ($uid == $user_uid)?0700:0007;
+		if(($mode & $req_perms) != $req_perms)
+		{
+			print "$username doesn't appear to have the necessary permissions on $d\n";
+			print "Either adjust permissions, or create a directory 'backup' inside the\n";
+			print "directory specified in raidfile.conf which is writable.\n";
+			exit(1);
+		}
+	}
+}
+close RAIDCONF;
+
+# ssl stuff
+my $private_key = "$config_dir/bbstored/$server-key.pem";
+my $certificate_request = "$config_dir/bbstored/$server-csr.pem";
+my $certificate = "$config_dir/bbstored/$server-cert.pem";
+my $ca_root_cert = "$config_dir/bbstored/clientCA.pem";
+
+# other files
+my $config_file = "$config_dir/bbstored.conf";
+my $accounts_file = "$config_dir/bbstored/accounts.txt";
+
+# summarise configuration
+
+print <<__E;
+
+Setup bbstored config utility.
+
+Configuration:
+   Writing configuration file: $config_file
+   Writing empty accounts file: $accounts_file
+   Server hostname: $server
+   RaidFile config: $raidfile_config
+
+__E
+
+# create directories
+if(!-d $config_dir)
+{
+	print "Creating $config_dir...\n";
+	mkdir $config_dir,0755 or die "Can't create $config_dir";
+}
+
+if(!-d "$config_dir/bbstored")
+{
+	print "Creating $config_dir/bbstored\n";
+	mkdir "$config_dir/bbstored",0755 or die "Can't create $config_dir/bbstored";
+}
+
+# create blank accounts file
+if(!-f $accounts_file)
+{
+	print "Creating blank accounts file\n";
+	open ACC,">$accounts_file";
+	close ACC;
+}
+
+# generate the private key for the server
+if(!-f $private_key)
+{
+	print "Generating private key...\n";
+	if(system("openssl genrsa -out $private_key 2048") != 0)
+	{
+		die "Couldn't generate private key."
+	}
+}
+
+# generate a certificate request
+if(!-f $certificate_request)
+{
+	die "Couldn't run openssl for CSR generation" unless
+		open(CSR,"|openssl req -new -key $private_key -sha1 -out $certificate_request");
+	print CSR <<__E;
+.
+.
+.
+.
+.
+$server
+.
+.
+.
+
+__E
+	close CSR;
+	print "\n\n";
+	die "Certificate request wasn't created.\n" unless -f $certificate_request
+}
+
+# write the configuration file
+print "Writing configuration file $config_file\n";
+open CONFIG,">$config_file" or die "Can't open config file for writing";
+print CONFIG <<__E;
+
+RaidFileConf = $raidfile_config
+AccountDatabase = $accounts_file
+
+# Uncomment this line to see exactly what commands are being received from clients.
+# ExtendedLogging = yes
+
+# scan all accounts for files which need deleting every 15 minutes.
+
+TimeBetweenHousekeeping = 900
+
+Server
+{
+	PidFile = /var/run/bbstored.pid
+	User = $username
+	ListenAddresses = inet:$server
+	CertificateFile = $certificate
+	PrivateKeyFile = $private_key
+	TrustedCAsFile = $ca_root_cert
+}
+
+
+__E
+
+close CONFIG;
+
+# explain to the user what they need to do next
+my $daemon_args = ($config_file eq $default_config_location)?'':" $config_file";
+
+print <<__E;
+
+===================================================================
+
+bbstored basic configuration complete.
+
+What you need to do now...
+
+1) Sign $certificate_request
+   using the bbstored-certs utility.
+
+2) Install the server certificate and root CA certificate as
+      $certificate
+      $ca_root_cert
+
+3) You may wish to read the configuration file
+      $config_file
+   and adjust as appropraite.
+
+4) Create accounts with bbstoreaccounts
+
+5) Start the backup store daemon with the command
+      /usr/local/bin/bbstored$daemon_args
+   in /etc/rc.local, or your local equivalent.
+
+===================================================================
+
+__E
+
+
+
diff --git a/bin/bbstored/bbstored.cpp b/bin/bbstored/bbstored.cpp
new file mode 100755
index 00000000..3eaf2639
--- /dev/null
+++ b/bin/bbstored/bbstored.cpp
@@ -0,0 +1,25 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    bbstored.cpp
+//		Purpose: main file for backup store daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreDaemon.h"
+#include "MainHelper.h"
+
+#include "MemLeakFindOn.h"
+
+int main(int argc, const char *argv[])
+{
+	MAINHELPER_START
+
+	BackupStoreDaemon daemon;
+	return daemon.Main(BOX_FILE_BBSTORED_DEFAULT_CONFIG, argc, argv);
+	
+	MAINHELPER_END
+}
+
diff --git a/cleanupforcvs.pl b/cleanupforcvs.pl
new file mode 100755
index 00000000..270ea525
--- /dev/null
+++ b/cleanupforcvs.pl
@@ -0,0 +1,196 @@
+#!/usr/bin/perl
+use strict;
+
+my @del_macos_files;
+my @bad_cpp;
+my @test_main;
+my @makefiles;
+my @autogen_cpp;
+my $cleaned = 1;
+my $dist_archives_exist = 0;
+my @bad_h;
+
+open EVERYTHING,'find . |' or die "Can't open find for file listing";
+
+my %exclude_from_memtest_checks = ('PollEmulator.cpp'=>1,'DebugMemLeakFinder.cpp'=>1,'MemLeakFinder.h'=>1,'MemLeakFindOn.h'=>1,'MemLeakFindOff.h'=>1,'Box.h'=>1);
+
+while(<EVERYTHING>)
+{
+	chomp;
+	next if -d;
+	if(m~/autogen_\w+\.(h|cpp)~)	
+	{
+		push @autogen_cpp,$_
+	}
+	if(m~/\._[^/]+\Z~ || m~/\.DS_Store\Z~)
+	{
+		# mac OS files we don't want
+		push @del_macos_files,$_
+	}
+	elsif(m/\/(\w+\.cpp)/)
+	{
+		my $leafname = $1;
+		# check that Box.h is first include
+		open CPP,$_ or die "Can't open $_ for reading";
+		
+		my $box_found = 0;
+		my $last_was_memteston = 0;
+		my $ok = 1;
+		
+		while(my $l = <CPP>)
+		{
+			if($l =~ m/#include\s+["<](.+?)[">]/)
+			{
+				my $inc_name = $1;
+				if($inc_name eq 'Box.h')
+				{
+					$box_found = 1;
+				}
+				else
+				{
+					# Box.h must be first include file in every cpp file
+					$ok = 0 unless $box_found;
+				}
+				# is it the mem test on thing? (ignoring the wire packing .h files)
+				if($inc_name ne 'BeginStructPackForWire.h' && $inc_name ne 'EndStructPackForWire.h')
+				{
+					$last_was_memteston = ($inc_name eq 'MemLeakFindOn.h');
+				}
+			}
+		}
+		if(!exists $exclude_from_memtest_checks{$leafname})
+		{
+			$ok = 0 unless $last_was_memteston;
+		}
+		push @bad_cpp,$_ unless $ok;
+				
+		close CPP;
+	}
+	elsif(m/\/(\w+\.h)/)
+	{
+		my $leafname = $1;
+
+		open H,$_ or die "Can't open $_ for reading";
+		
+		my $ok = 1;
+		my $memteston = 0;
+		
+		while(my $l = <H>)
+		{
+			if($l =~ m/#include\s+["<](.+?)[">]/)
+			{
+				if($1 eq 'MemLeakFindOn.h')
+				{
+					$memteston = 1;
+				}
+				elsif($1 eq 'MemLeakFindOff.h')
+				{
+					$memteston = 0;
+				}
+				else
+				{
+					# don't allow #include within mem test on
+					$ok = 0 unless !$memteston;
+				}
+			}
+			else
+			{
+				# strip comments
+				my $lsc = $l;
+				$lsc =~ s~//.+$~~;
+				if($lsc =~ m/\b(new|delete|malloc|free|realloc)\b/)
+				{
+					# only allow this if memory checking is ON
+					$ok = 0 unless $memteston;
+				}
+			}
+		}
+		# mem test must be off at the end of this .h file
+		$ok = 0 if $memteston;
+		
+		if($_ !~ /testfiles/ && !exists $exclude_from_memtest_checks{$leafname})
+		{
+			push @bad_h,$_ unless $ok;
+		}
+		close H;
+	}
+	elsif(m~/Makefile\Z~)
+	{
+		push @makefiles,$_
+	}
+	
+	if(m~/_(main\.cpp|t|t-gdb)\Z~)
+	{
+		push @test_main,$_
+	}
+	
+	if(m~\./boxbackup~)
+	{
+		$dist_archives_exist = 1;
+	}
+}
+
+close EVERYTHING;
+
+ask_about_delete(\@del_macos_files, "supurious MacOS X files");
+ask_about_delete(\@makefiles, "automatically generated Makefiles");
+ask_about_delete(\@test_main, "automatically generated test files");
+ask_about_delete(\@autogen_cpp, "automatically generated source files");
+
+if($#bad_cpp >= 0)
+{
+	print "\n";
+	print $_,"\n" for @bad_cpp;
+	print "There are some .cpp file where Box.h is not the first included file or MemLeakFindOn.h is not the last .h file included\n";
+	$cleaned = 0;
+}
+if($#bad_h >= 0)
+{
+	print "\n";
+	print $_,"\n" for @bad_h;
+	print "There are some .h files which use memory functions without memory leak finding on, or leave memory leak finding on at end\n";
+	$cleaned = 0;
+}
+
+if(-d 'debug') {print "debug directory exists\n"; $cleaned = 0;}
+if(-d 'release') {print "release directory exists\n"; $cleaned = 0;}
+if(-d 'parcels') {print "parcels directory exists\n"; $cleaned = 0;}
+if($dist_archives_exist) {print "boxbackup* files/dirs exist\n"; $cleaned = 0;}
+
+if(!$cleaned)
+{
+	print <<__E;
+
+========================================================
+   NOT CLEANED!
+========================================================
+__E
+}
+
+
+sub ask_about_delete
+{
+	my ($del_r, $name) = @_;
+	return if $#$del_r < 0;
+		
+	print "\n";
+	for(@$del_r)
+	{
+		print $_,"\n";
+	}
+	print "Delete these ",$#$del_r + 1, " $name?";
+	my $in = <STDIN>;
+	chomp $in;
+	if($in eq 'yes')
+	{
+		print "Deleting...\n";
+		unlink $_ for @$del_r
+	}
+	else
+	{
+		$cleaned = 0;
+	}
+}
+
+
+
diff --git a/configure b/configure
new file mode 100755
index 00000000..6de0d998
--- /dev/null
+++ b/configure
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+perl ./infrastructure/makebuildenv.pl $1 $2 $3 $4 $5 $6 $7 $8 $9
+if [ "$?" -ne "0" ]; then
+	exit 1
+fi
+perl ./infrastructure/makeparcels.pl
+
+echo
+echo Type \'make\' to build the installation parcels.
+echo
+echo Then use the install scripts inside 'parcels/<dir>' to install
+echo the components you need.
+echo
+echo
diff --git a/distribution/COMMON-MANIFEST.txt b/distribution/COMMON-MANIFEST.txt
new file mode 100644
index 00000000..5f734964
--- /dev/null
+++ b/distribution/COMMON-MANIFEST.txt
@@ -0,0 +1,24 @@
+lib/common
+lib/crypto
+lib/server
+lib/compress
+test/common
+test/common/testfiles
+test/basicserver
+test/basicserver/testfiles
+test/crypto
+test/compress
+docs/common notes
+docs/common/lib_common notes/lib_common
+docs/common/lib_crypto notes/lib_crypto
+docs/common/lib_server notes/lib_server
+MKDIR infrastructure
+infrastructure/buildenv-testmain-template.cpp
+infrastructure/makebuildenv.pl
+infrastructure/BoxPlatform.pm
+infrastructure/makeparcels.pl
+MKDIR infrastructure/tests
+infrastructure/tests/common_tests.pl
+parcels.txt
+configure
+runtest.pl
diff --git a/distribution/boxbackup/CONTACT.txt b/distribution/boxbackup/CONTACT.txt
new file mode 100644
index 00000000..849c1e76
--- /dev/null
+++ b/distribution/boxbackup/CONTACT.txt
@@ -0,0 +1,6 @@
+
+http://www.fluffy.co.uk/boxbackup/
+
+Ben Summers
+ben@fluffy.co.uk
+
diff --git a/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt
new file mode 100644
index 00000000..72b3866d
--- /dev/null
+++ b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt
@@ -0,0 +1,34 @@
+lib/raidfile
+test/raidfile
+test/raidfile/testfiles
+lib/backupclient
+lib/backupstore
+bin/bbstored
+bin/bbstoreaccounts
+bin/bbackupd
+bin/bbackupquery
+bin/bbackupctl
+bin/bbackupobjdump
+test/backupstore
+test/backupstore/testfiles
+test/backupstorefix
+test/backupstorefix/testfiles
+test/backupstorepatch
+test/bbackupd
+test/bbackupd/testfiles
+test/backupdiff
+docs/raidfile notes
+docs/raidfile/lib_raidfile notes/lib_raidfile
+docs/backup notes
+TODO.txt
+BUGS.txt
+distribution/boxbackup/contrib contrib
+NO-LICENSE-IN-DIR contrib/rpm
+distribution/boxbackup/contrib/rpm contrib/rpm
+REPLACE-VERSION-IN contrib/rpm/boxbackup.spec
+NO-LICENSE-IN-DIR contrib/redhat
+distribution/boxbackup/contrib/redhat contrib/redhat
+NO-LICENSE-IN-DIR contrib/suse
+distribution/boxbackup/contrib/suse contrib/suse
+NO-LICENSE-IN-DIR contrib/cygwin
+distribution/boxbackup/contrib/cygwin contrib/cygwin
diff --git a/distribution/boxbackup/DOCUMENTATION.txt b/distribution/boxbackup/DOCUMENTATION.txt
new file mode 100644
index 00000000..1cda3a2a
--- /dev/null
+++ b/distribution/boxbackup/DOCUMENTATION.txt
@@ -0,0 +1,6 @@
+
+For compilation and installation instructions, see the web site at
+
+	http://www.fluffy.co.uk/boxbackup/
+
+
diff --git a/distribution/boxbackup/LICENSE.txt b/distribution/boxbackup/LICENSE.txt
new file mode 100644
index 00000000..fc3f9f3e
--- /dev/null
+++ b/distribution/boxbackup/LICENSE.txt
@@ -0,0 +1,37 @@
+ 
+Copyright (c) 2003, 2005
+     Ben Summers.  All rights reserved.
+ 
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. All use of this software and associated advertising materials must 
+   display the following acknowledgement:
+       This product includes software developed by Ben Summers.
+4. The names of the Authors may not be used to endorse or promote
+   products derived from this software without specific prior written
+   permission.
+
+[Where legally impermissible the Authors do not disclaim liability for 
+direct physical injury or death caused solely by defects in the software 
+unless it is modified by a third party.]
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+ 
+ 
+ 
\ No newline at end of file
diff --git a/distribution/boxbackup/LINUX.txt b/distribution/boxbackup/LINUX.txt
new file mode 100755
index 00000000..3342ab2a
--- /dev/null
+++ b/distribution/boxbackup/LINUX.txt
@@ -0,0 +1,27 @@
+
+For instructions on building an RPM of Box Backup, see the contrib/rpm
+directory. This is primarily for RedHat style systems, but notes are provided
+on what needs to be modified for SUSE.
+
+
+Requirements:
+
+	OpenSSL 0.9.7
+
+Require zlib and openssl headers for compilation -- may not be included when
+installing the packages. (libssl-dev + libz-dev packages under debian)
+
+Bekerley DB v1 support is required. The configure script should find an
+appropriate db package -- and if not, use an in-memory version of the code.
+However, the in-memory version is not desirable as it will lose information
+over restarts of the daemon.
+
+Ideally, include the readline headers -- libreadline4-dev, probably.
+
+
+
+(OpenSSL 0.9.7 is required as it implements new interfaces which make encryption
+using the same key faster by avoiding key setup each time around. Configure it with
+./config shared , then copy the libs and headers into the required places.)
+
+
diff --git a/distribution/boxbackup/NETBSD.txt b/distribution/boxbackup/NETBSD.txt
new file mode 100644
index 00000000..03000791
--- /dev/null
+++ b/distribution/boxbackup/NETBSD.txt
@@ -0,0 +1,8 @@
+
+Install perl
+
+Install OpenSSL 0.9.7 or later.
+
+Not a completely working port -- symlinks don't get backed up or restored properly.
+(run test/bbackupd)
+
diff --git a/distribution/boxbackup/THANKS.txt b/distribution/boxbackup/THANKS.txt
new file mode 100644
index 00000000..42736a65
--- /dev/null
+++ b/distribution/boxbackup/THANKS.txt
@@ -0,0 +1,37 @@
+
+Many individuals have helped with the development of Box Backup by testing, reporting experiences, and making suggestions. In particular, thanks are due to
+
+Pascal Lalonde
+  - Comprehensive and accurate bug reports, and constructive feedback
+
+Paul Arch
+  - Cygwin client port
+
+Ben Lovett
+  - Help with odd architectures, suggesting small changes
+
+Martin Ebourne
+  - RPM specification for RedHat based Linux distributions
+  - Patch to fix problems on 64 bit architectures
+  - Patch to fix compilation after RedHat Fedora's latest changes
+
+Per Thomsen
+  - Cygwin Windows service install scripts and build notes
+
+Tim Fletcher
+David Harris
+Richard Eigenmann
+  - Testing many attempts at clean compiles on various Linux distributions
+
+Eduardo Alvarenga
+  - Valuable feedback and persuasion to include new features
+
+Joe Gillespie
+  - Web site design
+
+JŽr™me Schell
+  - Fixes to build+config problems on Linux
+
+John Pybus
+  - Ideas and feature requests
+  - Useful little patches to code
diff --git a/distribution/boxbackup/VERSION.txt b/distribution/boxbackup/VERSION.txt
new file mode 100644
index 00000000..88e195fb
--- /dev/null
+++ b/distribution/boxbackup/VERSION.txt
@@ -0,0 +1,2 @@
+0.08
+boxbackup
diff --git a/distribution/boxbackup/contrib/cygwin/README.txt b/distribution/boxbackup/contrib/cygwin/README.txt
new file mode 100644
index 00000000..83f32fd9
--- /dev/null
+++ b/distribution/boxbackup/contrib/cygwin/README.txt
@@ -0,0 +1,30 @@
+      Making boxbackup run as a Windows Service
+
+For most installations (with the default locations for config files,
+etc.) running the install-cygwin-service.pl script will complete the
+installation painlessly, and you will have a running bbackupd after
+completing the installation, and whenever you reboot.
+
+Simply run the script:
+
+perl install-cygwin-service.pl
+
+The service can be monitored in the Windows Service Manager. It is named
+boxbackup.
+
+For non-standard configurations, there are command-line options to point
+the script to the bbackupd.conf config file, and the bbackupd.exe
+executable:
+
+perl install-cygwin-service.pl [-c <path-to-bbackupd-config-file>] [-e
+<path-to-bbackupd-executable-file>]
+
+
+      Removing the Service
+
+If you decide not to run backups on a machine anymore, simply remove the
+service by running:
+
+sh remove-cygwin-service.sh
+
+
diff --git a/distribution/boxbackup/contrib/cygwin/install-cygwin-service.pl b/distribution/boxbackup/contrib/cygwin/install-cygwin-service.pl
new file mode 100755
index 00000000..65758067
--- /dev/null
+++ b/distribution/boxbackup/contrib/cygwin/install-cygwin-service.pl
@@ -0,0 +1,112 @@
+#!/usr/bin/perl -w
+
+
+# Contributed to the boxbackup project by Per Reedtz Thomsen. pthomsen@reedtz.com
+
+# This script reads the config file for boxbackup, and changes the mode
+# of the directory named by 'DataDirectory' and any files there. Also,
+# the files pointed to by the 'CommandSocket' and 'PidFile' configuration
+# parameters will be chmod'ed to be read-write by all.
+# The Windows services are created and started using the 'cygrunsrv' utility.
+
+# Date      Who                      Comments
+# 20041005  pthomsen@reedtz.com      Created 
+# 20041020  pthomsen@reedtz.com	     Switched to using Getopt::Std for cmd-line things.
+
+use strict;
+
+use Getopt::Std;
+getopt('ce');
+our ($opt_c, $opt_e);
+# Figure out the config file to use. Default is /etc/box/bbackupd.conf
+my $confFile = (defined($opt_c) ? $opt_c :  "/etc/box/bbackupd.conf");
+# Figure out the bbaackupd executable to use. Default is /usr/local/bin/bbackupd.exe
+my $exeFile = (defined($opt_e) ? $opt_e :  "/usr/local/bin/bbackupd.exe");
+
+die "File $confFile does not exist. Please provide the full path to the bbackupd configuration file.\n" if !(-f $confFile);
+die "Can't read $confFile. Permission denied. Please chmod the file so I can read it.\n" if !(-r $confFile);
+die "File $exeFile does not exist. Please provide the full path to the bbackupd.exe file.\n" if !(-f $exeFile);
+die "File $exeFile is not executable. Please provide the full path to the correct bbackupd.exe file.\n" if !(-x $exeFile);
+
+# print "Config: $confFile\n";
+
+my $dataDir;
+my $cmdSocket;
+my $pidFile;
+
+open (CONFIG, "<$confFile") or die "Can't open $confFile: $!\n";
+
+# Read the confgiguration file, and pull the DataDirectory, CommandSocket, and PidFile parameters.
+while (<CONFIG>)
+{
+    
+    if (/^\s*DataDirectory\s*=\s*([^\n\s]+)\s*\n/)
+    {
+	$dataDir = $1;
+	next;
+    }
+
+    if (/^\s*CommandSocket\s*=\s*([^\n\s]+)\s*\n/)
+    {
+	$cmdSocket = $1;
+	next;
+    }
+    if (/^\s*PidFile\s*=\s*([^\n\s]+)\s*\n/)
+    {
+	$pidFile = $1;
+	next;
+    }
+}
+
+# check that we got all the parameters from the file. If not, die.
+if ((!defined($dataDir)) || (!defined($cmdSocket)) || (!defined($pidFile)))
+{
+    die "Could not read config parameters from $confFile. Values retrieved:\n\tDataDirectory = $dataDir\n\tCommandSocket = $cmdSocket\n\tPidFile = $pidFile\n";
+}
+
+
+print  "Parameters retrieved from $confFile. Values:\n\tDataDirectory = $dataDir\n\tCommandSocket = $cmdSocket\n\tPidFile = $pidFile\n";
+print "chmod...";
+# change the mode of the files/dirs retrieved.
+chmod(0777, $dataDir) or die "Can't chmod $dataDir: $!\n";
+chmod(0666, "$dataDir/*") or die "Can't chmod $dataDir/*: $!\n";
+chmod(0666, $pidFile) or die "Can't chmod $pidFile: $!\n";
+chmod(0755, $cmdSocket) or die "Can't chmod $cmdSocket: $!\n";
+print " Done.\n";
+
+# Now install the service using cygrunsrv.
+# Details: 
+#   -I <svc_name>      Install a service. svc_name is the name under which the
+#                      service will appear in the Windows Service Manager
+#   -p <path_to_exe>   Path to the executable.
+#   -a <options>       Command line options to the executable.
+#   -f <description>   Description of the service.
+#   -o                 Attempt clean exit of service during system shutdown
+
+print "Installing boxbackup service...";
+my $sysCmd = "cygrunsrv.exe -I boxbackup -p " . $exeFile;
+$sysCmd .= " -a \"" . $confFile . " SINGLEPROCESS\"";
+$sysCmd .= " -o -f \"Online Backup System by Ben Summers\"";
+print "$sysCmd\n";
+my $output = qx($sysCmd);
+die "cygrunsrv failed to install service. Error Message: $output\n" if($output ne "");
+print " Done.\n";
+
+
+# Start the service
+# Details:
+#   -S <svc_name>     Start a service. svc_name is the name of the (installed)
+#                     service to start.
+
+print "Starting boxbackup service...";
+$sysCmd = "cygrunsrv.exe -S boxbackup";
+print "$sysCmd\n";
+$output = qx($sysCmd);
+die "cygrunsrv failed to start service. Error Message: $output\n" if($output ne "");
+print " Done.\n";
+
+print "\n\nService Installation complete. To test, reboot your machine, and make sure that\n";
+print "the boxbackup service is running. A good way to make sure, is to check that the account number\n";
+print "from this machine is connecting to the bbstored server. Check the bbstored logs for more info.\n\n";
+
+
diff --git a/distribution/boxbackup/contrib/cygwin/remove-cygwin-service.sh b/distribution/boxbackup/contrib/cygwin/remove-cygwin-service.sh
new file mode 100755
index 00000000..e766333d
--- /dev/null
+++ b/distribution/boxbackup/contrib/cygwin/remove-cygwin-service.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Contributed to the boxbackup project by Per Reedtz Thomsen. pthomsen@reedtz.com
+
+# This script removes the 'boxbackup' service from the Windows service manager
+# using the cygrunsrv utility. 
+
+# Date      Who                      Comments
+# 20041005  pthomsen@reedtz.com      Created 
+
+cygrunsrv -R boxbackup
+
+echo "Service \"boxbackup\" removed."
+
diff --git a/distribution/boxbackup/contrib/redhat/README.txt b/distribution/boxbackup/contrib/redhat/README.txt
new file mode 100644
index 00000000..cfc8d968
--- /dev/null
+++ b/distribution/boxbackup/contrib/redhat/README.txt
@@ -0,0 +1,7 @@
+These start scripts are for Fedora Core or RedHat Enterprise Linux. If
+installed manually they should be placed in /etc/rc.d/init.d.
+
+They may also work for Mandrake.
+
+Martin Ebourne
+martin@zepler.org
diff --git a/distribution/boxbackup/contrib/redhat/bbackupd b/distribution/boxbackup/contrib/redhat/bbackupd
new file mode 100644
index 00000000..63c61ff7
--- /dev/null
+++ b/distribution/boxbackup/contrib/redhat/bbackupd
@@ -0,0 +1,83 @@
+#! /bin/bash
+#
+# bbackupd          Start/Stop the box backup daemon.
+#
+# chkconfig: 345 93 07
+# description: bbackup is the client side deamon for Box Backup, a completely \
+#              automatic on-line backup system
+# processname: bbackupd
+# config: /etc/box
+# pidfile: /var/run/bbackupd.pid
+
+# Source function library.
+. /etc/init.d/functions
+
+RETVAL=0
+
+# See how we were called.
+
+prog="bbackupd"
+
+# Check that configuration exists.
+[ -f /etc/box/$prog.conf ] || exit 0
+
+start() {
+	echo -n $"Starting $prog: "
+	daemon $prog
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog
+	return $RETVAL
+}
+
+stop() {
+	echo -n $"Stopping $prog: "
+	killproc $prog
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog
+	return $RETVAL
+}
+
+rhstatus() {
+	status $prog
+}
+
+restart() {
+  	stop
+	start
+}
+
+reload() {
+        echo -n $"Reloading $prog daemon configuration: "
+        killproc $prog -HUP
+        retval=$?
+        echo
+        return $RETVAL
+}
+
+case "$1" in
+  start)
+  	start
+	;;
+  stop)
+  	stop
+	;;
+  restart)
+  	restart
+	;;
+  reload)
+        reload
+        ;;
+  status)
+  	rhstatus
+	;;
+  condrestart)
+  	[ -f /var/lock/subsys/$prog ] && restart || :
+	;;
+  *)
+	echo $"Usage: $0 {start|stop|status|reload|restart|condrestart}"
+	exit 1
+esac
+
+exit $?
diff --git a/distribution/boxbackup/contrib/redhat/bbstored b/distribution/boxbackup/contrib/redhat/bbstored
new file mode 100644
index 00000000..eadca1d1
--- /dev/null
+++ b/distribution/boxbackup/contrib/redhat/bbstored
@@ -0,0 +1,83 @@
+#! /bin/bash
+#
+# bbstored          Start/Stop the box backup daemon.
+#
+# chkconfig: 345 93 07
+# description: bbstore is the server side deamon for Box Backup, a completely \
+#              automatic on-line backup system
+# processname: bbstored
+# config: /etc/box
+# pidfile: /var/run/bbstored.pid
+
+# Source function library.
+. /etc/init.d/functions
+
+RETVAL=0
+
+# See how we were called.
+
+prog="bbstored"
+
+# Check that configuration exists.
+[ -f /etc/box/$prog.conf ] || exit 0
+
+start() {
+	echo -n $"Starting $prog: "
+	daemon $prog
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog
+	return $RETVAL
+}
+
+stop() {
+	echo -n $"Stopping $prog: "
+	killproc $prog
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog
+	return $RETVAL
+}
+
+rhstatus() {
+	status $prog
+}
+
+restart() {
+  	stop
+	start
+}
+
+reload() {
+        echo -n $"Reloading $prog daemon configuration: "
+        killproc $prog -HUP
+        retval=$?
+        echo
+        return $RETVAL
+}
+
+case "$1" in
+  start)
+  	start
+	;;
+  stop)
+  	stop
+	;;
+  restart)
+  	restart
+	;;
+  reload)
+        reload
+        ;;
+  status)
+  	rhstatus
+	;;
+  condrestart)
+  	[ -f /var/lock/subsys/$prog ] && restart || :
+	;;
+  *)
+	echo $"Usage: $0 {start|stop|status|reload|restart|condrestart}"
+	exit 1
+esac
+
+exit $?
diff --git a/distribution/boxbackup/contrib/rpm/README.txt b/distribution/boxbackup/contrib/rpm/README.txt
new file mode 100644
index 00000000..290a5252
--- /dev/null
+++ b/distribution/boxbackup/contrib/rpm/README.txt
@@ -0,0 +1,16 @@
+BUILDING AN RPM
+
+The easy way is to:
+
+rpmbuild -ta <tarfile>
+
+where <tarfile> is the archive you downloaded of Box Backup.
+
+This RPM should work on RedHat Enterprise, Fedora Core, Mandrake, SUSE, and
+any similar distributions. It has been developed and tested on Fedora Core.
+
+Changes for SUSE Linux were provided by Chris Smith
+(chris.smith@nothingbutnet.co.nz).
+
+Martin Ebourne
+martin@zepler.org
diff --git a/distribution/boxbackup/contrib/rpm/boxbackup.spec b/distribution/boxbackup/contrib/rpm/boxbackup.spec
new file mode 100644
index 00000000..2dd88cc1
--- /dev/null
+++ b/distribution/boxbackup/contrib/rpm/boxbackup.spec
@@ -0,0 +1,206 @@
+%define bb_user_id 171
+%define ident %{name}-%{version}
+
+# Detect distribution. So far we only special-case SUSE. If you need to make
+# any distro specific changes to get the package building on your system
+# please email them to martin@zepler.org
+#%define is_fc   %(test -e %{_sysconfdir}/fedora-release && echo 1 || echo 0)
+#%define is_mdk  %(test -e %{_sysconfdir}/mandrake-release && echo 1 || echo 0)
+#%define is_rh   %(test -e %{_sysconfdir}/redhat-release && echo 1 || echo 0)
+%define is_suse %(test -e %{_sysconfdir}/SuSE-release && echo 1 || echo 0)
+
+%if %{is_suse}
+%define init_dir %{_sysconfdir}/init.d
+%define dist suse
+%define rc_start rc
+%else
+%define init_dir %{_sysconfdir}/rc.d/init.d
+%define dist redhat
+%define rc_start "service "
+%endif
+
+Summary: An automatic on-line backup system for UNIX.
+Name: boxbackup
+Version: ###DISTRIBUTION-VERSION-NUMBER###
+Release: 1
+License: BSD
+Group: Applications/Archiving
+Packager: Martin Ebourne <martin@zepler.org>
+URL: http://www.fluffy.co.uk/boxbackup/
+Source0: %{ident}.tgz
+Requires: openssl >= 0.9.7a
+BuildRoot: %{_tmppath}/%{ident}-%{release}-root
+BuildRequires: openssl >= 0.9.7a, openssl-devel
+
+%description
+Box Backup is a completely automatic on-line backup system. Backed up files
+are stored encrypted on a filesystem on a remote server, which does not need
+to be trusted. The backup server runs as a daemon on the client copying only
+the changes within files, and old versions and deleted files are retained. It
+is designed to be easy and cheap to run a server and (optional) RAID is
+implemented in userland for ease of use.
+
+%package client
+Summary: An automatic on-line backup system for UNIX.
+Group: Applications/Archiving
+
+%description client
+Box Backup is a completely automatic on-line backup system. Backed up files
+are stored encrypted on a filesystem on a remote server, which does not need
+to be trusted. The backup server runs as a daemon on the client copying only
+the changes within files, and old versions and deleted files are retained. It
+is designed to be easy and cheap to run a server and (optional) RAID is
+implemented in userland for ease of use.
+
+This package contains the client.
+
+%package server
+Summary: An automatic on-line backup system for UNIX.
+Group: System Environment/Daemons
+
+%description server
+Box Backup is a completely automatic on-line backup system. Backed up files
+are stored encrypted on a filesystem on a remote server, which does not need
+to be trusted. The backup server runs as a daemon on the client copying only
+the changes within files, and old versions and deleted files are retained. It
+is designed to be easy and cheap to run a server and (optional) RAID is
+implemented in userland for ease of use.
+
+This package contains the server.
+
+%prep
+%setup -q
+
+%build
+./configure
+
+make
+
+%install
+rm -rf $RPM_BUILD_ROOT
+
+mkdir -p $RPM_BUILD_ROOT%{_docdir}/%{ident}
+mkdir -p $RPM_BUILD_ROOT%{_bindir}
+mkdir -p $RPM_BUILD_ROOT%{_sbindir}
+mkdir -p $RPM_BUILD_ROOT%{init_dir}
+mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/box/bbackupd
+mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/box/bbstored
+mkdir -p $RPM_BUILD_ROOT%{_var}/lib/box
+
+install -m 644 BUGS.txt $RPM_BUILD_ROOT%{_docdir}/%{ident}
+install -m 644 LINUX.txt $RPM_BUILD_ROOT%{_docdir}/%{ident}
+install -m 644 VERSION.txt $RPM_BUILD_ROOT%{_docdir}/%{ident}
+install -m 644 CONTACT.txt $RPM_BUILD_ROOT%{_docdir}/%{ident}
+install -m 644 DOCUMENTATION.txt $RPM_BUILD_ROOT%{_docdir}/%{ident}
+install -m 644 ExceptionCodes.txt $RPM_BUILD_ROOT%{_docdir}/%{ident}
+install -m 644 THANKS.txt $RPM_BUILD_ROOT%{_docdir}/%{ident}
+install -m 644 LICENSE.txt $RPM_BUILD_ROOT%{_docdir}/%{ident}
+install -m 644 TODO.txt $RPM_BUILD_ROOT%{_docdir}/%{ident}
+
+# Client
+touch $RPM_BUILD_ROOT%{_sysconfdir}/box/bbackupd.conf
+install -m 755 contrib/%{dist}/bbackupd $RPM_BUILD_ROOT%{init_dir}
+%if %{is_suse}
+ln -s ../../%{init_dir}/bbackupd $RPM_BUILD_ROOT%{_sbindir}/rcbbackupd
+%endif
+%define client_dir parcels/%{ident}-backup-client-Linux
+install %{client_dir}/bbackupd $RPM_BUILD_ROOT%{_sbindir}
+install %{client_dir}/bbackupquery $RPM_BUILD_ROOT%{_sbindir}
+install %{client_dir}/bbackupctl $RPM_BUILD_ROOT%{_sbindir}
+install %{client_dir}/bbackupd-config $RPM_BUILD_ROOT%{_sbindir}
+
+# Server
+touch $RPM_BUILD_ROOT%{_sysconfdir}/box/bbstored.conf
+touch $RPM_BUILD_ROOT%{_sysconfdir}/box/raidfile.conf
+install -m 755 contrib/%{dist}/bbstored $RPM_BUILD_ROOT%{init_dir}
+%if %{is_suse}
+ln -s ../../%{init_dir}/bbstored $RPM_BUILD_ROOT%{_sbindir}/rcbbstored
+%endif
+%define server_dir parcels/%{ident}-backup-server-Linux
+install %{server_dir}/bbstored $RPM_BUILD_ROOT%{_sbindir}
+install %{server_dir}/bbstoreaccounts $RPM_BUILD_ROOT%{_sbindir}
+install %{server_dir}/bbstored-certs $RPM_BUILD_ROOT%{_bindir}
+install %{server_dir}/bbstored-config $RPM_BUILD_ROOT%{_sbindir}
+install %{server_dir}/raidfile-config $RPM_BUILD_ROOT%{_sbindir}
+
+%pre server
+%{_sbindir}/useradd -c "Box Backup" -u %{bb_user_id} \
+	-s /sbin/nologin -r -d / box 2> /dev/null || :
+
+%post client
+/sbin/chkconfig --add bbackupd
+if [ ! -f %{_sysconfdir}/box/bbackupd.conf ]; then
+	echo "You should run the following to configure the client:"
+	echo "bbackupd-config %{_sysconfdir}/box lazy <account-number> <server-host>" \
+	     "%{_var}/lib/box <backup-directories>"
+	echo "Then follow the instructions. Use this to start the client:"
+	echo "%{rc_start}bbackupd start"
+fi
+
+%post server
+/sbin/chkconfig --add bbstored
+if [ ! -f %{_sysconfdir}/box/bbstored.conf ]; then
+	echo "You should run the following to configure the server:"
+	echo "raidfile-config %{_sysconfdir}/box 2048 <store-directory> [<raid-directories>]"
+	echo "bbstored-config %{_sysconfdir}/box" `hostname` box
+	echo "Then follow the instructions. Use this to start the server:"
+	echo "%{rc_start}bbstored start"
+fi
+
+%preun client
+if [ $1 = 0 ]; then
+	%{init_dir}/bbackupd stop > /dev/null 2>&1
+	/sbin/chkconfig --del bbackupd
+fi
+
+%preun server
+if [ $1 = 0 ]; then
+	%{init_dir}/bbstored stop > /dev/null 2>&1
+	/sbin/chkconfig --del bbstored
+fi
+
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files client
+%defattr(-,root,root,-)
+%dir %attr(700,root,root) %{_sysconfdir}/box/bbackupd
+%dir %attr(755,root,root) %{_var}/lib/box
+%doc %{_docdir}/%{ident}/*.txt
+%config %{init_dir}/bbackupd
+%if %{is_suse}
+%{_sbindir}/rcbbackupd
+%endif
+%config %ghost %{_sysconfdir}/box/bbackupd.conf
+%{_sbindir}/bbackupd
+%{_sbindir}/bbackupquery
+%{_sbindir}/bbackupctl
+%{_sbindir}/bbackupd-config
+
+%files server
+%defattr(-,root,root,-)
+%dir %attr(700,box,root) %{_sysconfdir}/box/bbstored
+%config %{init_dir}/bbstored
+%if %{is_suse}
+%{_sbindir}/rcbbstored
+%endif
+%config %ghost %{_sysconfdir}/box/bbstored.conf
+%config %ghost %{_sysconfdir}/box/raidfile.conf
+%{_sbindir}/bbstored
+%{_sbindir}/bbstoreaccounts
+%{_bindir}/bbstored-certs
+%{_sbindir}/bbstored-config
+%{_sbindir}/raidfile-config
+
+%changelog
+* Fri Oct  1 2004 Martin Ebourne <martin@zepler.org> - 0.08-3
+- Moved most of the exes to /usr/sbin
+- SUSE updates from Chris Smith
+
+* Fri Sep 24 2004 Martin Ebourne <martin@zepler.org> - 0.08-2
+- Added support for other distros
+- Changes for SUSE provided by Chris Smith <chris.smith@nothingbutnet.co.nz>
+
+* Mon Sep 16 2004 Martin Ebourne <martin@zepler.org> - 0.07-1
+- Initial build
diff --git a/distribution/boxbackup/contrib/suse/README.txt b/distribution/boxbackup/contrib/suse/README.txt
new file mode 100644
index 00000000..0f260b7a
--- /dev/null
+++ b/distribution/boxbackup/contrib/suse/README.txt
@@ -0,0 +1,5 @@
+These start scripts are for SUSE Linux. If installed manually they should be
+placed in /etc/init.d.
+
+Copyright (c)2004, Nothing But Net Limited
+<chris.smith@nothingbutnet.co.nz>
diff --git a/distribution/boxbackup/contrib/suse/bbackupd b/distribution/boxbackup/contrib/suse/bbackupd
new file mode 100644
index 00000000..4dd94154
--- /dev/null
+++ b/distribution/boxbackup/contrib/suse/bbackupd
@@ -0,0 +1,101 @@
+#!/bin/sh
+#
+# Copyright (c)2004, Nothing But Net Limited
+#	<chris.smith@nothingbutnet.co.nz>
+#
+######################################################################
+# RELEASED AND PROVIDED TO YOU UNDER THE SAME LICENCE AS THE BOXBACKUP
+# SUITE OF PROGRAMS. LICENCE MAY BE VIEWED HERE:
+#
+# http://www.fluffy.co.uk/boxbackup/license.html
+######################################################################
+#
+# /etc/init.d/bbackupd
+#   and its symbolic link
+# /(usr/)sbin/rcbbackupd
+#
+### BEGIN INIT INFO
+# Provides:          bbackupd
+# Required-Start:    $named $network $local_fs $syslog
+# X-UnitedLinux-Should-Start: $time ypbind sendmail
+# Required-Stop:     $named $network $localfs $syslog
+# X-UnitedLinux-Should-Stop: $time ypbind sendmail
+# Default-Start:     3 5
+# Default-Stop:      0 1 2 6
+# Short-Description: BoxBackup client side daemon
+# Description: Client daemon for the BoxBackup software       
+#	that allows you to communicate with a bbstored server.
+### END INIT INFO
+
+# Check for missing binaries (stale symlinks should not happen)
+BBACKUPD_BIN=/usr/sbin/bbackupd
+if [ ! -x $BBACKUPD_BIN ] ; then
+	echo "$BBACKUPD_BIN not installed"
+	exit 5
+fi
+
+. /etc/rc.status
+
+# Reset status of this service
+rc_reset
+
+case "$1" in
+	start)
+		echo -n "Starting bbackupd "
+		startproc $BBACKUPD_BIN
+		rc_status -v
+		;;
+
+    stop)
+		echo -n "Shutting down bbackupd "
+		killproc -TERM $BBACKUPD_BIN
+		rc_status -v
+		;;
+
+    try-restart|condrestart)
+		if test "$1" = "condrestart"; then
+			echo "${attn} Use try-restart ${done}(LSB)${attn} rather than condrestart ${warn}(RH)${norm}"
+		fi
+		$0 status
+		if test $? = 0; then
+			$0 restart
+		else
+			rc_reset	# Not running is not a failure.
+		fi
+		rc_status
+		;;
+
+    restart)
+		$0 stop
+		$0 start
+		rc_status
+		;;
+
+    force-reload)
+		echo -n "Reload service bbackupd "
+		killproc -HUP $BBACKUPD_BIN
+		rc_status -v
+		;;
+
+    reload)
+		echo -n "Reload service bbackupd  "
+		killproc -HUP $BBACKUPD_BIN
+		rc_status -v
+		;;
+
+    status)
+		echo -n "Checking for service bbackupd "
+		checkproc $BBACKUPD_BIN
+		rc_status -v
+		;;
+
+    probe)
+		test /etc/box/bbackupd.conf -nt /var/run/bbackupd.pid && echo reload
+		;;
+
+    *)
+		echo "Usage: $0 {start|stop|status|try-restart|restart|force-reload|reload|probe}"
+		exit 1
+
+esac
+rc_exit
diff --git a/distribution/boxbackup/contrib/suse/bbstored b/distribution/boxbackup/contrib/suse/bbstored
new file mode 100644
index 00000000..1824dda7
--- /dev/null
+++ b/distribution/boxbackup/contrib/suse/bbstored
@@ -0,0 +1,103 @@
+#!/bin/sh
+#
+# Copyright (c)2004, Nothing But Net Limited
+#   <chris.smith@nothingbutnet.co.nz>
+#
+######################################################################
+# RELEASED AND PROVIDED TO YOU UNDER THE SAME LICENCE AS THE BOXBACKUP
+# SUITE OF PROGRAMS. LICENCE MAY BE VIEWED HERE:
+#
+# http://www.fluffy.co.uk/boxbackup/license.html
+######################################################################
+#
+# /etc/init.d/bbackupd
+#   and its symbolic link
+# /(usr/)sbin/rcbbackupd
+#
+### BEGIN INIT INFO
+# Provides:          bbackupd
+# Required-Start:    $named $network $local_fs $syslog
+# X-UnitedLinux-Should-Start: $time ypbind sendmail
+# Required-Stop:     $named $network $localfs $syslog
+# X-UnitedLinux-Should-Stop: $time ypbind sendmail
+# Default-Start:     3 5
+# Default-Stop:      0 1 2 6
+# Short-Description: BoxBackup server side daemon
+# Description: Client daemon for the BoxBackup software       
+#	that allows you to communicate with a bbstored server.
+### END INIT INFO
+# 
+
+# Check for missing binaries (stale symlinks should not happen)
+BBACKUPD_BIN=/usr/sbin/bbstored
+if [ ! -x $BBACKUPD_BIN ] ; then
+	echo "$BBACKUPD_BIN not installed"
+	exit 5
+fi
+
+. /etc/rc.status
+
+# Reset status of this service
+rc_reset
+
+case "$1" in
+	start)
+	echo -n "Starting bbstored "
+	startproc $BBACKUPD_BIN
+	rc_status -v
+	;;
+
+    stop)
+	echo -n "Shutting down bstored "
+	killproc -TERM $BBACKUPD_BIN
+	rc_status -v
+	;;
+
+    try-restart|condrestart)
+	if test "$1" = "condrestart"; then
+		echo "${attn} Use try-restart ${done}(LSB)${attn} rather than condrestart ${warn}(RH)${norm}"
+	fi
+	$0 status
+	if test $? = 0; then
+		$0 restart
+	else
+		rc_reset	# Not running is not a failure.
+	fi
+	rc_status
+	;;
+
+    restart)
+	$0 stop
+	$0 start
+	rc_status
+	;;
+
+    force-reload)
+	echo -n "Reload service bbstored "
+	killproc -HUP $BBACKUPD_BIN
+	rc_status -v
+	;;
+
+    reload)
+	echo -n "Reload service bbstored  "
+	killproc -HUP $BBACKUPD_BIN
+	rc_status -v
+	;;
+
+    status)
+	echo -n "Checking for service bbstored "
+	checkproc $BBACKUPD_BIN
+	rc_status -v
+	;;
+
+    probe)
+	test /etc/box/bbstored.conf -nt /var/run/bbstored.pid && echo reload
+	;;
+
+    *)
+	echo "Usage: $0 {start|stop|status|try-restart|restart|force-reload|reload|probe}"
+	exit 1
+	;;
+
+esac
+rc_exit
diff --git a/docs/backup/INDEX.txt b/docs/backup/INDEX.txt
new file mode 100644
index 00000000..50406cac
--- /dev/null
+++ b/docs/backup/INDEX.txt
@@ -0,0 +1,61 @@
+TITLE Programmers Notes for Box Backup
+
+This directory contains the programmers notes for the Box Backup system. They will be of interest only if you want to review or modify the code.
+
+These notes are intended to be run through a script to produce HTML at some later stage, hence the marks such as 'TITLE'.
+
+
+SUBTITLE Organisation
+
+The project is split up into several modules. The modules within 'lib' are building blocks for creation of the actual executable programs within 'bin'. 'test' contains unit tests for lib and bin modules.
+
+The file modules.txt lists the modules which are to be built, and their dependencies. It also allows for platform differences.
+
+
+SUBTITLE Documentation Organisation
+
+In this directory, the files correspond to modules or areas of interest. Sub directories of the same name contain files documenting specific classes within that module.
+
+
+SUBTITLE Suggested reading order
+
+* lib_common
+* lib_server
+* bin_bbackupd
+* backup_encryption.txt
+* bin_bstored
+* lib_raidfile
+
+and refer to other sections as required.
+
+
+SUBTITLE Building
+
+The makefiles are generated by makebuildenv.pl. (The top level makefile is generated by makeparcels.pl, but this is for the end user to use, not a programmer.)
+
+To build a module, cd to it and type make. If the -DRELEASE option is specified (RELEASE=1 with GNU make) the release version will be built. The object files and exes are placed in a directory structure under 'release' or 'debug'.
+
+It is intended that a test will be written for everything, so in general make commands will be issued only within the test/* modules. Once it has been built, cd to debug/test/<testname> and run the test with ./t .
+
+
+SUBTITLE Programming style
+
+The code is written to be easy to write. Ease of programming is the primary concern, as this should lead to fewer bugs. Efficiency improvements can be made later when the system as a whole works.
+
+Much use is made of the STL.
+
+There is no common base class.
+
+All errors are reported using exceptions.
+
+Some of the boring code is generated by perl scripts from description files.
+
+There are a lot of modules and classes which can easily be used to build other projects in the future -- there is a lot of "framework" code.
+
+
+SUBTITLE Lots more documentation
+
+The files are extensively commented. Consider this notes as an overview, and then read the source files for detailed and definitive information.
+
+Each function and class has a very brief decsription of it's purpose in a standard header, and extensive efforts have been maed to comment the code itself.
+
diff --git a/docs/backup/backup_encryption.txt b/docs/backup/backup_encryption.txt
new file mode 100644
index 00000000..36580581
--- /dev/null
+++ b/docs/backup/backup_encryption.txt
@@ -0,0 +1,109 @@
+TITLE Encryption in the backup system
+
+This document explains how everything is encrypted in the backup system, and points to the various functions which need reviewing to ensure they do actually follow this scheme.
+
+
+SUBTITLE Security objectives
+
+The crpyto system is designed to keep the following things secret from an attacker who has full access to the server.
+
+* The names of the files and directories
+* The contents of files and directories
+* The exact size of files
+
+Things which are not secret are
+
+* Directory heirarchy and number of files in each directory
+* How the files change over time
+* Approximate size of files
+
+
+SUBTITLE Keys
+
+There are four separate keys used:
+
+* Filename
+* File attributes
+* File block index
+* File data
+
+and an additional secret for file attribute hashes.
+
+The Cipher is Blowfish in CBC mode in most cases, except for the file data. All keys are maximum length 448 bit keys, since the key size only affects the setup time and this is done very infrequently.
+
+The file data is encrypted with AES in CBC mode, with a 256 bit key (max length). Blowfish is used elsewhere because the larger block size of AES, while more secure, would be terribly space inefficient. Note that Blowfish may also be used when older versions of OpenSSL are in use, and for backwards compatibility with older versions.
+
+The keys are generated using "openssl rand", and a 1k file of key material is stored in /etc/box/bbackupd. The configuration scripts make this readable only by root.
+
+Code for review: BackupClientCryptoKeys_Setup()
+in lib/backupclient/BackupClientCryptoKeys.cpp
+
+
+SUBTITLE Filenames
+
+Filenames need to be secret from the attacker, but they need to be compared on the server so it can determine whether or not is it a new version of an old file.
+
+So, the same Initialisation Vector is used for every single filename, so the same filename encrypted twice will have the same binary representation.
+
+Filenames use standard PKCS padding implemented by OpenSSL. They are proceeded by two bytes of header which describe the length, and the encoding.
+
+Code for review: BackupStoreFilenameClear::EncryptClear()
+in lib/backupclient/BackupStoreFilenameClear.cpp
+
+
+SUBTITLE File attributes
+
+These are kept secret as well, since they reveal information. Especially as they contain the target name of symbolic links.
+
+To encrypt, a random Initialisation Vector is choosen. This is stored first, followed by the attribute data encrypted with PKCS padding.
+
+Code for review: BackupClientFileAttributes::EncryptAttr()
+in lib/backupclient/BackupClientFileAttributes.cpp
+
+
+SUBTITLE File attribute hashes
+
+To detect and update file attributes efficiently, the file status change time is not used, as this would give suprious results and result in unnecessary updates to the server. Instead, a hash of user id, group id, and mode is used.
+
+To avoid revealing details about attributes
+
+1) The filename is added to the hash, so that an attacker cannot determine whether or not two files have identical attributes
+
+2) A secret is added to the hash, so that an attacker cannot compare attributes between accounts.
+
+The hash used is the first 64 bits of an MD5 hash.
+
+
+SUBTITLE File block index
+
+Files are encoded in blocks, so that the rsync algorithm can be used on them. The data is compressed first before encryption. These small blocks don't give the best possible compression, but there is no alternative because the server can't see their contents.
+
+The file contains a number of blocks, which contain among other things
+
+* Size of the block when it's not compressed
+* MD5 checksum of the block
+* RollingChecksum of the block
+
+We don't want the attacker to know the size, so the first is bad. (Because of compression and padding, there's uncertainty on the size.)
+
+When the block is only a few bytes long, the latter two reveal it's contents with only a moderate amount of work. So these need to be encrypted.
+
+In the header of the index, a 64 bit number is chosen. The sensitive parts of the block are then encrypted, without padding, with an Initialisation Vector of this 64 bit number + the block index.
+
+If a block from an previous file is included in a new version of a file, the same checksum data will be encrypted again, but with a different IV. An eavesdropper will be able to easily find out which data has been re-encrypted, but the plaintext is not revealed.
+
+Code for review: BackupStoreFileEncodeStream::Read() (IV base choosen about half-way through)
+BackupStoreFileEncodeStream::EncodeCurrentBlock() (encrypt index entry)
+in lib/backupclient/BackupStoreFileEncodeStream.cpp
+
+
+SUBTITLE File data
+
+As above, the first is split into chunks and compressed.
+
+Then, a random initialisation vector is chosen, stored first, followed by the compressed file data encrypted using PKCS padding.
+
+Code for review: BackupStoreFileEncodeStream::EncodeCurrentBlock()
+in lib/backupclient/BackupStoreFileEncodeStream.cpp
+
+
diff --git a/docs/backup/bin_bbackupd.txt b/docs/backup/bin_bbackupd.txt
new file mode 100644
index 00000000..67ad9267
--- /dev/null
+++ b/docs/backup/bin_bbackupd.txt
@@ -0,0 +1,88 @@
+TITLE bin/bbackupd
+
+The backup client daemon.
+
+This aims to maintain as little information as possible to record which files have been uploaded to the server, while minimising the amount of queries which have to be made to the server.
+
+
+SUBTITLE Scanning
+
+The daemon is given a length of time, t, which files over this age should be uploaded to the server. This is to stop recently updated files being uploaded immediately to avoid uploading something repeatedly (on the assumption that if a file has been written, it is likely to be modified again shortly).
+
+It will scan the files at a configured interval, and connect to the server if it needs to upload files or make queries about files and directories.
+
+The scan interval is actually varied slightly between each run by adding a random number up to a 64th of the configured time. This is to reduce cyclic patterns of load on the backup servers -- otherwise if all the boxes are turned on at about 9am, every morning at 9am there will be a huge spike in load on the server.
+
+Each scan chooses a time interval, which ends at the current time - t. This will be from 0 to current time - t on the first run, then the next run takes the start time as the end time of the previous run. The scan is only performed if the difference between the start and end times is greater or equal to t.
+
+For each configured location, the client scans the directories on disc recursively.
+
+For each directory
+
+* If the directory has never been scanned before (in this invocation of the daemon) or the modified time on the directory is not that recorded, the listing on the server is downloaded.
+
+* For each file, if it's modified time is within the time period, it is uploaded. If the directory has been downloaded, it is compared against that, and only uploaded if it's changed.
+
+* Find all the new files, and upload them if they lie within the time interval.
+
+* Recurse to sub directories, creating them on the server if necessary.
+
+Hence, the first time it runs, it will download and compare the entries on the disc to those on the server, but in future runs it will use the file and directory modification times to work out if there is anything which needs uploading.
+
+If there aren't any changes, it won't even need to connect to the server.
+
+There are some extra details which allow this to work reliably, but they are documented in the source.
+
+
+SUBTITLE File attributes
+
+The backup client will update the file attributes on files as soon as it notices they are changed. It records most of the details from stat(), but only a few can be restored. Attributes will only be considered changed if the user id, group id or mode is changed. Detection is by a 64 bit hash, so detection is strictly speaking probablistic.
+
+
+SUBTITLE Encryption
+
+All the user data is encrypted. There is a separate file, backup_encryption.txt which describes this, and where in the code to look to verify it works as described.
+
+
+SUBTITLE Tracking files and directories
+
+Renaming files is a difficult problem under this minimal data scanning scheme, because you don't really know whether a file has been renamed, or another file deleted and new one created.
+
+The solution is to keep (on disc) a map of inode numbers to server object IDs for all directories and files over a certain user configurable threshold. Then, when a new file is discovered, it is first checked to see if it's in this map. If so, a rename is considered, which will take place if the local object corresponding to the name of the tracked object doesn't exist any more.
+
+Because of the renaming requirement, deletions of objects from the server are recorded and delayed until the end of the scan.
+
+
+SUBTITLE Running out of space
+
+If the store server indicates on login to the backup client, it will scan, but not upload anything nor adjust it's internal stored details of the local objects. However, deletions and renames happen.
+
+This is to allow deletions to still work and reduce the amount of storage space used on the server, in the hope that in the future there will be enough space.
+
+Just not doing anything would mean that one big file created and then deleted at the wrong time would stall the whole backup process.
+
+
+SUBTITLE BackupDaemon
+
+This is the daemon class for the backup daemon. It handles setting up of all the objects, and implements calulcation of the time intervals for the scanning.
+
+
+SUBTITLE BackupClientContext
+
+State information for the scans, including maintaining a connection to the store server if required.
+
+
+SUBTITLE BackupClientDirectoryRecord
+
+A record of state of a directory on the local filesystem. Containing the recursive scanning function, which is long and entertaining, but very necessary. It contains lots of comments which explain the exact details of what's going on.
+
+
+SUBTITLE BackupClientInodeToIDMap
+
+A implementation of a map of inode number to object ID on the server. If Berkeley DB is available on the platform, it is stored on disc, otherwise there is an in memory version which isn't so good.
+
+
+
+
+
+
diff --git a/docs/backup/bin_bbstored.txt b/docs/backup/bin_bbstored.txt
new file mode 100644
index 00000000..c9c4e229
--- /dev/null
+++ b/docs/backup/bin_bbstored.txt
@@ -0,0 +1,54 @@
+TITLE bin/bbstored
+
+The backup store daemon.
+
+Maintains a store of encrypted files, and every so often goes through deleting unnecessary data.
+
+Uses an implementation of Protocol to communicate with the backup client daemon. See bin/bbstored/backupprotocol.txt for details.
+
+
+SUBTITLE Data storage
+
+The data is arranged as a set of objects within a RaidFile disc set. Each object has a 64 bit object ID, which is turned into a filename in a mildly complex manner which ensure that directories don't have too many objects in them, but there is a minimal number of nested directories. See StoreStructure::MakeObjectFilename in lib/backupstore/StoreStructure.cpp for more details.
+
+An object can be a directory or a file. Directories contain files and other directories.
+
+Files in directories are supersceded by new versions when uploaded, but the old versions are flagged as such. A new version has a different object ID to the old version.
+
+Every so often, a housekeeping process works out what can be deleted, and deletes unnecessary files to take them below the storage limits set in the store info file.
+
+
+SUBTITLE Note about file storage and downloading
+
+There's one slight entertainment to file storage, in that the format of the file streamed depends on whether it's being downloaded or uploaded.
+
+The problem is that it contains an index of all the blocks. For efficiency in managing these blocks, they all need to be in the same place.
+
+Files are encoded and decoded as they are streamed to and from the server. With encoding, the index is only completely known at the end of the process, so it's sent last, and lives in the filesystem last.
+
+When it's downloaded, it can't be decoded without knowing the index. So the index is sent first, followed by the data.
+
+
+SUBTITLE BackupContext
+
+The context of the current connection, and the object which modifies the store.
+
+Maintains a cache of directories, to avoid reading them continuously, and keeps a track of a BackupStoreInfo object which is written back periodiocally.
+
+
+SUBTITLE BackupStoreDaemon
+
+A ServerTLS daemon which forks off a separate housekeeping process as it starts up.
+
+Handling connections is delegated to a Protocol implementation.
+
+
+SUBTITLE BackupCommands.cpp
+
+Implementation of all the commands. Work which requires writing is handled in the context, read only commands mainly in this file.
+
+
+SUBTITLE HousekeepStoreAccount
+
+A class which performs housekeeping on a single account.
+
diff --git a/docs/backup/encryt_rsync.txt b/docs/backup/encryt_rsync.txt
new file mode 100644
index 00000000..9e3427c1
--- /dev/null
+++ b/docs/backup/encryt_rsync.txt
@@ -0,0 +1,66 @@
+TITLE Encrypted rsync algorithm
+
+The backup system uses a modified version of the rsync algorithm. A description of the plain algorithm can be found here:
+
+	http://samba.anu.edu.au/rsync/tech_report/
+
+The algorithm is modified to allow the server side to be encrypted, yet still benefit from the reduced bandwidth usage. For a single file transfer, the result will be only slightly less efficient than plain rsync. For a backup of a large directory, the overall bandwidth may be less due to the way the backup client daemon detects changes.
+
+This document assumes you have read the rsync document.
+
+The code is in lib/backupclient/BackupStoreFile*.*.
+
+
+SUBTITLE Blocks
+
+Each file is broken up into small blocks. These are individually compressed and encrypted, and have an entry in an index which contains, encrypted, it's weak and strong checksums and decoded plaintext size. This is all done on the client.
+
+Why not just encrypt the file, and use the standard rsync algorithm?
+
+1) Compression cannot be used, since encryption turns the file into essentially random data. This is not very compressible.
+
+2) Any modification to the file will result in all data after that in the file having different ciphertext (in any cipher mode we might want to use). Therefore the rsync algorithm will only be able to detect "same" blocks up until the first modification. This significantly reduces the effectiveness of the process.
+
+Note that blocks are not all the same size. The last block in the file is unlikely to be a full block, and if data is inserted which is not a integral multiple of the block size, odd sized blocks need to be created. This is because the server cannot reassemble the blocks, because the contents are opaque to the server.
+
+
+SUBTITLE Modifed algorithm
+
+To produce a list of the changes to send the new version, the client requests the block index of the file. This is the same step as requesting the weak and strong checksums from the remote side with rsync.
+
+The client then decrypts the index, and builds a list of the 8 most used block sizes above a certain threshold size.
+
+The new version of the file is then scanned in exactly the same way as rsync for these 8 block sizes. If a block is found, then it is added to a list of found blocks, sorted by position in the file. If a block has already been found at that position, then the old entry is replaced by the new entry.
+
+The smallest block size is searched first, so that larger blocks replace smaller blocks in the found list.
+
+A "recipe" is then built from the found list, by trivially discarding overlapping blocks. Each entry consists of a number of bytes of "new" data, a block start number, and a number of blocks from the old file. The data is stored like this as a memory optimisation, assuming that files mostly stay the same rather than having all their blocks reordered.
+
+The file is then encoded, with new data being sent as blocks of data, and references to blocks in the old file. The new index is built completely, as the checksums and size need to be rencrypted to match their position in the index.
+
+
+SUBTITLE Combination on server
+
+The "diff" which is sent from the client is assembled into a full file on the server, simply by adding in blocks from the old file where they are specified in the block index.
+
+
+SUBTITLE Storage on server
+
+Given that the server will in general store several versions of a file, combining old and new files to form a new file is not terribly efficient on storage space. Particularly for large multi-Gb database files.
+
+An alternative scheme is outlined below, however, it is significantly more complex to implement, and so is not implemented in this version.
+
+1) In the block index of the files, store the file ID of the file which each block is source from. This allows a single file to reference blocks from many files.
+
+2) When the file is downloaded, the server combines the blocks from all the files into a new file as it is streamed to the client. (This is not particuarly complicated to do.)
+
+This all sounds fine, until housekeeping is considered. Old versions need to be deleted, without losing any blocks necessary for future versions.
+
+Instead of just deleting a file, the server works out which blocks are still required, and rebuilds the file omitting those blocks which aren't required.
+
+This complicates working out how much space a file will release when it is "deleted", and indeed, adds a whole new level of complexity to the housekeeping process. (And the tests!)
+
+The directory structure will need an additional flag, "Partial file", which specifies that the entry cannot be built as previous blocks are no longer available. Entries with this flag should never be sent to the client.
+
+
+
diff --git a/docs/backup/lib_backupclient.txt b/docs/backup/lib_backupclient.txt
new file mode 100644
index 00000000..3e4a079b
--- /dev/null
+++ b/docs/backup/lib_backupclient.txt
@@ -0,0 +1,46 @@
+TITLE lib/backupclient
+
+Classes used on the store and on the server.
+
+See documentation in the files for more details.
+
+
+SUBTITLE BackupStoreDirectory
+
+The directory listing class, containing a number of entries, representing files.
+
+
+SUBTITLE BackupStoreFile
+
+Handles compressing and encrypting files, and decoding files downloaded from the server.
+
+
+SUBTITLE BackupStoreFilename
+
+An encrypted filename.
+
+
+SUBTITLE BackupStoreFilenameClear
+
+Derived from BackupStoreFilename, but with the ability to encrypt and decrypt filenames. Client side only.
+
+
+SUBTITLE BackupClientFileAttributes
+
+Only used on the client -- the server treats attributes as blocks of opaque data.
+
+This reads attributes from files on discs, stores them, encrypts them, and applies them to new files.
+
+Also has a static function to generate filename attribute hashes given a struct stat and the filename.
+
+
+SUBTITLE BackupClientRestore
+
+Routines to restore files from the server onto the client filesystem.
+
+
+SUBTITLE BackupClientCryptoKeys
+
+This reads the key material from disc, and sets up the crypto for storing files, attributes and directories.
+
+
diff --git a/docs/backup/lib_backupstore.txt b/docs/backup/lib_backupstore.txt
new file mode 100644
index 00000000..8f24eb7b
--- /dev/null
+++ b/docs/backup/lib_backupstore.txt
@@ -0,0 +1,30 @@
+TITLE lib/backupstore
+
+Classes which are shared amongst the server side store utilities, bbstored and bbstoreaccounts. Note also depends on lib/backupclient, as a lot of code is shared between the client and server.
+
+
+SUBTITLE BackupStoreAccountDatabase
+
+A simple implementation of an account database. This will be replaced with a more suitable implementation.
+
+
+SUBTITLE BackupStoreAccounts
+
+An interface to the account database, and knowledge of how to initialise an account on disc.
+
+
+SUBTITLE BackupStoreConfigVerify
+
+The same configuration file is used by all the utilities. This is the Configuration verification structure for this file.
+
+
+SUBTITLE BackupStoreInfo
+
+The "header" information about an account, specifying current disc usage, space limits, etc.
+
+
+SUBTITLE StoreStructure
+
+Functions specifying how the files are laid out on disc in the store.
+
+
diff --git a/docs/backup/windows_porting.txt b/docs/backup/windows_porting.txt
new file mode 100644
index 00000000..ada3b857
--- /dev/null
+++ b/docs/backup/windows_porting.txt
@@ -0,0 +1,100 @@
+TITLE Notes on porting the backup client to Windows
+
+It should be relatively easy to port the backup client (bbackupd) to Windows. However, the server relies on unlink() behaviour of the UNIX filesystem which is different on the Windows platform, so it will not be so easy to port.
+
+An installation of perl is required to build the system. The ActiveState port is the easiest to install.
+
+
+SUBTITLE Build environment
+
+The build environment generation script, makebuildenv.pl, uses perl to scan all the files and generate makefiles. It's rather orientated towards UNIX systems with gcc.
+
+Probably the easiest way to handle this is to detect the Windows platform, set up a few variables appropriately (in BoxPlatform.pm) and then post-process the generated Makefiles to mould them into something more handy for the MS Windows compiler and toolchain.
+
+The script itself is a bit messy. It was fine at first, but then the multi-platform thing got in the way. I do intend to rewrite it at some point in the future.
+
+Make sure your new version defines PLATFORM_WIN32 on the compile line.
+
+All files #include "Box.h" as the first include file. Use this for pre-compiled headers. Edit BoxPlatform.h to include the Windows headers required, and include a PLATFORM_WIN32 section. The easiest start will be to leave this blank, apart from typedefs for the basic types and any "not supported" #defines you can find.
+
+Boring bits of the code, such as exceptions and protocol definitions, are autogenerated using perl scripts. This code should be portable without modification.
+
+I have tried to avoid the things I know won't work with the MS compiler, so hopefully the code will be fairly clean. However, it might be a little easier to use the MinGW compiler [ http://www.mingw.org/ ] just to be consistent with the UNIX version. But I hope this won't be necessary.
+
+You'll need the latest version of OpenSSL. This was slightly difficult to get to compile last time I tried -- especially if you're determined to use the optimised assembler version. The main difficulty was getting a version which would link properly with the options used in my project, the default libraries selected got in the way.
+
+
+SUBTITLE Porting as UNIX emulation
+
+Since the daemon uses so few UNIX system calls and with a limited set of options, it seems to make sense to port it by writing emulations of these functions. It's probably nicest to create a lib/win32 directory, and populate this with header files corresponding to the UNIX header files used. These just contain inline functions which map the UNIX calls to Win32 calls.
+
+File/socket handles may have to be translated. -1 is used as a failure return value and by the code internally to mark an invalid socket handle. (0 is a valid socket handle)
+
+Of course, some bits of code aren't relevant, so will just be #ifdefed out, or replaced. But this should be minimal. (Only perhaps the small bit relating to filesystem structure -- there aren't really mount points as such.)
+
+
+SUBTITLE File tracking
+
+The daemon uses the inode number of a file to keep track of files and directories, so when they're renamed they can be moved efficiently on the store. Some unique (per filesystem) number will have to be found and used instead.
+
+It uses the Berkeley DB to store these on disc. It's likely another storage system will need to be used. (It just has to map the file's unique number into to a 8 byte struct.)
+
+There is a in-memory implementation for platforms which don't support Berkeley DB, but this isn't so good when the daemon has to be restarted as all the tracking is lost. But it's an easy start.
+
+
+SUBTITLE Expected filesystem behaviour
+
+File and directories have (at least) two modification times, for contents and attributes.
+
+For files, the contents modification time must change when the contents change, and the attributes time when the attributes change (and may change when the contents change too.)
+
+For directories, the contents modification time must change when files or directories are deleted or added. If it changes any more frequently than this, then the client will be slightly less efficient -- it will download the store's directory listing whenever this time changes. The attributes modification time is less important, as the actual attributes are compared and only uploaded if different.
+
+
+SUBTITLE Attributes
+
+Attributes means file modification times, flags, and filesystem permissions.
+
+The BackupClientFileAttribute class will need to be extended. Allocate another "attribute type" for the Win32 attributes, and then serialise it in a compatible way -- put your new attribute type in the header, and then a serialised network byte order structure in the rest. The different size of block is handled for you, and the server never looks inside.
+
+Add code so that under UNIX, Win32 attributes are ignored, and UNIX attributes under Win32.
+
+It's probably not necessary to worry too much about these for the first version. Not many people seem to use these attributes anyway.
+
+
+SUBTITLE Times
+
+The system uses it's own 64 bit time type -- see BoxTime.h. Everything is translated to this from the various different system time types, and calculated and stored internally in this form.
+
+
+SUBTITLE Daemon as a Service
+
+The client is derived from the Daemon class, which implements a daemon. The interface is simple, and it shouldn't be hard to write a compatible class which implements a Windows Service instead.
+
+Or cheat and run it as a Win32 application.
+
+Note that the daemon expects to be able to read every file it wants, and will abort a scan and upload run if it gets an error. The daemon must therefore be run with sufficient privileges. It runs as root under UNIX.
+
+
+SUBTITLE Command Socket
+
+The backup daemon accepts commands from bbackupctl through a UNIX domain socket. When a connection is made, the user ID of the connecting process is checked to see if it's the same user ID as the daemon is running under.
+
+This may not have any exact analogue under Win32, so another communications scheme may have to be devised.
+
+This is only actually necessary if the client is to be run in snapshot mode. It can be safely left unimplemented if snapshot mode is not required, or the prompts for it to sync with the server are implemented some other way.
+
+
+SUBTITLE NTFS streams
+
+If you want to back up NTFS streams, then a generic solution should probably be defined, so that the Mac OS X resource forks can be backed up with the same mechanism.
+
+
+SUBTITLE Source code
+
+I work on a slightly different version of the source files. A make distribution script adds the license header and removes private sections of code. This means submitted diffs need a slight bit of translation.
+
+
+
+
+
diff --git a/docs/common/lib_common.txt b/docs/common/lib_common.txt
new file mode 100644
index 00000000..11d7b02d
--- /dev/null
+++ b/docs/common/lib_common.txt
@@ -0,0 +1,52 @@
+TITLE lib/common
+
+This module is the basic building block of the project. It is included implicitly as a dependency of all other modules.
+
+It provides basic services such as file and stream functionality, platform information, and various bits of utility code which doesn't fit naturally anywhere else but is useful to many other applications.
+
+
+SUBTITLE Box.h
+
+The main include file for the project. It should be the first file included in every cpp file, followed by any system headers, then other project headers. This order has been chosen so that eventually it can be used as a target for pre-compiled headers. (This will be important for the Windows port)
+
+
+SUBTITLE BoxPlatform.h
+
+This contains various bits of information on platform differences. The build scripts include a compile command line definition like PLATFORM_OPENBSD, which is then used to select the required options.
+
+Some of the stuff is not particularly pleasant, but it aims to produce a consistent compilation environment, and to have a abstracted way of setting other defines to tell the other files what things are available on this platform.
+
+GNU configure is not used, as it would be just another dependency. Although something does have to be done about compilation on Linux, which is just too different on each distribution to be caught by a common configuration here.
+
+
+SUBTITLE Streams
+
+Much use is made of streams -- files, connections, buffers, all sorts of things are implemented using streams.
+
+The abstract base class IOStream defines the interface, see the class file for more details.
+
+Streams are lib/common's basic contribution to the project. A standard interface for handling data, and a number of utility classes which make performing common stream functions easy.
+
+
+SUBTITLE Serialisation
+
+Note there is no "serialisable" class. Instead, objects which need to be serialised into Streams simply have two functions, ReadFromStream and WriteToStream. These perform as expected.
+
+As it turns out, there's no real need for a specific serialisation base class. If you do need to be able to serialise arbitary objects, there are two approaches.
+
+1) If you're serialising an arbitary object, you're going to know roughly what type it is. So it's likely to be a subclass of a known base class... in which case, you simply use virtual functions.
+
+2) If it really is an arbitary type, then you just write your function which accepts any type of object to serialise as a template.
+
+
+SUBTITLE BoxException
+
+The exception base class for the project. All exceptions thrown by this code are dervied from BoxException. In general, each module which has unique errors has it's own exception class.
+
+CommonException errors are thrown throughout the project.
+
+
+SUBTITLE Other bits
+
+There are some other extra useful bits which are documented only in the source files.
+
diff --git a/docs/common/lib_common/BoxTime.txt b/docs/common/lib_common/BoxTime.txt
new file mode 100644
index 00000000..18bef5a5
--- /dev/null
+++ b/docs/common/lib_common/BoxTime.txt
@@ -0,0 +1,7 @@
+TITLE BoxTime
+
+Not strictly a class, but time is presented in the project as 64 bit integers as microseconds since the UNIX Epoch.
+
+There are a number of utility functions in the BoxTime*.h files, which are useful.
+
+To get BoxTime values from a struct stat, use the FileModificationTime.h functions.
\ No newline at end of file
diff --git a/docs/common/lib_common/CollectInBufferStream.txt b/docs/common/lib_common/CollectInBufferStream.txt
new file mode 100644
index 00000000..5f9556a3
--- /dev/null
+++ b/docs/common/lib_common/CollectInBufferStream.txt
@@ -0,0 +1,26 @@
+CLASS CollectInBufferStream
+
+This class is essentially a buffer. Data is written to it, and stored in memory. Then when all data is written which will be written, it can be read out again.
+
+Useful for building streams in memory.
+
+
+FUNCTION CollectInBufferStream::SetForReading()
+
+After you've written all the data, call this to set it to read mode.
+
+
+FUNCTION CollectInBufferStream::Reset()
+
+Reset the stream to the state where is has no data, and is about to have data written.
+
+
+FUNCTION CollectInBufferStream::GetSize()
+
+Get the size of the buffered data -- the entire data. Use the interface function BytesLeftToRead() in most cases. 
+
+
+FUNCTION CollectInBufferStream::GetBuffer()
+
+Get the buffer, so you can do bad things with it if you're feeling naughty.
+
diff --git a/docs/common/lib_common/Configuration.txt b/docs/common/lib_common/Configuration.txt
new file mode 100644
index 00000000..ef5f38f6
--- /dev/null
+++ b/docs/common/lib_common/Configuration.txt
@@ -0,0 +1,102 @@
+CLASS Configuration
+
+Implements the reading of multi-level configuration files. This is intended to be a generic way of representing configuration implementation.
+
+Basic validation is performed on files as they are read, specified by a data structure.
+
+test/common has some examples of usage. 
+
+
+SUBTITLE Configuration file format
+
+The format is simple, a list of Key = Value pairs. For example
+
+Key1 = Value1
+Key2 = Value2
+
+Optionally, multiple values for the same key are allowed.
+
+Lists of configurations can be nested to any level. These are called Sub-configurations:
+
+SubConfig
+{
+	SubKey1 = ValueX
+	SubKey2 = ValueY
+}
+
+
+SUBTITLE Verification
+
+The verification structure specifies what keys are required, what are optional, and what sub-configurations are expected. Default values can be specified.
+
+See Configuration.h for the structures.
+
+RaidFileController::Initialise has a good example of a simple verification layout.
+
+Wildcards can be used for SubConfigurations, by specifying a name of "*". This allows you to use multiple sections to configure any number of items.
+
+Each item has a number of flags, which are combined to say whether an item is required, should be an integer or boolean, and rather importantly, whether it's the last item in the list.
+
+Verification is limited, so you may wish to do more verification the file. There are unimplemented hooks for custom verification functions to be included in the verification definitions. Should be done at some point.
+
+Boolean keys have possible values "true", "yes", "false", "no" (case insensitive). 
+
+
+FUNCTION Configuration::LoadAndVerify()
+
+Loads the configuration from disc, and verifies it. If there are problems with the verification, it returns some text which can be used to tell the user the problems with the file. These are fairly basic error messages, but do say what should be done to get it to parse properly.
+
+This returns a Configuration object, which can then be queries for Keys and Values. Sub-configurations are implemented through an interface which returns a reference to another Configuration object.
+
+
+
+FUNCTION Configuration::KeyExists()
+
+Does a specified key exist?
+
+Use this for optional key values only -- specify them in the verification structure if they are required.
+
+
+FUNCTION Configuration::GetKeyValue()
+
+Get the value of a key as a string.
+
+If ConfigTest_MultiValueAllowed is set in the relevant entry in the verify structure, this string may contain multiple values, separated by a single byte with value 0x01 (use Configuration::MultiValueSeparator in code). Use SplitString() defined in Utils.h to split it into components.
+
+This representation was chosen as multi-values are probably rare, and unlikely to justify writing a nicer (but more memory intensive and complex) solution.
+
+
+FUNCTION Configuration::GetKeyValueInt()
+
+Get the value of a key as an integer. Make sure the AsInt property is requried in the verification structure.
+
+
+FUNCTION Configuration::GetKeyValueBool()
+
+Get the value of a key as a boolean value. Make sure the AsBool property is requried in the verification structure.
+
+Default to "false", should this verification not be performed and an unknown value is specified.
+
+
+FUNCTION Configuration::GetKeyNames()
+
+Return a list of all the keys in this configuration.
+
+
+FUNCTION Configuration::SubConfigurationExists()
+
+Does a specified sub-configuration exist?
+
+
+FUNCTION Configuration::GetSubConfiguration()
+
+Return another Configuration object representing the sub section.
+
+
+FUNCTION Configuration::GetSubConfigurationNames()
+
+Get a list of all the sub configurations.
+
+As there isn't a particularly neat way that configurations can be iterated over, mSubConfigurations is public. (BAD!)
+
+
diff --git a/docs/common/lib_common/Conversion.txt b/docs/common/lib_common/Conversion.txt
new file mode 100644
index 00000000..62d1967a
--- /dev/null
+++ b/docs/common/lib_common/Conversion.txt
@@ -0,0 +1,14 @@
+TITLE Generic type conversion
+
+Conversion.h provides generic type conversion. Within the BoxConvert namespace, it defines the templated function
+
+	ToType Convert<ToType, FromType>(FromType From)
+
+which converts the data type as expected. In general, from and to types have to be specified explicitly.
+
+Templates rather than overloaded functions are used, firstly to be absolutely explicit about the conversion required, and secondly because overloaded functions can't have differing return types for the same argument type.
+
+The function is specialised for various types, and the generic version uses C++ type conversion.
+
+Exceptions may be thrown if the conversion is not possible. These are all of the ConversionException type.
+
diff --git a/docs/common/lib_common/ExcludeList.txt b/docs/common/lib_common/ExcludeList.txt
new file mode 100755
index 00000000..8a5bf36c
--- /dev/null
+++ b/docs/common/lib_common/ExcludeList.txt
@@ -0,0 +1,21 @@
+TITLE ExcludeList
+
+A class implementing a list of strings which are to be excluded in some way. Entries can be added as strings to be matched exactly, or as regular expressions.
+
+Multiple entries can be added in a single function call in a manner designed to work with the multi-value entries store in a Configuation object.
+
+
+FUNCTION ExcludeList::AddDefiniteEntries()
+
+Definite entries are exact strings to match.
+
+
+FUNCTION ExcludeList::AddRegexEntries()
+
+Regular expressions as defined by POSIX, and supported by the host platform. Will throw an exception if regular expressions are not supported by the platform.
+
+
+FUNCTION ExcludeList::IsExcluded()
+
+Test a string for being excluded by definite or regex entries.
+
diff --git a/docs/common/lib_common/FdGetLine.txt b/docs/common/lib_common/FdGetLine.txt
new file mode 100644
index 00000000..d92ff94d
--- /dev/null
+++ b/docs/common/lib_common/FdGetLine.txt
@@ -0,0 +1,11 @@
+CLASS FdGetLine
+
+See IOStreamGetLine, difference is that it works on basic UNIX file descriptors rather than full blown streams. Follows basic interface, except...
+
+
+FUNCTION FdGetLine::GetLine
+
+Returns a string containing the optionally preprocessed line.
+
+Do not call if IsEOF if true.
+
diff --git a/docs/common/lib_common/Guards.txt b/docs/common/lib_common/Guards.txt
new file mode 100644
index 00000000..6174fea6
--- /dev/null
+++ b/docs/common/lib_common/Guards.txt
@@ -0,0 +1,5 @@
+TITLE Guards.h
+
+This file contains a couple of classes for using file and memory. When the class goes out of scope, the underlying object is closed or freed, respectively.
+
+
diff --git a/docs/common/lib_common/IOStream.txt b/docs/common/lib_common/IOStream.txt
new file mode 100644
index 00000000..09460656
--- /dev/null
+++ b/docs/common/lib_common/IOStream.txt
@@ -0,0 +1,89 @@
+CLASS IOStream
+
+The base class for streams of data. See IOStream.h for specific details on functions, but the interface is described here.
+
+Most streams only implement one direction, but some both.
+
+
+SUBTITLE Reading
+
+
+FUNCTION IOStream::Read()
+
+Reads data from the stream. Returns 0 if there is no data available within the timeout requested.
+
+Unlike the UNIX API, a return of 0 does not mean that there is no more data to read. Use IOStream::StreamDataLeft() to find this out.
+
+
+FUNCTION IOStream::ReadFullBuffer()
+
+If you want to read a specific sized block of data from a stream, it is annoying ot use the Read function, because it might return less, so you have to loop.
+
+This implements that loop. Note that the timeout is the timeout for each individual read -- it might take a lot longer to complete.
+
+You must check the return value, which is whether or not it managed to get all that data.
+
+
+FUNCTION IOStream::BytesLeftToRead()
+
+How many bytes are left to read in the stream treated as a whole. May return IOStream::SizeOfStreamUnknown if it is something like a socket which doesn't know how much data is in the stream.
+
+
+FUNCTION IOStream::StreamDataLeft()
+
+Returns true if more data can be read. Or more specifically, if more data might be able to be read some time in the future.
+
+
+SUBTITLE Writing
+
+
+FUNCTION IOStream::Write()
+
+Write data to a stream. Writes the entire block, or exceptions.
+
+
+FUNCTION IOStream::WriteAllBuffered()
+
+Writes any buffered data to the underlying object.
+
+Call at the end of a series of writes to make sure the data is actually written. (Buffering is not yet implemented anywhere, but it should be soon.)
+
+
+FUNCTION IOStream::StreamClosed()
+
+Is the stream closed, and writing no longer possible?
+
+
+FUNCTION IOStream::CopyStreamTo()
+
+A utility function to copy the contents of one stream to another, until the reading stream has no data left.
+
+
+SUBTITLE Other interfaces
+
+These are slightly nasty interfaces, because they have to match the UNIX API to some extent.
+
+
+FUNCTION IOStream::Close()
+
+Close the stream -- intended to indicate that nothing more will be written.
+
+However, also closes reading in most cases. Stream dependent. Probably best just to delete the object and let it sort itself out.
+
+
+FUNCTION IOStream::Seek()
+
+Seeks within the stream -- mainly for using files within a stream interface, although some of the utility stream classes support it too.
+
+This is actually a bad interface, because it does not specify whether it applies to reading or writing. This matches the interface provided by files with a single file pointer, but does not map well onto other stream types.
+
+Should it be changed? Perhaps. But then it means that files would either have to have an inconsitent interface, or the class keep track of two separate file pointers. Which isn't nice.
+
+So use carefully, and remember whether you're using a file stream, a reading stream, or a writing stream.
+
+
+FUNCTION IOStream::GetPosition()
+
+Get the current file pointer. Subject to same problems as Seek with regard to semantics.
+
+
diff --git a/docs/common/lib_common/IOStreamGetLine.txt b/docs/common/lib_common/IOStreamGetLine.txt
new file mode 100644
index 00000000..04c56b57
--- /dev/null
+++ b/docs/common/lib_common/IOStreamGetLine.txt
@@ -0,0 +1,29 @@
+CLASS IOStreamGetLine
+
+This class provides a convenient way to read text from a file, line by line. It also can preprocess the line to remove leading and trailing whitespace and comments. Comments are started by the character # and run to the end of the line.
+
+Create an instance by passing a reference to a stream into the constructor.
+
+Note the class does internal buffering, so you can only detach it later if the stream supports seeking backwards.
+
+
+FUNCTION IOStreamGetLine::GetLine()
+
+Returns true if a line could be retreieved without a read timing out.
+
+
+FUNCTION IOStreamGetLine::IsEOF()
+
+Whether the end of the stream has been reached. Do not call GetLine if this is true.
+
+
+FUNCTION IOStreamGetLine::GetLineNumber()
+
+Returns the line number.
+
+
+FUNCTION IOStreamGetLine::DetachFile()
+
+Detaches the stream from the GetLine class. Will seek backwards to "replace" data it's buffered back in the stream.
+
+
diff --git a/docs/common/lib_common/MainHelper.txt b/docs/common/lib_common/MainHelper.txt
new file mode 100644
index 00000000..eb5b07f0
--- /dev/null
+++ b/docs/common/lib_common/MainHelper.txt
@@ -0,0 +1,4 @@
+TITLE MainHelper.h
+
+This header contains a couple of macros which add exception handling and reporting to main() functions for executable programs.
+
diff --git a/docs/common/lib_common/WaitForEvent.txt b/docs/common/lib_common/WaitForEvent.txt
new file mode 100644
index 00000000..0bc55726
--- /dev/null
+++ b/docs/common/lib_common/WaitForEvent.txt
@@ -0,0 +1,16 @@
+CLASS WaitForEvent
+
+This class implements a way to efficiently wait on one or more system objects, for example sockets and file descriptors. Where kqueue() is available, this is used, otherwise poll(). The poll() implementation is less comprehensive and rather more limited.
+
+To add a compatible object, call Add(). To remove it, call Remove(). To wait for an object to become ready, call Wait(), which returns a pointer to the first ready object, or 0 for a timeout.
+
+The timeout is set either in the constructor, or using the SetTimout() method. It is specified in milliseconds.
+
+The kqueue based implementation will automatically remove objects when they are closed (actually, the OS does this), but the poll implementation requires that Remove() be called.
+
+The flags passed to Add() or Remove() are passed to the object, and are ignored by this class.
+
+For an object to be able to be added to the list, it should implement FillInKEvent() and FillInPoll() for kqueue and poll implementations respectively. Use the PLATFORM_KQUEUE_NOT_SUPPORTED define to test which is necessary for the platform.
+
+It does not have to be derived off any particular class, as it uses templates to be compatible with any class.
+
diff --git a/docs/common/lib_common/xStream.txt b/docs/common/lib_common/xStream.txt
new file mode 100644
index 00000000..91e9c0ea
--- /dev/null
+++ b/docs/common/lib_common/xStream.txt
@@ -0,0 +1,40 @@
+TITLE Various stream types
+
+Some useful streams are implemented in lib/common.
+
+
+SUBTITLE CollectInBufferStream
+
+Described in it's own page.
+
+
+SUBTITLE FileStream
+
+Implements a stream from a file, allowing both reading and writing.
+
+
+SUBTITLE MemBlockStream
+
+Turns a memory block into a readable stream.
+
+Can also be constructed using StreamableMemBlock, CollectInBufferStream and MemBlockStream as sources of the buffer.
+
+
+SUBTITLE PartialReadStream
+
+Create a readable stream which will read a set number of bytes from another stream, and then declare itself as closed.
+
+Useful for extracting a small chunk of another stream to present to a function which expects to consume all of a stream.
+
+
+SUBTITLE ReadGatherStream
+
+Create a readable stream out of blocks from many streams -- so various sections of any number of streams are composed into a single stream.
+
+To use, register each stream with AddComponent, then use the returned 'handle' with AddBlock to add blocks to the composed stream.
+
+Optionally, the object will take ownership of the streams added, and delete them when itself is deleted.
+
+See the comments in the function blocks in the cpp file for more info.
+
+
diff --git a/docs/common/lib_compress.txt b/docs/common/lib_compress.txt
new file mode 100644
index 00000000..f3e26f20
--- /dev/null
+++ b/docs/common/lib_compress.txt
@@ -0,0 +1,8 @@
+TITLE lib/compress
+
+This is a horrid C++ interface to zlib. It is written this way to avoid having to buffer any data, and so be unnecessarily inefficient. But this does end up just being a wrapper to the zlib interface.
+
+See test/compress for an example of how to use it. But it should be very familiar.
+
+For an easier interface, use CompressStream.
+
diff --git a/docs/common/lib_compress/CompressStream.txt b/docs/common/lib_compress/CompressStream.txt
new file mode 100644
index 00000000..eca43eb6
--- /dev/null
+++ b/docs/common/lib_compress/CompressStream.txt
@@ -0,0 +1,27 @@
+TITLE CompressStream
+
+This implements a compressing stream class, which compresses and decompresses data sent to an underlying stream object. Data is likely to be buffered.
+
+
+WARNING: Use WriteAllBuffered() (after objects which need to be sent in their entirity) and Close() (after you have finished writing to the stream) otherwise the compression buffering will lose the last bit of data for you.
+
+
+The class works as a filter to an existing stream. Ownership can optionally be taken so that the source stream is deleted when this object is deleted.
+
+It can operate in one or two directions at any time. Data written to the stream is compressed, data read from the stream is decompressed. If a direction is not being (de)compressed, then it can act as a pass-through without touching the data.
+
+The constructor specifies the actions to be taken:
+
+CompressStream(IOStream *pStream, bool TakeOwnership,
+	bool DecompressRead, bool CompressWrite,
+	bool PassThroughWhenNotCompressed = false);
+
+pStream - stream to filter
+TakeOwnership - if true, delete the stream when this object is deleted.
+DecompressRead - reads pass through a decompress filter
+CompressWrite - writes pass through a compression filter
+PassThroughWhenNotCompressed - if not filtering a direction, pass through the data unaltered, otherwise exception if an attempt is made on that direction.
+
+
+Note that the buffering and compression will result in different behaviour than the underlying stream: data may not be written in one go, and data being read may come out in different sized chunks.
+
diff --git a/docs/common/lib_crypto.txt b/docs/common/lib_crypto.txt
new file mode 100644
index 00000000..9ddafe69
--- /dev/null
+++ b/docs/common/lib_crypto.txt
@@ -0,0 +1,28 @@
+TITLE lib/crypto
+
+Provides cryptographic primatives using the OpenSSL implementation.
+
+
+SUBTITLE CipherContexts and CipherDescriptions
+
+A CipherContext is the interface to encryption and decryption, providing a generic interface.
+
+It is constructed with a decrypt or encrypt flag, and a CipherDescription. The CipherDescription specifies which cipher is to be used, the key, and all other cipher specific paramteres.
+
+The CipherContext has support for padding and initialisation vectors.
+
+
+SUBTITLE Random numbers
+
+An interface to the OpenSSL psuedo random number generater is provided.
+
+
+SUBTITLE Digests
+
+An interface to the MD5 digest is provided.
+
+
+SUBTITLE RollingChecksum
+
+The rsync rolling checksum is implemented. The interface is a little tricky to be efficient as the caller must track the position and provide relevant bytes to advance the checksum.
+
diff --git a/docs/common/lib_crypto/CipherContext.txt b/docs/common/lib_crypto/CipherContext.txt
new file mode 100644
index 00000000..30fb4608
--- /dev/null
+++ b/docs/common/lib_crypto/CipherContext.txt
@@ -0,0 +1,28 @@
+CLASS CipherContext
+
+Encryption and decryption using OpenSSL EVP interface.
+
+See the cpp file for more documentation in the function headers, and test/crypto for examples.
+
+General notes below.
+
+
+SUBTITLE Construction
+
+Construct with the encryption direction, and a CipherDescription of the cipher and key required.
+
+
+SUBTITLE Encrypting or decrypting
+
+Begin() and Transform() allow piece by piece transformation of a block.
+
+TransformBlock() transforms an entire block.
+
+
+SUBTITLE Buffering
+
+All transforms expect to have enough space in the buffers for their entire output. Because of block boundaries and padding, it is necessary that the output buffer should be bigger than the input buffer. The amount of space depends on the cipher.
+
+InSizeForOutBufferSize() and MaxOutSizeForInBufferSize() perform these space calculations, returning the maximuim in size for a specified out size, and the reverse, respectively.
+
+
diff --git a/docs/common/lib_crypto/RollingChecksum.txt b/docs/common/lib_crypto/RollingChecksum.txt
new file mode 100644
index 00000000..cbee1454
--- /dev/null
+++ b/docs/common/lib_crypto/RollingChecksum.txt
@@ -0,0 +1,32 @@
+CLASS RollingChecksum
+
+Implementing the rsync rolling checksum algorithm. Read it's description first:
+
+http://samba.anu.edu.au/rsync/tech_report/node3.html
+
+
+SUBTITLE Construction and initial checksum calculation
+
+The constructor takes a pointer to a block of data and a size, and calculates the checksum of this block. It can now be "rolled forward" to find the checksum of the block of the same size, one byte forward, with minimal calculation.
+
+
+FUNCTION RollingChecksum::GetChecksum()
+
+Returns the checksum for the current block.
+
+
+FUNCTION RollingChecksum::RollForward()
+
+This function takes the byte at the start of the current block, and the last byte of the block it's rolling forward to, and moves the checksum on.
+
+If the block is
+
+	char *pBlock = <something>;
+
+with size s, then it should be called with
+
+	RollForward(pBlock[0], pBlock[s])
+
+and now GetChecksum will return the checksum of the block (pBlock+1) of size s.
+
+
diff --git a/docs/common/lib_server.txt b/docs/common/lib_server.txt
new file mode 100644
index 00000000..392f331d
--- /dev/null
+++ b/docs/common/lib_server.txt
@@ -0,0 +1,9 @@
+TITLE lib/server
+
+This provides various classes for implementing client/server applications (and UNIX daemons).
+
+The stars of the show are ServerTLS and Protocol, which allow client server applications which communicate using TLS (SSL successor) to be built with surprisingly few lines of code.
+
+All details in the respective class files.
+
+Look at test/basicserver for examples.
diff --git a/docs/common/lib_server/Daemon.txt b/docs/common/lib_server/Daemon.txt
new file mode 100644
index 00000000..6956ec2b
--- /dev/null
+++ b/docs/common/lib_server/Daemon.txt
@@ -0,0 +1,96 @@
+CLASS Daemon
+
+Implement a UNIX daemon which
+
+* Daemonises (detach from console, etc)
+* Sets up signal handlers, and does useful things with them
+* Reads configuration files
+* Writes PID file
+* Opens syslog
+
+all the usual UNIX basic daemon behaviour.
+
+The daemon exe takes optional arguments: The first is a configuration file filename which overrides the default. If the second argument is 'SINGLEPROCESS' the daemon will not daemonise.
+
+The configuration file must have a section like this
+
+Server
+{
+	PidFile = /var/run/daemon.pid
+	User = username
+}
+
+Use DAEMON_VERIFY_SERVER_KEYS (defined in Daemon.h) to include the necessary keys in your configuration file's verify structure.
+
+The "User" line is optional, and if it is present the Daemon will change user to this username just before it daemonises. Note that unless the directory the PID file is written to has write permission for this user, the PID file will not be deleted on exit.
+
+
+To implement a daemon, derive a class from Daemon, and override Run().
+
+Then in your main file, do something like
+
+int main(int argc, const char *argv[])
+{
+	MAINHELPER_START
+
+	BackupDaemon daemon;
+	return daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv);
+	
+	MAINHELPER_END
+}
+
+and that's it. Obviously it's worth doing a few more things as well, but that's it.
+
+
+FUNCTION Daemon::Run()
+
+Override with the main daemon code. It should behave like this
+
+void SomethingDaemon::Run()
+{
+	// Read configuration file
+
+	// Do lots of work until StopRun() returns true
+	while(!StopRun())
+	{
+		::sleep(10);
+	}
+
+	// Clean up nicely
+}
+
+which allows the base class to implement the standard start, terminate and -HUP behaviours correctly.
+
+
+FUNCTION Daemon::DaemonName()
+
+Returns the name of the daemon, for use in syslog.
+
+
+FUNCTION Daemon::DaemonBanner()
+
+Returns the banner to be displayed on startup, or 0 for no banner.
+
+
+FUNCTION Daemon::GetConfigVerify()
+
+Returns a configuration verify structure for verifying the config file. Note that this configuration structure should include a sub-configuration called "Server, and have entries defined by DAEMON_VERIFY_SERVER_KEYS. See one of the bin/bbackupd for an example.
+
+
+FUNCTION Daemon::StopRun()
+
+Returns true when the daemon needs to be terminated or restarted. Use IsReloadConfigWanted() and IsTerminateWanted() to find out which one, if you need to know.
+
+
+FUNCTION Daemon::SetupInInitialProcess()
+
+Override to perform additional functions in the initial process, before forking and detachment happens.
+
+
+FUNCTION Daemon::EnterChild()
+
+Called when a child is entered. If you override it, remember to call the base class.
+
+
+
+
diff --git a/docs/common/lib_server/Protocol.txt b/docs/common/lib_server/Protocol.txt
new file mode 100644
index 00000000..09d3c1f1
--- /dev/null
+++ b/docs/common/lib_server/Protocol.txt
@@ -0,0 +1,120 @@
+CLASS Protocol
+
+Protocol
+
+* serialises and deserialises data objects
+* sends arbitary streams
+
+through a bi-directional IOStream object, usually a socket.
+
+These data objects are auto-generated by a perl script, along with the logic to implement a simple protocol where one end is a client, and the other a server. The client sends a command object (and optional streams) and the server returns a reply object (and optional streams).
+
+The perl script uses a description file which specifies the data inside these objects (which can be extended to include any type which implements the standard serialisation functions), the required responses to the commands, and whether streams are involved.
+
+It then implements a server object, which given a stream and a user defined context object, waits for commands, processes them, and sends the results back. All you need to do is implement a DoCommand() function for each command object you define.
+
+On the client side, a Query() function is implemented which takes objects as parameters, and returns the right type of reply object (or throws an exception if it doesn't get it.) Short cut Query<CommandName>() functions are also implemented which don't require objects to be created manually.
+
+Thus, implementing a server is as simple as deriving a daemon off ServerStream or ServerTLS, and implementing the Connection() function as 
+
+	void TestProtocolServer::Connection(SocketStream &rStream)
+	{
+		TestProtocolServer server(rStream);
+		TestContext context;
+		server.DoServer(context);
+	}
+
+and that's it. TestContext is a user defined class which keeps track of all the state of the connection, and is passed to all the DoCommand() functions, which look like this:
+
+	std::auto_ptr<ProtocolObject>
+		TestProtocolServerSimple::DoCommand(TestProtocolServer &rProtocol,
+		TestContext &rContext)
+	{
+		return std::auto_ptr<ProtocolObject>
+			(new TestProtocolServerSimpleReply(mValue+1));
+	}
+
+(taken from test/basicserver)
+
+The client code looks like this
+
+	SocketStream conn;
+	conn.Open(Socket::TypeUNIX, "testfiles/srv4.sock");
+	
+	TestProtocolClient protocol(conn);
+	
+	// Query
+	{
+		std::auto_ptr<TestProtocolClientSimpleReply>
+			reply(protocol.QuerySimple(41));
+		TEST_THAT(reply->GetValuePlusOne() == 42);
+	}
+
+
+Finally, debug logging can be generated which allows a list of all commands and their parameters to be logged to syslog or a file.
+
+
+SUBTITLE Protocol Description File
+
+This file is passed to the lib/server/makeprotocol.pl script, which generates a h and cpp file to implement the protocol.
+
+It is in two sections, separated by a 'BEGIN_OBJECTS' on a line of it's own.
+
+In the top half, the following statements must be made.
+
+Name 			<name>
+	The name of the protocol, used in naming classes.
+
+IdentString		<string>
+	The idenfitifaction string sent over the IOStream to confirm it it
+	is talking to another Protocol object speaking the same Protocol.
+
+ServerContextClass	<class-name>	<header-file>
+	The user defined context class used for the server, and the header
+	file it is defined in.
+
+Additionally, the following optional commands can be made.
+
+ClientType <description-typename> <C++ typename> <headerfile>
+ServerType (similarly)
+	Extends the types used in the objects below. Server and client
+	can use different types for the same object type.
+
+ImplementLog	(Client|Server)	(syslog|file)
+	Implement command logging for client or server into syslog or a file.
+
+LogTypeToText (Client|Server) <description-typename> <printf-element>
+		<evaluate>
+	For extended types, optionally define how to convert them into printf
+	elements and the code to run to get the argument. Within the evaluate
+	parameter, VAR is replaced by the name of the variable to display.
+	If this is not specified for a given type, OPAQUE is output instead.
+
+
+In the object section, an object is defined by a line
+
+<name> <id number> <attributes>
+
+followed by lines beginning with whitespace defining the data transmitted in the object. The type may be list<type>, which specifies a list (implemented as a std::vector) of entries of that type.
+
+The attributes specify exactly how that object is used in the defined protocol.
+
+Reply
+	The object is a reply object, sent from the server to the client.
+
+Command(Reply-Type)
+	The object is a command, send from the client to the server, and the server
+	will send back an object of type Reply-Type.
+
+IsError(Type-Field,SubType-Field)
+	The command is an error object, and the two files specify the data member
+	which describes the error type and sub type.
+
+EndsConversation
+	When this command is received, the connection is to be terminated.
+	(ie a logout command)
+
+StreamWithCommand
+	When this command is sent as a command, a stream follows it.
+
+
diff --git a/docs/common/lib_server/ServerStream.txt b/docs/common/lib_server/ServerStream.txt
new file mode 100644
index 00000000..6c5932a0
--- /dev/null
+++ b/docs/common/lib_server/ServerStream.txt
@@ -0,0 +1,29 @@
+CLASS ServerStream
+
+ServerStream implementes a Daemon which accepts stream connections over sockets, and forks into a child process to handle them.
+
+To implement a daemon, derive from
+
+	ServerStream<SocketStream, SERVER_LISTEN_PORT>
+
+The type SocketStream specifies that incoming connections should be treated as normal sockets, and SERVER_LISTEN_PORT is the port to listen to (if it's a inet socket, and if not overridden in the config file). The actual addresses (or names) to bind to are specified in the configuration file by the user.
+
+Make sure SERVERSTREAM_VERIFY_SERVER_KEYS(0) is included in the configuration verification structure in the "Server" sub configuration. 0 could be replaced with a default address, for example "unix:/var/run/server.sock" to specific a default UNIX socket in the filesystem.
+
+See test/basicserver for a simple example.
+
+The ListenAddresses key in the Server subconfiguration is a comma separated list of addresses, specified as family:name. Internet sockets are family 'inet', for example 'inet:localhost' (or 'inet:localhost:1080' to specify a port number as well), and unix domain sockets are 'unix', example above.
+
+
+Override Connection to handle the connection.
+
+Remember to override Daemon functions like the server name, and start it up, just like a generic Daemon.
+
+
+FUNCTION ServerStream::Connection
+
+This function takes a connected stream as it's argument. It should then proceed to do whatever it needs to do to talk to the client.
+
+Using IOStreamGetLine or a Protocol class to communicate may be quick ways of implementing this functionality.
+
+
diff --git a/docs/common/lib_server/ServerTLS.txt b/docs/common/lib_server/ServerTLS.txt
new file mode 100644
index 00000000..dbde500f
--- /dev/null
+++ b/docs/common/lib_server/ServerTLS.txt
@@ -0,0 +1,6 @@
+CLASS ServerTLS
+
+Implements a server which uses TLS (SSL) to encrypt and authenticate connections.
+
+Very similar to ServerStream, except it reads the certificate files for the TLSContext out of the Server sub-configuration to set up a TLSContext ("CertificateFile", "PrivateKeyFile" and "TrustedCAsFile"). Otherwise works exactly the same.
+
diff --git a/docs/common/lib_server/SocketStream.txt b/docs/common/lib_server/SocketStream.txt
new file mode 100644
index 00000000..82813279
--- /dev/null
+++ b/docs/common/lib_server/SocketStream.txt
@@ -0,0 +1,8 @@
+CLASS SocketStream
+
+A implementation of IOStream which wraps a socket connection.
+
+It can either be created by attaching to an existing object, or use the Open() function to open a connection to a named host on a specific port (or a local UNIX socket in the filesystem).
+
+Follows stream interface.
+
diff --git a/docs/common/lib_server/SocketStreamTLS.txt b/docs/common/lib_server/SocketStreamTLS.txt
new file mode 100644
index 00000000..ebb3f233
--- /dev/null
+++ b/docs/common/lib_server/SocketStreamTLS.txt
@@ -0,0 +1,11 @@
+CLASS SocketStreamTLS
+
+An implementation of IOStream which wraps a TLS (SSL) connection over a socket.
+
+The Open function takes a TLSContext reference which specifies the parameters for the connection.
+
+
+FUNCTION GetPeerCommonName()
+
+Returns the common name of the certificate presented by the remote end.
+
diff --git a/docs/common/lib_server/TLSContext.txt b/docs/common/lib_server/TLSContext.txt
new file mode 100644
index 00000000..ff50d3e6
--- /dev/null
+++ b/docs/common/lib_server/TLSContext.txt
@@ -0,0 +1,16 @@
+CLASS TLSContext
+
+A wrapper over the OpenSSL context object.
+
+Note: you need to call SSLLib::Initialise at the beginning of your program to use these functions.
+
+
+SUBTITLE Construction
+
+The constuctor takes the following parameters
+
+* Boolean for whether this is acting as a server or a client
+* The .pem file containing the certificate which will be used
+* The .pem file containing the private key for this certificate
+* The .pem file containing the certificates which will certify the other end of the connection.
+
diff --git a/docs/common/memory_leaks.txt b/docs/common/memory_leaks.txt
new file mode 100755
index 00000000..9a9764ea
--- /dev/null
+++ b/docs/common/memory_leaks.txt
@@ -0,0 +1,44 @@
+TITLE Memory leak detection
+
+The build system contains a primative memory leak detection system in debug builds.
+
+It works by using #defines to replace calls to malloc, free, realloc, new and delete with debug versions which record their use. When a process ends, it dumps a list of all the blocks or objects which were allocated by not freed, and the file and line of the source where they are originally allocated.
+
+It's not perfect, but should catch most things and work on most platforms.
+
+If it doesn't work on your platform, define PLATFORM_DISABLE_MEM_LEAK_TESTING in BoxPlatform.h within the relevant section.
+
+
+SUBTITLE Requirements in source
+
+It does require some extra lines in the source file. The last included file in each .cpp file must be
+
+	#include "MemLeakFindOn.h"
+
+and if a .h file uses any of these functions, the contents of the .h file should be bounded with
+
+	#include "MemLeakFindOn.h"
+	
+	// ... some code, but absolutely no #includes
+	
+	#include "MemLeakFindOff.h"
+
+The cleanupforcvs.pl script checks for correct usage.
+
+
+SUBTITLE Use in tests
+
+Tests will automatically dump memory leaks and regard them as a failure for anything executing in the main test process.
+
+To test for memory leaks in programs run from the test, or daemons, use something like
+
+	TestRemoteProcessMemLeaks("bbackupd.memleaks");
+
+If memory leak detection is being used, it will check the specified file for memory leak reports (deleting it afterwards). Any leak is an error.
+
+The Daemon class automactically arranges for the memory leak reports to be written. Other exes should set this up themselves, preferably using the define in MainHelper.h, as the first thing in their main() function.
+
+	MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, name)
+
+File is the filename to write the report to, conventionally called "<programname>.memleaks", and name is the name of the exe.
+
diff --git a/docs/raidfile/lib_raidfile.txt b/docs/raidfile/lib_raidfile.txt
new file mode 100644
index 00000000..0c4244ce
--- /dev/null
+++ b/docs/raidfile/lib_raidfile.txt
@@ -0,0 +1,73 @@
+TITLE lib/raidfile
+
+Implements RAID type abilities in files in userland, along with simple transaction like semantics for writing and reading files.
+
+IOStream interfaces are used to read and write files.
+
+Eventually a raid file daemon will be written to "raidify" files in the background. This will allow fast streaming of data to a single disc, followed by splitting into separate raid files later.
+
+There is an option to disable raidification for use on systems with only one hard disc, or a hardware RAID array.
+
+
+SUBTITLE Controller
+
+The raid disc sets are managed by RaidFileController. This reads the configuration file, /etc/box/raidfile.conf, and then stores the sets of discs for use by the other classes.
+
+In the code, files are referenced using an integer set number, and a filename within that set. For example, (0, "dir/subdir/filename.ext"). The RAID file controller turns this into actual physcial filenames transparently.
+
+The files are devided into blocks, which should match the fragment/block size of the underlying filesystem for maximum space efficiency.
+
+If the directories specified in the configuration files are all equal, then userland RAID is disabled. The files will not be processed into the redundant parts.
+
+
+SUBTITLE Writing
+
+Files are written (and modified) using the RaidFileWrite class. This behaves as a seekable IOStream for writing.
+
+When all data has been written, the file is committed. This makes it available to be read. Processing to split it into the three files can be delayed.
+
+If the data is not commited, the temporary write file will be deleted.
+
+RaidFileWrite will optionally refuse to overwrite another file. If it is being overwritten, a read to that file will get the old file until it has been committed.
+
+
+SUBTITLE Reading
+
+File are opened and read with RaidFileRead::Open. This returns an IOStream (with some extras) which reads a file.
+
+If the file has been turned into the RAID representation, then if an EIO error occurs on any operation, it will switch transparently into recovery mode where the file will be regenerated from the remaining two files using the parity file. (and log an error to syslog)
+
+
+SUBTITLE Layout on disc
+
+The directory structure is replicated onto each of the three directories within the disc set.
+
+Given a filename x, within the disc set, one of the three discs is chosen as the 0th disc for this file. This is done by hashing the filename. This ensures that an equal load is placed on each disc -- otherwise one disc would only be accessed when a RAID file was being written.
+
+When the file is being written, the actual physical file is "x.rfwX". This just a striaght normal file -- nothing is done to the data.
+
+When it is commited, it is renamed to "x.rfw".
+
+Then, when it is turned into the raid representation, it is split across three discs, each file named "x.rf".
+
+When trying to open a file, the first attempt is for "x.rfw". If this exists, it is used as is.
+
+If this fails, two of the "x.rf" files are attempted to be opened. If this succeeds, the file is opened in RAID mode.
+
+This procedure guarenettes that a file will be opened either as a normal file, or as the correct three RAID file elements, even if a new version is being commited at the same time, or another process is working to turn it into a raid file.
+
+
+SUBTITLE Raid file representation
+
+The file is split into three files, stripe 1, stripe 2 and parity.
+
+Considering the file as a set of blocks of the size specified in the configuration, odd numbered blocks go in stripe 1, even blocks in stripe 2, and a XOR of the corresponding blocks in parity.
+
+Thus only two files are needed to recreate the original.
+
+The complexity comes because the size of the file is not stored inside the files at any point. This means if stripe 2 is missing, it is ambiguous as to how big the file is.
+
+Size is stored as a 64 bit integer. This is appended to the parity file, unless it can be stored by XORing it into the last 8 bytes of the parity file.
+
+This scheme is complex to implement (see the RaidFileRead.cpp!) but minimises the disc spaced wasted.
+
diff --git a/docs/raidfile/lib_raidfile/RaidFileRead.txt b/docs/raidfile/lib_raidfile/RaidFileRead.txt
new file mode 100644
index 00000000..0a5efbf7
--- /dev/null
+++ b/docs/raidfile/lib_raidfile/RaidFileRead.txt
@@ -0,0 +1,14 @@
+CLASS RaidFileRead
+
+Read a raid file.
+
+IOStream interface, plus a few extras, including reading directories and checking that files exist.
+
+
+FUNCTION RaidFileRead::Open
+
+Open a given raid file -- returns a pointer to a new RaidFileRead object.
+
+Note that one of two types could be returned, depending on the representation of the file.
+
+
diff --git a/docs/raidfile/lib_raidfile/RaidFileWrite.txt b/docs/raidfile/lib_raidfile/RaidFileWrite.txt
new file mode 100644
index 00000000..89500c37
--- /dev/null
+++ b/docs/raidfile/lib_raidfile/RaidFileWrite.txt
@@ -0,0 +1,36 @@
+CLASS RaidFileWrite
+
+Interface to writing raidfiles.
+
+See IOStream interface.
+
+Some other useful functions are available, see h and cpp files.
+
+
+FUNCTION RaidFileWrite::RaidFileWrite()
+
+The constructor takes the disc set number and filename of the file you're interested.
+
+
+FUNCTION RaidFileWrite::Open()
+
+Open() opens the file for writing, and takes an "allow overwrite" flag.
+
+
+FUNCTION RaidFileWrite::Commit()
+
+Commmit the file, and make it visible to RaidFileRead. If ConvertToRaidNow == true, it will be converted to raid file representation immediately.
+
+Setting it to false is not a good idea. Later on, it will tell a daemon to convert it in the background, but for now it simply won't be converted.
+
+
+FUNCTION RaidFileWrite::Discard()
+
+Abort the creation/update. Equivalent to just deleting the object without calling Commit().
+
+
+FUNCTION RaidFileWrite::Delete()
+
+Delete a file -- don't need to Open() it first.
+
+
diff --git a/infrastructure/BoxPlatform.pm b/infrastructure/BoxPlatform.pm
new file mode 100644
index 00000000..4c8280a3
--- /dev/null
+++ b/infrastructure/BoxPlatform.pm
@@ -0,0 +1,78 @@
+package BoxPlatform;
+use Exporter;
+@ISA = qw/Exporter/;
+@EXPORT = qw/$build_os $make_command $bsd_make $platform_define $gcc_v3 $gcc_v4 $product_version $product_name $install_into_dir $sub_make_options $platform_compile_line_extra $platform_link_line_extra/;
+
+BEGIN
+{
+
+	# which OS are we building under?
+	$build_os = `uname`;
+	chomp $build_os;
+	# Cygwin Builds usually something like CYGWIN_NT-5.0, CYGWIN_NT-5.1
+	# Box Backup tried on Win2000,XP only :)
+    $build_os = 'CYGWIN' if $build_os =~ m/CYGWIN/;
+
+	$make_command = ($build_os ne 'Darwin')?'make':'bsdmake';
+	$bsd_make = ($build_os ne 'Linux' && $build_os ne 'CYGWIN');
+    $platform_define = 'PLATFORM_'.uc($build_os);
+
+	# blank extra flags by default
+	$platform_compile_line_extra = '';
+	$platform_link_line_extra = '';
+	
+	# compiler version?
+	$gcc_v3 = 0;
+	$gcc_v4 = 0;
+	open COMPILER,"gcc -v 2>&1 |" or die "Can't open gcc -v";
+	while(<COMPILER>)
+	{
+		$gcc_v3 = 1 if (m/version gcc 3/ || m/gcc version 3/ || m/gcc \(GCC\) 3/i || m/gcc.Version\s+3/i);
+		$gcc_v4 = 1 if (m/version gcc 4/ || m/gcc version 4/ || m/gcc \(GCC\) 4/i || m/gcc.Version\s+4/i);
+	}
+	close COMPILER;
+	$gcc_v3 = 1 if $gcc_v4;		# hacks are about the same
+
+	# get version
+	open VERSION,"VERSION.txt";
+	$product_version = <VERSION>;
+	chomp $product_version;
+	$product_name = <VERSION>;
+	chomp $product_name;
+	close VERSION;
+	
+	# where to put the files
+	$install_into_dir = '/usr/local/bin';
+	
+	# if it's Darwin,
+	if($build_os eq 'Darwin')
+	{
+		# see how many processors there are, and set make flags accordingly
+		my $cpus = `sysctl hw.ncpu`;
+		if($cpus =~ m/hw.ncpu:\s(\d+)/ && $1 > 1)
+		{
+			print "$1 processors detected, will set make to perform concurrent jobs\n";
+			$sub_make_options = ' -j '.($1 + 1);
+		}
+		
+		# test for fink installation
+		if(-d '/sw/include' && -d '/sw/lib')
+		{
+			print "Fink installation detected, will use headers and libraries\n";
+			$platform_compile_line_extra = '-I/sw/include ';
+			$platform_link_line_extra = '-L/sw/lib ';
+		}
+	}
+}
+
+sub make_flag
+{
+	if($bsd_make)
+	{
+		return "-D $_[0]"
+	}
+	return $_[0].'=1';
+}
+
+1;
+
diff --git a/infrastructure/buildenv-testmain-template.cpp b/infrastructure/buildenv-testmain-template.cpp
new file mode 100755
index 00000000..36cc4da0
--- /dev/null
+++ b/infrastructure/buildenv-testmain-template.cpp
@@ -0,0 +1,144 @@
+//
+//	AUTOMATICALLY GENERATED FILE
+//		do not edit
+//
+
+
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testmain.template.h
+//		Purpose: Template file for running tests
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "stdio.h"
+#include <exception>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <syslog.h>
+#include <stdarg.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include "MemLeakFindOn.h"
+
+int test(int argc, const char *argv[]);
+
+#ifdef NDEBUG
+	#define MODE_TEXT	"release"
+#else
+	#define MODE_TEXT	"debug"
+#endif
+
+int failures = 0;
+
+int filedes_open_at_beginning = -1;
+
+int count_filedes()
+{
+	int c = 0;
+
+	// See how many file descriptors there are with values < 256
+	for(int d = 0; d < 256; ++d)
+	{
+		if(::fcntl(d, F_GETFD) != -1)
+		{
+			// File descriptor obviously exists
+			++c;
+		}
+	}
+	
+	return c;
+}
+
+bool checkfilesleftopen()
+{
+	if(filedes_open_at_beginning == -1)
+	{
+		// Not used correctly, pretend that there were things left open so this gets invesitgated
+		return true;
+	}
+
+	// make sure syslog log file is closed, if it was opened
+	::closelog();
+
+	// Count the file descriptors open
+	return filedes_open_at_beginning != count_filedes();
+}
+
+int main(int argc, const char *argv[])
+{
+	// Start memory leak testing
+	MEMLEAKFINDER_START
+
+	// If there is more than one argument, then the test is doing something advanced, so leave it alone
+	bool fulltestmode = (argc == 1);
+
+	if(fulltestmode)
+	{
+		// Count open file descriptors for a very crude "files left open" test
+		filedes_open_at_beginning = count_filedes();
+
+		// banner
+		printf("Running test TEST_NAME in " MODE_TEXT " mode...\n");
+	}
+	try
+	{
+		int returncode = test(argc, argv);
+		
+		// check for memory leaks, if enabled
+		#ifdef BOX_MEMORY_LEAK_TESTING
+			if(memleakfinder_numleaks() != 0)
+			{
+				failures++;
+				printf("FAILURE: Memory leaks detected\n");
+				printf("==== MEMORY LEAKS =================================\n");
+				memleakfinder_reportleaks();
+				printf("===================================================\n");
+			}
+		#endif
+		
+		if(fulltestmode)
+		{
+			bool filesleftopen = checkfilesleftopen();
+			if(filesleftopen)
+			{
+				failures++;
+				printf("IMPLICIT TEST FAILED: Something left files open\n");
+			}
+			if(failures > 0)
+			{
+				printf("FAILED: %d tests failed\n", failures);
+			}
+			else
+			{
+				printf("PASSED\n");
+			}
+		}
+		
+		return returncode;
+	}
+	catch(std::exception &e)
+	{
+		printf("FAILED: Exception caught: %s\n", e.what());
+		return 1;
+	}
+	catch(...)
+	{
+		printf("FAILED: Unknown exception caught\n");
+		return 1;
+	}
+	if(fulltestmode)
+	{
+		if(checkfilesleftopen())
+		{
+			printf("WARNING: Files were left open\n");
+		}
+	}
+}
+
diff --git a/infrastructure/makebuildenv.pl b/infrastructure/makebuildenv.pl
new file mode 100755
index 00000000..99455cc2
--- /dev/null
+++ b/infrastructure/makebuildenv.pl
@@ -0,0 +1,1126 @@
+#!/usr/bin/perl
+use strict;
+use Symbol;
+
+my @modules;
+my %module_dependency;
+my %module_library_link_opts;
+my %header_dependency;
+
+$|=1;
+
+
+# note: Mac OS X resource forks and .DS_Store files are explicity ignored
+
+print "Box build environment setup.\n\nChecking environment...\n";
+
+
+my $implicit_dep = 'lib/common';
+
+# work out platform variables
+use lib 'infrastructure';
+use BoxPlatform;
+
+# don't allow old versions of openssl by default.
+my $old_version_of_openssl_ok = 0;
+
+# keep copy of command line args
+my $makebuildenv_args = join(' ',@ARGV);
+
+# do command line arguments
+my $compile_line_extra = $platform_compile_line_extra;
+my $link_line_extra = $platform_link_line_extra;
+for(@ARGV)
+{
+	if($_ eq 'allow-old-openssl')
+	{
+		$old_version_of_openssl_ok = 1;
+		next;
+	}
+	my ($k,$v) = split /:/,$_,2;
+	if($k eq 'compile')
+	{
+		$compile_line_extra .= $v . ' ';
+	}
+	elsif($k eq 'link')
+	{
+		$link_line_extra .= $v . ' ';
+	}
+	elsif($k eq 'openssl')
+	{
+		# assume that the bin/lib/include dirs are under the specified path
+		chop $v if ($v =~ /\/$/);
+		$compile_line_extra = "-I$v/include $compile_line_extra"; 
+		$link_line_extra    = "-L$v/lib $link_line_extra";
+	}
+	else
+	{
+		die "invalid option $_ specified on command line"
+	}
+}
+
+# make sure local files directory exists
+unless(-d 'local')
+{
+	mkdir 'local',0755;
+}
+
+
+# flags about the environment
+my %env_flags;
+# messages on test failure
+my $test_failure_text;
+
+# run all tests
+{
+	opendir DIR,'infrastructure/tests' or die "Can't read the tests directory";
+	my @tests = grep {m/_tests\.pl\Z/} readdir DIR;
+	closedir DIR;
+	for(@tests)
+	{
+		require "infrastructure/tests/$_";
+	}
+}
+
+
+# ---- LINUX start --------------------------------------------------------------------------
+# do configuration on Linux... find out what's available, and configure appropraitely.
+if($build_os eq 'Linux')
+{
+	# setup the basic library translation flags -- these will omit those libraries from the compile line
+	$env_flags{'LIBTRANS_-lreadline'} = '';
+	$env_flags{'LIBTRANS_-lLINUX_DB'} = '';
+
+	my $curses_lib = '-lcurses';
+	if((!exists($env_flags{'curses_PRESENT'})) && exists($env_flags{'ncurses_PRESENT'}))
+	{
+		# got to use ncurses instead...
+		$env_flags{'LIBTRANS_-lcurses'} = '-lncurses';
+		$curses_lib = '-lncurses';
+	}
+
+	my ($linux_readline_h, $linux_db_h);
+	my $db_ver = -1;
+	
+	open H_FILES,"find /usr/include |" or die "Can't open find";
+	while(<H_FILES>)
+	{
+		chomp;
+		
+		if(m~/readline.h\Z~)
+		{
+			my $rlh = strip_h_name($_);
+			if(check_readline_h($rlh,$curses_lib))
+			{
+				$linux_readline_h = $rlh;
+				# allow readline library to be used
+				delete $env_flags{'LIBTRANS_-lreadline'};
+			}
+		}
+		if(m~/db(|_1\d+).h\Z~)
+		{
+			# version?
+			my $dv = 0;
+			$dv = $1 if m~(\d+)/db~;
+			my $fn = $_;
+
+			# check this file mentions dbopen
+			open DB_H,$_ or die "Can't open $_";
+			my $found = 0;
+			while(<DB_H>)
+			{
+				if(m/dbopen/)
+				{
+					$found = 1;
+					last;
+				}
+			}
+			close DB_H;
+			next unless $found;
+
+			# see if this version works
+			print("Checking db version...\n");
+			my $db_h = strip_h_name($fn);
+			my $db_lib = 'db';
+			$db_lib .= $db_ver if $db_ver > 0;
+			if(!check_db_lib($db_h, $db_lib))
+			{
+				# try another
+				$db_lib = 'db';
+				next unless check_db_lib($db_h, $db_lib)
+			}
+
+	 		# good version?
+			if($dv > $db_ver)
+			{
+				$linux_db_h = $db_h;
+				$db_ver = $dv;
+				$env_flags{'LIBTRANS_-lLINUX_DB'} = '-l'.$db_lib;
+			}
+		}
+	}
+	print "Finished checking headers\n";
+	close H_FILES;
+
+	# write the platform file accordingly and tell the user what's happened.
+	open LIN,">local/_linux_platform.h" or die "Can't open file for writing";
+	if($linux_readline_h eq '')
+	{
+		print("---------------------\nWARNING: readline isn't installed\n---------------------\n");
+		print LIN "#define PLATFORM_READLINE_NOT_SUPPORTED\n";
+
+	}
+	else
+	{
+		open RL_H,">local/_linux_readline.h" or die "Can't open file for writing";
+		print RL_H "#include <$linux_readline_h>\n";
+		my $hist = $linux_readline_h;
+		$hist =~ s/readline\.h/history.h/;
+		print RL_H "#include <$hist>\n";
+		close RL_H;
+	}
+	if($linux_db_h eq '')
+	{
+		print("---------------------\nWARNING: db is not installed -- will run in reduced efficiency mode without it.\n---------------------\n");
+		print LIN "#define PLATFORM_BERKELEY_DB_NOT_SUPPORTED\n";
+	}	
+	else
+	{
+		open DB_H,">local/_linux_db.h" or die "Can't open file for writing";
+		print DB_H "#include <$linux_db_h>\n";
+		close DB_H;
+	}
+	close LIN;
+}
+sub strip_h_name
+{
+	my $i = $_[0];
+	$i =~ s~\A/usr/include/~~;
+	return $i;
+}
+sub check_readline_h
+{
+	my ($h,$curses_lib) = @_;
+	print "Check readline from $h...\n";
+	my $hist = $h;
+	$hist =~ s/readline\.h/history.h/;
+	open READLINEH,">readlineh.cpp" or die "Can't open readline test file for writing";
+	print READLINEH <<__E;
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <$h>
+#include <$hist>
+int main(int argc, char *argv[])
+{
+	using_history();
+	add_history(0);
+	readline(0);
+	return 0;
+}
+__E
+	close READLINEH;
+	# attempt to compile
+	my $r = system("g++ readlineh.cpp -lreadline $curses_lib -o readlineh $compile_line_extra $link_line_extra 2>/dev/null >/dev/null");
+	# delete test files
+	unlink 'readlineh';
+	unlink 'readlineh.cpp';
+	return $r == 0; # compilation succeeded
+}
+sub check_db_lib
+{
+	my ($h,$db) = @_;
+	open TESTDBLIB,">testdblib.cpp" or die "Can't open db test file for writing";
+	print TESTDBLIB <<__E;
+#include <$h>
+int main(int argc, char *argv[])
+{
+	DB *dbp = 0;
+	dbp = dbopen(0, 0, 0, DB_HASH, 0);
+	dbp->close(dbp);
+	DBT data;
+	dbp->put(dbp, &data, &data, 0);
+	dbp->get(dbp, &data, &data, 0);
+	return 0;
+}
+__E
+	close TESTDBLIB;
+	# attempt to compile
+	my $r = system("g++ testdblib.cpp -l$db -o testdblib $compile_line_extra $link_line_extra 2>/dev/null >/dev/null");
+	# delete test files
+	unlink 'testdblib';
+	unlink 'testdblib.cpp';
+	return $r == 0; # compilation succeeded
+}
+# ---- LINUX end --------------------------------------------------------------------------
+
+# print "Flag: $_\n" for(keys %env_flags);
+
+# check the version of openssl installed
+if(!exists $env_flags{'OPENSSL_OK'})
+{
+	# failed somehow...
+
+	# is an old version present?
+	my $old_version = (exists $env_flags{'OLD_OPENSSL_OK'});
+
+	if(!$old_version && !-e '/usr/include/openssl/evp.h')
+	{
+		# headers not installed?
+		print "\n\nERROR: OpenSSL library and headers need to be installed.\n\nSee documentation on web site if you need to add extra search paths.\n\n";
+		exit(1);
+	}
+	else
+	{
+		if($old_version)
+		{
+			if($old_version_of_openssl_ok)
+			{
+				print "\n\nWARNING: Configuring for old version of OpenSSL.\nPerformance will be lower than with version 0.9.7\n\n";
+				$compile_line_extra .= '-DPLATFORM_OLD_OPENSSL ';
+			}
+			else
+			{
+				# tell user how they might try anyway with an old version of openssl
+				print <<__E;
+
+===============================================================================
+
+You appear to have a version of OpenSSL installed which is less than 0.9.7.
+
+It is possible to configure to use this old version, but performance will be
+lower than if you have 0.9.7 -- the later version is recommended.
+
+If you wish to use this old version, repeat the configuration with the extra
+argument 'allow-old-openssl', like this:
+
+   ./configure allow-old-openssl
+
+* Please do not distribute binary packages with this option enabled.
+
+* Please do not distribute ports which set this option by default.
+
+IMPORTANT: Support for older versions of OpenSSL should be considered
+experimental. It is not recommended for production use, but provided to allow
+easier evaluation of this software before installing the latest OpenSSL.
+
+===============================================================================
+
+__E
+				exit(1);
+			}
+		}
+		else
+		{
+			print "\n\nERROR: You need to install OpenSSL, preferably at least version 0.9.7.\n\n";
+			print "If you believe you have installed OpenSSL, check that the headers are installed\nas well ('dev' packages?)\n\nSee documentation on web site if you need to add extra search paths.\n\n";
+			exit(1);
+		}
+	}
+}
+
+
+# finished checking the environment
+print "done\n\n";
+
+
+# seed autogen code
+print "Seeding autogen code...\n";
+open FINDAUTOGEN,"find . -follow -name Makefile.extra |" or die "Can't use find for locating files";
+while(<FINDAUTOGEN>)
+{
+	chomp;
+	my $file = $_;
+	$file =~ m~\A(.+)/[^/]+\Z~;
+	my $dir = $1;
+	open FL,$file or die "Can't open $_ for reading";
+	my %vars;
+	my $do_cmds = 0;
+	while(<FL>)
+	{
+		chomp;
+		if(m/\A(.+)\s+=\s+(.+)\Z/)
+		{
+			# is a variable
+			$vars{$1} = $2;
+			next;
+		}
+		next unless m/\S/;
+		if(m/AUTOGEN SEEDING/)
+		{
+			$do_cmds = 1;
+		}
+		elsif(m/\A\S/)
+		{
+			$do_cmds = 0 if $do_cmds == 2;
+		}
+		else
+		{
+			# command, run it?
+			if($do_cmds)
+			{
+				$do_cmds = 2; # flag something has been done
+				
+				# subsitute variables, repeatedly
+				my $c = $_;
+				$c =~ s/\A\s+//;
+				while(1)
+				{
+					my $did_subst = 0;
+					
+					for my $k (keys %vars)
+					{
+						$did_subst = 1 if $c =~ s/\$\($k\)/$vars{$k}/g;
+					}
+					
+					last unless $did_subst;
+				}
+				
+				# run command
+				die "Couldn't run command $c" unless (0 == system("(cd $dir; $c)"))
+			}
+		}
+	}
+	close FL;
+}
+close FINDAUTOGEN;
+print "done\n\n";
+
+
+# open test mail program template file
+my $test_template_file = 'infrastructure/buildenv-testmain-template.cpp';
+open FL,$test_template_file or die "Can't open test template file\n";
+my $test_template;
+read FL,$test_template,-s $test_template_file;
+close FL;
+
+
+# extra platform defines
+my $extra_platform_defines = '';
+if($gcc_v3 && !$gcc_v4)
+{
+	$extra_platform_defines .= ' -DPLATFORM_GCC3'
+}
+if($gcc_v4)
+{
+	$extra_platform_defines .= ' -DPLATFORM_GCC4'
+}
+
+# read in module definitions file, and any files it includes
+my @modules_files;
+sub read_modules_file
+{
+	my ($mf) = @_;
+	my $f = gensym;
+	open $f,$mf or die "Can't open modules file '$mf'\n";
+	while(<$f>)
+	{
+		if(m/\AINCLUDE\s+(\S+)\Z/)
+		{
+			# include another file
+			read_modules_file($1)
+		}
+		else
+		{
+			push @modules_files,$_
+		}
+	}
+	close $f;
+}
+read_modules_file('modules.txt');
+
+# prepare directories...
+mkdir "release",0755;
+mkdir "debug",0755;
+
+# is the library code in another directory?
+my $external_lib = readlink('lib');
+if($external_lib ne '')
+{
+	# adjust to root of the library distribution
+	$external_lib =~ s!/lib\Z!!;
+	$external_lib = '../'.$external_lib;
+	# make symlinks
+	make_obj_symlink('debug');
+	make_obj_symlink('release');
+}
+sub make_obj_symlink
+{
+	my $m = $_[0];
+	my $target = $external_lib."/$m/lib/";
+	my $link = "$m/lib";
+	# check link
+	if(-e $link)
+	{
+		if(-l $link)
+		{
+			if(readlink($link) ne $target)
+			{
+				print "Warning: replacing $link with new link to $target\n";
+				unlink $link;
+			}
+		}
+		else
+		{
+			die "$link already exists, but it isn't a symbolic link"
+		}
+	}
+	if(!-e $link)
+	{
+		symlink $target,$link or die "Can't make $m/lib symlink";
+	}
+}
+
+print "Scanning code...\n";
+
+my $modules_omitted = 0;
+
+# process lines in flattened modules files
+for(@modules_files)
+{
+	# clean up line
+	chomp; s/\A\s+//; s/#.*\Z//; s/\s+\Z//; s/\s+/ /g;
+	next unless m/\S/;
+	
+	# omit bits on some platforms?
+	next if m/\AEND-OMIT/;
+	if(m/\AOMIT:(.+)/)
+	{
+		if($1 eq $build_os)
+		{
+			$modules_omitted = 1;
+			while(<MODULES>)
+			{
+				last if m/\AEND-OMIT/;	
+			}
+		}
+		next;
+	}
+	
+	# split up...
+	my ($mod, @deps_i) = split / /;
+	
+	# ignore this module?
+	next if ignore_module($mod);
+	
+	# deps for this platform
+	my @deps;
+	for(@deps_i)
+	{
+		my ($dep,$exclude_from) = split /!/;
+		# generic library translation
+		$dep = $env_flags{'LIBTRANS_'.$dep} if exists($env_flags{'LIBTRANS_'.$dep});
+		next if $dep eq '';
+		if($exclude_from =~ m/\A\+(.+)\Z/)
+		{
+			$exclude_from = $1;
+			my $inc = 0;
+			for(split /,/,$exclude_from)
+			{
+				$inc = 1 if $_ eq $build_os
+			}
+			push @deps,$dep if $inc
+		}
+		else
+		{
+			my $inc = 1;
+			for(split /,/,$exclude_from)
+			{
+				$inc = 0 if $_ eq $build_os
+			}
+			push @deps,$dep if $inc
+		}
+	}
+	
+	# check directory exists
+	die "Module $mod can't be found\n" unless -d $mod;
+	
+	# and put in lists
+	push @modules,$mod;
+	my @md;	# module dependencies
+	my @lo; # link line options
+	for(@deps)
+	{
+		if(/\A-l/)
+		{
+			push @lo,$_
+		}
+		else
+		{
+			push @md,$_ unless ignore_module($_)
+		}
+	}
+	$module_dependency{$mod} = [$implicit_dep,@md];
+	$module_library_link_opts{$mod} = [@lo];
+	
+	# make directories, but not if we're using an external library and this a library module
+	my ($s,$d) = split /\//,$mod;
+	if($s ne 'lib' || $external_lib eq '')
+	{
+		mkdir "release/$s",0755;
+		mkdir "release/$s/$d",0755;
+		mkdir "debug/$s",0755;
+		mkdir "debug/$s/$d",0755;
+	}
+}
+
+# make dirs for implicit dep
+mkdir "release/$implicit_dep",0755;
+mkdir "debug/$implicit_dep",0755;
+
+# write a list of all the modules we've configured to use
+open CONFIGURED_MODS,'>local/modules.h' or die "Can't write configured modules list";
+print CONFIGURED_MODS <<__E;
+// automatically generated file, do not edit
+#ifndef _CONFIGURED_MODULES__H
+#define _CONFIGURED_MODULES__H
+__E
+for($implicit_dep,@modules)
+{
+	my $m = $_;
+	$m =~ s~/~_~;
+	print CONFIGURED_MODS "#define MODULE_$m\n";
+}
+print CONFIGURED_MODS <<__E;
+#endif // _CONFIGURED_MODULES__H
+__E
+close CONFIGURED_MODS;
+
+
+# now make a list of all the .h files we can find, recording which module they're in
+my %hfiles;
+for my $mod (@modules, $implicit_dep)
+{
+	opendir DIR,$mod;
+	my @items = readdir DIR;
+	closedir DIR;
+
+	# add in items from autogen directories, and create output directories
+	{
+		my @autogen_items;
+		
+		for my $di (@items)
+		{
+			if($di =~ m/\Aautogen/ && -d "$mod/$di")
+			{
+				# Read items
+				my $d = "$mod/$di";
+				opendir DIR,$d;
+				my @i = readdir DIR;
+				closedir DIR;
+				for(@i)
+				{
+					next if m/\A\./;
+					push @autogen_items,"$di/$_"
+				}
+			}
+		}
+		@items = (@items, @autogen_items);
+	}
+
+	for(grep /\.h\Z/i, @items)
+	{
+		next if /\A\._/;	# Temp Mac OS Resource hack
+		die "Header file $_ already used in module ".$hfiles{$_}."\n" if exists $hfiles{$_};
+		$hfiles{$_} = $mod
+	}
+}
+
+for my $mod (@modules, $implicit_dep)
+{
+	opendir DIR,$mod;
+	for my $h (grep /\.h\Z/i, readdir DIR)
+	{	
+		next if /\A\._/;	# Temp Mac OS Resource hack
+
+		open FL,"$mod/$h" or die "can't open $mod/$h";
+		my $f;
+		read FL,$f,-s "$mod/$h";
+		close FL;
+
+		while($f =~ m/\#include\s+"([^"]+?)"/g)
+		{
+			my $i = $1;
+			# ignore autogen exceptions
+			next if $i =~ m/\Aautogen_.+?Exception.h\Z/;
+			# record dependency
+			${$header_dependency{$h}}{$i} = 1 if exists $hfiles{$i};
+		}
+	}
+	closedir DIR;
+}
+
+print "done\n\nGenerating Makefiles...\n";
+
+
+# Then write a makefile for each module
+for my $mod (@modules, $implicit_dep)
+{
+	print $mod,"\n";
+	
+	my ($type,$name) = split /\//,$mod;
+	
+	# add additional files for tests
+	if($type eq 'test')
+	{
+		my $testmain = $test_template;
+		$testmain =~ s/TEST_NAME/$name/g;
+		open TESTMAIN,">$mod/_main.cpp" or die "Can't open test main file for $mod for writing\n";
+		print TESTMAIN $testmain;
+		close TESTMAIN;
+		
+		# test file...
+		sub writetestfile
+		{
+			my ($filename,$runcmd,$module) = @_;		
+			open TESTFILE,">$filename" or die "Can't open test script file for $module for writing\n";
+			print TESTFILE "#!/bin/sh\necho TEST: $module\n";
+			if(-d "$module/testfiles")
+			{
+				print TESTFILE <<__E;
+echo Removing old test files...
+rm -rf testfiles
+echo Copying new test files...
+cp -p -R ../../../$module/testfiles .
+__E
+			}
+			if(-e "$module/testextra")
+			{
+				open FL,"$module/testextra" or die "Can't open $module/testextra";
+				while(<FL>) {print TESTFILE}
+				close FL;
+			}
+			print TESTFILE "$runcmd\n";
+			close TESTFILE;
+		}
+		
+		writetestfile("$mod/_t", './test $1 $2 $3 $4 $5', $mod);
+		writetestfile("$mod/_t-gdb", 'gdb ./test', $mod);
+		
+	}
+	
+	my @all_deps_for_module;
+	{
+		# work out what dependencies need to be run
+		my @deps_raw;
+		sub add_mod_deps
+		{
+			my ($arr_r,$nm) = @_;
+			if($#{$module_dependency{$nm}} >= 0)
+			{
+				push @$arr_r,@{$module_dependency{$nm}};
+				for(@{$module_dependency{$nm}})
+				{
+					add_mod_deps($arr_r,$_)
+				}
+			}
+		}
+		add_mod_deps(\@deps_raw, $mod);
+		# and then dedup and reorder them
+		my %d_done;
+		for(my $a = $#deps_raw; $a >= 0; $a--)
+		{
+			if(!exists $d_done{$deps_raw[$a]})
+			{
+				# insert
+				push @all_deps_for_module, $deps_raw[$a];
+				# mark as done
+				$d_done{$deps_raw[$a]} = 1;
+			}
+		}
+	}	
+	
+
+	# make include path
+	my $include_paths = join(' ',map {'-I../../'.$_} @all_deps_for_module);
+
+	# is target a library?
+	my $target_is_library = ($type ne 'bin' && $type ne 'test');
+
+	# make target name
+	my $end_target = $name;
+	$end_target .= '.a' if $target_is_library;
+	$end_target = 'test' if $type eq 'test';
+	# adjust for outdir
+	$end_target = '$(OUTDIR)/' . $end_target;
+
+	# start the makefile
+	my $mk_name_extra = ($bsd_make)?'':'X';
+	open MAKE,">$mod/Makefile".$mk_name_extra or die "Can't open Makefile for $mod\n";
+	my $debug_link_extra = ($target_is_library)?'':'../../debug/lib/debug/debug.a';
+	print MAKE <<__E;
+#
+# AUTOMATICALLY GENERATED FILE
+#    do not edit!
+#
+#
+CXX = g++
+AR = ar
+RANLIB = ranlib
+.ifdef RELEASE
+CXXFLAGS = -DNDEBUG -O2 -Wall $include_paths -D$platform_define$extra_platform_defines -DBOX_VERSION="\\"$product_version\\""
+OUTBASE = ../../release
+OUTDIR = ../../release/$mod
+DEPENDMAKEFLAGS = -D RELEASE
+VARIENT = RELEASE
+.else
+CXXFLAGS = -g -Wall $include_paths -D$platform_define$extra_platform_defines -DBOX_VERSION="\\"$product_version\\""
+OUTBASE = ../../debug
+OUTDIR = ../../debug/$mod
+DEPENDMAKEFLAGS =
+VARIENT = DEBUG
+.endif
+
+__E
+	
+	# read directory
+	opendir DIR,$mod;
+	my @items = readdir DIR;
+	closedir DIR;
+	
+	# add in items from autogen directories, and create output directories
+	{
+		my @autogen_items;
+		for my $di (@items)
+		{
+			if($di =~ m/\Aautogen/ && -d "$mod/$di")
+			{
+				# Read items
+				my $d = "$mod/$di";
+				opendir DIR,$d;
+				my @i = readdir DIR;
+				closedir DIR;
+				for(@i)
+				{
+					next if m/\A\./;
+					push @autogen_items,"$di/$_"
+				}
+				
+				# output directories
+				mkdir "release/$mod/$di",0755;
+				mkdir "debug/$mod/$di",0755;
+			}
+		}
+		@items = (@items, @autogen_items);
+	}
+	
+	# first, obtain a list of depenencies within the .h files
+	my %headers;
+	for my $h (grep /\.h\Z/i, @items)
+	{
+		open FL,"$mod/$h";
+		my $f;
+		read FL,$f,-s "$mod/$h";
+		close FL;
+
+		while($f =~ m/\#include\s+"([^"]+?)"/g)
+		{
+			${$headers{$h}}{$1} = 1 if exists $hfiles{$1};
+		}
+	}
+	
+	# ready for the rest of the details...
+	my $make;
+	
+	# then... do the cpp files...
+	my @obj_base;
+	for my $cpp (@items)
+	{
+		next unless $cpp =~ m/\A(.+)\.cpp\Z/i;
+		next if $cpp =~ /\A\._/;	# Temp Mac OS Resource hack
+
+		# store for later
+		my $base = $1;
+		push @obj_base,$base;
+	
+		# get the file...
+		open FL,"$mod/$cpp";
+		my $f;
+		read FL,$f,-s "$mod/$cpp";
+		close FL;
+		
+		my %dep;
+
+		while($f =~ m/\#include\s+"([^"]+?)"/g)
+		{
+			insert_dep($1, \%dep) if exists $hfiles{$1};
+		}
+		
+		# output filename
+		my $out_name = '$(OUTDIR)/'.$base.'.o';
+		
+		# write the line for this cpp file
+		$make .= $out_name.': '.join(' ',$cpp,map
+			{ ($hfiles{$_} eq $mod)?$_:'../../'.$hfiles{$_}."/$_" } keys %dep)."\n";
+		$make .= "\t\$(CXX) \$(CXXFLAGS) $compile_line_extra -c $cpp -o $out_name\n\n";
+
+	}
+
+	my $has_deps = ($#{$module_dependency{$mod}} >= 0);
+# ----- # always has dependencies with debug library
+	$has_deps = 1;
+
+	# Depenency stuff
+	my $deps_makeinfo;
+	if($has_deps)
+	{
+		if($bsd_make)
+		{
+			$deps_makeinfo = <<'__E';
+.BEGIN::
+.ifndef NODEPS
+.	if $(.TARGETS) == ""
+__E
+		}
+		else
+		{
+			# gnu make
+			$deps_makeinfo = <<'__E';
+.PHONY:	dep_modules
+dep_modules:
+ifndef NODEPS
+ifeq ($(strip $(.TARGETS)),)
+__E
+		}
+		
+		# run make for things we require
+		for my $dep (@all_deps_for_module)
+		{
+			$deps_makeinfo .= "\t\t(cd ../../$dep; $make_command$sub_make_options \$(DEPENDMAKEFLAGS) -D NODEPS)\n";
+		}
+		$deps_makeinfo .= ".\tendif\n.endif\n\n";
+	}
+	print MAKE $deps_makeinfo if $bsd_make;
+	
+	# get the list of library things to add -- in order of dependency so things link properly
+	my $lib_files = join(' ',map {($_ =~ m/lib\/(.+)\Z/)?('$(OUTBASE)/'.$_.'/'.$1.'.a'):undef} (reverse(@all_deps_for_module)));
+
+	# need to see if the extra makefile fragments require extra object files
+	# or include any more makefiles
+	my @objs = @obj_base;
+	my @makefile_includes;
+	
+	additional_objects_from_make_fragment("$mod/Makefile.extra", \@objs, \@makefile_includes);
+	additional_objects_from_make_fragment("$mod/Makefile.extra.$build_os", \@objs, \@makefile_includes);
+
+	my $o_file_list = join(' ',map {'$(OUTDIR)/'.$_.'.o'} @objs);
+	print MAKE $end_target,': ',$o_file_list;
+	print MAKE ' dep_modules' if !$bsd_make;
+	print MAKE " ",$lib_files unless $target_is_library;
+	print MAKE "\n";
+	
+	# stuff to make the final target...
+	if($target_is_library)
+	{
+		# make a library archive...
+		print MAKE "\t(echo -n > $end_target; rm $end_target)\n";
+		print MAKE "\t\$(AR) -q $end_target $o_file_list\n";
+		print MAKE "\t\$(RANLIB) $end_target\n";
+	}
+	else
+	{
+		# work out library options
+		# need to be... least used first, in absolute order they appear in the modules.txt file
+		my @libops;
+		sub libops_fill
+		{
+			my ($m,$r) = @_;
+			push @$r,$_ for(@{$module_library_link_opts{$m}});
+			libops_fill($_,$r) for(@{$module_dependency{$m}});
+		}
+		libops_fill($mod,\@libops);
+		my $lo = '';
+		my %ldone;
+		for(@libops)
+		{
+			next if exists $ldone{$_};
+			$lo .= ' '.$_;
+			$ldone{$_} = 1;
+		}
+	
+		# link line...
+		print MAKE "\t\$(CXX) $link_line_extra -o $end_target $o_file_list $lib_files$lo\n";
+	}
+	# tests need to copy the test file over
+	if($type eq 'test')
+	{
+		print MAKE "\tcp _t \$(OUTDIR)/t\n\tchmod u+x \$(OUTDIR)/t\n";
+		print MAKE "\tcp _t-gdb \$(OUTDIR)/t-gdb\n\tchmod u+x \$(OUTDIR)/t-gdb\n";
+	}
+	# dependency line?
+	print MAKE "\n";
+
+	# module dependcies for GNU make?
+	print MAKE $deps_makeinfo if !$bsd_make;
+	
+	# print the rest of the file
+	print MAKE $make,"\n";
+	
+	# and a clean target
+	print MAKE "clean:\n\t-rm -rf \$(OUTDIR)/*\n.\tifndef SUBCLEAN\n";
+	for my $dep (@all_deps_for_module)
+	{
+		print MAKE "\t(cd ../../$dep; $make_command \$(DEPENDMAKEFLAGS) -D SUBCLEAN clean)\n";
+	}
+	print MAKE ".\tendif\n";
+	
+	# include any extra stuff
+	print MAKE "\n\n";
+	if(-e "$mod/Makefile.extra")
+	{
+		print MAKE ".include <Makefile.extra>\n\n";
+	}
+	if(-e "$mod/Makefile.extra.$build_os")
+	{
+		print MAKE ".include <Makefile.extra.$build_os>\n\n";
+	}
+	for(@makefile_includes)
+	{
+		print MAKE ".include <$_>\n\n";
+	}
+	
+	# and finally a target for rebuilding the build system
+	print MAKE "\nbuildsystem:\n\t(cd ../..; perl ./infrastructure/makebuildenv.pl $makebuildenv_args)\n\n";
+	
+	close MAKE;
+
+	if(!$bsd_make)
+	{
+		# need to post process this into a GNU makefile
+		open MAKE,">$mod/Makefile";
+		open MAKEB,"$mod/MakefileX";
+
+		while(<MAKEB>)
+		{
+			s/\A\.\s*(ifdef|else|endif|ifndef)/$1/;
+			s/\A\.\s*include\s+<(.+?)>/include $1/;
+			s/-D\s+(\w+)/$1=1/;
+			print MAKE;
+		}
+
+		close MAKEB;
+		close MAKE;
+		unlink "$mod/MakefileX";
+	}
+}
+
+print "\nType 'cd <module_dir>; $make_command' to build a module\n\n";
+
+print $test_failure_text;
+
+if($modules_omitted)
+{
+	print "\nNOTE: Some modules have been omitted on this platform\n\n"
+}
+
+sub insert_dep
+{
+	my ($h,$dep_r) = @_;
+	
+	# stop random recusion
+	return if exists $$dep_r{$h};
+	
+	# insert more depencies
+	insert_dep($_,$dep_r) for keys %{$header_dependency{$h}};
+
+	# mark this one as a dependency
+	$$dep_r{$h} = 1;
+}
+
+
+sub additional_objects_from_make_fragment
+{
+	my ($fn,$objs_r,$include_r) = @_;
+	
+	if(-e $fn)
+	{
+		open FL,$fn or die "Can't open $fn";
+		
+		while(<FL>)
+		{
+			chomp;
+			if(m/link-extra:\s*(.+)\Z/)
+			{
+				my @o = split /\s+/,$1;
+				for(@o)
+				{
+					push @$objs_r,$1 if m/\A(.+)\.o\Z/;
+				}
+			}
+			elsif(m/include-makefile:\s*(\S+)/)
+			{
+				push @$include_r,$1
+			}
+		}
+		
+		close FL;
+	}
+}
+
+
+sub ignore_module
+{
+	exists $env_flags{'IGNORE_'.$_[0]}
+}
+
+# how to run a test
+sub do_test
+{
+	my %t = @_;
+	
+	print $t{'Name'},':';
+	open TEST_CODE,">envtest.cpp" or die "Can't open envtest.cpp for writing";
+	print TEST_CODE $t{'Code'};
+	close TEST_CODE;
+	my $result = (system("g++ envtest.cpp ".$t{'TestCompileFlags'}." -o envtest $compile_line_extra $link_line_extra 2>/dev/null >/dev/null") == 0);
+	if($result && exists $t{'RunCode'})
+	{
+		$result = 0 unless (system('./envtest') == 0);
+	}
+	unlink 'envtest.cpp';
+	unlink 'envtest';
+	print $result?" yes\n":" no\n";
+	if($result)
+	{
+		# success
+		if(exists $t{'SuccessFlags'})
+		{
+			for(@{$t{'SuccessFlags'}})
+			{
+				my ($k,$v) = split /=>/,$_;
+				$v = 1 if $v eq '';
+				$env_flags{$k} = $v
+			}
+		}
+		$compile_line_extra .= $t{'SuccessCompileFlags'}.' ' if exists $t{'SuccessCompileFlags'};
+		$link_line_extra .= $t{'SuccessLinkFlags'}.' ' if exists $t{'SuccessLinkFlags'};
+	}
+	else
+	{
+		# aborting failure?
+		if(exists $t{'AbortOnFailure'})
+		{
+			print $t{'FailureText'};
+			print "\nAborting configuration, cannot build in this environment.\n";
+			exit(1);
+		}
+	
+		# failure
+		if(exists $t{'FailureFlags'})
+		{
+			for(@{$t{'FailureFlags'}})
+			{
+				my ($k,$v) = split /=>/,$_;
+				$v = 1 if $v eq '';
+				$env_flags{$k} = $v
+			}
+		}
+		$compile_line_extra .= $t{'FailureCompileFlags'}.' ' if exists $t{'FailureCompileFlags'};
+		$link_line_extra .= $t{'FailureLinkFlags'}.' ' if exists $t{'FailureLinkFlags'};
+		$test_failure_text .= $t{'FailureText'} if exists $t{'FailureText'};
+	}
+	$result
+}
+
+
+
diff --git a/infrastructure/makedistribution.pl b/infrastructure/makedistribution.pl
new file mode 100755
index 00000000..712b068c
--- /dev/null
+++ b/infrastructure/makedistribution.pl
@@ -0,0 +1,312 @@
+#!/usr/bin/perl
+use strict;
+use Symbol;
+
+# comment string for various endings
+my %comment_chars = ('cpp' => '// ', 'h' => '// ', 'pl' => '# ', 'pm' => '# ', '' => '# ');
+
+# other extensions which need text copying, just to remove the private stuff
+my %text_files = ('txt' => 1);
+
+# files which don't get the license added
+my %no_license = (); # 'filename' => 1
+
+# ----------------------------------------------
+
+# filled in from the manifest file
+my %no_license_dir = ();
+
+# distribution name
+my $distribution = $ARGV[0];
+die "No distribution name specified on the command line" if $distribution eq '';
+my $dist_root = "distribution/$distribution";
+
+# check distribution exists
+die "Distribution '$distribution' does not exist" unless -d $dist_root;
+
+# get version
+open VERSION,"$dist_root/VERSION.txt" or die "Can't open $dist_root/VERSION.txt";
+my $version = <VERSION>;
+chomp $version;
+my $archive_name = <VERSION>;
+chomp $archive_name;
+close VERSION;
+
+# consistency check
+die "Archive name '$archive_name' is not equal to the distribution name '$distribution'"
+	unless $archive_name eq $distribution;
+
+# make initial directory
+my $base_name = "$archive_name-$version";
+system "rm -rf $base_name";
+system "rm $base_name.tgz";
+mkdir $base_name,0755;
+
+# get license file
+open LICENSE,"$dist_root/LICENSE.txt" or die "Can't open $dist_root/LICENSE.txt";
+my $license_f;
+read LICENSE,$license_f,100000;
+close LICENSE;
+my @license = ('distribution '.$base_name,'',split(/\n/,$license_f));
+
+# copy files, make a note of all the modules included
+my %modules_included;
+my $private_sections_removed = 0;
+my $non_distribution_sections_removed = 0;
+sub copy_from_list
+{
+	my $list = $_[0];
+	open LIST,$list or die "Can't open $list";
+	
+	while(<LIST>)
+	{
+		next unless m/\S/;
+		chomp;
+		my ($src,$dst) = split /\s+/;
+		$dst = $src if $dst eq '';
+		if($src eq 'MKDIR')
+		{
+			# actually we just need to make a directory here
+			mkdir "$base_name/$dst",0755;
+		}
+		elsif($src eq 'NO-LICENSE-IN-DIR')
+		{
+			# record that this directory shouldn't have the license added
+			$no_license_dir{$dst} = 1;
+		}
+		elsif($src eq 'REPLACE-VERSION-IN')
+		{
+			replace_version_in($dst);
+		}
+		elsif(-d $src)
+		{
+			$modules_included{$_} = 1;
+			copy_dir($src,$dst);
+		}
+		else
+		{
+			copy_file($src,$dst);
+		}
+	}
+	
+	close LIST;
+}
+copy_from_list("distribution/COMMON-MANIFEST.txt");
+copy_from_list("$dist_root/DISTRIBUTION-MANIFEST.txt");
+
+# Copy in the root directory and delete the DISTRIBUTION-MANIFEST file
+(system("cp $dist_root/*.* $base_name/") == 0)
+	or die "Copy of root extra files failed";
+unlink "$base_name/DISTRIBUTION-MANIFEST.txt"
+	or die "Delete of DISTRIBUTION-MANIFEST.txt file failed";
+
+# produce a new modules file
+my $modules = gensym;
+open $modules,"modules.txt" or die "Can't open modules.txt for reading";
+open MODULES_OUT,">$base_name/modules.txt";
+
+while(<$modules>)
+{
+	# skip lines for modules which aren't included
+	next if m/\A(\w+\/\w+)\s/ && !exists $modules_included{$1};
+	
+	# skip private sections
+	unless(skip_non_applicable_section($_, $modules, 'modules.txt'))
+	{
+		# copy line to out files
+		print MODULES_OUT
+	}
+}
+
+close MODULES_OUT;
+close $modules;
+
+# report on how many private sections were removed
+print "Private sections removed: $private_sections_removed\nNon-distribution sections removed: $non_distribution_sections_removed\n";
+
+# tar it up
+system "tar cf - $base_name | gzip -9 - > $base_name.tgz";
+
+sub copy_file
+{
+	my ($fn,$dst_fn) = @_;
+	
+	my $ext;
+	$ext = $1 if $fn =~ m/\.(\w+)\Z/;
+	
+	# licenses not used in this directory?
+	my $license_in_dir = 1;
+	$dst_fn =~ m~\A(.+)/[^/]+?\Z~;
+	$license_in_dir = 0 if exists $no_license_dir{$1};
+	
+	# licensed or not?
+	if(exists $comment_chars{$ext} && !exists $no_license{$fn} && $license_in_dir)
+	{
+		my $b = $comment_chars{$ext};
+
+		# copy as text, inserting license
+		my $in = gensym;
+		open $in,$fn;
+		open OUT,">$base_name/$dst_fn";
+		
+		my $first = <$in>;
+		if($first =~ m/\A#!/)
+		{
+			print OUT $first;
+			$first = '';
+		}
+		
+		# write license
+		for(@license)
+		{
+			print OUT $b,$_,"\n"
+		}
+		
+		if($first ne '')
+		{
+			print OUT $first;
+		}
+		
+		while(<$in>)
+		{
+			unless(skip_non_applicable_section($_, $in, $fn))
+			{
+				print OUT
+			}
+		}	
+		
+		close OUT;
+		close $in;
+	}
+	else
+	{
+		if(exists $text_files{$ext})
+		{
+			# copy this as text, to remove private stuff
+			my $in = gensym;
+			open $in,$fn;
+			open OUT,">$base_name/$dst_fn";
+
+			while(<$in>)
+			{
+				unless(skip_non_applicable_section($_, $in, $fn))
+				{
+					print OUT
+				}
+			}	
+
+			close OUT;
+			close $in;
+		}
+		else
+		{
+			# copy as binary
+			system 'cp',$fn,"$base_name/$dst_fn"
+		}
+	}
+	
+	# make sure perl scripts are marked as executable, and other things aren't
+	if($ext eq 'pl' || $ext eq '')
+	{
+		system 'chmod','a+x',"$base_name/$dst_fn"
+	}
+	else
+	{
+		system 'chmod','a-x',"$base_name/$dst_fn"
+	}
+}
+
+sub skip_non_applicable_section
+{
+	my ($l, $filehandle, $filename) = @_;
+	if($l =~ m/BOX_PRIVATE_BEGIN/)
+	{
+		# skip private section
+		print "Removing private section from $filename\n";
+		$private_sections_removed++;
+		while(<$filehandle>) {last if m/BOX_PRIVATE_END/}
+		
+		# skipped something
+		return 1;
+	}
+	elsif($l =~ m/IF_DISTRIBUTION\((.+?)\)/)
+	{
+		# which distributions does this apply to?
+		my $applies = 0;
+		for(split /,/,$1)
+		{
+			$applies = 1 if $_ eq $distribution
+		}
+		unless($applies)
+		{
+			# skip section?
+			print "Removing distribution specific section from $filename\n";
+			$non_distribution_sections_removed++;
+			while(<$filehandle>) {last if m/END_IF_DISTRIBUTION/}
+		}
+		# hide this line
+		return 1;
+	}
+	elsif($l =~ m/END_IF_DISTRIBUTION/)
+	{
+		# hide these lines
+		return 1;
+	}
+	else
+	{
+		# no skipping, return this line
+		return 0;
+	}
+}
+
+sub copy_dir
+{
+	my ($dir,$dst_dir) = @_;
+
+	# copy an entire directory... first make sure it exists
+	my @n = split /\//,$dst_dir;
+	my $d = $base_name;
+	for(@n)
+	{
+		$d .= '/';
+		$d .= $_;
+		mkdir $d,0755;
+	}
+	
+	# then do each of the files within in
+	opendir DIR,$dir;
+	my @items = readdir DIR;
+	closedir DIR;
+	
+	for(@items)
+	{
+		next if m/\A\./;
+		next if m/\A_/;
+		next if m/\AMakefile\Z/;
+		next if m/\Aautogen/;
+		next if !-f "$dir/$_";
+		
+		copy_file("$dir/$_","$dst_dir/$_");
+	}
+}
+
+sub replace_version_in
+{
+	my ($file) = @_;
+	
+	my $fn = $base_name . '/' . $file;
+	open IN,$fn or die "Can't open $fn";
+	open OUT,'>'.$fn.'.new' or die "Can't open $fn.new for writing";
+	
+	while(<IN>)
+	{
+		s/###DISTRIBUTION-VERSION-NUMBER###/$version/g;
+		print OUT
+	}
+	
+	close OUT;
+	close IN;
+
+	rename($fn.'.new', $fn) or die "Can't rename in place $fn";
+}
+
diff --git a/infrastructure/makeparcels.pl b/infrastructure/makeparcels.pl
new file mode 100755
index 00000000..6cd3e8f2
--- /dev/null
+++ b/infrastructure/makeparcels.pl
@@ -0,0 +1,166 @@
+#!/usr/bin/perl
+
+use strict;
+use lib 'infrastructure';
+use BoxPlatform;
+
+my $os_suffix = '';
+if($build_os eq 'OpenBSD')
+{
+	$os_suffix = `uname -r`;
+	$os_suffix =~ tr/0-9//cd;
+}
+
+my @parcels;
+my %parcel_contents;
+
+open PARCELS,"parcels.txt" or die "Can't open parcels file";
+{
+	my $cur_parcel = '';
+	while(<PARCELS>)
+	{
+		chomp; s/#.+\Z//; s/\s+\Z//; s/\s+/ /g;
+		next unless m/\S/;
+		
+		# omit bits on some platforms?
+		next if m/\AEND-OMIT/;
+		if(m/\AOMIT:(.+)/)
+		{
+			if($1 eq $build_os)
+			{
+				while(<PARCELS>)
+				{
+					last if m/\AEND-OMIT/;	
+				}
+			}
+			next;
+		}
+		
+		# new parcel, or a new parcel definition?
+		if(m/\A\s+(.+)\Z/)
+		{
+			push @{$parcel_contents{$cur_parcel}},$1
+		}
+		else
+		{
+			$cur_parcel = $_;
+			push @parcels,$_;
+		}
+	}
+}
+close PARCELS;
+
+# create parcels directory
+mkdir "parcels",0755;
+mkdir "parcels/scripts",0755;
+
+# write master makefile
+
+open MAKE,">Makefile" or die "Can't open master Makefile for writing";
+
+print MAKE <<__E;
+#
+# AUTOMATICALLY GENERATED FILE
+#    do not edit!
+#
+#
+
+__E
+
+print MAKE "all:\t",join(' ',map {parcel_target($_)} @parcels),"\n\n";
+
+print MAKE "clean:\n";
+for my $parcel (@parcels)
+{
+	print MAKE "\trm -rf ",parcel_dir($parcel),"\n";
+	print MAKE "\trm -f ",parcel_target($parcel),"\n";
+}
+print MAKE "\n";
+
+print MAKE "test:\trelease/common/test\n\nrelease/common/test:\n\t./runtest.pl ALL release\n\n";
+
+my $release_flag = BoxPlatform::make_flag('RELEASE');
+
+for my $parcel (@parcels)
+{
+	my $target = parcel_target($parcel);
+	print MAKE $target,":\n";
+	
+	my $dir = parcel_dir($parcel);
+	print MAKE "\tmkdir $dir\n";
+	
+	open SCRIPT,">parcels/scripts/install-$parcel" or die "Can't open installer script for $parcel for writing";
+	print SCRIPT "#!/bin/sh\n\n";
+	
+	for(@{$parcel_contents{$parcel}})
+	{
+		my ($type,$name) = split /\s+/;
+		
+		if($type eq 'bin')
+		{
+			my $exeext = ($build_os eq 'CYGWIN')?'.exe':'';
+			print MAKE "\t(cd bin/$name; $make_command $release_flag)\n";
+			print MAKE "\tcp release/bin/$name/$name$exeext $dir\n";
+		}
+		elsif ($type eq 'script')
+		{
+			print MAKE "\tcp $name $dir\n";
+			# remove path from script name
+			$name =~ m~/([^/]+)\Z~;
+			$name = $1;
+		}
+
+		print SCRIPT "install $name $install_into_dir\n";
+	}
+	
+	close SCRIPT;
+	
+	chmod 0755,"parcels/scripts/install-$parcel";
+	
+	my $root = parcel_root($parcel);
+	print MAKE "\tcp parcels/scripts/install-$parcel $dir\n";
+	print MAKE "\t(cd parcels; tar cf - $root | gzip -9 - > $root.tgz )\n";
+	
+	print MAKE "\n";
+	
+	print MAKE "install-$parcel:\n";
+	print MAKE "\t(cd $dir; ./install-$parcel)\n\n";
+}
+
+print MAKE <<__E;
+install:
+	cat local/install.msg
+
+__E
+
+close MAKE;
+
+open INSTALLMSG,">local/install.msg" or die "Can't open install message file for writing";
+print INSTALLMSG <<__E;
+
+Parcels need to be installed separately, and as root. Type one of the following:
+
+__E
+
+for(@parcels)
+{
+	print INSTALLMSG "    make install-".$_."\n";
+}
+print INSTALLMSG "\n";
+
+close INSTALLMSG;
+
+sub parcel_root
+{
+	$product_name.'-'.$product_version.'-'.$_[0].'-'.$build_os.$os_suffix
+}
+
+sub parcel_dir
+{
+	'parcels/'.parcel_root($_[0])
+}
+
+sub parcel_target
+{
+	parcel_dir($_[0]).'.tgz'
+}
diff --git a/infrastructure/setupexternal.pl b/infrastructure/setupexternal.pl
new file mode 100755
index 00000000..e21403e1
--- /dev/null
+++ b/infrastructure/setupexternal.pl
@@ -0,0 +1,55 @@
+#!/usr/bin/perl
+use strict;
+
+# This script links in the essential directories and processes various
+# files to allow the Box libraries to be used in projects outside the main
+# box library tree.
+
+# directories to link through
+my @linkdirs = qw/lib infrastructure/;
+
+# ----------------------------------------------------
+
+my $libdir = $ARGV[0];
+die "Provided library dir $libdir does not exist" unless -d $libdir;
+
+# Check and remove links from the directory, then add new symlinks
+for my $d (@linkdirs)
+{
+	if(-e $d)
+	{
+		die "In project, $d is not a symbolic link"
+			unless -l $d;
+		print "Removing existing symlink $d\n";
+		unlink $d;
+	}
+	my $link_target = "$libdir/$d";
+	print "Add symlink $d -> $link_target\n";
+	die "Can't create symlink $d" unless
+		symlink $link_target, $d;
+}
+
+# Copy and create a base modules file which includes all the libraries
+print "Create new modules_base.txt file\n";
+open OUT,">modules_base.txt" or die "Can't open modules_base.txt file for writing";
+print OUT <<__E;
+#
+# Automatically generated file, do not edit
+#
+# Source: $libdir/modules.txt
+#
+
+__E
+
+open IN,"$libdir/modules.txt" or die "Can't open $libdir/modules.txt for reading";
+
+while(<IN>)
+{
+	if(m/\A(lib\/.+?)\s/)
+	{
+		print OUT
+	}
+}
+
+close IN;
+close OUT;
diff --git a/infrastructure/tests/common_tests.pl b/infrastructure/tests/common_tests.pl
new file mode 100644
index 00000000..454f09c7
--- /dev/null
+++ b/infrastructure/tests/common_tests.pl
@@ -0,0 +1,178 @@
+
+# perl fragment, not directly runnable
+
+{
+	# test for a C++ compiler
+	do_test('Name' => 'Compiler 1',
+		'AbortOnFailure' => 1,
+		'FailureText' => <<__E,
+
+================
+
+You do not appear to have the g++ compiler installed. Please fix and try again.
+(Tested for C++ compilation and use of standard STL C++ library.)
+
+Some distributions rename the g++ compiler to something including the version number,
+and fail to create a symlink to the expected name. Investigate this if you believe
+you have the C++ compiler installed.
+
+================
+
+__E
+		'Code' => <<__E);
+#include <string>
+int main(int argc, char *argv[])
+{
+	std::string str;
+	str = "Test";
+	str += "_test";
+}
+__E
+
+	# test for a properly working C++ compiler
+	do_test('Name' => 'Compiler 2',
+		'AbortOnFailure' => 1,
+		'RunCode' => 1,
+		'FailureText' => <<__E,
+
+================
+
+The C++ compiler fails basic C++ tests. It is impossible to compile and deploy this
+software on this platform.
+
+Some less common platforms do not have a working C++ implementation, especially
+regarding modern language features such as exceptions. A basic test failed. It is
+unlikely that you will be able to use this software without fixing the compiler.
+
+You could try a later version of the compiler, if available.
+
+================
+
+__E
+		'Code' => <<__E);
+#include <string>
+class test_class
+{
+public:
+	test_class(const char *str) : mString(str) {}
+	~test_class() {}
+private:
+	std::string mString;
+};
+int main(int argc, char *argv[])
+{
+	try
+	{
+		throw test_class("Test exception");
+	}
+	catch(test_class &e)
+	{
+		return 0;
+	}
+	return 1;
+}
+__E
+
+	# test for any version of OpenSSL
+	do_test('Name' => 'OpenSSL 1',
+		'SuccessFlags' => ['OLD_OPENSSL_OK'],
+		'TestCompileFlags' => '-lcrypto ',
+		'Code' => <<__E);
+#include <openssl/evp.h>
+int main(int argc, char *argv[])
+{
+	EVP_CipherInit(0, 0, 0, 0, 0);
+	return 0;
+}
+__E
+
+	# test for new version of OpenSSL
+	do_test('Name' => 'OpenSSL 2',
+		'SuccessFlags' => ['OPENSSL_OK'],
+		'TestCompileFlags' => '-lcrypto ',
+		'Code' => <<__E);
+#include <openssl/evp.h>
+int main(int argc, char *argv[])
+{
+	EVP_CipherInit_ex(0, 0, 0, 0, 0, 0);
+	return 0;
+}
+__E
+
+	# Linux is always more fun
+	if($build_os eq 'Linux')
+	{
+		# see if curses is available
+		sub curses_test
+		{
+			my $c = $_[0];
+			do_test('Name' => $c,
+				'SuccessFlags' => [$c.'_PRESENT'],
+				'TestCompileFlags' => '-l'.$c.' ',
+				'Code' => <<__E);
+#include <curses.h>
+int main(int argc, char *argv[])
+{
+	initscr(); cbreak(); noecho();
+	nonl();
+	intrflush(stdscr, 0);
+	keypad(stdscr, 0);
+	return 0;
+}
+__E
+		}
+		curses_test('curses');
+		curses_test('ncurses');
+
+		# see if LFS support is available
+		do_test('Name' => 'Linux LFS support',
+			'RunCode' => 1,
+			'TestCompileFlags' => '-D_FILE_OFFSET_BITS=64 ',
+			'SuccessCompileFlags' => '-D_FILE_OFFSET_BITS=64',
+			'Code' => <<__E);
+#include <sys/types.h>
+#include <sys/stat.h>
+int main(int argc, char *argv[])
+{
+	struct stat st;
+	if(sizeof(st.st_size) == 8 && sizeof(off_t) == 8)
+	{
+		return 0;
+	}
+	return 1;
+}
+__E
+
+	}
+
+	# USE_MALLOC doesn't work on some < gcc3 platforms
+	if(!$gcc_v3)
+	{
+		do_test('Name' => 'USE_MALLOC',
+			'FailureCompileFlags' => '-DPLATFORM_STL_USE_MALLOC_BROKEN',
+			'FailureText' => <<__E,
+
+================
+WARNING: The implementation of the C++ STL on this platform may have a flaw
+which causes it to apparently leak memory, and this flaw cannot be worked
+around.
+
+When running the daemons, check their memory usage does not constantly
+increase. The STL flaw can cause excessive memory use.
+================
+
+__E
+			'Code' => <<__E);
+#define __USE_MALLOC
+#include <string>
+int main(int argc, char *argv[])
+{
+        std::string s;
+        s = "test";
+}
+__E
+	}
+}
+
+
+1;
diff --git a/lib/backupclient/BackupClientCryptoKeys.cpp b/lib/backupclient/BackupClientCryptoKeys.cpp
new file mode 100755
index 00000000..015dadd2
--- /dev/null
+++ b/lib/backupclient/BackupClientCryptoKeys.cpp
@@ -0,0 +1,67 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientCryptoKeys.cpp
+//		Purpose: function for setting up all the backup client keys
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "BackupClientCryptoKeys.h"
+#include "FileStream.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreException.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFile.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientCryptoKeys_Setup(const char *)
+//		Purpose: Read in the key material file, and set keys to all the backup elements required.
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename)
+{
+	// Read in the key material
+	unsigned char KeyMaterial[BACKUPCRYPTOKEYS_FILE_SIZE];
+	
+	// Open the file
+	FileStream file(KeyMaterialFilename);
+	// Read in data
+	if(!file.ReadFullBuffer(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE, 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldntLoadClientKeyMaterial)
+	}
+	
+	// Tell the filename how to encrypt
+	BackupStoreFilenameClear::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH,
+		KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH);
+	BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish);
+
+	// Tell the attributes how to encrypt
+	BackupClientFileAttributes::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH);
+	// and the secret for hashing
+	BackupClientFileAttributes::SetAttributeHashSecret(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH);
+
+	// Tell the files how to encrypt
+	BackupStoreFile::SetBlowfishKeys(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH,
+		KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH);
+#ifndef PLATFORM_OLD_OPENSSL
+	// Use AES where available
+	BackupStoreFile::SetAESKey(KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH);
+#endif
+
+	// Wipe the key material from memory
+	::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE);
+}
+
+
+
diff --git a/lib/backupclient/BackupClientCryptoKeys.h b/lib/backupclient/BackupClientCryptoKeys.h
new file mode 100755
index 00000000..5e3a7df2
--- /dev/null
+++ b/lib/backupclient/BackupClientCryptoKeys.h
@@ -0,0 +1,55 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientCryptoKeys.h
+//		Purpose: Format of crypto keys file, and function for setting everything up
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTCRYTOKEYS__H
+#define BACKUPCLIENTCRYTOKEYS__H
+
+
+// All keys are the maximum size that Blowfish supports. Since only the
+// setup time is affected by key length (encryption same speed whatever)
+// there is no disadvantage to using long keys as they are never
+// transmitted and are static over long periods of time.
+
+
+// All sizes in bytes. Some gaps deliberately left in the used material.
+
+// How long the key material file is expected to be
+#define BACKUPCRYPTOKEYS_FILE_SIZE						1024
+
+// key for encrypting filenames (448 bits)
+#define BACKUPCRYPTOKEYS_FILENAME_KEY_START				0
+#define BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH			56
+#define BACKUPCRYPTOKEYS_FILENAME_IV_START				(0 + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH)
+#define BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH				8
+
+// key for encrypting attributes (448 bits)
+#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START			(BACKUPCRYPTOKEYS_FILENAME_KEY_START+64)
+#define BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH			56
+
+// Blowfish key for encrypting file data (448 bits (max blowfish key length))
+#define BACKUPCRYPTOKEYS_FILE_KEY_START					(BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START+64)
+#define BACKUPCRYPTOKEYS_FILE_KEY_LENGTH				56
+
+// key for encrypting file block index entries
+#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START		(BACKUPCRYPTOKEYS_FILE_KEY_START+64)
+#define BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH	56
+
+// Secret for hashing attributes
+#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START	(BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START+64)
+#define BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH	128
+
+// AES key for encrypting file data (256 bits (max AES key length))
+#define BACKUPCRYPTOKEYS_FILE_AES_KEY_START				(BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START+128)
+#define BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH			32
+
+
+void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename);
+
+#endif // BACKUPCLIENTCRYTOKEYS__H
+
diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp
new file mode 100755
index 00000000..4aa4edba
--- /dev/null
+++ b/lib/backupclient/BackupClientFileAttributes.cpp
@@ -0,0 +1,773 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientFileAttributes.cpp
+//		Purpose: Storage of file attributes
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <unistd.h>
+#include <limits.h>
+
+#include "BackupClientFileAttributes.h"
+#include "CommonException.h"
+#include "FileModificationTime.h"
+#include "BoxTimeToUnix.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "MD5Digest.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#define ATTRIBUTETYPE_GENERIC_UNIX	1
+
+#define ATTRIBUTE_ENCODING_BLOWFISH	2
+
+typedef struct 
+{
+	int32_t		AttributeType;
+	u_int32_t	UID;
+	u_int32_t	GID;
+	u_int64_t	ModificationTime;
+	u_int64_t	AttrModificationTime;
+	u_int32_t	UserDefinedFlags;
+	u_int32_t	FileGenerationNumber;
+	u_int16_t	Mode;
+	// Symbolic link filename may follow
+} attr_StreamFormat;
+
+// This has wire packing so it's compatible across platforms
+// Use wider than necessary sizes, just to be careful.
+typedef struct
+{
+	int32_t uid, gid, mode;
+} attributeHashData;
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+#define MAX_ATTRIBUTE_HASH_SECRET_LENGTH	256
+
+// Hide private static variables from the rest of the world
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace
+{
+	CipherContext sBlowfishEncrypt;
+	CipherContext sBlowfishDecrypt;
+	uint8_t sAttributeHashSecret[MAX_ATTRIBUTE_HASH_SECRET_LENGTH];
+	int sAttributeHashSecretLength = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::BackupClientFileAttributes()
+//		Purpose: Default constructor
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes()
+	: mpClearAttributes(0)
+{
+	ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &)
+//		Purpose: Copy constructor
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy)
+	: StreamableMemBlock(rToCopy), // base class does the hard work
+	  mpClearAttributes(0)
+{
+}
+BackupClientFileAttributes::BackupClientFileAttributes(const StreamableMemBlock &rToCopy)
+	: StreamableMemBlock(rToCopy), // base class does the hard work
+	  mpClearAttributes(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::~BackupClientFileAttributes()
+//		Purpose: Destructor
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes::~BackupClientFileAttributes()
+{
+	if(mpClearAttributes)
+	{
+		delete mpClearAttributes;
+		mpClearAttributes = 0;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes &operator=(const BackupClientFileAttributes &)
+//		Purpose: Assignment operator
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const BackupClientFileAttributes &rAttr)
+{
+	StreamableMemBlock::Set(rAttr);
+	RemoveClear();	// make sure no decrypted version held
+	return *this;
+}
+// Assume users play nice
+BackupClientFileAttributes &BackupClientFileAttributes::operator=(const StreamableMemBlock &rAttr)
+{
+	StreamableMemBlock::Set(rAttr);
+	RemoveClear();	// make sure no decrypted version held
+	return *this;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::operator==(const BackupClientFileAttributes &)
+//		Purpose: Comparison operator
+//		Created: 2003/10/09
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rAttr) const
+{
+	EnsureClearAvailable();
+	rAttr.EnsureClearAvailable();
+
+	return mpClearAttributes->operator==(*rAttr.mpClearAttributes);
+}
+// Too dangerous to allow -- put the two names the wrong way round, and it compares encrypted data.
+/*bool BackupClientFileAttributes::operator==(const StreamableMemBlock &rAttr) const
+{
+	StreamableMemBlock *pDecoded = 0;
+
+	try
+	{
+		EnsureClearAvailable();
+		StreamableMemBlock *pDecoded = MakeClear(rAttr);
+
+		// Compare using clear version
+		bool compared = mpClearAttributes->operator==(rAttr);
+		
+		// Delete temporary
+		delete pDecoded;
+	
+		return compared;
+	}
+	catch(...)
+	{
+		delete pDecoded;
+		throw;
+	}
+}*/
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool)
+//		Purpose: Compare, optionally ignoring the attribute modification time and/or modification time, and some data which is
+//				 irrelevant in practise (eg file generation number)
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime, bool IgnoreModTime) const
+{
+	EnsureClearAvailable();
+	rAttr.EnsureClearAvailable();
+
+	// Check sizes are the same, as a first check
+	if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize())
+	{
+		return false;
+	}
+	
+	// Then check the elements of the two things
+	// Bytes are checked in network order, but this doesn't matter as we're only checking for equality.
+	attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+	attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer();
+	
+	if(a1->AttributeType != a2->AttributeType
+		|| a1->UID != a2->UID
+		|| a1->GID != a2->GID
+		|| a1->UserDefinedFlags != a2->UserDefinedFlags
+		|| a1->Mode != a2->Mode)
+	{
+		return false;
+	}
+	
+	if(!IgnoreModTime)
+	{
+		if(a1->ModificationTime != a2->ModificationTime)
+		{
+			return false;
+		}
+	}
+
+	if(!IgnoreAttrModTime)
+	{
+		if(a1->AttrModificationTime != a2->AttrModificationTime)
+		{
+			return false;
+		}
+	}
+	
+	// Check symlink string?
+	unsigned int size = mpClearAttributes->GetSize();
+	if(size > sizeof(attr_StreamFormat))
+	{
+		// Symlink strings don't match
+		if(::memcmp(a1 + 1, a2 + 1, size - sizeof(attr_StreamFormat)) != 0)
+		{
+			return false;
+		}
+	}
+	
+	// Passes all test, must be OK
+	return true;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::ReadAttributes(const char *)
+//		Purpose: Read the attributes of the file, and store them ready for streaming.
+//				 Optionally retrieve the modification time and attribute modification time.
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroModificationTimes, box_time_t *pModTime,
+	box_time_t *pAttrModTime, int64_t *pFileSize, ino_t *pInodeNumber, bool *pHasMultipleLinks)
+{
+	StreamableMemBlock *pnewAttr = 0;
+	try
+	{
+		struct stat st;
+		if(::lstat(Filename, &st) != 0)
+		{
+			THROW_EXCEPTION(CommonException, OSFileError)
+		}
+		
+		// Modification times etc
+		if(pModTime) {*pModTime = FileModificationTime(st);}
+		if(pAttrModTime) {*pAttrModTime = FileAttrModificationTime(st);}
+		if(pFileSize) {*pFileSize = st.st_size;}
+		if(pInodeNumber) {*pInodeNumber = st.st_ino;}
+		if(pHasMultipleLinks) {*pHasMultipleLinks = (st.st_nlink > 1);}
+		
+		// Is it a link?
+		if((st.st_mode & S_IFMT) == S_IFLNK)
+		{
+			ReadAttributesLink(Filename, &st, ZeroModificationTimes);
+			return;
+		}
+		ASSERT((st.st_mode & S_IFMT) != S_IFLNK);
+	
+		// Now, can allocate the block
+		pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat));
+		
+		// Fill in the entries
+		attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer();
+		
+#define FILL_IN_ATTRIBUTES	\
+		::memset(pattr, 0, sizeof(attr_StreamFormat));				\
+		ASSERT(pattr != 0);											\
+		pattr->AttributeType = htonl(ATTRIBUTETYPE_GENERIC_UNIX);	\
+		pattr->UID = htonl(st.st_uid);								\
+		pattr->GID = htonl(st.st_gid);								\
+		if(ZeroModificationTimes)									\
+		{															\
+			pattr->ModificationTime = 0;							\
+			pattr->AttrModificationTime = 0;						\
+		}															\
+		else														\
+		{															\
+			pattr->ModificationTime = hton64(FileModificationTime(st));	\
+			pattr->AttrModificationTime = hton64(FileAttrModificationTime(st));	\
+		}															\
+		pattr->Mode = htons(st.st_mode);
+#ifdef PLATFORM_stat_NO_st_flags
+	#define FILL_IN_ATTRIBUTES_2	\
+		pattr->UserDefinedFlags = 0;				\
+		pattr->FileGenerationNumber = 0;
+#else
+	#define FILL_IN_ATTRIBUTES_2	\
+		pattr->UserDefinedFlags = htonl(st.st_flags);				\
+		pattr->FileGenerationNumber = htonl(st.st_gen);
+#endif
+		
+		FILL_IN_ATTRIBUTES
+		FILL_IN_ATTRIBUTES_2
+		
+		// Attributes ready. Encrypt into this block
+		EncryptAttr(*pnewAttr);
+		
+		// Store the new attributes
+		RemoveClear();
+		mpClearAttributes = pnewAttr;
+		pnewAttr = 0;
+	}
+	catch(...)
+	{
+		// clean up
+		delete pnewAttr;
+		pnewAttr = 0;
+		throw;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::ReadAttributesLink()
+//		Purpose: Private function, handles the case where a symbolic link is needed
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes)
+{
+	StreamableMemBlock *pnewAttr = 0;
+	try
+	{
+		// Avoid needing to have struct stat available when including header file
+		struct stat &st = *((struct stat *)pst);
+		// Make sure we're only called for symbolic links
+		ASSERT((st.st_mode & S_IFMT) == S_IFLNK);
+	
+		// Get the filename the link is linked to
+		char linkedTo[PATH_MAX+4];
+		int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX);
+		if(linkedToSize == -1)
+		{
+			THROW_EXCEPTION(CommonException, OSFileError)
+		}
+	
+		// Now, can allocate the block
+		pnewAttr = new StreamableMemBlock(sizeof(attr_StreamFormat) + linkedToSize + 1);
+		
+		// Fill in the entries
+		attr_StreamFormat *pattr = (attr_StreamFormat*)pnewAttr->GetBuffer();
+	
+		FILL_IN_ATTRIBUTES
+		FILL_IN_ATTRIBUTES_2
+	
+		// Add the path name for the symbolic link
+		::memcpy(pattr + 1, linkedTo, linkedToSize);
+		// Make sure it's neatly terminated
+		((char*)(pattr + 1))[linkedToSize] = '\0';
+		
+		// Attributes ready. Encrypt into this block
+		EncryptAttr(*pnewAttr);
+		
+		// Store the new attributes
+		RemoveClear();
+		mpClearAttributes = pnewAttr;
+		pnewAttr = 0;
+	}
+	catch(...)
+	{
+		// clean up
+		delete pnewAttr;
+		pnewAttr = 0;
+		throw;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::WriteAttributes(const char *)
+//		Purpose: Apply the stored attributes to the file
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::WriteAttributes(const char *Filename) const
+{
+	// Got something loaded
+	if(GetSize() <= 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+	}
+	
+	// Make sure there are clear attributes to use
+	EnsureClearAvailable();
+	ASSERT(mpClearAttributes != 0);
+
+	// Check if the decrypted attributes are small enough, and the type of attributes stored
+	if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+	{
+		THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+	}
+	int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+	ASSERT(type != 0);
+	if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX)
+	{
+		// Don't know what to do with these
+		THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood);
+	}
+	
+	// Check there is enough space for an attributes block
+	if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat))
+	{
+		// Too small
+		THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+	}
+
+	// Get pointer to structure
+	attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+	
+	// is it a symlink?
+	int16_t mode = ntohs(pattr->Mode);
+	if((mode & S_IFMT) == S_IFLNK)
+	{
+		// Check things are sensible
+		if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat) + 1)
+		{
+			// Too small
+			THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+		}
+	
+		// Make a symlink, first deleting anything in the way
+		::unlink(Filename);
+		if(::symlink((char*)(pattr + 1), Filename) != 0)
+		{
+			THROW_EXCEPTION(CommonException, OSFileError)
+		}
+	}
+	
+	// If working as root, set user IDs
+	if(::geteuid() == 0)
+	{
+		#ifdef PLATFORM_LCHOWN_NOT_SUPPORTED
+			// only if not a link, can't set their owner on this platform
+			if((mode & S_IFMT) != S_IFLNK)
+			{
+				// Not a link, use normal chown
+				if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0)
+				{
+					THROW_EXCEPTION(CommonException, OSFileError)
+				}
+			}
+		#else
+			if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0)	// use the version which sets things on symlinks
+			{
+				THROW_EXCEPTION(CommonException, OSFileError)
+			}
+		#endif
+	}
+	
+	// Stop now if symlink, because otherwise it'll just be applied to the target
+	if((mode & S_IFMT) == S_IFLNK)
+	{
+		return;
+	}
+
+	// Set modification time?
+	box_time_t modtime = ntoh64(pattr->ModificationTime);
+	if(modtime != 0)
+	{
+		// Work out times as timevals
+		struct timeval times[2];
+		BoxTimeToTimeval(modtime, times[1]);
+		// Copy access time as well, why not, got to set it to something
+		times[0] = times[1];
+		// Attr modification time will be changed anyway, nothing that can be done about it
+		
+		// Try to apply
+		if(::utimes(Filename, times) != 0)
+		{
+			THROW_EXCEPTION(CommonException, OSFileError)
+		}
+	}
+	
+	// Apply everything else... (allowable mode flags only)
+	if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX)) != 0)	// mode must be done last (think setuid)
+	{
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::IsSymLink()
+//		Purpose: Do these attributes represent a symbolic link?
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+bool BackupClientFileAttributes::IsSymLink() const
+{
+	EnsureClearAvailable();
+
+	// Got the right kind of thing?
+	if(mpClearAttributes->GetSize() < (int)sizeof(int32_t))
+	{
+		THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded);
+	}
+	
+	// Get the type of attributes stored
+	int32_t *type = (int32_t*)mpClearAttributes->GetBuffer();
+	ASSERT(type != 0);
+	if(ntohl(*type) == ATTRIBUTETYPE_GENERIC_UNIX && mpClearAttributes->GetSize() > (int)sizeof(attr_StreamFormat))
+	{
+		// Check link
+		attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer();
+		return ((ntohs(pattr->Mode)) & S_IFMT) == S_IFLNK;
+	}
+	
+	return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::RemoveClear()
+//		Purpose: Private. Deletes any clear version of the attributes that may be held
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::RemoveClear() const
+{
+	if(mpClearAttributes)
+	{
+		delete mpClearAttributes;
+	}
+	mpClearAttributes = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::EnsureClearAvailable()
+//		Purpose: Private. Makes sure the clear version is available
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EnsureClearAvailable() const
+{
+	if(mpClearAttributes == 0)
+	{
+		mpClearAttributes = MakeClear(*this);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::MakeClear(const StreamableMemBlock &)
+//		Purpose: Static. Decrypts stored attributes.
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock *BackupClientFileAttributes::MakeClear(const StreamableMemBlock &rEncrypted)
+{
+	// New block
+	StreamableMemBlock *pdecrypted = 0;
+
+	try
+	{
+		// Check the block is big enough for IV and header
+		int ivSize = sBlowfishEncrypt.GetIVLength();
+		if(rEncrypted.GetSize() <= (ivSize + 1))
+		{
+			THROW_EXCEPTION(BackupStoreException, BadEncryptedAttributes);
+		}
+		
+		// How much space is needed for the output?
+		int maxDecryptedSize = sBlowfishDecrypt.MaxOutSizeForInBufferSize(rEncrypted.GetSize() - ivSize);
+		
+		// Allocate it
+		pdecrypted = new StreamableMemBlock(maxDecryptedSize);
+	
+		// ptr to block	
+		uint8_t *encBlock = (uint8_t*)rEncrypted.GetBuffer();
+
+		// Check that the header has right type
+		if(encBlock[0] != ATTRIBUTE_ENCODING_BLOWFISH)
+		{
+			THROW_EXCEPTION(BackupStoreException, EncryptedAttributesHaveUnknownEncoding);
+		}
+
+		// Set IV
+		sBlowfishDecrypt.SetIV(encBlock + 1);
+		
+		// Decrypt
+		int decryptedSize = sBlowfishDecrypt.TransformBlock(pdecrypted->GetBuffer(), maxDecryptedSize, encBlock + 1 + ivSize, rEncrypted.GetSize() - (ivSize + 1));
+
+		// Resize block to fit
+		pdecrypted->ResizeBlock(decryptedSize);
+	}
+	catch(...)
+	{
+		delete pdecrypted;
+		pdecrypted = 0;
+	}
+
+	return pdecrypted;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::SetBlowfishKey(const void *, int)
+//		Purpose: Static. Sets the key to use for encryption and decryption.
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetBlowfishKey(const void *pKey, int KeyLength)
+{
+	// IVs set later
+	sBlowfishEncrypt.Reset();
+	sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+	sBlowfishDecrypt.Reset();
+	sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &)
+//		Purpose: Private. Encrypt the given attributes into this block. 
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::EncryptAttr(const StreamableMemBlock &rToEncrypt)
+{
+	// Free any existing block
+	FreeBlock();
+	
+	// Work out the maximum amount of space we need
+	int maxEncryptedSize = sBlowfishEncrypt.MaxOutSizeForInBufferSize(rToEncrypt.GetSize());
+	// And the size of the IV
+	int ivSize = sBlowfishEncrypt.GetIVLength();
+	
+	// Allocate this space
+	AllocateBlock(maxEncryptedSize + ivSize + 1);
+	
+	// Store the encoding byte
+	uint8_t *block = (uint8_t*)GetBuffer();
+	block[0] = ATTRIBUTE_ENCODING_BLOWFISH;
+	
+	// Generate and store an IV for this attribute block
+	int ivSize2 = 0;
+	const void *iv = sBlowfishEncrypt.SetRandomIV(ivSize2);
+	ASSERT(ivSize == ivSize2);
+	
+	// Copy into the encrypted block
+	::memcpy(block + 1, iv, ivSize);
+	
+	// Do the transform
+	int encrytedSize = sBlowfishEncrypt.TransformBlock(block + 1 + ivSize, maxEncryptedSize, rToEncrypt.GetBuffer(), rToEncrypt.GetSize());
+
+	// Resize this block
+	ResizeBlock(encrytedSize + ivSize + 1);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::SetAttributeHashSecret(const void *, int)
+//		Purpose: Set the secret for the filename attribute hash
+//		Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int SecretLength)
+{
+	if(SecretLength > (int)sizeof(sAttributeHashSecret))
+	{
+		SecretLength = sizeof(sAttributeHashSecret);
+	}
+	if(SecretLength < 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+	
+	// Copy
+	::memcpy(sAttributeHashSecret, pSecret, SecretLength);
+	sAttributeHashSecretLength = SecretLength;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientFileAttributes::GenerateAttributeHash(struct stat &, const std::string &)
+//		Purpose: Generate a 64 bit hash from the attributes, used to detect changes.
+//				 Include filename in the hash, so that it changes from one file to another,
+//				 so don't reveal identical attributes.
+//		Created: 25/4/04
+//
+// --------------------------------------------------------------------------
+uint64_t BackupClientFileAttributes::GenerateAttributeHash(struct stat &st, const std::string &rFilename)
+{
+	if(sAttributeHashSecretLength == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, AttributeHashSecretNotSet)
+	}
+	
+	// Assemble stuff we're interested in
+	attributeHashData hashData;
+	memset(&hashData, 0, sizeof(hashData));
+	// Use network byte order and large sizes to be cross platform
+	hashData.uid = htonl(st.st_uid);
+	hashData.gid = htonl(st.st_gid);
+	hashData.mode = htonl(st.st_mode);
+
+	// Create a MD5 hash of the data, filename, and secret
+	MD5Digest digest;
+	digest.Add(&hashData, sizeof(hashData));
+	digest.Add(rFilename.c_str(), rFilename.size());
+	digest.Add(sAttributeHashSecret, sAttributeHashSecretLength);
+	digest.Finish();
+	
+	// Return the first 64 bits of the hash
+	uint64_t result = *((uint64_t *)(digest.DigestAsData()));
+	return result;
+}
+
+
+
diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h
new file mode 100755
index 00000000..dd930684
--- /dev/null
+++ b/lib/backupclient/BackupClientFileAttributes.h
@@ -0,0 +1,70 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientFileAttributes.h
+//		Purpose: Storage of file attributes
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTFILEATTRIBUTES__H
+#define BACKUPCLIENTFILEATTRIBUTES__H
+
+#include <string>
+
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+struct stat;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupClientFileAttributes
+//		Purpose: Storage, streaming and application of file attributes
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+class BackupClientFileAttributes : public StreamableMemBlock
+{
+public:
+	BackupClientFileAttributes();
+	BackupClientFileAttributes(const BackupClientFileAttributes &rToCopy);
+	BackupClientFileAttributes(const StreamableMemBlock &rToCopy);
+	~BackupClientFileAttributes();
+	BackupClientFileAttributes &operator=(const BackupClientFileAttributes &rAttr);
+	BackupClientFileAttributes &operator=(const StreamableMemBlock &rAttr);
+	bool operator==(const BackupClientFileAttributes &rAttr) const;
+//	bool operator==(const StreamableMemBlock &rAttr) const; // too dangerous?
+
+	bool Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime = false, bool IgnoreModTime = false) const;
+	
+	// Prevent access to base class members accidently
+	void Set();
+
+	void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false,
+		box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0,
+		ino_t *pInodeNumber = 0, bool *pHasMultipleLinks = 0);
+	void WriteAttributes(const char *Filename) const;
+
+	bool IsSymLink() const;
+
+	static void SetBlowfishKey(const void *pKey, int KeyLength);
+	static void SetAttributeHashSecret(const void *pSecret, int SecretLength);
+	
+	static uint64_t GenerateAttributeHash(struct stat &st, const std::string &rFilename);
+
+private:
+	void ReadAttributesLink(const char *Filename, void *pst, bool ZeroModificationTimes);
+
+	void RemoveClear() const;
+	void EnsureClearAvailable() const;
+	static StreamableMemBlock *MakeClear(const StreamableMemBlock &rEncrypted);
+	void EncryptAttr(const StreamableMemBlock &rToEncrypt);
+
+private:
+	mutable StreamableMemBlock *mpClearAttributes;
+};
+
+#endif // BACKUPCLIENTFILEATTRIBUTES__H
+
diff --git a/lib/backupclient/BackupClientMakeExcludeList.cpp b/lib/backupclient/BackupClientMakeExcludeList.cpp
new file mode 100755
index 00000000..0615faaa
--- /dev/null
+++ b/lib/backupclient/BackupClientMakeExcludeList.cpp
@@ -0,0 +1,75 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientMakeExcludeList.cpp
+//		Purpose: Makes exclude lists from bbbackupd config location entries
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupClientMakeExcludeList.h"
+#include "Configuration.h"
+#include "ExcludeList.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientMakeExcludeList(const Configuration &, const char *, const char *)
+//		Purpose: Given a Configuration object corresponding to a bbackupd Location, and the
+//				 two names of the keys for definite and regex entries, return a ExcludeList.
+//				 Or 0 if it isn't required.
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName,
+	const char *AlwaysIncludeDefiniteName, const char *AlwaysIncludeRegexName)
+{
+	// Check that at least one of the entries exists
+	if(!rConfig.KeyExists(DefiniteName) && !rConfig.KeyExists(RegexName))
+	{
+		// Neither exists -- return 0 as an Exclude list isn't required.
+		return 0;
+	}
+	
+	// Create the exclude list
+	ExcludeList *pexclude = new ExcludeList;
+
+	try
+	{
+		// Definite names to add?
+		if(rConfig.KeyExists(DefiniteName))
+		{
+			pexclude->AddDefiniteEntries(rConfig.GetKeyValue(DefiniteName));
+		}
+		// Regular expressions to add?
+		if(rConfig.KeyExists(RegexName))
+		{
+			pexclude->AddRegexEntries(rConfig.GetKeyValue(RegexName));
+		}
+		
+		// Add a "always include" list?
+		if(AlwaysIncludeDefiniteName != 0 && AlwaysIncludeRegexName != 0)
+		{
+			// This will accept NULL as a valid argument, so safe to do this.
+			pexclude->SetAlwaysIncludeList(
+					BackupClientMakeExcludeList(rConfig, AlwaysIncludeDefiniteName, AlwaysIncludeRegexName)
+				);
+		}
+	}
+	catch(...)
+	{
+		// Clean up
+		delete pexclude;
+		throw;
+	}
+
+	return pexclude;
+}
+
+
+
diff --git a/lib/backupclient/BackupClientMakeExcludeList.h b/lib/backupclient/BackupClientMakeExcludeList.h
new file mode 100755
index 00000000..d5dd8af4
--- /dev/null
+++ b/lib/backupclient/BackupClientMakeExcludeList.h
@@ -0,0 +1,48 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientMakeExcludeList.h
+//		Purpose: Makes exclude lists from bbbackupd config location entries
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPCLIENTMAKEEXCLUDELIST__H
+#define BACKUPCLIENTMAKEEXCLUDELIST__H
+
+class ExcludeList;
+class Configuration;
+
+ExcludeList *BackupClientMakeExcludeList(const Configuration &rConfig, const char *DefiniteName, const char *RegexName,
+	const char *AlwaysIncludeDefiniteName = 0, const char *AlwaysIncludeRegexName = 0);
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientMakeExcludeList_Files(const Configuration &)
+//		Purpose: Create a exclude list from config file entries for files. May return 0.
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+inline ExcludeList *BackupClientMakeExcludeList_Files(const Configuration &rConfig)
+{
+	return BackupClientMakeExcludeList(rConfig, "ExcludeFile", "ExcludeFilesRegex", "AlwaysIncludeFile", "AlwaysIncludeFilesRegex");
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientMakeExcludeList_Dirs(const Configuration &)
+//		Purpose: Create a exclude list from config file entries for directories. May return 0.
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+inline ExcludeList *BackupClientMakeExcludeList_Dirs(const Configuration &rConfig)
+{
+	return BackupClientMakeExcludeList(rConfig, "ExcludeDir", "ExcludeDirsRegex", "AlwaysIncludeDir", "AlwaysIncludeDirsRegex");
+}
+
+
+#endif // BACKUPCLIENTMAKEEXCLUDELIST__H
+
diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp
new file mode 100755
index 00000000..ebe1d365
--- /dev/null
+++ b/lib/backupclient/BackupClientRestore.cpp
@@ -0,0 +1,468 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientRestore.cpp
+//		Purpose: 
+//		Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string>
+#include <set>
+#include <unistd.h>
+#include <limits.h>
+#include <stdio.h>
+
+#include "BackupClientRestore.h"
+#include "autogen_BackupProtocolClient.h"
+#include "CommonException.h"
+#include "BackupClientFileAttributes.h"
+#include "IOStream.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "CollectInBufferStream.h"
+#include "FileStream.h"
+#include "Utils.h"
+
+#include "MemLeakFindOn.h"
+
+#define MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES (128*1024)
+
+class RestoreResumeInfo
+{
+public:
+	// constructor
+	RestoreResumeInfo()
+		: mNextLevelID(0),
+		  mpNextLevel(0)
+	{
+	}
+	
+	// destructor
+	~RestoreResumeInfo()
+	{
+		delete mpNextLevel;
+		mpNextLevel = 0;
+	}
+	
+	// Get a next level object
+	RestoreResumeInfo &AddLevel(int64_t ID, const std::string &rLocalName)
+	{
+		ASSERT(mpNextLevel == 0 && mNextLevelID == 0);
+		mpNextLevel = new RestoreResumeInfo;
+		mNextLevelID = ID;
+		mNextLevelLocalName = rLocalName;
+		return *mpNextLevel;
+	}
+	
+	// Remove the next level info
+	void RemoveLevel()
+	{
+		ASSERT(mpNextLevel != 0 && mNextLevelID != 0);
+		delete mpNextLevel;
+		mpNextLevel = 0;
+		mNextLevelID = 0;
+		mNextLevelLocalName.erase();
+	}
+	
+	void Save(const std::string &rFilename) const
+	{
+		// TODO: use proper buffered streams when they're done
+		// Build info in memory buffer
+		CollectInBufferStream write;
+		
+		// Save this level
+		SaveLevel(write);
+	
+		// Store in file
+		write.SetForReading();
+		FileStream file(rFilename.c_str(), O_WRONLY | O_CREAT);
+		write.CopyStreamTo(file, IOStream::TimeOutInfinite, 8*1024 /* large buffer */);
+	}
+
+	void SaveLevel(IOStream &rWrite) const
+	{
+		// Write the restored objects
+		int64_t numObjects = mRestoredObjects.size();
+		rWrite.Write(&numObjects, sizeof(numObjects));
+		for(std::set<int64_t>::const_iterator i(mRestoredObjects.begin()); i != mRestoredObjects.end(); ++i)
+		{
+			int64_t id = *i;
+			rWrite.Write(&id, sizeof(id));
+		}
+		
+		// Next level?
+		if(mpNextLevel != 0)
+		{
+			// ID
+			rWrite.Write(&mNextLevelID, sizeof(mNextLevelID));
+			// Name string
+			int32_t nsize = mNextLevelLocalName.size();
+			rWrite.Write(&nsize, sizeof(nsize));
+			rWrite.Write(mNextLevelLocalName.c_str(), nsize);
+			// And then the level itself
+			mpNextLevel->SaveLevel(rWrite);
+		}
+		else
+		{
+			// Just write a zero
+			int64_t zero = 0;
+			rWrite.Write(&zero, sizeof(zero));
+		}
+	}
+
+	// Not written to be efficient -- shouldn't be called very often.
+	bool Load(const std::string &rFilename)
+	{
+		// Delete and reset if necessary
+		if(mpNextLevel != 0)
+		{
+			RemoveLevel();
+		}
+		
+		// Open file
+		FileStream file(rFilename.c_str());
+		
+		// Load this level
+		return LoadLevel(file);
+	}
+	
+	#define CHECKED_READ(x, s) if(!rRead.ReadFullBuffer(x, s, 0)) {return false;}
+	bool LoadLevel(IOStream &rRead)
+	{
+		// Load the restored objects list
+		mRestoredObjects.clear();
+		int64_t numObjects = 0;
+		CHECKED_READ(&numObjects, sizeof(numObjects));
+		for(int64_t o = 0; o < numObjects; ++o)
+		{
+			int64_t id;
+			CHECKED_READ(&id, sizeof(id));
+			mRestoredObjects.insert(id);
+		}
+
+		// ID of next level?
+		int64_t nextID = 0;
+		CHECKED_READ(&nextID, sizeof(nextID));
+		if(nextID != 0)
+		{
+			// Load the next level!
+			std::string name;
+			int32_t nsize = 0;
+			CHECKED_READ(&nsize, sizeof(nsize));
+			char n[PATH_MAX];
+			if(nsize > PATH_MAX) return false;
+			CHECKED_READ(n, nsize);
+			name.assign(n, nsize);
+			
+			// Create a new level
+			mpNextLevel = new RestoreResumeInfo;
+			mNextLevelID = nextID;
+			mNextLevelLocalName = name;
+			
+			// And ask it to read itself in
+			if(!mpNextLevel->LoadLevel(rRead))
+			{
+				return false;
+			}
+		}
+
+		return true;
+	}
+
+	// List of objects at this level which have been done already
+	std::set<int64_t> mRestoredObjects;
+	// Next level ID
+	int64_t mNextLevelID;
+	// Pointer to next level
+	RestoreResumeInfo *mpNextLevel;
+	// Local filename of next level
+	std::string mNextLevelLocalName;
+};
+
+// parameters structure
+typedef struct
+{
+	bool PrintDots;
+	bool RestoreDeleted;
+	std::string mRestoreResumeInfoFilename;
+	RestoreResumeInfo mResumeInfo;
+} RestoreParams;
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientRestoreDir(BackupProtocolClient &, int64_t, const char *, bool)
+//		Purpose: Restore a directory
+//		Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName,
+	RestoreParams &Params, RestoreResumeInfo &rLevel)
+{
+	// If we're resuming... check that we haven't got a next level to look at
+	if(rLevel.mpNextLevel != 0)
+	{
+		// Recurse immediately
+		std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName);
+		BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, localDirname, Params, *rLevel.mpNextLevel);
+		
+		// Add it to the list of done itmes
+		rLevel.mRestoredObjects.insert(rLevel.mNextLevelID);
+
+		// Remove the level for the recursed directory
+		rLevel.RemoveLevel();		
+	}
+	
+	// Save the resumption information
+	Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+
+	// Create the local directory (if not already done) -- path and owner set later, just use restrictive owner mode
+	switch(ObjectExists(rLocalDirectoryName.c_str()))
+	{
+		case ObjectExists_Dir:
+			// Do nothing
+			break;
+		case ObjectExists_File:
+			{
+				// File exists with this name, which is fun. Get rid of it.
+				::printf("WARNING: File present with name '%s', removing out of the way of restored directory. Use specific restore with ID to restore this object.", rLocalDirectoryName.c_str());
+				if(::unlink(rLocalDirectoryName.c_str()) != 0)
+				{
+					THROW_EXCEPTION(CommonException, OSFileError);
+				}
+				TRACE1("In restore, directory name collision with file %s", rLocalDirectoryName.c_str());
+			}
+			// follow through to... (no break)
+		case ObjectExists_NoObject:
+			if(::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0)
+			{
+				THROW_EXCEPTION(CommonException, OSFileError);
+			}
+			break;
+		default:
+			ASSERT(false);
+			break;
+	}
+	
+	// Fetch the directory listing from the server -- getting a list of files which is approparite to the restore type
+	rConnection.QueryListDirectory(
+			DirectoryID,
+			Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING),
+			BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)),
+			true /* want attributes */);
+
+	// Retrieve the directory from the stream following
+	BackupStoreDirectory dir;
+	std::auto_ptr<IOStream> dirstream(rConnection.ReceiveStream());
+	dir.ReadFromStream(*dirstream, rConnection.GetTimeout());
+
+	// Apply attributes to the directory
+	const StreamableMemBlock &dirAttrBlock(dir.GetAttributes());
+	BackupClientFileAttributes dirAttr(dirAttrBlock);
+	dirAttr.WriteAttributes(rLocalDirectoryName.c_str());
+
+	int64_t bytesWrittenSinceLastRestoreInfoSave = 0;
+	
+	// Process files
+	{
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0)
+		{
+			// Check ID hasn't already been done
+			if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end())
+			{
+				// Local name
+				BackupStoreFilenameClear nm(en->GetName());
+				std::string localFilename(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename());
+				
+				// Unlink anything which already exists -- for resuming restores, we can't overwrite files already there.
+				::unlink(localFilename.c_str());
+				
+				// Request it from the store
+				rConnection.QueryGetFile(DirectoryID, en->GetObjectID());
+		
+				// Stream containing encoded file
+				std::auto_ptr<IOStream> objectStream(rConnection.ReceiveStream());
+		
+				// Decode the file -- need to do different things depending on whether 
+				// the directory entry has additional attributes
+				if(en->HasAttributes())
+				{
+					// Use these attributes
+					const StreamableMemBlock &storeAttr(en->GetAttributes());
+					BackupClientFileAttributes attr(storeAttr);
+					BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout(), &attr);
+				}
+				else
+				{
+					// Use attributes stored in file
+					BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout());
+				}
+				
+				// Progress display?
+				if(Params.PrintDots)
+				{
+					printf(".");
+					fflush(stdout);
+				}
+
+				// Add it to the list of done itmes
+				rLevel.mRestoredObjects.insert(en->GetObjectID());
+				
+				// Save restore info?
+				int64_t fileSize;
+				if(FileExists(localFilename.c_str(), &fileSize, true /* treat links as not existing */))
+				{
+					// File exists...
+					bytesWrittenSinceLastRestoreInfoSave += fileSize;
+					
+					if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES)
+					{
+						// Save the restore info, in case it's needed later
+						Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+						bytesWrittenSinceLastRestoreInfoSave = 0;
+					}
+				}
+			}
+		}
+	}
+
+	// Make sure the restore info has been saved	
+	if(bytesWrittenSinceLastRestoreInfoSave != 0)
+	{
+		// Save the restore info, in case it's needed later
+		Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename);
+		bytesWrittenSinceLastRestoreInfoSave = 0;
+	}
+
+	
+	// Recuse to directories
+	{
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+		while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0)
+		{
+			// Check ID hasn't already been done
+			if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end())
+			{
+				// Local name
+				BackupStoreFilenameClear nm(en->GetName());
+				std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename());
+				
+				// Add the level for the next entry
+				RestoreResumeInfo &rnextLevel(rLevel.AddLevel(en->GetObjectID(), nm.GetClearFilename()));
+				
+				// Recurse
+				BackupClientRestoreDir(rConnection, en->GetObjectID(), localDirname, Params, rnextLevel);
+				
+				// Remove the level for the above call
+				rLevel.RemoveLevel();
+				
+				// Add it to the list of done itmes
+				rLevel.mRestoredObjects.insert(en->GetObjectID());
+			}
+		}
+	}	
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupClientRestore(BackupProtocolClient &, int64_t, const char *, bool, bool)
+//		Purpose: Restore a directory on the server to a local directory on the disc.
+//
+//				 The local directory must not already exist.
+//
+//				 If a restore is aborted for any reason, then it may be resumed if
+//				 Resume == true. If Resume == false and resumption is possible, then
+//				 Restore_ResumePossible is returned.
+//
+//				 Set RestoreDeleted to restore a deleted directory. This may not give the
+//				 directory structure when it was deleted, because files may have been deleted
+//				 within it before it was deleted.
+//
+//				 Returns Restore_TargetExists if the target directory exists, but
+//				 there is no restore possible. (Won't attempt to overwrite things.)
+//
+//				 Returns Restore_Complete on success. (Exceptions on error.)
+//		Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName,
+	bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume)
+{
+	// Parameter block
+	RestoreParams params;
+	params.PrintDots = PrintDots;
+	params.RestoreDeleted = RestoreDeleted;
+	params.mRestoreResumeInfoFilename = LocalDirectoryName;
+	params.mRestoreResumeInfoFilename += ".boxbackupresume";
+
+	// Target exists?
+	int targetExistance = ObjectExists(LocalDirectoryName);
+
+	// Does any resumption information exist?
+	bool doingResume = false;
+	if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && targetExistance == ObjectExists_Dir)
+	{
+		if(!Resume)
+		{
+			// Caller didn't specify that resume should be done, so refuse to do it
+			// but say why.
+			return Restore_ResumePossible;
+		}
+		
+		// Attempt to load the resume info file
+		if(!params.mResumeInfo.Load(params.mRestoreResumeInfoFilename))
+		{
+			// failed -- bad file, so things have gone a bit wrong
+			return Restore_TargetExists;
+		}
+		
+		// Flag as doing resume so next check isn't actually performed
+		doingResume = true;
+	}
+
+	// Does the directory already exist?
+	if(targetExistance != ObjectExists_NoObject && !doingResume)
+	{
+		// Don't do anything in this case!
+		return Restore_TargetExists;
+	}
+	
+	// Restore the directory
+	std::string localName(LocalDirectoryName);
+	BackupClientRestoreDir(rConnection, DirectoryID, localName, params, params.mResumeInfo);
+
+	// Undelete the directory on the server?
+	if(RestoreDeleted && UndeleteAfterRestoreDeleted)
+	{
+		// Send the command
+		rConnection.QueryUndeleteDirectory(DirectoryID);
+	}
+
+	// Finish progress display?
+	if(PrintDots)
+	{
+		printf("\n");
+		fflush(stdout);
+	}
+	
+	// Delete the resume information file
+	::unlink(params.mRestoreResumeInfoFilename.c_str());
+
+	return Restore_Complete;
+}
+
+
+
+
diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h
new file mode 100755
index 00000000..4b382771
--- /dev/null
+++ b/lib/backupclient/BackupClientRestore.h
@@ -0,0 +1,26 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupClientRestore.h
+//		Purpose: Functions to restore files from a backup store
+//		Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSCLIENTRESTORE_H
+#define BACKUPSCLIENTRESTORE__H
+
+class BackupProtocolClient;
+
+enum
+{
+	Restore_Complete = 0,
+	Restore_ResumePossible = 1,
+	Restore_TargetExists = 2
+};
+
+int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName,
+	bool PrintDots = false, bool RestoreDeleted = false, bool UndeleteAfterRestoreDeleted = false, bool Resume = false);
+
+#endif // BACKUPSCLIENTRESTORE__H
+
diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp
new file mode 100755
index 00000000..eec2ceaf
--- /dev/null
+++ b/lib/backupclient/BackupDaemonConfigVerify.cpp
@@ -0,0 +1,105 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupDaemonConfigVerify.cpp
+//		Purpose: Configuration file definition for bbackupd
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupDaemonConfigVerify.h"
+#include "Daemon.h"
+#include "BoxPortsAndFiles.h"
+
+#include "MemLeakFindOn.h"
+
+
+static const ConfigurationVerifyKey backuplocationkeys[] = 
+{
+	{"ExcludeFile", 0, ConfigTest_MultiValueAllowed, 0},
+	{"ExcludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0},
+	{"ExcludeDir", 0, ConfigTest_MultiValueAllowed, 0},
+	{"ExcludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0},
+	{"AlwaysIncludeFile", 0, ConfigTest_MultiValueAllowed, 0},
+	{"AlwaysIncludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0},
+	{"AlwaysIncludeDir", 0, ConfigTest_MultiValueAllowed, 0},
+	{"AlwaysIncludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0},
+	{"Path", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+static const ConfigurationVerify backuplocations[] = 
+{
+	{
+		"*",
+		0,
+		backuplocationkeys,
+		ConfigTest_LastEntry,
+		0
+	}
+};
+
+static const ConfigurationVerifyKey verifyserverkeys[] = 
+{
+	DAEMON_VERIFY_SERVER_KEYS
+};
+
+static const ConfigurationVerify verifyserver[] = 
+{
+	{
+		"Server",
+		0,
+		verifyserverkeys,
+		ConfigTest_Exists,
+		0
+	},
+	{
+		"BackupLocations",
+		backuplocations,
+		0,
+		ConfigTest_Exists | ConfigTest_LastEntry,
+		0
+	}
+};
+
+static const ConfigurationVerifyKey verifyrootkeys[] = 
+{
+	{"AccountNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+
+	{"UpdateStoreInterval", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"MinimumFileAge", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"MaxUploadWait", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"MaxFileTimeInFuture", "172800", ConfigTest_IsInt, 0},		// file is uploaded if the file is this much in the future (2 days default)
+
+	{"AutomaticBackup", "yes", ConfigTest_IsBool, 0},
+	
+	{"SyncAllowScript", 0, 0, 0},			// optional script to run to see if the sync should be started now
+				// return "now" if it's allowed, or a number of seconds if it's not
+
+	{"MaximumDiffingTime", 0, ConfigTest_IsInt, 0},
+
+	{"FileTrackingSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"DiffingUploadSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"StoreHostname", 0, ConfigTest_Exists, 0},
+	{"ExtendedLogging",	"no", ConfigTest_IsBool, 0},			// make value "yes" to enable in config file
+
+	{"CommandSocket", 0, 0, 0},				// not compulsory to have this
+
+	{"NotifyScript", 0, 0, 0},				// optional script to run when backup needs attention, eg store full
+	
+	{"CertificateFile", 0, ConfigTest_Exists, 0},
+	{"PrivateKeyFile", 0, ConfigTest_Exists, 0},
+	{"TrustedCAsFile", 0, ConfigTest_Exists, 0},
+	{"KeysFile", 0, ConfigTest_Exists, 0},
+	{"DataDirectory", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+const ConfigurationVerify BackupDaemonConfigVerify =
+{
+	"root",
+	verifyserver,
+	verifyrootkeys,
+	ConfigTest_Exists | ConfigTest_LastEntry,
+	0
+};
+
diff --git a/lib/backupclient/BackupDaemonConfigVerify.h b/lib/backupclient/BackupDaemonConfigVerify.h
new file mode 100755
index 00000000..fefa703c
--- /dev/null
+++ b/lib/backupclient/BackupDaemonConfigVerify.h
@@ -0,0 +1,18 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupDaemonConfigVerify.h
+//		Purpose: Configuration file definition for bbackupd
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPDAEMONCONFIGVERIFY__H
+#define BACKUPDAEMONCONFIGVERIFY__H
+
+#include "Configuration.h"
+
+extern const ConfigurationVerify BackupDaemonConfigVerify;
+
+#endif // BACKUPDAEMONCONFIGVERIFY__H
+
diff --git a/lib/backupclient/BackupStoreConstants.h b/lib/backupclient/BackupStoreConstants.h
new file mode 100755
index 00000000..8254e918
--- /dev/null
+++ b/lib/backupclient/BackupStoreConstants.h
@@ -0,0 +1,53 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreContants.h
+//		Purpose: constants for the backup system
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECONSTANTS__H
+#define BACKUPSTORECONSTANTS__H
+
+#define BACKUPSTORE_ROOT_DIRECTORY_ID	1
+
+#define BACKUP_STORE_SERVER_VERSION		1
+
+// Minimum size for a chunk to be compressed
+#define BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE	256
+
+// min and max sizes for blocks
+#define BACKUP_FILE_MIN_BLOCK_SIZE				4096
+#define BACKUP_FILE_MAX_BLOCK_SIZE				(512*1024)
+
+// Increase the block size if there are more than this number of blocks
+#define BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER 	4096
+
+// Avoid creating blocks smaller than this
+#define	BACKUP_FILE_AVOID_BLOCKS_LESS_THAN		128
+
+// Maximum number of sizes to do an rsync-like scan for
+#define BACKUP_FILE_DIFF_MAX_BLOCK_SIZES		8
+
+// When doing rsync scans, do not scan for blocks smaller than
+#define BACKUP_FILE_DIFF_MIN_BLOCK_SIZE			256
+
+// A limit to stop diffing running out of control: If more than this
+// times the number of blocks in the original index are found, stop
+// looking. This stops really bad cases of diffing files containing
+// all the same byte using huge amounts of memory and processor time.
+// This is a multiple of the number of blocks in the diff from file.
+#define BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE	4096
+
+// How many seconds to wait before deleting unused root directory entries?
+#ifndef NDEBUG
+	// Debug: 30 seconds (easier to test)
+	#define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER		30
+#else
+	// Release: 2 days (plenty of time for sysadmins to notice, or change their mind)
+	#define BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER		172800
+#endif
+
+#endif // BACKUPSTORECONSTANTS__H
+
diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp
new file mode 100755
index 00000000..31fed78d
--- /dev/null
+++ b/lib/backupclient/BackupStoreDirectory.cpp
@@ -0,0 +1,563 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreDirectory.h
+//		Purpose: Representation of a backup directory
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+
+#include "BackupStoreDirectory.h"
+#include "IOStream.h"
+#include "BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+	int32_t mMagicValue;	// also the version number
+	int32_t mNumEntries;
+	int64_t mObjectID;		// this object ID
+	int64_t mContainerID;	// ID of container
+	uint64_t mAttributesModTime;
+	int32_t mOptionsPresent;	// bit mask of optional sections / features present
+	// Then a StreamableMemBlock for attributes
+} dir_StreamFormat;
+
+enum
+{
+	Option_DependencyInfoPresent = 1
+} dir_StreamFormatOptions;
+
+typedef struct
+{
+	uint64_t mModificationTime;
+	int64_t mObjectID;
+	int64_t mSizeInBlocks;
+	uint64_t mAttributesHash;
+	int16_t mFlags;				// order smaller items after bigger ones (for alignment)
+	// Then a BackupStoreFilename
+	// Then a StreamableMemBlock for attributes
+} en_StreamFormat;
+
+typedef struct
+{
+	int64_t mDependsNewer;
+	int64_t mDependsOlder;
+} en_StreamFormatDepends;
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::BackupStoreDirectory()
+//		Purpose: Constructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory()
+	: mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0)
+{
+	ASSERT(sizeof(u_int64_t) == sizeof(box_time_t));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreDirectory::BackupStoreDirectory(int64_t, int64_t)
+//		Purpose: Constructor giving object and container IDs
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID)
+	: mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::~BackupStoreDirectory()
+//		Purpose: Destructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::~BackupStoreDirectory()
+{
+	for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+	{
+		delete (*i);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::ReadFromStream(IOStream &, int)
+//		Purpose: Reads the directory contents from a stream. Exceptions will yeild incomplete reads.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout)
+{
+	// Get the header
+	dir_StreamFormat hdr;
+	if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+
+	// Check magic value...
+	if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue))
+	{
+		THROW_EXCEPTION(BackupStoreException, BadDirectoryFormat)
+	}
+	
+	// Get data
+	mObjectID = ntoh64(hdr.mObjectID);
+	mContainerID = ntoh64(hdr.mContainerID);
+	mAttributesModTime = ntoh64(hdr.mAttributesModTime);
+	
+	// Options
+	int32_t options = ntohl(hdr.mOptionsPresent);
+	
+	// Get attributes
+	mAttributes.ReadFromStream(rStream, Timeout);
+	
+	// Decode count
+	int count = ntohl(hdr.mNumEntries);
+	
+	// Clear existing list
+	mEntries.clear();
+	
+	// Read them in!
+	for(int c = 0; c < count; ++c)
+	{
+		Entry *pen = new Entry;
+		try
+		{
+			// Read from stream
+			pen->ReadFromStream(rStream, Timeout);
+			
+			// Add to list
+			mEntries.push_back(pen);
+		}
+		catch(...)
+		{
+			delete pen;
+			throw;
+		}
+	}
+	
+	// Read in dependency info?
+	if(options & Option_DependencyInfoPresent)
+	{
+		// Read in extra dependency data
+		for(int c = 0; c < count; ++c)
+		{
+			mEntries[c]->ReadFromStreamDependencyInfo(rStream, Timeout);
+		}
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::WriteToStream(IOStream &, int16_t, int16_t, bool, bool)
+//		Purpose: Writes a selection of entries to a stream
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const
+{
+	// Get count of entries
+	int32_t count = mEntries.size();
+	if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING)
+	{
+		// Need to count the entries
+		count = 0;
+		Iterator i(*this);
+		while(i.Next(FlagsMustBeSet, FlagsNotToBeSet) != 0)
+		{
+			count++;
+		}
+	}
+	
+	// Check that sensible IDs have been set
+	ASSERT(mObjectID != 0);
+	ASSERT(mContainerID != 0);
+	
+	// Need dependency info?
+	bool dependencyInfoRequired = false;
+	if(StreamDependencyInfo)
+	{
+		Iterator i(*this);
+		Entry *pen = 0;
+		while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+		{
+			if(pen->HasDependencies())
+			{
+				dependencyInfoRequired = true;
+			}
+		}	
+	}
+	
+	// Options
+	int32_t options = 0;
+	if(dependencyInfoRequired) options |= Option_DependencyInfoPresent;
+
+	// Build header
+	dir_StreamFormat hdr;
+	hdr.mMagicValue = htonl(OBJECTMAGIC_DIR_MAGIC_VALUE);
+	hdr.mNumEntries = htonl(count);
+	hdr.mObjectID = hton64(mObjectID);
+	hdr.mContainerID = hton64(mContainerID);
+	hdr.mAttributesModTime = hton64(mAttributesModTime);
+	hdr.mOptionsPresent = htonl(options);
+	
+	// Write header
+	rStream.Write(&hdr, sizeof(hdr));
+	
+	// Write the attributes?
+	if(StreamAttributes)
+	{
+		mAttributes.WriteToStream(rStream);
+	}
+	else
+	{
+		// Write a blank header instead
+		StreamableMemBlock::WriteEmptyBlockToStream(rStream);
+	}
+
+	// Then write all the entries
+	Iterator i(*this);
+	Entry *pen = 0;
+	while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+	{
+		pen->WriteToStream(rStream);
+	}
+	
+	// Write dependency info?
+	if(dependencyInfoRequired)
+	{
+		Iterator i(*this);
+		Entry *pen = 0;
+		while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0)
+		{
+			pen->WriteToStreamDependencyInfo(rStream);
+		}	
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::AddEntry(const Entry &)
+//		Purpose: Adds entry to directory (no checking)
+//		Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy)
+{
+	Entry *pnew = new Entry(rEntryToCopy);
+	try
+	{
+		mEntries.push_back(pnew);
+	}
+	catch(...)
+	{
+		delete pnew;
+		throw;
+	}
+	
+	return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::AddEntry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+//		Purpose: Adds entry to directory (no checking)
+//		Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime)
+{
+	Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesModTime);
+	try
+	{
+		mEntries.push_back(pnew);
+	}
+	catch(...)
+	{
+		delete pnew;
+		throw;
+	}
+	
+	return pnew;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::DeleteEntry(int64_t)
+//		Purpose: Deletes entry with given object ID (uses linear search, maybe a little inefficient)
+//		Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::DeleteEntry(int64_t ObjectID)
+{
+	for(std::vector<Entry*>::iterator i(mEntries.begin());
+		i != mEntries.end(); ++i)
+	{
+		if((*i)->mObjectID == ObjectID)
+		{
+			// Delete
+			delete (*i);
+			// Remove from list
+			mEntries.erase(i);
+			// Done
+			return;
+		}
+	}
+	
+	// Not found
+	THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::FindEntryByID(int64_t)
+//		Purpose: Finds a specific entry. Returns 0 if the entry doesn't exist.
+//		Created: 12/11/03
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const
+{
+	for(std::vector<Entry*>::const_iterator i(mEntries.begin());
+		i != mEntries.end(); ++i)
+	{
+		if((*i)->mObjectID == ObjectID)
+		{
+			// Found
+			return (*i);
+		}
+	}
+
+	// Not found
+	return 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::Entry::Entry()
+//		Purpose: Constructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry()
+	: mModificationTime(0),
+	  mObjectID(0),
+	  mSizeInBlocks(0),
+	  mFlags(0),
+	  mAttributesHash(0),
+	  mMinMarkNumber(0),
+	  mMarkNumber(0),
+	  mDependsNewer(0),
+	  mDependsOlder(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::Entry::~Entry()
+//		Purpose: Destructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::~Entry()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::Entry::Entry(const Entry &)
+//		Purpose: Copy constructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const Entry &rToCopy)
+	: mName(rToCopy.mName),
+	  mModificationTime(rToCopy.mModificationTime),
+	  mObjectID(rToCopy.mObjectID),
+	  mSizeInBlocks(rToCopy.mSizeInBlocks),
+	  mFlags(rToCopy.mFlags),
+	  mAttributesHash(rToCopy.mAttributesHash),
+	  mAttributes(rToCopy.mAttributes),
+	  mMinMarkNumber(rToCopy.mMinMarkNumber),
+	  mMarkNumber(rToCopy.mMarkNumber),
+	  mDependsNewer(rToCopy.mDependsNewer),
+	  mDependsOlder(rToCopy.mDependsOlder)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &, int64_t, int64_t, int16_t)
+//		Purpose: Constructor from values
+//		Created: 2003/08/27
+//
+// --------------------------------------------------------------------------
+BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash)
+	: mName(rName),
+	  mModificationTime(ModificationTime),
+	  mObjectID(ObjectID),
+	  mSizeInBlocks(SizeInBlocks),
+	  mFlags(Flags),
+	  mAttributesHash(AttributesHash),
+	  mMinMarkNumber(0),
+	  mMarkNumber(0),
+	  mDependsNewer(0),
+	  mDependsOlder(0)
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::Entry::TryReading(IOStream &, int)
+//		Purpose: Read an entry from a stream
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout)
+{
+	// Grab the raw bytes from the stream which compose the header
+	en_StreamFormat entry;
+	if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+
+	// Do reading first before modifying the variables, to be more exception safe
+	
+	// Get the filename
+	BackupStoreFilename name;
+	name.ReadFromStream(rStream, Timeout);
+	
+	// Get the attributes
+	mAttributes.ReadFromStream(rStream, Timeout);
+
+	// Store the rest of the bits
+	mModificationTime =		ntoh64(entry.mModificationTime);
+	mObjectID = 			ntoh64(entry.mObjectID);
+	mSizeInBlocks = 		ntoh64(entry.mSizeInBlocks);
+	mAttributesHash =		ntoh64(entry.mAttributesHash);
+	mFlags = 				ntohs(entry.mFlags);
+	mName =					name;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::Entry::WriteToStream(IOStream &)
+//		Purpose: Writes the entry to a stream
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const
+{
+	// Build a structure
+	en_StreamFormat entry;
+	entry.mModificationTime = 	hton64(mModificationTime);
+	entry.mObjectID = 			hton64(mObjectID);
+	entry.mSizeInBlocks = 		hton64(mSizeInBlocks);
+	entry.mAttributesHash =		hton64(mAttributesHash);
+	entry.mFlags = 				htons(mFlags);
+	
+	// Write it
+	rStream.Write(&entry, sizeof(entry));
+	
+	// Write the filename
+	mName.WriteToStream(rStream);
+	
+	// Write any attributes
+	mAttributes.WriteToStream(rStream);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &, int)
+//		Purpose: Read the optional dependency info from a stream
+//		Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout)
+{
+	// Grab the raw bytes from the stream which compose the header
+	en_StreamFormatDepends depends;
+	if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+
+	// Store the data
+	mDependsNewer = ntoh64(depends.mDependsNewer);
+	mDependsOlder = ntoh64(depends.mDependsOlder);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &)
+//		Purpose: Write the optional dependency info to a stream
+//		Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const
+{
+	// Build structure
+	en_StreamFormatDepends depends;	
+	depends.mDependsNewer = hton64(mDependsNewer);
+	depends.mDependsOlder = hton64(mDependsOlder);
+	// Write
+	rStream.Write(&depends, sizeof(depends));
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreDirectory.h b/lib/backupclient/BackupStoreDirectory.h
new file mode 100755
index 00000000..958eee81
--- /dev/null
+++ b/lib/backupclient/BackupStoreDirectory.h
@@ -0,0 +1,268 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreDirectory.h
+//		Purpose: Representation of a backup directory
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREDIRECTORY__H
+#define BACKUPSTOREDIRECTORY__H
+
+#include <string>
+#include <vector>
+
+#include "BackupStoreFilenameClear.h"
+#include "StreamableMemBlock.h"
+#include "BoxTime.h"
+
+class IOStream;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreDirectory
+//		Purpose: In memory representation of a directory
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreDirectory
+{
+public:
+	BackupStoreDirectory();
+	BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID);
+private:
+	// Copying not allowed
+	BackupStoreDirectory(const BackupStoreDirectory &rToCopy);
+public:
+	~BackupStoreDirectory();
+
+	class Entry
+	{
+	public:
+		friend class BackupStoreDirectory;
+
+		Entry();
+		~Entry();
+		Entry(const Entry &rToCopy);
+		Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash);
+		
+		void ReadFromStream(IOStream &rStream, int Timeout);
+		void WriteToStream(IOStream &rStream) const;
+		
+		const BackupStoreFilename &GetName() const {return mName;}
+		box_time_t GetModificationTime() const {return mModificationTime;}
+		int64_t GetObjectID() const {return mObjectID;}
+		int64_t GetSizeInBlocks() const {return mSizeInBlocks;}
+		int16_t GetFlags() const {return mFlags;}
+		void AddFlags(int16_t Flags) {mFlags |= Flags;}
+		void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;}
+
+		// Some things can be changed
+		void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;}
+		void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;}
+
+		// Attributes
+		bool HasAttributes() const {return !mAttributes.IsEmpty();}
+		void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;}
+		const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+		uint64_t GetAttributesHash() const {return mAttributesHash;}
+		
+		// Marks
+		// The lowest mark number a version of a file of this name has ever had
+		uint32_t GetMinMarkNumber() const {return mMinMarkNumber;}
+		// The mark number on this file
+		uint32_t GetMarkNumber() const {return mMarkNumber;}
+
+		// Make sure these flags are synced with those in backupprocotol.txt
+		// ListDirectory command
+		enum
+		{
+			Flags_INCLUDE_EVERYTHING 	= -1,
+			Flags_EXCLUDE_NOTHING 		= 0,
+			Flags_EXCLUDE_EVERYTHING	= 31,	// make sure this is kept as sum of ones below!
+			Flags_File					= 1,
+			Flags_Dir					= 2,
+			Flags_Deleted				= 4,
+			Flags_OldVersion			= 8,
+			Flags_RemoveASAP			= 16	// if this flag is set, housekeeping will remove it as it is marked Deleted or OldVersion
+		};
+		// characters for textual listing of files -- see bbackupquery/BackupQueries
+		#define BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES "fdXoR"
+		
+		bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
+		{
+			return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet))
+				&& ((mFlags & FlagsNotToBeSet) == 0);
+		};
+
+		// Get dependency info
+		// new version this depends on
+		int64_t GetDependsNewer() const {return mDependsNewer;}
+		void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;}
+		// older version which depends on this
+		int64_t GetDependsOlder() const {return mDependsOlder;}
+		void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;}
+
+		// Dependency info saving
+		bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;}
+		void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout);
+		void WriteToStreamDependencyInfo(IOStream &rStream) const;
+
+	private:
+		BackupStoreFilename	mName;
+		box_time_t mModificationTime;
+		int64_t mObjectID;
+		int64_t mSizeInBlocks;
+		int16_t mFlags;
+		uint64_t mAttributesHash;
+		StreamableMemBlock mAttributes;
+		uint32_t mMinMarkNumber;
+		uint32_t mMarkNumber;
+		
+		uint64_t mDependsNewer;	// new version this depends on
+		uint64_t mDependsOlder;	// older version which depends on this
+	};
+	
+	void ReadFromStream(IOStream &rStream, int Timeout);
+	void WriteToStream(IOStream &rStream,
+			int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING,
+			int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING,
+			bool StreamAttributes = true, bool StreamDependencyInfo = true) const;
+			
+	Entry *AddEntry(const Entry &rEntryToCopy);
+	Entry *AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, box_time_t AttributesModTime);
+	void DeleteEntry(int64_t ObjectID);
+	Entry *FindEntryByID(int64_t ObjectID) const;
+	
+	int64_t GetObjectID() const {return mObjectID;}
+	int64_t GetContainerID() const {return mContainerID;}
+	
+	// Need to be able to update the container ID when moving objects
+	void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;}
+
+	// Purely for use of server -- not serialised into streams	
+	int64_t GetRevisionID() const {return mRevisionID;}
+	void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;}
+	
+	unsigned int GetNumberOfEntries() const {return mEntries.size();}
+
+	// User info -- not serialised into streams
+	int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;}
+	void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;}
+
+	// Attributes
+	bool HasAttributes() const {return !mAttributes.IsEmpty();}
+	void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;}
+	const StreamableMemBlock &GetAttributes() const {return mAttributes;}
+	box_time_t GetAttributesModTime() const {return mAttributesModTime;}
+
+	class Iterator
+	{
+	public:
+		Iterator(const BackupStoreDirectory &rDir)
+			: mrDir(rDir), i(rDir.mEntries.begin())
+		{
+		}
+		
+		BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+		{
+			// Skip over things which don't match the required flags
+			while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+			{
+				++i;
+			}
+			// Not the last one?
+			if(i == mrDir.mEntries.end())
+			{
+				return 0;
+			}
+			// Return entry, and increment
+			return (*(i++));
+		}
+
+		// WARNING: This function is really very inefficient.
+		// Only use when you want to look up ONE filename, not in a loop looking up lots.
+		// In a looping situation, cache the decrypted filenames in another memory structure.
+		BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+		{
+			// Skip over things which don't match the required flags or filename
+			while( (i != mrDir.mEntries.end())
+				&& ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+					|| (BackupStoreFilenameClear((*i)->GetName()).GetClearFilename() != rFilename.GetClearFilename()) ) )
+			{
+				++i;
+			}
+			// Not the last one?
+			if(i == mrDir.mEntries.end())
+			{
+				return 0;
+			}
+			// Return entry, and increment
+			return (*(i++));
+		}
+
+	private:
+		const BackupStoreDirectory &mrDir;
+		std::vector<Entry*>::const_iterator i;
+	};
+		
+	friend class Iterator;
+
+	class ReverseIterator
+	{
+	public:
+		ReverseIterator(const BackupStoreDirectory &rDir)
+			: mrDir(rDir), i(rDir.mEntries.rbegin())
+		{
+		}
+		
+		BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING)
+		{
+			// Skip over things which don't match the required flags
+			while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet))
+			{
+				++i;
+			}
+			// Not the last one?
+			if(i == mrDir.mEntries.rend())
+			{
+				return 0;
+			}
+			// Return entry, and increment
+			return (*(i++));
+		}
+	
+	private:
+		const BackupStoreDirectory &mrDir;
+		std::vector<Entry*>::const_reverse_iterator i;
+	};
+		
+	friend class ReverseIterator;
+
+	// For recovery of the store
+	// Implemented in BackupStoreCheck2.cpp
+	bool CheckAndFix();
+	void AddUnattactedObject(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags);
+	bool NameInUse(const BackupStoreFilename &rName);
+	// Don't use these functions in normal code!
+
+	// For testing
+	void TESTONLY_SetObjectID(int64_t ObjectID) {mObjectID = ObjectID;}
+
+	// Debug and diagonistics
+	void Dump(void *clibFileHandle, bool ToTrace); // first arg is FILE *, but avoid including stdio.h everywhere
+
+private:
+	int64_t mRevisionID;
+	int64_t mObjectID;
+	int64_t mContainerID;
+	std::vector<Entry*> mEntries;
+	box_time_t mAttributesModTime;
+	StreamableMemBlock mAttributes;
+	int64_t mUserInfo1;
+};
+
+#endif // BACKUPSTOREDIRECTORY__H
+
diff --git a/lib/backupclient/BackupStoreException.h b/lib/backupclient/BackupStoreException.h
new file mode 100755
index 00000000..981dfa60
--- /dev/null
+++ b/lib/backupclient/BackupStoreException.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreException.h
+//		Purpose: Exception
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREEXCEPTION__H
+#define BACKUPSTOREEXCEPTION__H
+
+// Compatibility
+#include "autogen_BackupStoreException.h"
+
+#endif // BACKUPSTOREEXCEPTION__H
+
diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt
new file mode 100644
index 00000000..50c615b2
--- /dev/null
+++ b/lib/backupclient/BackupStoreException.txt
@@ -0,0 +1,70 @@
+EXCEPTION BackupStore 4
+
+Internal						0
+BadAccountDatabaseFile			1
+AccountDatabaseNoSuchEntry		2
+InvalidBackupStoreFilename		3
+UnknownFilenameEncoding			4
+CouldntReadEntireStructureFromStream	5
+BadDirectoryFormat				6
+CouldNotFindEntryInDirectory	7
+OutputFileAlreadyExists			8
+OSFileError						9
+StreamDoesntHaveRequiredFeatures		10
+BadBackupStoreFile				11
+CouldNotLoadStoreInfo			12
+BadStoreInfoOnLoad				13
+StoreInfoIsReadOnly				14
+StoreInfoDirNotInList			15
+StoreInfoBlockDeltaMakesValueNegative	16
+DirectoryHasBeenDeleted			17
+StoreInfoNotInitialised			18
+StoreInfoAlreadyLoaded			19
+StoreInfoNotLoaded				20
+ReadFileFromStreamTimedOut		21
+FileWrongSizeAfterBeingStored	22
+AddedFileDoesNotVerify			23
+StoreInfoForWrongAccount		24
+ContextIsReadOnly				25
+AttributesNotLoaded				26
+AttributesNotUnderstood			27
+WrongServerVersion				28			# client side
+ClientMarkerNotAsExpected		29	Another process logged into the store and modified it while this process was running. Check you're not running two or more clients on the same account.
+NameAlreadyExistsInDirectory	30
+BerkelyDBFailure				31			# client side
+InodeMapIsReadOnly				32			# client side
+InodeMapNotOpen					33			# client side
+FilenameEncryptionKeyNotKnown	34
+FilenameEncryptionNoKeyForSpecifiedMethod	35
+FilenameEncryptionNotSetup		36
+CouldntLoadClientKeyMaterial	37
+BadEncryptedAttributes			38
+EncryptedAttributesHaveUnknownEncoding	39
+OutputSizeTooSmallForChunk		40
+BadEncodedChunk					41
+NotEnoughSpaceToDecodeChunk		42
+ChunkHasUnknownEncoding			43
+ChunkContainsBadCompressedData	44
+CantWriteToEncodedFileStream	45
+Temp_FileEncodeStreamDidntReadBuffer	46
+CantWriteToDecodedFileStream	47
+WhenDecodingExpectedToReadButCouldnt	48
+BackupStoreFileFailedIntegrityCheck		49
+ThereIsNoDataInASymLink			50
+IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements	51
+BlockEntryEncodingDidntGiveExpectedLength	52
+CouldNotFindUnusedIDDuringAllocation	53
+AddedFileExceedsStorageLimit	54
+CannotDiffAnIncompleteStoreFile	55
+CannotDecodeDiffedFilesWithoutCombining	56
+FailedToReadBlockOnCombine		57
+OnCombineFromFileIsIncomplete	58
+BadNotifySysadminEventCode		59
+InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing	60
+CouldNotLockStoreAccount		61	Another process is accessing this account -- is a client connected to the server?
+AttributeHashSecretNotSet		62
+AEScipherNotSupportedByInstalledOpenSSL	63	The system needs to be compiled with support for OpenSSL 0.9.7 or later to be able to decode files encrypted with AES
+SignalReceived					64	A signal was received by the process, restart or terminate needed. Exception thrown to abort connection.
+IncompatibleFromAndDiffFiles	65	Attempt to use a diff and a from file together, when they're not related
+DiffFromIDNotFoundInDirectory	66	When uploading via a diff, the diff from file must be in the same directory
+PatchChainInfoBadInDirectory	67	A directory contains inconsistent information. Run bbstoreaccounts check to fix it.
diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp
new file mode 100755
index 00000000..2ea877b4
--- /dev/null
+++ b/lib/backupclient/BackupStoreFile.cpp
@@ -0,0 +1,1499 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFile.cpp
+//		Purpose: Utils for manipulating files
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <string.h>
+#include <new>
+#include <string.h>
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+	#include <syslog.h>
+	#include <stdio.h>
+#endif
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFilename.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+#include "FileModificationTime.h"
+#include "FileStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreObjectMagic.h"
+#include "Compress.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "CipherAES.h"
+#include "BackupStoreConstants.h"
+#include "CollectInBufferStream.h"
+#include "RollingChecksum.h"
+#include "MD5Digest.h"
+#include "ReadGatherStream.h"
+#include "Random.h"
+#include "BackupStoreFileEncodeStream.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+// How big a buffer to use for copying files
+#define COPY_BUFFER_SIZE	(8*1024)
+
+// Statistics
+BackupStoreFileStats BackupStoreFile::msStats = {0,0,0};
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+	bool sWarnedAboutBackwardsCompatiblity = false;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::EncodeFile(IOStream &, IOStream &)
+//		Purpose: Encode a file into something for storing on file server.
+//				 Requires a real filename so full info can be stored.
+//
+//				 Returns a stream. Most of the work is done by the stream
+//				 when data is actually requested -- the file will be held
+//				 open until the stream is deleted or the file finished.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime)
+{
+	// Create the stream
+	std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
+
+	// Do the initial setup
+	((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, 0 /* no recipe, just encode */,
+		ContainerID, rStoreFilename, pModificationTime);
+	
+	// Return the stream for the caller
+	return stream;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::VerifyEncodedFileFormat(IOStream &)
+//		Purpose: Verify that an encoded file meets the format requirements.
+//				 Doesn't verify that the data is intact and can be decoded.
+//				 Optionally returns the ID of the file which it is diffed from,
+//				 and the (original) container ID.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut)
+{
+	// Get the size of the file
+	int64_t fileSize = rFile.BytesLeftToRead();
+	if(fileSize == IOStream::SizeOfStreamUnknown)
+	{
+		THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+	}
+
+	// Get the header...
+	file_StreamFormat hdr;
+	if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+	{
+		// Couldn't read header
+		return false;
+	}
+	
+	// Check magic number
+	if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		&& ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+		)
+	{
+		return false;
+	}
+	
+	// Get a filename, see if it loads OK
+	try
+	{
+		BackupStoreFilename fn;
+		fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+	}
+	catch(...)
+	{
+		// an error occured while reading it, so that's not good
+		return false;
+	}
+	
+	// Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not
+	try
+	{
+		int32_t size_s;
+		if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+		{
+			THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+		}
+		int size = ntohl(size_s);
+		// Skip forward the size
+		rFile.Seek(size, IOStream::SeekType_Relative);
+	}
+	catch(...)
+	{
+		// an error occured while reading it, so that's not good
+		return false;
+	}
+
+	// Get current position in file -- the end of the header
+	int64_t headerEnd = rFile.GetPosition();
+	
+	// Get number of blocks
+	int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+	
+	// Calculate where the block index will be, check it's reasonable
+	int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+	if(blockIndexLoc < headerEnd)
+	{
+		// Not enough space left for the block index, let alone the blocks themselves
+		return false;
+	}
+
+	// Load the block index header
+	rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute);
+	file_BlockIndexHeader blkhdr;
+	if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */))
+	{
+		// Couldn't read block index header -- assume bad file
+		return false;
+	}
+	
+	// Check header
+	if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		&& ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+		)
+		|| (int64_t)ntoh64(blkhdr.mNumBlocks) != numBlocks)
+	{
+		// Bad header -- either magic value or number of blocks is wrong
+		return false;
+	}
+	
+	// Flag for recording whether a block is referenced from another file
+	bool blockFromOtherFileReferenced = false;
+	
+	// Read the index, checking that the length values all make sense
+	int64_t currentBlockStart = headerEnd;
+	for(int64_t b = 0; b < numBlocks; ++b)
+	{
+		// Read block entry
+		file_BlockIndexEntry blk;
+		if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */))
+		{
+			// Couldn't read block index entry -- assume bad file
+			return false;
+		}
+		
+		// Check size and location
+		int64_t blkSize = ntoh64(blk.mEncodedSize);
+		if(blkSize <= 0)
+		{
+			// Mark that this file references another file
+			blockFromOtherFileReferenced = true;
+		}
+		else
+		{
+			// This block is actually in this file
+			if((currentBlockStart + blkSize) > blockIndexLoc)
+			{
+				// Encoded size makes the block run over the index
+				return false;
+			}
+			
+			// Move the current block start ot the end of this block
+			currentBlockStart += blkSize;
+		}
+	}
+	
+	// Check that there's no empty space
+	if(currentBlockStart != blockIndexLoc)
+	{
+		return false;
+	}
+	
+	// Check that if another block is references, then the ID is there, and if one isn't there is no ID.
+	int64_t otherID = ntoh64(blkhdr.mOtherFileID);
+	if((otherID != 0 && blockFromOtherFileReferenced == false)
+		|| (otherID == 0 && blockFromOtherFileReferenced == true))
+	{
+		// Doesn't look good!
+		return false;
+	}
+	
+	// Does the caller want the other ID?
+	if(pDiffFromObjectIDOut)
+	{
+		*pDiffFromObjectIDOut = otherID;
+	}
+	
+	// Does the caller want the container ID?
+	if(pContainerIDOut)
+	{
+		*pContainerIDOut = ntoh64(hdr.mContainerID);
+	}
+
+	// Passes all tests
+	return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodeFile(IOStream &, const char *)
+//		Purpose: Decode a file. Will set file attributes. File must not exist.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+	// Does file exist?
+	struct stat st;
+	if(::stat(DecodedFilename, &st) == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists)
+	}
+	
+	// Try, delete output file if error
+	try
+	{
+		// Make a stream for outputting this file
+		FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL);
+
+		// Get the decoding stream
+		std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr));
+		
+		// Is it a symlink?
+		if(!stream->IsSymLink())
+		{
+			// Copy it out to the file
+			stream->CopyStreamTo(out);
+		}
+		
+		// Write the attributes
+		stream->GetAttributes().WriteAttributes(DecodedFilename);
+	}
+	catch(...)
+	{
+		::unlink(DecodedFilename);
+		throw;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *)
+//		Purpose: Return a stream which will decode the encrypted file data on the fly.
+//				 Accepts streams in block index first, or main header first, order. In the latter case,
+//				 the stream must be Seek()able.
+//
+//				 Before you use the returned stream, call IsSymLink() -- symlink streams won't allow
+//				 you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation.
+//		Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
+{
+	// Create stream
+	std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout));
+	
+	// Get it ready
+	stream->Setup(pAlterativeAttr);
+	
+	// Return to caller
+	return stream;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int)
+//		Purpose: Constructor
+//		Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout)
+	: mrEncodedFile(rEncodedFile),
+	  mTimeout(Timeout),
+	  mNumBlocks(0),
+	  mpBlockIndex(0),
+	  mpEncodedData(0),
+	  mpClearData(0),
+	  mClearDataSize(0),
+	  mCurrentBlock(-1),
+	  mCurrentBlockClearSize(0),
+	  mPositionInCurrentBlock(0),
+	  mEntryIVBase(42)	// different to default value in the encoded stream!
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+	  , mIsOldVersion(false)
+#endif
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodedStream::~DecodedStream()
+//		Purpose: Desctructor
+//		Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::DecodedStream::~DecodedStream()
+{
+	// Free any allocated memory
+	if(mpBlockIndex)
+	{
+		::free(mpBlockIndex);
+	}
+	if(mpEncodedData)
+	{
+		BackupStoreFile::CodingChunkFree(mpEncodedData);
+	}
+	if(mpClearData)
+	{
+		::free(mpClearData);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *)
+//		Purpose: Get the stream ready to decode -- reads in headers
+//		Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr)
+{
+	// Get the size of the file
+	int64_t fileSize = mrEncodedFile.BytesLeftToRead();
+
+	// Get the magic number to work out which order the stream is in
+	int32_t magic;
+	if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout))
+	{
+		// Couldn't read magic value
+		THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+	}
+
+	bool inFileOrder = true;	
+	switch(ntohl(magic))
+	{
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+	case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+		mIsOldVersion = true;
+		// control flows on
+#endif
+	case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+		inFileOrder = true;
+		break;
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+	case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0:
+		mIsOldVersion = true;
+		// control flows on
+#endif
+	case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1:
+		inFileOrder = false;
+		break;
+
+	default:
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// If not in file order, then the index list must be read now
+	if(!inFileOrder)
+	{
+		ReadBlockIndex(true /* have already read and verified the magic number */);
+	}
+
+	// Get header
+	file_StreamFormat hdr;
+	if(inFileOrder)
+	{
+		// Read the header, without the magic number
+		if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic),
+			0 /* not interested in bytes read if this fails */, mTimeout))
+		{
+			// Couldn't read header
+			THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+		}
+		// Put in magic number
+		hdr.mMagicValue = magic;
+	}
+	else
+	{
+		// Not in file order, so need to read the full header
+		if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+		{
+			// Couldn't read header
+			THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+		}
+	}	
+
+	// Check magic number
+	if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		&& ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+		)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+
+	// Get the filename
+	mFilename.ReadFromStream(mrEncodedFile, mTimeout);
+	
+	// Get the attributes (either from stream, or supplied attributes)
+	if(pAlterativeAttr != 0)
+	{
+		// Read dummy attributes
+		BackupClientFileAttributes attr;
+		attr.ReadFromStream(mrEncodedFile, mTimeout);
+
+		// Set to supplied attributes
+		mAttributes = *pAlterativeAttr;
+	}
+	else
+	{
+		// Read the attributes from the stream
+		mAttributes.ReadFromStream(mrEncodedFile, mTimeout);
+	}
+	
+	// If it is in file order, go and read the file attributes
+	// Requires that the stream can seek
+	if(inFileOrder)
+	{
+		// Make sure the file size is known
+		if(fileSize == IOStream::SizeOfStreamUnknown)
+		{
+			THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+		}
+	
+		// Store current location (beginning of encoded blocks)
+		int64_t endOfHeaderPos = mrEncodedFile.GetPosition();
+		
+		// Work out where the index is
+		int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+		int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+		
+		// Seek to that position
+		mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute);
+		
+		// Read the block index
+		ReadBlockIndex(false /* magic number still to be read */);		
+		
+		// Seek back to the end of header position, ready for reading the chunks
+		mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute);
+	}
+	
+	// Check view of blocks from block header and file header match
+	if(mNumBlocks != (int64_t)ntoh64(hdr.mNumBlocks))
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Need to allocate some memory for the two blocks for reading encoded data, and clear data
+	if(mNumBlocks > 0)
+	{
+		// Find the maximum encoded data size
+		int32_t maxEncodedDataSize = 0;
+		const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+		ASSERT(entry != 0);
+		for(int64_t e = 0; e < mNumBlocks; e++)
+		{
+			// Get the clear and encoded size
+			int32_t encodedSize = ntoh64(entry[e].mEncodedSize);
+			ASSERT(encodedSize > 0);
+			
+			// Larger?
+			if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize;
+		}
+		
+		// Allocate those blocks!
+		mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32);
+
+		// Allocate the block for the clear data, using the hint from the header.
+		// If this is wrong, things will exception neatly later on, so it can't be used
+		// to do anything more than cause an error on downloading.
+		mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32;
+		mpClearData = (uint8_t*)::malloc(mClearDataSize);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodedStream::ReadBlockIndex(bool)
+//		Purpose: Read the block index from the stream, and store in internal buffer (minus header)
+//		Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead)
+{
+	// Header
+	file_BlockIndexHeader blkhdr;
+	
+	// Read it in -- way depends on how whether the magic number has already been read
+	if(MagicAlreadyRead)
+	{
+		// Read the header, without the magic number
+		if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue),
+			0 /* not interested in bytes read if this fails */, mTimeout))
+		{
+			// Couldn't read header
+			THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+		}
+	}
+	else
+	{
+		// Magic not already read, so need to read the full header
+		if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout))
+		{
+			// Couldn't read header
+			THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+		}
+		
+		// Check magic value
+		if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+			&& ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
+#endif
+			)
+		{
+			THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+		}
+	}
+	
+	// Get the number of blocks out of the header
+	mNumBlocks = ntoh64(blkhdr.mNumBlocks);
+	
+	// Read the IV base
+	mEntryIVBase = ntoh64(blkhdr.mEntryIVBase);
+	
+	// Load the block entries in?
+	if(mNumBlocks > 0)
+	{
+		// How big is the index?
+		int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks;
+		
+		// Allocate some memory
+		mpBlockIndex = ::malloc(indexSize);
+		if(mpBlockIndex == 0)
+		{
+			throw std::bad_alloc();
+		}
+		
+		// Read it in
+		if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+		{
+			// Couldn't read header
+			THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodedStream::Read(void *, int, int)
+//		Purpose: As interface. Reads decrpyted data.
+//		Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// Symlinks don't have data. So can't read it. Not even zero bytes.
+	if(IsSymLink())
+	{
+		// Don't allow reading in this case
+		THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink);
+	}
+
+	// Already finished?
+	if(mCurrentBlock >= mNumBlocks)
+	{
+		// At end of stream, nothing to do
+		return 0;
+	}
+
+	int bytesToRead = NBytes;
+	uint8_t *output = (uint8_t*)pBuffer;
+	
+	while(bytesToRead > 0 && mCurrentBlock < mNumBlocks)
+	{
+		// Anything left in the current block?
+		if(mPositionInCurrentBlock < mCurrentBlockClearSize)
+		{
+			// Copy data out of this buffer
+			int s = mCurrentBlockClearSize - mPositionInCurrentBlock;
+			if(s > bytesToRead) s = bytesToRead;	// limit to requested data
+			
+			// Copy
+			::memcpy(output, mpClearData + mPositionInCurrentBlock, s);
+			
+			// Update positions
+			output += s;
+			mPositionInCurrentBlock += s;
+			bytesToRead -= s;
+		}
+		
+		// Need to get some more data?
+		if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize)
+		{
+			// Number of next block
+			++mCurrentBlock;
+			if(mCurrentBlock >= mNumBlocks)
+			{
+				// Stop now!
+				break;
+			}
+		
+			// Get the size from the block index
+			const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
+			int32_t encodedSize = ntoh64(entry[mCurrentBlock].mEncodedSize);
+			if(encodedSize <= 0)
+			{
+				// The caller is attempting to decode a file which is the direct result of a diff
+				// operation, and so does not contain all the data.
+				// It needs to be combined with the previous version first.
+				THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining)
+			}
+			
+			// Load in next block
+			if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout))
+			{
+				// Couldn't read header
+				THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+			}
+			
+			// Decode the data
+			mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize);
+
+			// Calculate IV for this entry
+			uint64_t iv = mEntryIVBase;
+			iv += mCurrentBlock;
+			// Convert to network byte order before encrypting with it, so that restores work on
+			// platforms with different endiannesses.
+			iv = hton64(iv);
+			sBlowfishDecryptBlockEntry.SetIV(&iv);
+			
+			// Decrypt the encrypted section
+			file_BlockIndexEntryEnc entryEnc;
+			int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+					entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+			if(sectionSize != sizeof(entryEnc))
+			{
+				THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+			}
+
+			// Make sure this is the right size
+			if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+			{
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+				if(!mIsOldVersion)
+				{
+					THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+				}
+				// Versions 0.05 and previous of Box Backup didn't properly handle endianess of the
+				// IV for the encrypted section. Try again, with the thing the other way round
+				iv = box_swap64(iv);
+				sBlowfishDecryptBlockEntry.SetIV(&iv);
+				int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+						entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
+				if(sectionSize != sizeof(entryEnc))
+				{
+					THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+				}
+				if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
+				{
+					THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+				}
+				else
+				{
+					// Warn and log this issue
+					if(!sWarnedAboutBackwardsCompatiblity)
+					{
+						::printf("WARNING: Decoded one or more files using backwards compatibility mode for block index.\n");
+						::syslog(LOG_ERR, "WARNING: Decoded one or more files using backwards compatibility mode for block index.\n");
+						sWarnedAboutBackwardsCompatiblity = true;
+					}
+				}
+#else
+				THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+#endif
+			}
+			
+			// Check the digest
+			MD5Digest md5;
+			md5.Add(mpClearData, mCurrentBlockClearSize);
+			md5.Finish();
+			if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum))
+			{
+				THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck)
+			}
+			
+			// Set vars to say what's happening
+			mPositionInCurrentBlock = 0;
+		}
+	}
+	
+	ASSERT(bytesToRead >= 0);
+	ASSERT(bytesToRead <= NBytes);
+
+	return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodedStream::IsSymLink()
+//		Purpose: Is the unencoded file actually a symlink?
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::IsSymLink()
+{
+	// First, check in with the attributes
+	if(!mAttributes.IsSymLink())
+	{
+		return false;
+	}
+	
+	// So the attributes think it is a symlink.
+	// Consistency check...
+	if(mNumBlocks != 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodedStream::Write(const void *, int)
+//		Purpose: As interface. Throws exception, as you can't write to this stream.
+//		Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes)
+{
+	THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodedStream::StreamDataLeft()
+//		Purpose: As interface. Any data left?
+//		Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamDataLeft()
+{
+	return mCurrentBlock < mNumBlocks;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodedStream::StreamClosed()
+//		Purpose: As interface. Always returns true, no writing allowed.
+//		Created: 9/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::DecodedStream::StreamClosed()
+{
+	// Can't write to this stream!
+	return true;
+}
+
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::SetBlowfishKey(const void *, int)
+//		Purpose: Static. Sets the key to use for encryption and decryption.
+//		Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength)
+{
+	// IVs set later
+	sBlowfishEncrypt.Reset();
+	sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+	sBlowfishDecrypt.Reset();
+	sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+
+	sBlowfishEncryptBlockEntry.Reset();
+	sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+	sBlowfishEncryptBlockEntry.UsePadding(false);
+	sBlowfishDecryptBlockEntry.Reset();
+	sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
+	sBlowfishDecryptBlockEntry.UsePadding(false);
+}
+
+
+#ifndef PLATFORM_OLD_OPENSSL
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::SetAESKey(const void *, int)
+//		Purpose: Sets the AES key to use for file data encryption. Will select AES as
+//				 the cipher to use when encrypting.
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength)
+{
+	// Setup context
+	sAESEncrypt.Reset();
+	sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+	sAESDecrypt.Reset();
+	sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
+	
+	// Set encryption to use this key, instead of the "default" blowfish key
+	spEncrypt = &sAESEncrypt;
+	sEncryptCipherType = HEADER_AES_ENCODING;
+}
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::MaxBlockSizeForChunkSize(int)
+//		Purpose: The maximum output size of a block, given the chunk size
+//		Created: 7/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize)
+{
+	// Calculate... the maximum size of output by first the largest it could be after compression,
+	// which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck
+	// And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing
+	// an problem where a rather non-compressable file didn't fit in a block buffer.)
+	return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1
+		+ sBlowfishEncrypt.GetIVLength() + 128;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &)
+//		Purpose: Encodes a chunk (encryption, possible compressed beforehand)
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput)
+{
+	ASSERT(spEncrypt != 0);
+
+	// Check there's some space in the output block
+	if(rOutput.mBufferSize < 256)
+	{
+		rOutput.Reallocate(256);
+	}
+	
+	// Check alignment of the block
+	ASSERT((((uint32_t)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+	// Want to compress it?
+	bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE);
+
+	// Build header
+	uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT;
+	if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED;
+
+	// Store header
+	rOutput.mpBuffer[0] = header;
+	int outOffset = 1;
+
+	// Setup cipher, and store the IV
+	int ivLen = 0;
+	const void *iv = spEncrypt->SetRandomIV(ivLen);
+	::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen);
+	outOffset += ivLen;
+	
+	// Start encryption process
+	spEncrypt->Begin();
+	
+	#define ENCODECHUNK_CHECK_SPACE(ToEncryptSize)									\
+		{																			\
+			if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128))			\
+			{																		\
+				rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128);	\
+			}																		\
+		}
+	
+	// Encode the chunk
+	if(compressChunk)
+	{
+		// buffer to compress into
+		uint8_t buffer[2048];
+		
+		// Set compressor with all the chunk as an input
+		Compress<true> compress;
+		compress.Input(Chunk, ChunkSize);
+		compress.FinishInput();
+
+		// Get and encrypt output
+		while(!compress.OutputHasFinished())
+		{
+			int s = compress.Output(buffer, sizeof(buffer));
+			if(s > 0)
+			{
+				ENCODECHUNK_CHECK_SPACE(s)
+				outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s);				
+			}
+			else
+			{
+				// Should never happen, as we put all the input in in one go.
+				// So if this happens, it means there's a logical problem somewhere
+				THROW_EXCEPTION(BackupStoreException, Internal)
+			}
+		}
+		ENCODECHUNK_CHECK_SPACE(16)
+		outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+	}
+	else
+	{
+		// Straight encryption
+		ENCODECHUNK_CHECK_SPACE(ChunkSize)
+		outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize);
+		ENCODECHUNK_CHECK_SPACE(16)
+		outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
+	}
+	
+	ASSERT(outOffset < rOutput.mBufferSize);		// first check should have sorted this -- merely logic check
+
+	return outOffset;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DecodeChunk(const void *, int, void *, int)
+//		Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find
+//				 the extra output buffer size needed before calling.
+//				 See notes in EncodeChunk() for notes re alignment of the 
+//				 encoded data.
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize)
+{
+	// Check alignment of the encoded block
+	ASSERT((((uint32_t)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+
+	// First check
+	if(EncodedSize < 1)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+	}
+
+	const uint8_t *input = (uint8_t*)Encoded;
+	
+	// Get header, make checks, etc
+	uint8_t header = input[0];
+	bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED;
+	uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT);
+	if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING)
+	{
+		THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding)
+	}
+	
+#ifndef PLATFORM_OLD_OPENSSL
+	// Choose cipher
+	CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt);
+#else
+	// AES not supported with this version of OpenSSL
+	if(encodingType == HEADER_AES_ENCODING)
+	{
+		THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL)
+	}
+	CipherContext &cipher(sBlowfishDecrypt);
+#endif
+	
+	// Check enough space for header, an IV and one byte of input
+	int ivLen = cipher.GetIVLength();
+	if(EncodedSize < (1 + ivLen + 1))
+	{
+		THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
+	}
+
+	// Set IV in decrypt context, and start
+	cipher.SetIV(input + 1);
+	cipher.Begin();
+	
+	// Setup vars for code
+	int inOffset = 1 + ivLen;
+	uint8_t *output = (uint8_t*)Output;
+	int outOffset = 0;
+
+	// Do action
+	if(chunkCompressed)
+	{
+		// Do things in chunks
+		uint8_t buffer[2048];
+		int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer));
+		
+		// Decompressor
+		Compress<false> decompress;
+		
+		while(inOffset < EncodedSize)
+		{
+			// Decrypt a block
+			int bl = inputBlockLen;
+			if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset;	// not too long
+			int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl);
+			inOffset += bl;
+			
+			// Decompress the decrypted data
+			if(s > 0)
+			{
+				decompress.Input(buffer, s);
+				int os = 0;
+				do
+				{
+					os = decompress.Output(output + outOffset, OutputSize - outOffset);
+					outOffset += os;
+				} while(os > 0);
+				
+				// Check that there's space left in the output buffer -- there always should be
+				if(outOffset >= OutputSize)
+				{
+					THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+				}
+			}
+		}
+		
+		// Get any compressed data remaining in the cipher context and compression
+		int s = cipher.Final(buffer, sizeof(buffer));
+		decompress.Input(buffer, s);
+		decompress.FinishInput();
+		while(!decompress.OutputHasFinished())
+		{
+			int os = decompress.Output(output + outOffset, OutputSize - outOffset);
+			outOffset += os;
+
+			// Check that there's space left in the output buffer -- there always should be
+			if(outOffset >= OutputSize)
+			{
+				THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
+			}
+		}
+	}
+	else
+	{
+		// Easy decryption
+		outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset);
+		outOffset += cipher.Final(output + outOffset, OutputSize - outOffset);
+	}
+	
+	return outOffset;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool)
+//		Purpose: Returns a stream which gives a Stream order version of the encoded file.
+//				 If TakeOwnership == true, then the input stream will be deleted when the
+//				 returned stream is deleted.
+//				 The input stream must be seekable.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership)
+{
+	ASSERT(pStream != 0);
+
+	// Get the size of the file
+	int64_t fileSize = pStream->BytesLeftToRead();
+	if(fileSize == IOStream::SizeOfStreamUnknown)
+	{
+		THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+	}
+
+	// Read the header
+	int bytesRead = 0;
+	file_StreamFormat hdr;
+	bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead);
+
+	// Seek backwards to put the file pointer back where it was before we started this
+	pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative);
+
+	// Check we got a block
+	if(!readBlock)
+	{
+		// Couldn't read header -- assume file bad
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+
+	// Check magic number
+	if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		&& ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+		)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Get number of blocks
+	int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+	
+	// Calculate where the block index will be, check it's reasonable
+	int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+	int64_t blockIndexLoc = fileSize - blockIndexSize;
+	if(blockIndexLoc < 0)
+	{
+		// Doesn't look good!
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Build a reordered stream
+	std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership));
+	
+	// Set it up...
+	ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get()));
+	int component = rreordered.AddComponent(pStream);
+	// Send out the block index
+	rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc);
+	// And then the rest of the file
+	rreordered.AddBlock(component, blockIndexLoc, true, 0);
+		
+	return reordered;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::ResetStats()
+//		Purpose: Reset the gathered statistics
+//		Created: 20/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ResetStats()
+{
+	msStats.mBytesInEncodedFiles = 0;
+	msStats.mBytesAlreadyOnServer = 0;
+	msStats.mTotalFileStreamSize = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &)
+//		Purpose: Compares the contents of a file against the checksums contained in the
+//				 block index. Returns true if the checksums match, meaning the file is
+//				 extremely likely to match the original. Will always consume the entire index.
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout)
+{
+	// is it a symlink?
+	bool sourceIsSymlink = false;
+	{
+		struct stat st;
+		if(::lstat(Filename, &st) == -1)
+		{
+			THROW_EXCEPTION(CommonException, OSFileError)
+		}
+		if((st.st_mode & S_IFMT) == S_IFLNK)
+		{
+			sourceIsSymlink = true;
+		}
+	}
+
+	// Open file, if it's not a symlink
+	std::auto_ptr<FileStream> in;
+	if(!sourceIsSymlink)
+	{
+		in.reset(new FileStream(Filename));
+	}
+	
+	// Read header
+	file_BlockIndexHeader hdr;
+	if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+	{
+		// Couldn't read header
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+
+	// Check magic
+	if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		&& hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)
+#endif
+		)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+	bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0);
+#endif
+
+	// Get basic information
+	int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+	uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase);
+	
+	//TODO: Verify that these sizes look reasonable
+	
+	// setup
+	void *data = 0;
+	int32_t dataSize = -1;
+	bool matches = true;
+	int64_t totalSizeInBlockIndex = 0;
+	
+	try
+	{	
+		for(int64_t b = 0; b < numBlocks; ++b)
+		{
+			// Read an entry from the stream
+			file_BlockIndexEntry entry;
+			if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+			{
+				// Couldn't read entry
+				THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+			}	
+		
+			// Calculate IV for this entry
+			uint64_t iv = entryIVBase;
+			iv += b;
+			iv = hton64(iv);
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+			if(isOldVersion)
+			{
+				// Reverse the IV for compatibility
+				iv = box_swap64(iv);
+			}
+#endif
+			sBlowfishDecryptBlockEntry.SetIV(&iv);			
+			
+			// Decrypt the encrypted section
+			file_BlockIndexEntryEnc entryEnc;
+			int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+					entry.mEnEnc, sizeof(entry.mEnEnc));
+			if(sectionSize != sizeof(entryEnc))
+			{
+				THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+			}
+
+			// Size of block
+			int32_t blockClearSize = ntohl(entryEnc.mSize);
+			if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
+			{
+				THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+			}
+			totalSizeInBlockIndex += blockClearSize;
+
+			// Make sure there's enough memory allocated to load the block in
+			if(dataSize < blockClearSize)
+			{
+				// Too small, free the block if it's already allocated
+				if(data != 0)
+				{
+					::free(data);
+					data = 0;
+				}
+				// Allocate a block
+				data = ::malloc(blockClearSize + 128);
+				if(data == 0)
+				{
+					throw std::bad_alloc();
+				}
+				dataSize = blockClearSize + 128;
+			}
+			
+			// Load in the block from the file, if it's not a symlink
+			if(!sourceIsSymlink)
+			{
+				if(in->Read(data, blockClearSize) != blockClearSize)
+				{
+					// Not enough data left in the file, can't possibly match
+					matches = false;
+				}
+				else
+				{
+					// Check the checksum
+					MD5Digest md5;
+					md5.Add(data, blockClearSize);
+					md5.Finish();
+					if(!md5.DigestMatches(entryEnc.mStrongChecksum))
+					{
+						// Checksum didn't match
+						matches = false;
+					}
+				}
+			}
+			
+			// Keep on going regardless, to make sure the entire block index stream is read
+			// -- must always be consistent about what happens with the stream.
+		}
+	}
+	catch(...)
+	{
+		// clean up in case of errors
+		if(data != 0)
+		{
+			::free(data);
+			data = 0;
+		}
+		throw;
+	}
+	
+	// free block
+	if(data != 0)
+	{
+		::free(data);
+		data = 0;
+	}
+	
+	// Check for data left over if it's not a symlink
+	if(!sourceIsSymlink)
+	{
+		// Anything left to read in the file?
+		if(in->BytesLeftToRead() != 0)
+		{
+			// File has extra data at the end
+			matches = false;
+		}
+	}
+	
+	// Symlinks must have zero size on server
+	if(sourceIsSymlink)
+	{
+		matches = (totalSizeInBlockIndex == 0);
+	}
+	
+	return matches;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::EncodingBuffer::EncodingBuffer()
+//		Purpose: Constructor
+//		Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::EncodingBuffer()
+	: mpBuffer(0),
+	  mBufferSize(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+//		Purpose: Destructor
+//		Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFile::EncodingBuffer::~EncodingBuffer()
+{
+	if(mpBuffer != 0)
+	{
+		BackupStoreFile::CodingChunkFree(mpBuffer);
+		mpBuffer = 0;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::EncodingBuffer::Allocate(int)
+//		Purpose: Do initial allocation of block
+//		Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Allocate(int Size)
+{
+	ASSERT(mpBuffer == 0);
+	uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size);
+	if(buffer == 0)
+	{
+		throw std::bad_alloc();
+	}
+	mpBuffer = buffer;
+	mBufferSize = Size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::EncodingBuffer::Reallocate(int)
+//		Purpose: Reallocate the block. Try not to call this, it has to copy
+//				 the entire contents as the block can't be reallocated straight.
+//		Created: 25/11/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize)
+{
+	TRACE2("Reallocating EncodingBuffer from %d to %d\n", mBufferSize, NewSize);
+	ASSERT(mpBuffer != 0);
+	uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize);
+	if(buffer == 0)
+	{
+		throw std::bad_alloc();
+	}
+	// Copy data
+	::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize);
+	
+	// Free old
+	BackupStoreFile::CodingChunkFree(mpBuffer);
+	
+	// Store new buffer
+	mpBuffer = buffer;
+	mBufferSize = NewSize;
+}
+
+
diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h
new file mode 100755
index 00000000..0274107b
--- /dev/null
+++ b/lib/backupclient/BackupStoreFile.h
@@ -0,0 +1,194 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFile.h
+//		Purpose: Utils for manipulating files
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILE__H
+#define BACKUPSTOREFILE__H
+
+#include "IOStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreFilename.h"
+
+#include <memory>
+
+typedef struct 
+{
+	int64_t mBytesInEncodedFiles;
+	int64_t mBytesAlreadyOnServer;
+	int64_t mTotalFileStreamSize;
+} BackupStoreFileStats;
+
+// Uncomment to disable backwards compatibility
+//#define BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+
+
+// Output buffer to EncodeChunk and input data to DecodeChunk must
+// have specific alignment, see function comments.
+#define BACKUPSTOREFILE_CODING_BLOCKSIZE		16
+#define BACKUPSTOREFILE_CODING_OFFSET			15
+
+// Have some memory allocation commands, note closing "Off" at end of file.
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreFile
+//		Purpose: Class to hold together utils for maniplating files.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class BackupStoreFile
+{
+public:
+	class DecodedStream : public IOStream
+	{
+		friend class BackupStoreFile;
+	private:
+		DecodedStream(IOStream &rEncodedFile, int Timeout);
+		DecodedStream(const DecodedStream &); // not allowed
+		DecodedStream &operator=(const DecodedStream &); // not allowed
+	public:
+		~DecodedStream();
+
+		// Stream functions		
+		virtual int Read(void *pBuffer, int NBytes, int Timeout);
+		virtual void Write(const void *pBuffer, int NBytes);
+		virtual bool StreamDataLeft();
+		virtual bool StreamClosed();
+		
+		// Accessor functions
+		const BackupClientFileAttributes &GetAttributes() {return mAttributes;}
+		const BackupStoreFilename &GetFilename() {return mFilename;}
+		int64_t GetNumBlocks() {return mNumBlocks;}	// primarily for tests
+		
+		bool IsSymLink();
+		
+	private:
+		void Setup(const BackupClientFileAttributes *pAlterativeAttr);
+		void ReadBlockIndex(bool MagicAlreadyRead);
+			
+	private:
+		IOStream &mrEncodedFile;
+		int mTimeout;
+		BackupClientFileAttributes mAttributes;
+		BackupStoreFilename mFilename;
+		int64_t mNumBlocks;
+		void *mpBlockIndex;
+		uint8_t *mpEncodedData;
+		uint8_t *mpClearData;
+		int mClearDataSize;
+		int mCurrentBlock;
+		int mCurrentBlockClearSize;
+		int mPositionInCurrentBlock;
+		uint64_t mEntryIVBase;
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		bool mIsOldVersion;
+#endif
+	};
+
+
+	// Main interface
+	static std::auto_ptr<IOStream> EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0);
+	static std::auto_ptr<IOStream> EncodeFileDiff(const char *Filename, int64_t ContainerID,
+		const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
+		int Timeout, int64_t *pModificationTime = 0, bool *pIsCompletelyDifferent = 0);
+	static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0);
+	static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut);
+	static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut);
+	static void ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent = 0);
+	static void DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+	static std::auto_ptr<BackupStoreFile::DecodedStream> DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr = 0);
+	static bool CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout);
+	static std::auto_ptr<IOStream> CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly = false, bool FromIsIndexOnly = false);
+
+	// Stream manipulation
+	static std::auto_ptr<IOStream> ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership);
+	static void MoveStreamPositionToBlockIndex(IOStream &rStream);
+
+	// Crypto setup
+	static void SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength);
+#ifndef PLATFORM_OLD_OPENSSL
+	static void SetAESKey(const void *pKey, int KeyLength);
+#endif
+
+	// Allocation of properly aligning chunks for decoding and encoding chunks
+	inline static void *CodingChunkAlloc(int Size)
+	{
+		uint8_t *a = (uint8_t*)malloc((Size) + (BACKUPSTOREFILE_CODING_BLOCKSIZE * 3));
+		if(a == 0) return 0;
+		// Align to main block size
+		ASSERT(sizeof(uint32_t) == sizeof(void*));	// make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers
+		uint32_t adjustment = BACKUPSTOREFILE_CODING_BLOCKSIZE - (((uint32_t)a) % BACKUPSTOREFILE_CODING_BLOCKSIZE);
+		uint8_t *b = (a + adjustment);
+		// Store adjustment
+		*b = (uint8_t)adjustment;
+		// Return offset
+		return b + BACKUPSTOREFILE_CODING_OFFSET;
+	}
+	inline static void CodingChunkFree(void *Block)
+	{
+		// Check alignment is as expected
+		ASSERT(sizeof(uint32_t) == sizeof(void*));	// make sure casting the right pointer size, will need to fix on platforms with 64 bit pointers
+		ASSERT((((uint32_t)Block) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
+		uint8_t *a = (uint8_t*)Block;
+		a -= BACKUPSTOREFILE_CODING_OFFSET;
+		// Adjust downwards...
+		a -= *a;
+		free(a);
+	}
+
+	// Limits
+	static void SetMaximumDiffingTime(int Seconds);
+
+	// Building blocks
+	class EncodingBuffer
+	{
+	public:
+		EncodingBuffer();
+		~EncodingBuffer();
+	private:
+		// No copying
+		EncodingBuffer(const EncodingBuffer &);
+		EncodingBuffer &operator=(const EncodingBuffer &);
+	public:
+		void Allocate(int Size);
+		void Reallocate(int NewSize);
+		
+		uint8_t *mpBuffer;
+		int mBufferSize;
+	};
+	static int MaxBlockSizeForChunkSize(int ChunkSize);
+	static int EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput);
+
+	// Caller should know how big the output size is, but also allocate a bit more memory to cover various
+	// overheads allowed for in checks
+	static inline int OutputBufferSizeForKnownOutputSize(int KnownChunkSize)
+	{
+		// Plenty big enough
+		return KnownChunkSize + 256;
+	}
+	static int DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize);
+
+	// Statisitics, not designed to be completely reliable	
+	static void ResetStats();
+	static BackupStoreFileStats msStats;
+	
+	// For debug
+#ifndef NDEBUG
+	static bool TraceDetailsOfDiffProcess;
+#endif
+
+	// For decoding encoded files
+	static void DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile);
+};
+
+#include "MemLeakFindOff.h"
+
+#endif // BACKUPSTOREFILE__H
+
diff --git a/lib/backupclient/BackupStoreFileCmbDiff.cpp b/lib/backupclient/BackupStoreFileCmbDiff.cpp
new file mode 100644
index 00000000..039b00eb
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCmbDiff.cpp
@@ -0,0 +1,326 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileCmbDiff.cpp
+//		Purpose: Combine two diffs together
+//		Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::CombineDiffs(IOStream &, IOStream &, IOStream &rOut)
+//		Purpose: Given two diffs, combine them into a single diff, to produce a diff
+//				 which, combined with the original file, creates the result of applying
+//				 rDiff, then rDiff2. Two opens of rDiff2 are required
+//		Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut)
+{
+	// Skip header of first diff, record where the data starts, and skip to the index
+	int64_t diff1DataStarts = 0;
+	{
+		// Read the header for the From file
+		file_StreamFormat diff1Hdr;
+		if(!rDiff1.ReadFullBuffer(&diff1Hdr, sizeof(diff1Hdr), 0))
+		{
+			THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+		}
+		if(ntohl(diff1Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+		{
+			THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+		}
+		// Skip over the filename and attributes of the From file
+		// BLOCK
+		{
+			BackupStoreFilename filename2;
+			filename2.ReadFromStream(rDiff1, IOStream::TimeOutInfinite);
+			int32_t size_s;
+			if(!rDiff1.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+			{
+				THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+			}
+			int size = ntohl(size_s);
+			// Skip forward the size
+			rDiff1.Seek(size, IOStream::SeekType_Relative);		
+		}
+		// Record position
+		diff1DataStarts = rDiff1.GetPosition();
+		// Skip to index
+		rDiff1.Seek(0 - (((ntoh64(diff1Hdr.mNumBlocks)) * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+	}
+
+	// Read the index of the first diff
+	// Header first
+	file_BlockIndexHeader diff1IdxHdr;
+	if(!rDiff1.ReadFullBuffer(&diff1IdxHdr, sizeof(diff1IdxHdr), 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+	if(ntohl(diff1IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	int64_t diff1NumBlocks = ntoh64(diff1IdxHdr.mNumBlocks);
+	// Allocate some memory
+	int64_t *diff1BlockStartPositions = (int64_t*)::malloc((diff1NumBlocks + 1) * sizeof(int64_t));
+	if(diff1BlockStartPositions == 0)
+	{
+		throw std::bad_alloc();
+	}
+
+	// Buffer data
+	void *buffer = 0;
+	int bufferSize = 0;	
+	
+	try
+	{
+		// Then the entries:
+		// For each entry, want to know if it's in the file, and if so, how big it is.
+		// We'll store this as an array of file positions in the file, with an additioal
+		// entry on the end so that we can work out the length of the last block.
+		// If an entry isn't in the file, then store 0 - (position in other file).
+		int64_t diff1Position = diff1DataStarts;
+		for(int64_t b = 0; b < diff1NumBlocks; ++b)
+		{
+			file_BlockIndexEntry e;
+			if(!rDiff1.ReadFullBuffer(&e, sizeof(e), 0))
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+			}
+	
+			// Where's the block?
+			int64_t blockEn = ntoh64(e.mEncodedSize);
+			if(blockEn <= 0)
+			{
+				// Just store the negated block number
+				diff1BlockStartPositions[b] = blockEn;
+			}
+			else
+			{
+				// Block is present in this file
+				diff1BlockStartPositions[b] = diff1Position;
+				diff1Position += blockEn;
+			}
+		}
+		
+		// Finish off the list, so the last entry can have it's size calcuated.
+		diff1BlockStartPositions[diff1NumBlocks] = diff1Position;
+
+		// Now read the second diff's header, copying it to the out file
+		file_StreamFormat diff2Hdr;
+		if(!rDiff2.ReadFullBuffer(&diff2Hdr, sizeof(diff2Hdr), 0))
+		{
+			THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+		}
+		if(ntohl(diff2Hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+		{
+			THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+		}
+		// Copy
+		rOut.Write(&diff2Hdr, sizeof(diff2Hdr));
+		// Copy over filename and attributes
+		// BLOCK
+		{
+			BackupStoreFilename filename;
+			filename.ReadFromStream(rDiff2, IOStream::TimeOutInfinite);
+			filename.WriteToStream(rOut);
+			StreamableMemBlock attr;
+			attr.ReadFromStream(rDiff2, IOStream::TimeOutInfinite);
+			attr.WriteToStream(rOut);
+		}
+		
+		// Get to the index of rDiff2b, and read the header
+		MoveStreamPositionToBlockIndex(rDiff2b);
+		file_BlockIndexHeader diff2IdxHdr;
+		if(!rDiff2b.ReadFullBuffer(&diff2IdxHdr, sizeof(diff2IdxHdr), 0))
+		{
+			THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+		}
+		if(ntohl(diff2IdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+		{
+			THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+		}
+		int64_t diff2NumBlocks = ntoh64(diff2IdxHdr.mNumBlocks);
+		int64_t diff2IndexEntriesStart = rDiff2b.GetPosition();
+		
+		// Then read all the entries
+		int64_t diff2FilePosition = rDiff2.GetPosition();
+		for(int64_t b = 0; b < diff2NumBlocks; ++b)
+		{
+			file_BlockIndexEntry e;
+			if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0))
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+			}
+
+			// What do to next about copying data
+			bool copyBlock = false;
+			int copySize = 0;
+			int64_t copyFrom = 0;
+			bool fromFileDiff1 = false;
+	
+			// Where's the block?
+			int64_t blockEn = ntoh64(e.mEncodedSize);
+			if(blockEn > 0)
+			{
+				// Block is present in this file -- copy to out
+				copyBlock = true;
+				copyFrom = diff2FilePosition;
+				copySize = (int)blockEn;
+				
+				// Move pointer onwards
+				diff2FilePosition += blockEn;
+			}
+			else
+			{
+				// Block isn't present here -- is it present in the old one?
+				int64_t blockIndex = 0 - blockEn;
+				if(blockIndex < 0 || blockIndex > diff1NumBlocks)
+				{
+					THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+				}
+				if(diff1BlockStartPositions[blockIndex] > 0)
+				{
+					// Block is in the old diff file, copy it across
+					copyBlock = true;
+					copyFrom = diff1BlockStartPositions[blockIndex];
+					int nb = blockIndex + 1;
+					while(diff1BlockStartPositions[nb] <= 0)
+					{
+						// This is safe, because the last entry will terminate it properly!
+						++nb;
+						ASSERT(nb <= diff1NumBlocks);
+					}
+					copySize = diff1BlockStartPositions[nb] - copyFrom;
+					fromFileDiff1 = true;
+				}
+			}
+			//TRACE4("%d %d %lld %d\n", copyBlock, copySize, copyFrom, fromFileDiff1);
+						
+			// Copy data to the output file?
+			if(copyBlock)
+			{
+				// Allocate enough space
+				if(bufferSize < copySize || buffer == 0)
+				{
+					// Free old block
+					if(buffer != 0)
+					{
+						::free(buffer);
+						buffer = 0;
+						bufferSize = 0;
+					}
+					// Allocate new block
+					buffer = ::malloc(copySize);
+					if(buffer == 0)
+					{
+						throw std::bad_alloc();
+					}
+					bufferSize = copySize;
+				}
+				ASSERT(bufferSize >= copySize);
+				
+				// Load in the data
+				if(fromFileDiff1)
+				{
+					rDiff1.Seek(copyFrom, IOStream::SeekType_Absolute);
+					if(!rDiff1.ReadFullBuffer(buffer, copySize, 0))
+					{
+						THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+					}
+				}
+				else
+				{
+					rDiff2.Seek(copyFrom, IOStream::SeekType_Absolute);
+					if(!rDiff2.ReadFullBuffer(buffer, copySize, 0))
+					{
+						THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+					}
+				}
+				// Write out data
+				rOut.Write(buffer, copySize);
+			}
+		}
+		
+		// Write the modified header
+		diff2IdxHdr.mOtherFileID = diff1IdxHdr.mOtherFileID;
+		rOut.Write(&diff2IdxHdr, sizeof(diff2IdxHdr));
+		
+		// Then we'll write out the index, reading the data again
+		rDiff2b.Seek(diff2IndexEntriesStart, IOStream::SeekType_Absolute);
+		for(int64_t b = 0; b < diff2NumBlocks; ++b)
+		{
+			file_BlockIndexEntry e;
+			if(!rDiff2b.ReadFullBuffer(&e, sizeof(e), 0))
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+			}
+	
+			// Where's the block?
+			int64_t blockEn = ntoh64(e.mEncodedSize);
+		
+			// If it's not in this file, it needs modification...
+			if(blockEn <= 0)
+			{
+				int64_t blockIndex = 0 - blockEn;
+				// In another file. Need to translate this against the other diff
+				if(diff1BlockStartPositions[blockIndex] > 0)
+				{
+					// Block is in the first diff file, stick in size
+					int nb = blockIndex + 1;
+					while(diff1BlockStartPositions[nb] <= 0)
+					{
+						// This is safe, because the last entry will terminate it properly!
+						++nb;
+						ASSERT(nb <= diff1NumBlocks);
+					}
+					int64_t size = diff1BlockStartPositions[nb] - diff1BlockStartPositions[blockIndex];
+					e.mEncodedSize = hton64(size);
+				}
+				else
+				{
+					// Block in the original file, use translated value
+					e.mEncodedSize = hton64(diff1BlockStartPositions[blockIndex]);
+				}
+			}
+			
+			// Write entry
+			rOut.Write(&e, sizeof(e));
+		}
+	}
+	catch(...)
+	{
+		// clean up
+		::free(diff1BlockStartPositions);
+		if(buffer != 0)
+		{
+			::free(buffer);
+		}
+		throw;
+	}
+	
+	// Clean up allocated memory
+	::free(diff1BlockStartPositions);
+	if(buffer != 0)
+	{
+		::free(buffer);
+	}
+}
+
diff --git a/lib/backupclient/BackupStoreFileCmbIdx.cpp b/lib/backupclient/BackupStoreFileCmbIdx.cpp
new file mode 100644
index 00000000..253001c2
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCmbIdx.cpp
@@ -0,0 +1,324 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileCmbIdx.cpp
+//		Purpose: Combine indicies of a delta file and the file it's a diff from.
+//		Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <string.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+// Hide from outside world
+namespace
+{
+
+class BSFCombinedIndexStream : public IOStream
+{
+public:
+	BSFCombinedIndexStream(IOStream *pDiff);
+	~BSFCombinedIndexStream();
+	
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+	virtual void Initialise(IOStream &rFrom);
+	
+private:
+	IOStream *mpDiff;
+	bool mIsInitialised;
+	bool mHeaderWritten;
+	file_BlockIndexHeader mHeader;
+	int64_t mNumEntriesToGo;
+	int64_t mNumEntriesInFromFile;
+	int64_t *mFromBlockSizes;		// NOTE: Entries in network byte order
+};
+
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::CombineFileIndices(IOStream &, IOStream &, bool)
+//		Purpose: Given a diff file and the file it's a diff from, return a stream from which
+//				 can be read the index of the combined file, without actually combining them.
+//				 The stream of the diff must have a lifetime greater than or equal to the
+//				 lifetime of the returned stream object. The full "from" file stream
+//				 only needs to exist during the actual function call.
+//				 If you pass in dodgy files which aren't related, then you will either
+//				 get an error or bad results. So don't do that.
+//				 If DiffIsIndexOnly is true, then rDiff is assumed to be a stream positioned
+//				 at the beginning of the block index. Similarly for FromIsIndexOnly.
+//				 WARNING: Reads of the returned streams with buffer sizes less than 64 bytes
+//				 will not return any data.
+//		Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::CombineFileIndices(IOStream &rDiff, IOStream &rFrom, bool DiffIsIndexOnly, bool FromIsIndexOnly)
+{
+	// Reposition file pointers?
+	if(!DiffIsIndexOnly)
+	{
+		MoveStreamPositionToBlockIndex(rDiff);
+	}
+	if(!FromIsIndexOnly)
+	{
+		MoveStreamPositionToBlockIndex(rFrom);
+	}
+
+	// Create object
+	std::auto_ptr<IOStream> stream(new BSFCombinedIndexStream(&rDiff));
+
+	// Initialise it
+	((BSFCombinedIndexStream *)stream.get())->Initialise(rFrom);
+
+	// And return the stream
+	return stream;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BSFCombinedIndexStream::BSFCombinedIndexStream()
+//		Purpose: Private class. Constructor.
+//		Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+BSFCombinedIndexStream::BSFCombinedIndexStream(IOStream *pDiff)
+	: mpDiff(pDiff),
+	  mIsInitialised(false),
+	  mHeaderWritten(false),
+	  mNumEntriesToGo(0),
+	  mNumEntriesInFromFile(0),
+	  mFromBlockSizes(0)
+{
+	ASSERT(mpDiff != 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BSFCombinedIndexStream::~BSFCombinedIndexStream()
+//		Purpose: Private class. Destructor.
+//		Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+BSFCombinedIndexStream::~BSFCombinedIndexStream()
+{
+	if(mFromBlockSizes != 0)
+	{
+		::free(mFromBlockSizes);
+		mFromBlockSizes = 0;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BSFCombinedIndexStream::Initialise(IOStream &)
+//		Purpose: Private class. Initalise from the streams (diff passed in constructor).
+//				 Both streams must have file pointer positioned at the block index.
+//		Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+void BSFCombinedIndexStream::Initialise(IOStream &rFrom)
+{
+	// Paranoia is good.
+	if(mIsInitialised)
+	{
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+	
+	// Look at the diff file: Read in the header
+	if(!mpDiff->ReadFullBuffer(&mHeader, sizeof(mHeader), 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+	if(ntohl(mHeader.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Read relevant data.
+	mNumEntriesToGo = ntoh64(mHeader.mNumBlocks);
+	
+	// Adjust a bit to reflect the fact it's no longer a diff
+	mHeader.mOtherFileID = hton64(0);
+	
+	// Now look at the from file: Read header
+	file_BlockIndexHeader fromHdr;
+	if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+	if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Then... allocate memory for the list of sizes
+	mNumEntriesInFromFile = ntoh64(fromHdr.mNumBlocks);
+	mFromBlockSizes = (int64_t*)::malloc(mNumEntriesInFromFile * sizeof(int64_t));
+	if(mFromBlockSizes == 0)
+	{
+		throw std::bad_alloc();
+	}
+	
+	// And read them all in!
+	for(int64_t b = 0; b < mNumEntriesInFromFile; ++b)
+	{
+		file_BlockIndexEntry e;
+		if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
+		{
+			THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+		}
+		
+		// Check that the from file isn't a delta in itself
+		if(ntoh64(e.mEncodedSize) <= 0)
+		{
+			THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+		}
+
+		// Store size (in network byte order)
+		mFromBlockSizes[b] = e.mEncodedSize;
+	}
+	
+	// Flag as initialised
+	mIsInitialised = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BSFCombinedIndexStream::Read(void *, int, int)
+//		Purpose: Private class. As interface.
+//		Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// Paranoia is good.
+	if(!mIsInitialised || mFromBlockSizes == 0 || mpDiff == 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+	
+	int written = 0;
+	
+	// Header output yet?
+	if(!mHeaderWritten)
+	{
+		// Enough space?
+		if(NBytes < (int)sizeof(mHeader)) return 0;
+		
+		// Copy in
+		::memcpy(pBuffer, &mHeader, sizeof(mHeader));
+		NBytes -= sizeof(mHeader);
+		written += sizeof(mHeader);
+	
+		// Flag it's done
+		mHeaderWritten = true;
+	}
+
+	// How many entries can be written?
+	int entriesToWrite = NBytes / sizeof(file_BlockIndexEntry);
+	if(entriesToWrite > mNumEntriesToGo)
+	{
+		entriesToWrite = mNumEntriesToGo;
+	}
+	
+	// Setup ready to go
+	file_BlockIndexEntry *poutput = (file_BlockIndexEntry*)(((uint8_t*)pBuffer) + written);
+
+	// Write entries
+	for(int b = 0; b < entriesToWrite; ++b)
+	{
+		if(!mpDiff->ReadFullBuffer(&(poutput[b]), sizeof(file_BlockIndexEntry), 0))
+		{
+			THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+		}
+		
+		// Does this need adjusting?
+		int s = ntoh64(poutput[b].mEncodedSize);
+		if(s <= 0)
+		{
+			// A reference to a block in the from file
+			int block = 0 - s;
+			ASSERT(block >= 0);
+			if(block >= mNumEntriesInFromFile)
+			{
+				// That's not good, the block doesn't exist
+				THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+			}
+			
+			// Adjust the entry in the buffer
+			poutput[b].mEncodedSize = mFromBlockSizes[block];	// stored in network byte order, no translation necessary
+		}
+	}
+	
+	// Update written count
+	written += entriesToWrite * sizeof(file_BlockIndexEntry);
+	mNumEntriesToGo -= entriesToWrite;
+	
+	return written;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BSFCombinedIndexStream::Write(const void *, int)
+//		Purpose: Private class. As interface.
+//		Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes)
+{
+	THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BSFCombinedIndexStream::StreamDataLeft()
+//		Purpose: Private class. As interface
+//		Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+bool BSFCombinedIndexStream::StreamDataLeft()
+{
+	return (!mHeaderWritten) || (mNumEntriesToGo > 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BSFCombinedIndexStream::StreamClosed()
+//		Purpose: Private class. As interface.
+//		Created: 8/7/04
+//
+// --------------------------------------------------------------------------
+bool BSFCombinedIndexStream::StreamClosed()
+{
+	return true;	// doesn't do writing
+}
+
diff --git a/lib/backupclient/BackupStoreFileCombine.cpp b/lib/backupclient/BackupStoreFileCombine.cpp
new file mode 100755
index 00000000..562a32d9
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCombine.cpp
@@ -0,0 +1,410 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileCombine.cpp
+//		Purpose: File combining for BackupStoreFile
+//		Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+#include "FileStream.h"
+
+#include "MemLeakFindOn.h"
+
+typedef struct
+{
+	int64_t mFilePosition;
+} FromIndexEntry;
+
+static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries);
+static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks, IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut);
+static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut);
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::CombineFile(IOStream &, IOStream &, IOStream &)
+//		Purpose: Where rDiff is a store file which is incomplete as a result of a
+//				 diffing operation, rFrom is the file it is diffed from, and 
+//				 rOut is the stream in which to place the result, the old file
+//				 and new file are combined into a file containing all the data.
+//				 rDiff2 is the same file as rDiff, opened again to get two
+//				 independent streams to the same file.
+//		Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut)
+{
+	// Read and copy the header.
+	file_StreamFormat hdr;
+	if(!rDiff.ReadFullBuffer(&hdr, sizeof(hdr), 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+	}
+	if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	// Copy
+	rOut.Write(&hdr, sizeof(hdr));
+	// Copy over filename and attributes
+	// BLOCK
+	{
+		BackupStoreFilename filename;
+		filename.ReadFromStream(rDiff, IOStream::TimeOutInfinite);
+		filename.WriteToStream(rOut);
+		StreamableMemBlock attr;
+		attr.ReadFromStream(rDiff, IOStream::TimeOutInfinite);
+		attr.WriteToStream(rOut);
+	}
+	
+	// Read the header for the From file
+	file_StreamFormat fromHdr;
+	if(!rFrom.ReadFullBuffer(&fromHdr, sizeof(fromHdr), 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+	}
+	if(ntohl(fromHdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	// Skip over the filename and attributes of the From file
+	// BLOCK
+	{
+		BackupStoreFilename filename2;
+		filename2.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+		int32_t size_s;
+		if(!rFrom.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+		{
+			THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+		}
+		int size = ntohl(size_s);
+		// Skip forward the size
+		rFrom.Seek(size, IOStream::SeekType_Relative);		
+	}
+	
+	// Allocate memory for the block index of the From file
+	int64_t fromNumBlocks = ntoh64(fromHdr.mNumBlocks);
+	// NOTE: An extra entry is required so that the length of the last block can be calculated
+	FromIndexEntry *pFromIndex = (FromIndexEntry*)::malloc((fromNumBlocks+1) * sizeof(FromIndexEntry));
+	if(pFromIndex == 0)
+	{
+		throw std::bad_alloc();
+	}
+	
+	try
+	{
+		// Load the index from the From file, calculating the offsets in the
+		// file as we go along, and enforce that everything should be present.
+		LoadFromIndex(rFrom, pFromIndex, fromNumBlocks);
+		
+		// Read in the block index of the Diff file in small chunks, and output data
+		// for each block, either from this file, or the other file.
+		int64_t diffNumBlocks = ntoh64(hdr.mNumBlocks);
+		CopyData(rDiff /* positioned at start of data */, rDiff2, diffNumBlocks, rFrom, pFromIndex, fromNumBlocks, rOut);
+		
+		// Read in the block index again, and output the new block index, simply
+		// filling in the sizes of blocks from the old file.
+		WriteNewIndex(rDiff, diffNumBlocks, pFromIndex, fromNumBlocks, rOut);
+		
+		// Free buffers
+		::free(pFromIndex);
+		pFromIndex = 0;
+	}
+	catch(...)
+	{
+		// Clean up
+		if(pFromIndex != 0)
+		{
+			::free(pFromIndex);
+			pFromIndex = 0;
+		}	
+		throw;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static LoadFromIndex(IOStream &, FromIndexEntry *, int64_t)
+//		Purpose: Static. Load the index from the From file
+//		Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void LoadFromIndex(IOStream &rFrom, FromIndexEntry *pIndex, int64_t NumEntries)
+{
+	ASSERT(pIndex != 0);
+	ASSERT(NumEntries >= 0);
+
+	// Get the starting point in the file
+	int64_t filePos = rFrom.GetPosition();
+	
+	// Jump to the end of the file to read the index
+	rFrom.Seek(0 - ((NumEntries * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+	
+	// Read block index header
+	file_BlockIndexHeader blkhdr;
+	if(!rFrom.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+	}
+	if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+		|| (int64_t)ntoh64(blkhdr.mNumBlocks) != NumEntries)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// And then the block entries
+	for(int64_t b = 0; b < NumEntries; ++b)
+	{
+		// Read
+		file_BlockIndexEntry en;
+		if(!rFrom.ReadFullBuffer(&en, sizeof(en), 0))
+		{
+			THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+		}
+		
+		// Add to list
+		pIndex[b].mFilePosition = filePos;
+
+		// Encoded size?
+		int64_t encodedSize = ntoh64(en.mEncodedSize);
+		// Check that the block is actually there
+		if(encodedSize <= 0)
+		{
+			THROW_EXCEPTION(BackupStoreException, OnCombineFromFileIsIncomplete)
+		}
+
+		// Move file pointer on
+		filePos += encodedSize;
+	}
+	
+	// Store the position in the very last entry, so the size of the last entry can be calculated
+	pIndex[NumEntries].mFilePosition = filePos;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static CopyData(IOStream &, IOStream &, int64_t, IOStream &, FromIndexEntry *, int64_t, IOStream &)
+//		Purpose: Static. Copy data from the Diff and From file to the out file.
+//				 rDiffData is at beginning of data.
+//				 rDiffIndex at any position.
+//				 rFrom is at any position.
+//				 rOut is after the header, ready for data
+//		Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void CopyData(IOStream &rDiffData, IOStream &rDiffIndex, int64_t DiffNumBlocks,
+	IOStream &rFrom, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut)
+{
+	// Jump to the end of the diff file to read the index
+	rDiffIndex.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+	
+	// Read block index header
+	file_BlockIndexHeader diffBlkhdr;
+	if(!rDiffIndex.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+	}
+	if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+		|| (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Record where the From file is
+	int64_t fromPos = rFrom.GetPosition();
+	
+	// Buffer data
+	void *buffer = 0;
+	int bufferSize = 0;
+	
+	try
+	{
+		// Read the blocks in!
+		for(int64_t b = 0; b < DiffNumBlocks; ++b)
+		{
+			// Read
+			file_BlockIndexEntry en;
+			if(!rDiffIndex.ReadFullBuffer(&en, sizeof(en), 0))
+			{
+				THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+			}
+			
+			// What's the size value stored in the entry
+			int64_t encodedSize = ntoh64(en.mEncodedSize);
+			
+			// How much data will be read?
+			int32_t blockSize = 0;
+			if(encodedSize > 0)
+			{
+				// The block is actually in the diff file
+				blockSize = encodedSize;
+			}
+			else
+			{
+				// It's in the from file. First, check to see if it's valid
+				int64_t blockIdx = (0 - encodedSize);
+				if(blockIdx > FromNumBlocks)
+				{
+					// References a block which doesn't actually exist
+					THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+				}
+				// Calculate size. This operation is safe because of the extra entry at the end
+				blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition;
+			}
+			ASSERT(blockSize > 0);
+			
+			// Make sure there's memory available to copy this
+			if(bufferSize < blockSize || buffer == 0)
+			{
+				// Free old block
+				if(buffer != 0)
+				{
+					::free(buffer);
+					buffer = 0;
+					bufferSize = 0;
+				}
+				// Allocate new block
+				buffer = ::malloc(blockSize);
+				if(buffer == 0)
+				{
+					throw std::bad_alloc();
+				}
+				bufferSize = blockSize;
+			}
+			ASSERT(bufferSize >= blockSize);
+			
+			// Load in data from one of the files
+			if(encodedSize > 0)
+			{
+				// Load from diff file
+				if(!rDiffData.ReadFullBuffer(buffer, blockSize, 0))
+				{
+					THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+				}				
+			}
+			else
+			{
+				// Locate and read the data from the from file
+				int64_t blockIdx = (0 - encodedSize);
+				// Seek if necessary
+				if(fromPos != pFromIndex[blockIdx].mFilePosition)
+				{
+					rFrom.Seek(pFromIndex[blockIdx].mFilePosition, IOStream::SeekType_Absolute);
+					fromPos = pFromIndex[blockIdx].mFilePosition;
+				}
+				// Read
+				if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
+				{
+					THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+				}
+				
+				// Update fromPos to current position
+				fromPos += blockSize;
+			}
+			
+			// Write data to out file
+			rOut.Write(buffer, blockSize);
+		}
+		
+		// Free buffer, if allocated
+		if(buffer != 0)
+		{
+			::free(buffer);
+			buffer = 0;
+		}
+	}
+	catch(...)
+	{
+		if(buffer != 0)
+		{
+			::free(buffer);
+			buffer = 0;
+		}
+		throw;
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static WriteNewIndex(IOStream &, int64_t, FromIndexEntry *, int64_t, IOStream &)
+//		Purpose: Write the index to the out file, just copying from the diff file and
+//				 adjusting the entries.
+//		Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+static void WriteNewIndex(IOStream &rDiff, int64_t DiffNumBlocks, FromIndexEntry *pFromIndex, int64_t FromNumBlocks, IOStream &rOut)
+{
+	// Jump to the end of the diff file to read the index
+	rDiff.Seek(0 - ((DiffNumBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)), IOStream::SeekType_End);
+	
+	// Read block index header
+	file_BlockIndexHeader diffBlkhdr;
+	if(!rDiff.ReadFullBuffer(&diffBlkhdr, sizeof(diffBlkhdr), 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+	}
+	if(ntohl(diffBlkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+		|| (int64_t)ntoh64(diffBlkhdr.mNumBlocks) != DiffNumBlocks)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Write it out with a blanked out other file ID
+	diffBlkhdr.mOtherFileID = hton64(0);
+	rOut.Write(&diffBlkhdr, sizeof(diffBlkhdr));
+	
+	// Rewrite the index
+	for(int64_t b = 0; b < DiffNumBlocks; ++b)
+	{
+		file_BlockIndexEntry en;
+		if(!rDiff.ReadFullBuffer(&en, sizeof(en), 0))
+		{
+			THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+		}
+		
+		// What's the size value stored in the entry
+		int64_t encodedSize = ntoh64(en.mEncodedSize);
+		
+		// Need to adjust it?
+		if(encodedSize <= 0)
+		{
+			// This actually refers to a block in the from file. So rewrite this.
+			int64_t blockIdx = (0 - encodedSize);
+			if(blockIdx > FromNumBlocks)
+			{
+				// References a block which doesn't actually exist
+				THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+			}
+			// Calculate size. This operation is safe because of the extra entry at the end
+			int32_t blockSize = pFromIndex[blockIdx + 1].mFilePosition - pFromIndex[blockIdx].mFilePosition;
+			// Then replace entry
+			en.mEncodedSize = hton64(((uint64_t)blockSize));
+		}
+		
+		// Write entry
+		rOut.Write(&en, sizeof(en));
+	}
+}
+
+
+
+
+
diff --git a/lib/backupclient/BackupStoreFileCryptVar.cpp b/lib/backupclient/BackupStoreFileCryptVar.cpp
new file mode 100755
index 00000000..eeed64e4
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCryptVar.cpp
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileCryptVar.cpp
+//		Purpose: Cryptographic keys for backup store files
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreFileWire.h"
+
+#include "MemLeakFindOn.h"
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncrypt;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecrypt;
+
+#ifndef PLATFORM_OLD_OPENSSL
+	CipherContext BackupStoreFileCryptVar::sAESEncrypt;
+	CipherContext BackupStoreFileCryptVar::sAESDecrypt;
+#endif
+
+// Default to blowfish
+CipherContext *BackupStoreFileCryptVar::spEncrypt = &BackupStoreFileCryptVar::sBlowfishEncrypt;
+uint8_t BackupStoreFileCryptVar::sEncryptCipherType = HEADER_BLOWFISH_ENCODING;
+
+CipherContext BackupStoreFileCryptVar::sBlowfishEncryptBlockEntry;
+CipherContext BackupStoreFileCryptVar::sBlowfishDecryptBlockEntry;
+
diff --git a/lib/backupclient/BackupStoreFileCryptVar.h b/lib/backupclient/BackupStoreFileCryptVar.h
new file mode 100755
index 00000000..00a34f71
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileCryptVar.h
@@ -0,0 +1,39 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileCryptVar.h
+//		Purpose: Cryptographic keys for backup store files
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILECRYPTVAR__H
+#define BACKUPSTOREFILECRYPTVAR__H
+
+#include "CipherContext.h"
+
+// Hide private static variables from the rest of the world by putting them
+// as static variables in a namespace.
+// -- don't put them as static class variables to avoid openssl/evp.h being
+// included all over the project.
+namespace BackupStoreFileCryptVar
+{
+	// Keys for the main file data
+	extern CipherContext sBlowfishEncrypt;
+	extern CipherContext sBlowfishDecrypt;
+	// Use AES when available
+#ifndef PLATFORM_OLD_OPENSSL
+	extern CipherContext sAESEncrypt;
+	extern CipherContext sAESDecrypt;
+#endif
+	// How encoding will be done
+	extern CipherContext *spEncrypt;
+	extern uint8_t sEncryptCipherType;
+
+	// Keys for the block indicies
+	extern CipherContext sBlowfishEncryptBlockEntry;
+	extern CipherContext sBlowfishDecryptBlockEntry;
+}
+
+#endif // BACKUPSTOREFILECRYPTVAR__H
+
diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp
new file mode 100755
index 00000000..b72ce328
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileDiff.cpp
@@ -0,0 +1,973 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileDiff.cpp
+//		Purpose: Functions relating to diffing BackupStoreFiles
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <map>
+#include <signal.h>
+#include <sys/time.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreFileEncodeStream.h"
+#include "BackupStoreConstants.h"
+#include "FileStream.h"
+#include "RollingChecksum.h"
+#include "MD5Digest.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+using namespace BackupStoreFileCreation;
+
+// By default, don't trace out details of the diff as we go along -- would fill up logs significantly.
+// But it's useful for the test.
+#ifndef NDEBUG
+	bool BackupStoreFile::TraceDetailsOfDiffProcess = false;
+#endif
+
+static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis);
+static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
+static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]);
+static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable);
+static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks);
+static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile);
+
+// Avoid running on too long with diffs
+static int sMaximumDiffTime = 10;		// maximum time to spend diffing
+static bool sDiffTimedOut = false;
+static bool sSetTimerSignelHandler = false;
+static void TimerSignalHandler(int signal);
+static void StartDiffTimer();
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::SetMaximumDiffingTime(int)
+//		Purpose: Sets the maximum time to spend diffing, in seconds. Time is
+//				 process virutal time.
+//		Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::SetMaximumDiffingTime(int Seconds)
+{
+	sMaximumDiffTime = Seconds;
+	TRACE1("Set maximum diffing time to %d seconds\n", Seconds);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &)
+//		Purpose: Move the file pointer in this stream to just before the block index.
+//				 Assumes that the stream is at the beginning, seekable, and
+//				 reading from the stream is OK.
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream)
+{
+	// Size of file
+	int64_t fileSize = rStream.BytesLeftToRead();
+
+	// Get header
+	file_StreamFormat hdr;
+
+	// Read the header
+	if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
+	{
+		// Couldn't read header
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+
+	// Check magic number
+	if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		&& ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+		)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Work out where the index is
+	int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+	int64_t blockHeaderPosFromEnd = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
+	
+	// Sanity check
+	if(blockHeaderPosFromEnd > static_cast<int64_t>(fileSize - sizeof(file_StreamFormat)))
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Seek to that position
+	rStream.Seek(0 - blockHeaderPosFromEnd, IOStream::SeekType_End);
+	
+	// Done. Stream now in right position (as long as the file is formatted correctly)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *)
+//		Purpose: Similar to EncodeFile, but takes the object ID of the file it's
+//				 diffing from, and the index of the blocks in a stream. It'll then
+//				 calculate which blocks can be reused from that old file.
+//				 The timeout is the timeout value for reading the diff block index.
+//				 If pIsCompletelyDifferent != 0, it will be set to true if the
+//				 the two files are completely different (do not share any block), false otherwise.
+//				 
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff(const char *Filename, int64_t ContainerID,
+	const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex,
+	int Timeout, int64_t *pModificationTime, bool *pIsCompletelyDifferent)
+{
+	// Is it a symlink?
+	{
+		struct stat st;
+		if(::lstat(Filename, &st) != 0)
+		{
+			THROW_EXCEPTION(CommonException, OSFileError)
+		}
+		if((st.st_mode & S_IFLNK) == S_IFLNK)
+		{
+			// Don't do diffs for symlinks
+			if(pIsCompletelyDifferent != 0)
+			{
+				*pIsCompletelyDifferent = true;
+			}
+			return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+		}
+	}
+
+	// Load in the blocks
+	BlocksAvailableEntry *pindex = 0;
+	int64_t blocksInIndex = 0;
+	bool canDiffFromThis = false;
+	LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis);
+	//TRACE1("Diff: Blocks in index: %lld\n", blocksInIndex);
+	
+	if(!canDiffFromThis)
+	{
+		// Don't do diffing...
+		if(pIsCompletelyDifferent != 0)
+		{
+			*pIsCompletelyDifferent = true;
+		}
+		return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+	}
+	
+	// Pointer to recipe we're going to create
+	BackupStoreFileEncodeStream::Recipe *precipe = 0;
+
+	// Start the timeout timer, so that the operation doesn't continue for ever
+	StartDiffTimer();
+	
+	try
+	{
+		// Find which sizes should be scanned
+		int32_t sizesToScan[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+		FindMostUsedSizes(pindex, blocksInIndex, sizesToScan);
+		
+		// Flag for reporting to the user
+		bool completelyDifferent;
+			
+		// BLOCK
+		{
+			// Search the file to find matching blocks
+			std::map<int64_t, int64_t> foundBlocks; // map of offset in file to index in block index
+			int64_t sizeOfInputFile = 0;
+			// BLOCK
+			{
+				FileStream file(Filename);
+				// Get size of file
+				sizeOfInputFile = file.BytesLeftToRead();
+				// Find all those lovely matching blocks
+				SearchForMatchingBlocks(file, foundBlocks, pindex, blocksInIndex, sizesToScan);
+				
+				// Is it completely different?
+				completelyDifferent = (foundBlocks.size() == 0);
+			}
+			
+			// Create a recipe -- if the two files are completely different, don't put the from file ID in the recipe.
+			precipe = new BackupStoreFileEncodeStream::Recipe(pindex, blocksInIndex, completelyDifferent?(0):(DiffFromObjectID));
+			BlocksAvailableEntry *pindexKeptRef = pindex;	// we need this later, but must set pindex == 0 now, because of exceptions
+			pindex = 0;		// Recipe now has ownership
+			
+			// Fill it in
+			GenerateRecipe(*precipe, pindexKeptRef, blocksInIndex, foundBlocks, sizeOfInputFile);
+		}
+		// foundBlocks no longer required
+		
+		// Create the stream
+		std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
+	
+		// Do the initial setup
+		((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime);
+		precipe = 0;	// Stream has taken ownership of this
+		
+		// Tell user about completely different status?
+		if(pIsCompletelyDifferent != 0)
+		{
+			*pIsCompletelyDifferent = completelyDifferent;
+		}
+	
+		// Return the stream for the caller
+		return stream;
+	}
+	catch(...)
+	{
+		// cleanup
+		if(pindex != 0)
+		{
+			::free(pindex);
+			pindex = 0;
+		}
+		if(precipe != 0)
+		{
+			delete precipe;
+			precipe = 0;
+		}
+		throw;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static LoadIndex(IOStream &, int64_t, BlocksAvailableEntry **, int64_t, bool &)
+//		Purpose: Read in an index, and decrypt, and store in the in memory block format.
+//				 rCanDiffFromThis is set to false if the version of the from file is too old.
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis)
+{
+	// Reset
+	rNumBlocksOut = 0;
+	rCanDiffFromThis = false;
+	
+	// Read header
+	file_BlockIndexHeader hdr;
+	if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
+	{
+		// Couldn't read header
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+	// Check against backwards comptaibility stuff
+	if(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0))
+	{
+		// Won't diff against old version
+		
+		// Absorb rest of stream
+		char buffer[2048];
+		while(rBlockIndex.StreamDataLeft())
+		{
+			rBlockIndex.Read(buffer, sizeof(buffer), 1000 /* 1 sec timeout */);
+		}
+		
+		// Tell caller
+		rCanDiffFromThis = false;
+		return;
+	}
+#endif
+
+	// Check magic
+	if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1))
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// Check that we're not trying to diff against a file which references blocks from another file
+	if(((int64_t)ntoh64(hdr.mOtherFileID)) != 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
+	}
+
+	// Mark as an acceptable diff.
+	rCanDiffFromThis = true;
+
+	// Get basic information
+	int64_t numBlocks = ntoh64(hdr.mNumBlocks);
+	uint64_t entryIVBase = ntoh64(hdr.mEntryIVBase);
+	
+	//TODO: Verify that these sizes look reasonable
+	
+	// Allocate space for the index
+	BlocksAvailableEntry *pindex = (BlocksAvailableEntry*)::malloc(sizeof(BlocksAvailableEntry) * numBlocks);
+	if(pindex == 0)
+	{
+		throw std::bad_alloc();
+	}
+	
+	try
+	{	
+		for(int64_t b = 0; b < numBlocks; ++b)
+		{
+			// Read an entry from the stream
+			file_BlockIndexEntry entry;
+			if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
+			{
+				// Couldn't read entry
+				THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+			}	
+		
+			// Calculate IV for this entry
+			uint64_t iv = entryIVBase;
+			iv += b;
+			// Network byte order
+			iv = hton64(iv);
+			sBlowfishDecryptBlockEntry.SetIV(&iv);			
+			
+			// Decrypt the encrypted section
+			file_BlockIndexEntryEnc entryEnc;
+			int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+					entry.mEnEnc, sizeof(entry.mEnEnc));
+			if(sectionSize != sizeof(entryEnc))
+			{
+				THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+			}
+
+			// Check that we're not trying to diff against a file which references blocks from another file
+			if(((int64_t)ntoh64(entry.mEncodedSize)) <= 0)
+			{
+				THROW_EXCEPTION(BackupStoreException, CannotDiffAnIncompleteStoreFile)
+			}
+			
+			// Store all the required information
+			pindex[b].mpNextInHashList = 0;	// hash list not set up yet
+			pindex[b].mSize = ntohl(entryEnc.mSize);
+			pindex[b].mWeakChecksum = ntohl(entryEnc.mWeakChecksum);
+			::memcpy(pindex[b].mStrongChecksum, entryEnc.mStrongChecksum, sizeof(pindex[b].mStrongChecksum));
+		}
+		
+		// Store index pointer for called
+		ASSERT(ppIndex != 0);
+		*ppIndex = pindex;
+
+		// Store number of blocks for caller
+		rNumBlocksOut = numBlocks;	
+
+	}
+	catch(...)
+	{
+		// clean up and send the exception along its way
+		::free(pindex);
+		throw;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static FindMostUsedSizes(BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+//		Purpose: Finds the most commonly used block sizes in the index
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+{
+	// Array for lengths
+	int64_t sizeCounts[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES];
+
+	// Set arrays to lots of zeros (= unused entries)
+	for(int l = 0; l < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++l)
+	{
+		Sizes[l] = 0;
+		sizeCounts[l] = 0;
+	}
+
+	// Array for collecting sizes
+	std::map<int32_t, int64_t> foundSizes;
+	
+	// Run through blocks and make a count of the entries
+	for(int64_t b = 0; b < NumBlocks; ++b)
+	{
+		// Only if the block size is bigger than the minimum size we'll scan for
+		if(pIndex[b].mSize > BACKUP_FILE_DIFF_MIN_BLOCK_SIZE)
+		{
+			// Find entry?
+			std::map<int32_t, int64_t>::const_iterator f(foundSizes.find(pIndex[b].mSize));
+			if(f != foundSizes.end())
+			{
+				// Increment existing entry
+				foundSizes[pIndex[b].mSize] = foundSizes[pIndex[b].mSize] + 1;
+			}
+			else
+			{
+				// New entry
+				foundSizes[pIndex[b].mSize] = 1;
+			}
+		}
+	}
+	
+	// Make the block sizes
+	for(std::map<int32_t, int64_t>::const_iterator i(foundSizes.begin()); i != foundSizes.end(); ++i)
+	{
+		// Find the position of the size in the array
+		for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
+		{
+			if(i->second > sizeCounts[t])
+			{
+				// Then this size belong before this entry -- shuffle them up
+				for(int s = (BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1); s >= t; --s)
+				{
+					Sizes[s] = Sizes[s-1];
+					sizeCounts[s] = sizeCounts[s-1];
+				}
+				
+				// Insert this size
+				Sizes[t] = i->first;
+				sizeCounts[t] = i->second;
+				
+				// Shouldn't do any more searching
+				break;
+			}
+		}
+	}
+	
+	// trace the size table in debug builds
+#ifndef NDEBUG
+	if(BackupStoreFile::TraceDetailsOfDiffProcess)
+	{
+		for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t)
+		{
+			TRACE3("Diff block size %d: %d (count = %lld)\n", t, Sizes[t], sizeCounts[t]);
+		}
+	}
+#endif
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static SearchForMatchingBlocks(IOStream &, std::map<int64_t, int64_t> &, BlocksAvailableEntry *, int64_t, int32_t[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+//		Purpose: Find the matching blocks within the file.
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks,
+	BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES])
+{
+	// Allocate the hash lookup table
+	BlocksAvailableEntry **phashTable = (BlocksAvailableEntry **)::malloc(sizeof(BlocksAvailableEntry *) * (64*1024));
+
+	// Choose a size for the buffer, just a little bit more than the maximum block size
+	int32_t bufSize = Sizes[0];
+	for(int z = 1; z < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++z)
+	{
+		if(Sizes[z] > bufSize) bufSize = Sizes[z];
+	}
+	bufSize += 4;
+	ASSERT(bufSize > Sizes[0]);
+	ASSERT(bufSize > 0);
+	if(bufSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	
+	// TODO: Use buffered file class
+	// Because we read in the file a scanned block size at a time, it is likely to be
+	// inefficient. Probably will be much better to use a buffering IOStream class which
+	// reads data in at the size of the filesystem block size.
+
+	// Allocate the buffers.
+	uint8_t *pbuffer0 = (uint8_t *)::malloc(bufSize);
+	uint8_t *pbuffer1 = (uint8_t *)::malloc(bufSize);
+	try
+	{
+		// Check buffer allocation
+		if(pbuffer0 == 0 || pbuffer1 == 0 || phashTable == 0)
+		{
+			// If a buffer got alloocated, it will be cleaned up in the catch block
+			throw std::bad_alloc();
+		}
+		
+		// Flag to abort the run, if too many blocks are found -- avoid using
+		// huge amounts of processor time when files contain many similar blocks.
+		bool abortSearch = false;
+		
+		// Search for each block size in turn
+		// NOTE: Do the smallest size first, so that the scheme for adding
+		// entries in the found list works as expected and replaces smallers block
+		// with larger blocks when it finds matches at the same offset in the file.
+		for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s)
+		{
+			ASSERT(Sizes[s] <= bufSize);
+			//TRACE2("Diff pass %d, for block size %d\n", s, Sizes[s]);
+			
+			// Check we haven't finished
+			if(Sizes[s] == 0)
+			{
+				// empty entry, try next size
+				continue;
+			}
+			
+			// Set up the hash table entries
+			SetupHashTable(pIndex, NumBlocks, Sizes[s], phashTable);
+		
+			// Shift file position to beginning
+			rFile.Seek(0, IOStream::SeekType_Absolute);
+			
+			// Read first block
+			if(rFile.Read(pbuffer0, Sizes[s]) != Sizes[s])
+			{
+				// Size of file too short to match -- do next size
+				continue;
+			}
+			
+			// Setup block pointers
+			uint8_t *beginnings = pbuffer0;
+			uint8_t *endings = pbuffer1;
+			int offset = 0;
+			
+			// Calculate the first checksum, ready for rolling
+			RollingChecksum rolling(beginnings, Sizes[s]);
+			
+			// Then roll, until the file is exhausted
+			int64_t fileBlockNumber = 0;
+			int rollOverInitialBytes = 0;
+			while(true)
+			{
+				// Load in another block of data, and record how big it is
+				int bytesInEndings = rFile.Read(endings, Sizes[s]);
+
+				// Skip any bytes from a previous matched block
+				while(rollOverInitialBytes > 0 && offset < bytesInEndings)
+				{
+					rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+					++offset;
+					--rollOverInitialBytes;
+				}
+
+				while(offset < bytesInEndings)
+				{
+					// Put is current checksum in hash list?
+					uint16_t hash = rolling.GetComponentForHashing();
+					if(phashTable[hash] != 0)
+					{
+						if(SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks))
+						{
+							// Block matched, roll the checksum forward to the next block without doing
+							// any more comparisons, because these are pointless (as any more matches will be ignored when
+							// the receipe is generated) and just take up valuable processor time. Edge cases are
+							// especially nasty, using huge amounts of time and memory.
+							int skip = Sizes[s];
+							while(offset < bytesInEndings && skip > 0)
+							{
+								rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+								++offset;
+								--skip;
+							}
+							// Not all the bytes necessary will have been skipped, so get them
+							// skipped after the next block is loaded.
+							rollOverInitialBytes = skip;
+							
+							// End this loop, so the final byte isn't used again
+							break;
+						}
+						
+						if(static_cast<int64_t>(rFoundBlocks.size()) > (NumBlocks * BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE)
+							|| sDiffTimedOut)
+						{
+							abortSearch = true;
+							break;
+						}
+					}
+					
+					// Roll checksum forward
+					rolling.RollForward(beginnings[offset], endings[offset], Sizes[s]);
+				
+					// Increment offset
+					++offset;
+				}
+				
+				if(abortSearch) break;
+				
+				// Finished?
+				if(bytesInEndings != Sizes[s])
+				{
+					// No more data in file -- check the final block
+					// (Do a copy and paste of 5 lines of code instead of introducing a comparison for
+					// each byte of the file)
+					uint16_t hash = rolling.GetComponentForHashing();
+					if(phashTable[hash] != 0)
+					{
+						SecondStageMatch(phashTable[hash], hash, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks);
+					}
+
+					// finish
+					break;
+				}
+				
+				// Switch buffers, reset offset
+				beginnings = endings;
+				endings = (beginnings == pbuffer0)?(pbuffer1):(pbuffer0);	// ie the other buffer
+				offset = 0;
+
+				// And count the blocks which have been done
+				++fileBlockNumber;
+			}
+
+			if(abortSearch) break;
+		}
+		
+		// Free buffers and hash table
+		::free(pbuffer1);
+		pbuffer1 = 0;
+		::free(pbuffer0);
+		pbuffer0 = 0;
+		::free(phashTable);
+		phashTable = 0;
+	}
+	catch(...)
+	{
+		// Cleanup and throw
+		if(pbuffer1 != 0) ::free(pbuffer1);
+		if(pbuffer0 != 0) ::free(pbuffer0);
+		if(phashTable != 0) ::free(phashTable);
+		throw;
+	}
+	
+#ifndef NDEBUG
+	if(BackupStoreFile::TraceDetailsOfDiffProcess)
+	{
+		// Trace out the found blocks in debug mode
+		TRACE0("Diff: list of found blocks\n======== ======== ======== ========\n  Offset   BlkIdx     Size Movement\n");
+		for(std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin()); i != rFoundBlocks.end(); ++i)
+		{
+			int64_t orgLoc = 0;
+			for(int64_t b = 0; b < i->second; ++b)
+			{
+				orgLoc += pIndex[b].mSize;
+			}
+			TRACE4("%8lld %8lld %8lld %8lld\n", i->first, i->second,
+				(int64_t)(pIndex[i->second].mSize), i->first - orgLoc);
+		}
+		TRACE0("======== ======== ======== ========\n");
+	}
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static SetupHashTable(BlocksAvailableEntry *, int64_t, in32_t, BlocksAvailableEntry **)
+//		Purpose: Set up the hash table ready for a scan
+//		Created: 14/1/04
+//
+// --------------------------------------------------------------------------
+static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable)
+{
+	// Set all entries in the hash table to zero
+	::memset(pHashTable, 0, (sizeof(BlocksAvailableEntry *) * (64*1024)));
+
+	// Scan through the blocks, building the hash table
+	for(int64_t b = 0; b < NumBlocks; ++b)
+	{
+		// Only look at the required block size
+		if(pIndex[b].mSize == BlockSize)
+		{
+			// Get the value under which to hash this entry
+			uint16_t hash = RollingChecksum::ExtractHashingComponent(pIndex[b].mWeakChecksum);
+			
+			// Already present in table?
+			if(pHashTable[hash] != 0)
+			{
+				//TRACE1("Another hash entry for %d found\n", hash);
+				// Yes -- need to set the pointer in this entry to the current entry to build the linked list
+				pIndex[b].mpNextInHashList = pHashTable[hash];
+			}
+
+			// Put a pointer to this entry in the hash table
+			pHashTable[hash] = pIndex + b;
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static bool SecondStageMatch(xxx)
+//		Purpose: When a match in the hash table is found, scan for second stage match using strong checksum.
+//		Created: 14/1/04
+//
+// --------------------------------------------------------------------------
+static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, uint16_t Hash, uint8_t *pBeginnings, uint8_t *pEndings,
+	int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks)
+{
+	// Check parameters
+	ASSERT(pBeginnings != 0);
+	ASSERT(pEndings != 0);
+	ASSERT(Offset >= 0);
+	ASSERT(BlockSize > 0);
+	ASSERT(pFirstInHashList != 0);
+	ASSERT(pIndex != 0);
+
+	// Calculate the strong MD5 digest for this block
+	MD5Digest strong;
+	// Add the data from the beginnings
+	strong.Add(pBeginnings + Offset, BlockSize - Offset);
+	// Add any data from the endings
+	if(Offset > 0)
+	{
+		strong.Add(pEndings, Offset);
+	}
+	strong.Finish();
+	
+	// Then go through the entries in the hash list, comparing with the strong digest calculated
+	BlocksAvailableEntry *scan = pFirstInHashList;
+	//TRACE0("second stage match\n");
+	while(scan != 0)
+	{
+		//TRACE3("scan size %d, block size %d, hash %d\n", scan->mSize, BlockSize, Hash);
+		ASSERT(scan->mSize == BlockSize);
+		ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == Hash);
+	
+		// Compare?
+		if(strong.DigestMatches(scan->mStrongChecksum))
+		{
+			//TRACE0("Match!\n");
+			// Found! Add to list of found blocks...
+			int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset;
+			int64_t blockIndex = (scan - pIndex);	// pointer arthmitic is frowed apon. But most efficient way of doing it here -- alternative is to use more memory
+			
+			// Because we search for smallest blocks first, if there's an entry there
+			// we'll just be replacing it with a larger one, which is great news.
+			rFoundBlocks[fileOffset] = blockIndex;
+			
+			// No point in searching further, report success
+			return true;
+		}
+	
+		// Next
+		scan = scan->mpNextInHashList;
+	}
+	
+	// Not matched
+	return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static GenerateRecipe(BackupStoreFileEncodeStream::Recipe &, BlocksAvailableEntry *, int64_t, std::map<int64_t, int64_t> &)
+//		Purpose: Fills in the recipe from the found block list
+//		Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex,
+		int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile)
+{
+	// NOTE: This function could be a lot more sophisiticated. For example, if
+	// a small block overlaps a big block like this
+	//   ****
+	//      *******************************
+	// then the small block will be used, not the big one. But it'd be better to
+	// just ignore the small block and keep the big one. However, some stats should
+	// be gathered about real world files before writing complex code which might
+	// go wrong.
+
+	// Initialise a blank instruction
+	BackupStoreFileEncodeStream::RecipeInstruction instruction;
+	#define RESET_INSTRUCTION			\
+	instruction.mSpaceBefore = 0;		\
+	instruction.mBlocks = 0;			\
+	instruction.mpStartBlock = 0;
+	RESET_INSTRUCTION
+
+	// First, a special case for when there are no found blocks
+	if(rFoundBlocks.size() == 0)
+	{
+		// No blocks, just a load of space
+		instruction.mSpaceBefore = SizeOfInputFile;
+		rRecipe.push_back(instruction);	
+	
+		#ifndef NDEBUG
+		if(BackupStoreFile::TraceDetailsOfDiffProcess)
+		{
+			TRACE1("Diff: Default recipe generated, %lld bytes of file\n", SizeOfInputFile);
+		}
+		#endif
+		
+		// Don't do anything
+		return;
+	}
+
+	// Current location
+	int64_t loc = 0;
+
+	// Then iterate through the list, generating the recipe
+	std::map<int64_t, int64_t>::const_iterator i(rFoundBlocks.begin());
+	ASSERT(i != rFoundBlocks.end());	// check logic
+
+	// Counting for debug tracing
+#ifndef NDEBUG
+	int64_t debug_NewBytesFound = 0;
+	int64_t debug_OldBlocksUsed = 0;
+#endif
+	
+	for(; i != rFoundBlocks.end(); ++i)
+	{
+		// Remember... map is (position in file) -> (index of block in pIndex)
+		
+		if(i->first < loc)
+		{
+			// This block overlaps the last one
+			continue;
+		}
+		else if(i->first > loc)
+		{
+			// There's a gap between the end of the last thing and this block.
+			// If there's an instruction waiting, push it onto the list
+			if(instruction.mSpaceBefore != 0 || instruction.mpStartBlock != 0)
+			{
+				rRecipe.push_back(instruction);
+			}
+			// Start a new instruction, with the gap ready
+			RESET_INSTRUCTION
+			instruction.mSpaceBefore = i->first - loc;
+			// Move location forward to match
+			loc += instruction.mSpaceBefore;
+#ifndef NDEBUG
+			debug_NewBytesFound += instruction.mSpaceBefore;
+#endif
+		}
+		
+		// First, does the current instruction need pushing back, because this block is not
+		// sequential to the last one?
+		if(instruction.mpStartBlock != 0 && (pIndex + i->second) != (instruction.mpStartBlock + instruction.mBlocks))
+		{
+			rRecipe.push_back(instruction);
+			RESET_INSTRUCTION
+		}
+		
+		// Add in this block
+		if(instruction.mpStartBlock == 0)
+		{
+			// This block starts a new instruction
+			instruction.mpStartBlock = pIndex + i->second;
+			instruction.mBlocks = 1;
+		}
+		else
+		{
+			// It continues the previous section of blocks
+			instruction.mBlocks += 1;
+		}
+
+#ifndef NDEBUG
+		debug_OldBlocksUsed++;
+#endif
+
+		// Move location forward
+		loc += pIndex[i->second].mSize;
+	}
+	
+	// Push the last instruction generated
+	rRecipe.push_back(instruction);
+	
+	// Is there any space left at the end which needs sending?
+	if(loc != SizeOfInputFile)
+	{
+		RESET_INSTRUCTION
+		instruction.mSpaceBefore = SizeOfInputFile - loc;
+#ifndef NDEBUG
+		debug_NewBytesFound += instruction.mSpaceBefore;
+#endif
+		rRecipe.push_back(instruction);		
+	}
+	
+	// dump out the recipe
+#ifndef NDEBUG
+	TRACE2("Diff: %lld new bytes found, %lld old blocks used\n", debug_NewBytesFound, debug_OldBlocksUsed);
+	if(BackupStoreFile::TraceDetailsOfDiffProcess)
+	{
+		TRACE1("Diff: Recipe generated (size %d)\n======== ========= ========\nSpace b4 FirstBlk  NumBlks\n", rRecipe.size());
+		{
+			for(unsigned int e = 0; e < rRecipe.size(); ++e)
+			{
+				char b[64];
+				sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex));
+				TRACE3("%8lld %s %8lld\n", rRecipe[e].mSpaceBefore, (rRecipe[e].mpStartBlock == 0)?"       -":b, (int64_t)rRecipe[e].mBlocks);
+			}
+		}
+		TRACE0("======== ========= ========\n");
+	}
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static TimerSignalHandler(int)
+//		Purpose: Signal handler
+//		Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+void TimerSignalHandler(int signal)
+{
+	sDiffTimedOut = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static StartDiffTimer()
+//		Purpose: Starts the diff timeout timer
+//		Created: 19/3/04
+//
+// --------------------------------------------------------------------------
+void StartDiffTimer()
+{
+	// Set timer signal handler
+	if(!sSetTimerSignelHandler)
+	{
+		::signal(SIGVTALRM, TimerSignalHandler);
+		sSetTimerSignelHandler = true;
+	}
+
+	struct itimerval timeout;
+	// Don't want this to repeat
+	timeout.it_interval.tv_sec = 0;
+	timeout.it_interval.tv_usec = 0;
+	// Single timeout after the specified number of seconds
+	timeout.it_value.tv_sec = sMaximumDiffTime;
+	timeout.it_value.tv_usec = 0;
+	// Set timer
+	if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0)
+	{
+		TRACE0("WARNING: couldn't set diff timeout\n");
+	}
+	
+	// Unset flag (last thing)
+	sDiffTimedOut = false;
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp
new file mode 100755
index 00000000..20e1fd80
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp
@@ -0,0 +1,675 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileEncodeStream.cpp
+//		Purpose: Implement stream-based file encoding for the backup store
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "BackupStoreFileEncodeStream.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BoxTime.h"
+#include "BackupClientFileAttributes.h"
+#include "FileStream.h"
+#include "RollingChecksum.h"
+#include "Random.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::BackupStoreFileEncodeStream
+//		Purpose: Constructor (opens file)
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::BackupStoreFileEncodeStream()
+	: mpRecipe(0),
+	  mpFile(0),
+	  mStatus(Status_Header),
+	  mSendData(true),
+	  mTotalBlocks(0),
+	  mAbsoluteBlockNumber(-1),
+	  mInstructionNumber(-1),
+	  mNumBlocks(0),
+	  mCurrentBlock(-1),
+	  mCurrentBlockEncodedSize(0),
+	  mPositionInCurrentBlock(0),
+	  mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
+	  mLastBlockSize(0),
+	  mpRawBuffer(0),
+	  mAllocatedBufferSize(0),
+	  mEntryIVBase(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+//		Purpose: Destructor
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
+{
+	// Free buffers
+	if(mpRawBuffer)
+	{
+		::free(mpRawBuffer);
+		mpRawBuffer = 0;
+	}
+	
+	// Close the file, which we might have open
+	if(mpFile)
+	{
+		delete mpFile;
+		mpFile = 0;
+	}
+	
+	// Free the recipe
+	if(mpRecipe != 0)
+	{
+		delete mpRecipe;
+		mpRecipe = 0;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *)
+//		Purpose: Reads file information, and builds file header reading for sending.
+//				 Takes ownership of the Recipe.
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEncodeStream::Recipe *pRecipe,
+	int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime)
+{
+	// Pointer to a blank recipe which we might create
+	BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0;
+
+	try
+	{
+		// Get file attributes
+		box_time_t modTime = 0;
+		int64_t fileSize = 0;
+		BackupClientFileAttributes attr;
+		attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime,
+			0 /* not interested in attr mod time */, &fileSize);
+	
+		// Might need to create a blank recipe...
+		if(pRecipe == 0)
+		{
+			pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0);
+			
+			BackupStoreFileEncodeStream::RecipeInstruction instruction;
+			instruction.mSpaceBefore = fileSize; 	// whole file
+			instruction.mBlocks = 0;				// no blocks
+			instruction.mpStartBlock = 0;			// no block
+			pblankRecipe->push_back(instruction);		
+
+			pRecipe = pblankRecipe;
+		}
+	
+		// Tell caller?
+		if(pModificationTime != 0)
+		{
+			*pModificationTime = modTime;
+		}
+		
+		// Go through each instruction in the recipe and work out how many blocks
+		// it will add, and the max clear size of these blocks
+		int maxBlockClearSize = 0;
+		for(uint64_t inst = 0; inst < pRecipe->size(); ++inst)
+		{
+			if((*pRecipe)[inst].mSpaceBefore > 0)
+			{
+				// Calculate the number of blocks the space before requires
+				int64_t numBlocks;
+				int32_t blockSize, lastBlockSize;
+				CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize);
+				// Add to accumlated total
+				mTotalBlocks += numBlocks;
+				// Update maximum clear size
+				if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize;
+				if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize;
+			}
+			
+			// Add number of blocks copied from the previous file
+			mTotalBlocks += (*pRecipe)[inst].mBlocks;
+			
+			// Check for bad things
+			if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0))
+			{
+				THROW_EXCEPTION(BackupStoreException, Internal)
+			}
+
+			// Run through blocks to get the max clear size
+			for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b)
+			{
+				if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize;
+			}
+		}
+		
+		// Send data? (symlinks don't have any data in them)
+		mSendData = !attr.IsSymLink();
+
+		// If not data is being sent, then the max clear block size is zero
+		if(!mSendData)
+		{
+			maxBlockClearSize = 0;
+		}
+		
+		// Header
+		file_StreamFormat hdr;
+		hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1);
+		hdr.mNumBlocks = (mSendData)?(hton64(mTotalBlocks)):(0);
+		hdr.mContainerID = hton64(ContainerID);
+		hdr.mModificationTime = hton64(modTime);
+		// add a bit to make it harder to tell what's going on -- try not to give away too much info about file size
+		hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128);
+		hdr.mOptions = 0;		// no options defined yet
+		
+		// Write header to stream
+		mData.Write(&hdr, sizeof(hdr));
+		
+		// Write filename to stream
+		rStoreFilename.WriteToStream(mData);
+		
+		// Write attributes to stream
+		attr.WriteToStream(mData);
+	
+		// Allocate some buffers for writing data
+		if(mSendData)
+		{
+			// Open the file
+			mpFile = new FileStream(Filename);
+		
+			// Work out the largest possible block required for the encoded data
+			mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize);
+			
+			// Then allocate two blocks of this size
+			mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize);
+			if(mpRawBuffer == 0)
+			{
+				throw std::bad_alloc();
+			}
+#ifndef NDEBUG
+			// In debug builds, make sure that the reallocation code is exercised.
+			mEncodedBuffer.Allocate(mAllocatedBufferSize / 4);
+#else
+			mEncodedBuffer.Allocate(mAllocatedBufferSize);
+#endif
+		}
+		else
+		{
+			// Write an empty block index for the symlink
+			file_BlockIndexHeader blkhdr;
+			blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+			blkhdr.mOtherFileID = hton64(0);	// not other file ID
+			blkhdr.mEntryIVBase = hton64(0);
+			blkhdr.mNumBlocks = hton64(0);
+			mData.Write(&blkhdr, sizeof(blkhdr));
+		}
+	
+		// Ready for reading
+		mData.SetForReading();
+		
+		// Update stats
+		BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize;
+		
+		// Finally, store the pointer to the recipe, when we know exceptions won't occur
+		mpRecipe = pRecipe;
+	}
+	catch(...)
+	{
+		// Clean up any blank recipe
+		if(pblankRecipe != 0)
+		{
+			delete pblankRecipe;
+			pblankRecipe = 0;
+		}
+		throw;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &)
+//		Purpose: Calculates the sizes of blocks in a section of the file
+//		Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut)
+{
+	// How many blocks, and how big?
+	rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2;
+	do
+	{
+		rBlockSizeOut *= 2;
+		
+		rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut;
+		
+	} while(rBlockSizeOut <= BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER);
+	
+	// Last block size
+	rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut);
+	
+	// Avoid small blocks?
+	if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN
+		&& rNumBlocksOut > 1)
+	{
+		// Add the small bit of data to the last block
+		--rNumBlocksOut;
+		rLastBlockSizeOut += rBlockSizeOut;
+	}
+	
+	// checks!
+	ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize);
+	//TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut);
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::Read(void *, int, int)
+//		Purpose: As interface -- generates encoded file data on the fly from the raw file
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// Check there's something to do.
+	if(mStatus == Status_Finished)
+	{
+		return 0;
+	}
+
+	int bytesToRead = NBytes;
+	uint8_t *buffer = (uint8_t*)pBuffer;
+	
+	while(bytesToRead > 0 && mStatus != Status_Finished)
+	{
+		if(mStatus == Status_Header || mStatus == Status_BlockListing)
+		{
+			// Header or block listing phase -- send from the buffered stream
+		
+			// Send bytes from the data buffer
+			int b = mData.Read(buffer, bytesToRead, Timeout);
+			bytesToRead -= b;
+			buffer += b;
+			
+			// Check to see if all the data has been used from this stream
+			if(!mData.StreamDataLeft())
+			{
+				// Yes, move on to next phase (or finish, if there's no file data)
+				if(!mSendData)
+				{
+					mStatus = Status_Finished;
+				}
+				else
+				{
+					// Reset the buffer so it can be used for the next phase
+					mData.Reset();
+		
+					// Get buffer ready for index?
+					if(mStatus == Status_Header)
+					{
+						// Just finished doing the stream header, create the block index header
+						file_BlockIndexHeader blkhdr;
+						blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
+						ASSERT(mpRecipe != 0);
+						blkhdr.mOtherFileID = hton64(mpRecipe->GetOtherFileID());
+						blkhdr.mNumBlocks = hton64(mTotalBlocks);
+						
+						// Generate the IV base
+						Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase));
+						blkhdr.mEntryIVBase = hton64(mEntryIVBase);
+						
+						mData.Write(&blkhdr, sizeof(blkhdr));
+					}
+				
+					++mStatus;
+				}
+			}
+		}
+		else if(mStatus == Status_Blocks)
+		{
+			// Block sending phase
+			
+			if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize)
+			{
+				// Next block!
+				++mCurrentBlock;
+				++mAbsoluteBlockNumber;
+				if(mCurrentBlock >= mNumBlocks)
+				{
+					// Output extra blocks for this instruction and move forward in file
+					if(mInstructionNumber >= 0)
+					{
+						SkipPreviousBlocksInInstruction();
+					}
+				
+					// Is there another instruction to go?
+					++mInstructionNumber;
+					
+					// Skip instructions which don't contain any data
+					while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size())
+						&& (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0)
+					{
+						SkipPreviousBlocksInInstruction();
+						++mInstructionNumber;
+					}
+					
+					if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size()))
+					{
+						// End of blocks, go to next phase
+						++mStatus;
+						
+						// Set the data to reading so the index can be written
+						mData.SetForReading();
+					}
+					else
+					{
+						// Get ready for this instruction
+						SetForInstruction();
+					}
+				}
+
+				// Can't use 'else' here as SetForInstruction() will change this
+				if(mCurrentBlock < mNumBlocks)
+				{
+					EncodeCurrentBlock();
+				}
+			}
+			
+			// Send data from the current block (if there's data to send)
+			if(mPositionInCurrentBlock < mCurrentBlockEncodedSize)
+			{
+				// How much data to put in the buffer?
+				int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock;
+				if(s > bytesToRead) s = bytesToRead;
+				
+				// Copy it in
+				::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s);
+				
+				// Update variables
+				bytesToRead -= s;
+				buffer += s;
+				mPositionInCurrentBlock += s;
+			}
+		}
+		else
+		{
+			// Should never get here, as it'd be an invalid status
+			ASSERT(false);
+		}
+	}
+	
+	// Add encoded size to stats
+	BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead);
+	
+	// Return size of data to caller
+	return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction()
+//		Purpose: Private. Stores the blocks of the old file referenced in the current
+//				 instruction into the index and skips over the data in the file
+//		Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction()
+{
+	// Check something is necessary
+	if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0)
+	{
+		return;
+	}
+
+	// Index of the first block in old file (being diffed from)
+	int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock);
+	
+	int64_t sizeToSkip = 0;
+
+	for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b)
+	{
+		// Update stats
+		BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+	
+		// Store the entry
+		StoreBlockIndexEntry(0 - (firstIndex + b),
+			(*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize,
+			(*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum,
+			(*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum);	
+
+		// Increment the absolute block number -- kept encryption IV in sync
+		++mAbsoluteBlockNumber;
+		
+		// Add the size of this block to the size to skip
+		sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
+	}
+	
+	// Move forward in the stream
+	mpFile->Seek(sizeToSkip, IOStream::SeekType_Relative);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::SetForInstruction()
+//		Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe
+//		Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::SetForInstruction()
+{
+	// Calculate block sizes
+	CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize);
+	
+	// Set variables
+	mCurrentBlock = 0;
+	mCurrentBlockEncodedSize = 0;
+	mPositionInCurrentBlock = 0;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::EncodeCurrentBlock()
+//		Purpose: Private. Encodes the current block, and writes the block data to the index
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::EncodeCurrentBlock()
+{
+	// How big is the block, raw?
+	int blockRawSize = mBlockSize;
+	if(mCurrentBlock == (mNumBlocks - 1))
+	{
+		blockRawSize = mLastBlockSize;
+	}
+	ASSERT(blockRawSize < mAllocatedBufferSize);
+
+	// Check file open
+	if(mpFile == 0)
+	{
+		// File should be open, but isn't. So logical error.
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+	
+	// Read the data in
+	if(!mpFile->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */))
+	{
+		// TODO: Do something more intelligent, and abort this upload because the file
+		// has changed
+		THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer)
+	}
+	
+	// Encode it
+	mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer);
+	
+	//TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize);
+	
+	// Create block listing data -- generate checksums
+	RollingChecksum weakChecksum(mpRawBuffer, blockRawSize);
+	MD5Digest strongChecksum;
+	strongChecksum.Add(mpRawBuffer, blockRawSize);
+	strongChecksum.Finish();
+
+	// Add entry to the index
+	StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData());
+	
+	// Set vars to reading this block
+	mPositionInCurrentBlock = 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *)
+//		Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream.
+//		Created: 16/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum)
+{
+	// First, the encrypted section
+	file_BlockIndexEntryEnc entryEnc;
+	entryEnc.mSize = htonl(ClearSize);
+	entryEnc.mWeakChecksum = htonl(WeakChecksum);
+	::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum));
+
+	// Then the clear section
+	file_BlockIndexEntry entry;
+	entry.mEncodedSize = hton64(((uint64_t)EncSizeOrBlkIndex));
+	
+	// Then encrypt the encryted section
+	// Generate the IV from the block number
+	if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase))
+	{
+		THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements)
+	}
+	uint64_t iv = mEntryIVBase;
+	iv += mAbsoluteBlockNumber;
+	// Convert to network byte order before encrypting with it, so that restores work on
+	// platforms with different endiannesses.
+	iv = hton64(iv);
+	sBlowfishEncryptBlockEntry.SetIV(&iv);
+
+	// Encode the data
+	int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc));
+	if(encodedSize != sizeof(entry.mEnEnc))
+	{
+		THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
+	}
+
+	// Save to data block for sending at the end of the stream
+	mData.Write(&entry, sizeof(entry));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::Write(const void *, int)
+//		Purpose: As interface. Exceptions.
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes)
+{
+	THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::StreamDataLeft()
+//		Purpose: As interface -- end of stream reached?
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamDataLeft()
+{
+	return (mStatus != Status_Finished);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::StreamClosed()
+//		Purpose: As interface
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFileEncodeStream::StreamClosed()
+{
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t)
+//		Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted
+//		Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex,
+		int64_t NumBlocksInIndex, int64_t OtherFileID)
+	: mpBlockIndex(pBlockIndex),
+	  mNumBlocksInIndex(NumBlocksInIndex),
+	  mOtherFileID(OtherFileID)
+{
+	ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0))
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFileEncodeStream::Recipe::~Recipe()
+//		Purpose: Destructor
+//		Created: 15/1/04
+//
+// --------------------------------------------------------------------------
+BackupStoreFileEncodeStream::Recipe::~Recipe()
+{
+	// Free the block index, if there is one
+	if(mpBlockIndex != 0)
+	{
+		::free(mpBlockIndex);
+	}
+}
+
+
+
+
diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h
new file mode 100755
index 00000000..1c748798
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileEncodeStream.h
@@ -0,0 +1,127 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileEncodeStream.h
+//		Purpose: Implement stream-based file encoding for the backup store
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEENCODESTREAM__H
+#define BACKUPSTOREFILEENCODESTREAM__H
+
+#include <vector>
+
+#include "IOStream.h"
+#include "BackupStoreFilename.h"
+#include "CollectInBufferStream.h"
+#include "MD5Digest.h"
+#include "BackupStoreFile.h"
+
+namespace BackupStoreFileCreation
+{
+	// Diffing and creation of files share some implementation details.
+	typedef struct _BlocksAvailableEntry
+	{
+		struct _BlocksAvailableEntry *mpNextInHashList;
+		int32_t mSize;			// size in clear
+		uint32_t mWeakChecksum;	// weak, rolling checksum
+		uint8_t mStrongChecksum[MD5Digest::DigestLength];	// strong digest based checksum
+	} BlocksAvailableEntry;
+
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreFileEncodeStream
+//		Purpose: Encode a file into a stream
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+class BackupStoreFileEncodeStream : public IOStream
+{
+public:
+	BackupStoreFileEncodeStream();
+	~BackupStoreFileEncodeStream();
+	
+	typedef struct
+	{
+		int64_t mSpaceBefore;				// amount of bytes which aren't taken out of blocks which go
+		int32_t mBlocks;					// number of block to reuse, starting at this one
+		BackupStoreFileCreation::BlocksAvailableEntry *mpStartBlock;	// may be null
+	} RecipeInstruction;
+	
+	class Recipe : public std::vector<RecipeInstruction>
+	{
+		// NOTE: This class is rather tied in with the implementation of diffing.
+	public:
+		Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, int64_t NumBlocksInIndex,
+			int64_t OtherFileID = 0);
+		~Recipe();
+	
+		int64_t GetOtherFileID() {return mOtherFileID;}
+		int64_t BlockPtrToIndex(BackupStoreFileCreation::BlocksAvailableEntry *pBlock)
+		{
+			return pBlock - mpBlockIndex;
+		}
+	
+	private:
+		BackupStoreFileCreation::BlocksAvailableEntry *mpBlockIndex;
+		int64_t mNumBlocksInIndex;
+		int64_t mOtherFileID;
+	};
+	
+	void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime);
+
+	virtual int Read(void *pBuffer, int NBytes, int Timeout);
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+private:
+	enum
+	{
+		Status_Header = 0,
+		Status_Blocks = 1,
+		Status_BlockListing = 2,
+		Status_Finished = 3
+	};
+	
+private:
+	void EncodeCurrentBlock();
+	void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut);
+	void SkipPreviousBlocksInInstruction();
+	void SetForInstruction();
+	void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum);
+
+private:
+	Recipe *mpRecipe;
+	IOStream *mpFile;					// source file
+	CollectInBufferStream mData;		// buffer for header and index entries
+	int mStatus;
+	bool mSendData;						// true if there's file data to send (ie not a symlink)
+	int64_t mTotalBlocks;				// Total number of blocks in the file
+	int64_t mAbsoluteBlockNumber;		// The absolute block number currently being output
+	// Instruction number
+	int64_t mInstructionNumber;
+	// All the below are within the current instruction
+	int64_t mNumBlocks;					// number of blocks. Last one will be a different size to the rest in most cases
+	int64_t mCurrentBlock;
+	int32_t mCurrentBlockEncodedSize;
+	int32_t mPositionInCurrentBlock;	// for reading out
+	int32_t mBlockSize;					// Basic block size of most of the blocks in the file
+	int32_t mLastBlockSize;				// the size (unencoded) of the last block in the file
+	// Buffers
+	uint8_t *mpRawBuffer;				// buffer for raw data
+	BackupStoreFile::EncodingBuffer mEncodedBuffer;
+										// buffer for encoded data
+	int32_t mAllocatedBufferSize;		// size of above two allocated blocks
+	uint64_t mEntryIVBase;				// base for block entry IV
+};
+
+
+
+#endif // BACKUPSTOREFILEENCODESTREAM__H
+
diff --git a/lib/backupclient/BackupStoreFileRevDiff.cpp b/lib/backupclient/BackupStoreFileRevDiff.cpp
new file mode 100644
index 00000000..f1dc52d8
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileRevDiff.cpp
@@ -0,0 +1,258 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileRevDiff.cpp
+//		Purpose: Reverse a patch, to build a new patch from new to old files
+//		Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreException.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreFilename.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::ReverseDiffFile(IOStream &, IOStream &, IOStream &, IOStream &, int64_t)
+//		Purpose: Reverse a patch, to build a new patch from new to old files. Takes
+//				 two independent copies to the From file, for efficiency.
+//		Created: 12/7/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::ReverseDiffFile(IOStream &rDiff, IOStream &rFrom, IOStream &rFrom2, IOStream &rOut, int64_t ObjectIDOfFrom, bool *pIsCompletelyDifferent)
+{
+	// Read and copy the header from the from file to the out file -- beginnings of the patch
+	file_StreamFormat hdr;
+	if(!rFrom.ReadFullBuffer(&hdr, sizeof(hdr), 0))
+	{
+		THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+	}
+	if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+	}
+	// Copy
+	rOut.Write(&hdr, sizeof(hdr));
+	// Copy over filename and attributes
+	// BLOCK
+	{
+		BackupStoreFilename filename;
+		filename.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+		filename.WriteToStream(rOut);
+		StreamableMemBlock attr;
+		attr.ReadFromStream(rFrom, IOStream::TimeOutInfinite);
+		attr.WriteToStream(rOut);
+	}
+	
+	// Build an index of common blocks.
+	// For each block in the from file, we want to know it's index in the 
+	// diff file. Allocate memory for this information.
+	int64_t fromNumBlocks = ntoh64(hdr.mNumBlocks);
+	int64_t *pfromIndexInfo = (int64_t*)::malloc(fromNumBlocks * sizeof(int64_t));
+	if(pfromIndexInfo == 0)
+	{
+		throw std::bad_alloc();
+	}
+
+	// Buffer data
+	void *buffer = 0;
+	int bufferSize = 0;	
+	
+	// flag
+	bool isCompletelyDifferent = true;
+	
+	try
+	{
+		// Initialise the index to be all 0, ie not filled in yet
+		for(int64_t i = 0; i < fromNumBlocks; ++i)
+		{
+			pfromIndexInfo[i] = 0;
+		}
+	
+		// Within the from file, skip to the index
+		MoveStreamPositionToBlockIndex(rDiff);
+
+		// Read in header of index
+		file_BlockIndexHeader diffIdxHdr;
+		if(!rDiff.ReadFullBuffer(&diffIdxHdr, sizeof(diffIdxHdr), 0))
+		{
+			THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+		}
+		if(ntohl(diffIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+		{
+			THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+		}
+
+		// And then read in each entry
+		int64_t diffNumBlocks = ntoh64(diffIdxHdr.mNumBlocks);
+		for(int64_t b = 0; b < diffNumBlocks; ++b)
+		{
+			file_BlockIndexEntry e;
+			if(!rDiff.ReadFullBuffer(&e, sizeof(e), 0))
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+			}
+
+			// Where's the block?
+			int64_t blockEn = ntoh64(e.mEncodedSize);
+			if(blockEn > 0)
+			{
+				// Block is in the delta file, is ignored for now -- not relevant to rebuilding the from file
+			}
+			else
+			{
+				// Block is in the original file, store which block it is in this file
+				int64_t fromIndex = 0 - blockEn;
+				if(fromIndex < 0 || fromIndex >= fromNumBlocks)
+				{
+					THROW_EXCEPTION(BackupStoreException, IncompatibleFromAndDiffFiles)
+				}
+				
+				// Store information about where it is in the new file
+				// NOTE: This is slight different to how it'll be stored in the final index.
+				pfromIndexInfo[fromIndex] = -1 - b;
+			}
+		}
+		
+		// Open the index for the second copy of the from file
+		MoveStreamPositionToBlockIndex(rFrom2);
+
+		// Read in header of index
+		file_BlockIndexHeader fromIdxHdr;
+		if(!rFrom2.ReadFullBuffer(&fromIdxHdr, sizeof(fromIdxHdr), 0))
+		{
+			THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+		}
+		if(ntohl(fromIdxHdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
+			|| ntoh64(fromIdxHdr.mOtherFileID) != 0)
+		{
+			THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+		}
+
+		// So, we can now start building the data in the file
+		int64_t filePosition = rFrom.GetPosition();
+		for(int64_t b = 0; b < fromNumBlocks; ++b)
+		{
+			// Read entry from from index
+			file_BlockIndexEntry e;
+			if(!rFrom2.ReadFullBuffer(&e, sizeof(e), 0))
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+			}
+
+			// Get size
+			int64_t blockSize = hton64(e.mEncodedSize);
+			if(blockSize < 0)
+			{
+				THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
+			}
+		
+			// Copy this block?
+			if(pfromIndexInfo[b] == 0)
+			{
+				// Copy it, first move to file location
+				rFrom.Seek(filePosition, IOStream::SeekType_Absolute);
+				
+				// Make sure there's memory available to copy this
+				if(bufferSize < blockSize || buffer == 0)
+				{
+					// Free old block
+					if(buffer != 0)
+					{
+						::free(buffer);
+						buffer = 0;
+						bufferSize = 0;
+					}
+					// Allocate new block
+					buffer = ::malloc(blockSize);
+					if(buffer == 0)
+					{
+						throw std::bad_alloc();
+					}
+					bufferSize = blockSize;
+				}
+				ASSERT(bufferSize >= blockSize);
+				
+				// Copy the block
+				if(!rFrom.ReadFullBuffer(buffer, blockSize, 0))
+				{
+					THROW_EXCEPTION(BackupStoreException, FailedToReadBlockOnCombine)
+				}
+				rOut.Write(buffer, blockSize);
+
+				// Store the size
+				pfromIndexInfo[b] = blockSize;
+			}
+			else
+			{
+				// Block isn't needed, so it's not completely different
+				isCompletelyDifferent = false;
+			}
+			filePosition += blockSize;
+		}
+		
+		// Then write the index, modified header first
+		fromIdxHdr.mOtherFileID = isCompletelyDifferent?0:(hton64(ObjectIDOfFrom));
+		rOut.Write(&fromIdxHdr, sizeof(fromIdxHdr));
+
+		// Move to start of index entries
+		rFrom.Seek(filePosition + sizeof(file_BlockIndexHeader), IOStream::SeekType_Absolute);
+		
+		// Then copy modified entries
+		for(int64_t b = 0; b < fromNumBlocks; ++b)
+		{
+			// Read entry from from index
+			file_BlockIndexEntry e;
+			if(!rFrom.ReadFullBuffer(&e, sizeof(e), 0))
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+			}
+			
+			// Modify...
+			int64_t s = pfromIndexInfo[b];
+			// Adjust to reflect real block index (remember 0 has a different meaning here)
+			if(s < 0) ++s;
+			// Insert
+			e.mEncodedSize = hton64(s);
+			// Write
+			rOut.Write(&e, sizeof(e));
+		}
+	}
+	catch(...)
+	{
+		::free(pfromIndexInfo);
+		if(buffer != 0)
+		{
+			::free(buffer);
+		}
+		throw;
+	}
+
+	// Free memory used (oh for finally {} blocks)
+	::free(pfromIndexInfo);
+	if(buffer != 0)
+	{
+		::free(buffer);
+	}
+	
+	// return completely different flag
+	if(pIsCompletelyDifferent != 0)
+	{
+		*pIsCompletelyDifferent = isCompletelyDifferent;
+	}
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFileWire.h b/lib/backupclient/BackupStoreFileWire.h
new file mode 100755
index 00000000..5b1bc819
--- /dev/null
+++ b/lib/backupclient/BackupStoreFileWire.h
@@ -0,0 +1,74 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFileWire.h
+//		Purpose: On the wire / disc formats for backup store files
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILEWIRE__H
+#define BACKUPSTOREFILEWIRE__H
+
+#include "MD5Digest.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+	int32_t mMagicValue;	// also the version number
+	int64_t mNumBlocks;		// number of blocks contained in the file
+	int64_t mContainerID;
+	int64_t mModificationTime;
+	int32_t mMaxBlockClearSize;		// Maximum clear size that can be expected for a block
+	int32_t mOptions;		// bitmask of options used
+	// Then a BackupStoreFilename
+	// Then a BackupClientFileAttributes
+} file_StreamFormat;
+
+typedef struct
+{
+	int32_t mMagicValue;	// different magic value
+	int64_t mOtherFileID;	// the file ID of the 'other' file which may be referenced by the index
+	uint64_t mEntryIVBase;	// base value for block IV
+	int64_t mNumBlocks;		// repeat of value in file header
+} file_BlockIndexHeader;
+
+typedef struct
+{
+	int32_t mSize;			// size in clear
+	uint32_t mWeakChecksum;	// weak, rolling checksum
+	uint8_t mStrongChecksum[MD5Digest::DigestLength];	// strong digest based checksum
+} file_BlockIndexEntryEnc;
+
+typedef struct
+{
+	union
+	{
+		int64_t mEncodedSize;		// size encoded, if > 0
+		int64_t mOtherBlockIndex;	// 0 - block number in other file, if <= 0
+	};
+	uint8_t mEnEnc[sizeof(file_BlockIndexEntryEnc)];	// Encoded section
+} file_BlockIndexEntry;
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// header for blocks of compressed data in files
+#define HEADER_CHUNK_IS_COMPRESSED		1	// bit
+#define HEADER_ENCODING_SHIFT			1	// shift value
+#define HEADER_BLOWFISH_ENCODING		1	// value stored in bits 1 -- 7
+#define HEADER_AES_ENCODING				2	// value stored in bits 1 -- 7
+
+
+#endif // BACKUPSTOREFILEWIRE__H
+
diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp
new file mode 100755
index 00000000..fbfe3313
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilename.cpp
@@ -0,0 +1,279 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFilename.cpp
+//		Purpose: Filename for the backup store
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilename.h"
+#include "Protocol.h"
+#include "BackupStoreException.h"
+#include "IOStream.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::BackupStoreFilename()
+//		Purpose: Default constructor -- creates an invalid filename
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &)
+//		Purpose: Copy constructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy)
+	: BackupStoreFilename_base(rToCopy)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::~BackupStoreFilename()
+//		Purpose: Destructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilename::~BackupStoreFilename()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::CheckValid(bool)
+//		Purpose: Checks the encoded filename for validity
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const
+{
+	bool ok = true;
+	
+	if(size() < 2)
+	{
+		// Isn't long enough to have a header
+		ok = false;
+	}
+	else
+	{
+		// Check size is consistent
+		unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(*this);
+		if(dsize != size())
+		{
+			ok = false;
+		}
+		
+		// And encoding is an accepted value
+		unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this);
+		if(encoding < Encoding_Min || encoding > Encoding_Max)
+		{
+			ok = false;
+		}
+	}
+	
+	// Exception?
+	if(!ok && ExceptionIfInvalid)
+	{
+		THROW_EXCEPTION(BackupStoreException, InvalidBackupStoreFilename)
+	}
+
+	return ok;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::ReadFromProtocol(Protocol &)
+//		Purpose: Reads the filename from the protocol object
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol)
+{
+	// Read the header
+	char hdr[2];
+	rProtocol.Read(hdr, 2);
+	
+	// How big is it?
+	int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+	
+	// Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+	std::string data;
+	rProtocol.Read(data, dsize - 2);
+	
+	// assign to this string, storing the header and the extra data
+	assign(hdr, 2);
+	append(data.c_str(), data.size());
+	
+	// Check it
+	CheckValid();
+	
+	// Alert derived classes
+	EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::WriteToProtocol(Protocol &)
+//		Purpose: Writes the filename to the protocol object
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const
+{
+	CheckValid();
+	
+	rProtocol.Write(c_str(), size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::ReadFromStream(IOStream &)
+//		Purpose: Reads the filename from a stream
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout)
+{
+	// Read the header
+	char hdr[2];
+	if(!rStream.ReadFullBuffer(hdr, 2, 0 /* not interested in bytes read if this fails */, Timeout))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+	}
+	
+	// How big is it?
+	unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(hdr);
+	
+	// Assume most filenames are small
+	char buf[256];
+	if(dsize < sizeof(buf))
+	{
+		// Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+		if(!rStream.ReadFullBuffer(buf + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+		{
+			THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+		}
+		// Copy in header
+		buf[0] = hdr[0]; buf[1] = hdr[1];
+
+		// assign to this string, storing the header and the extra data
+		assign(buf, dsize);
+	}
+	else
+	{
+		// Block of memory to hold it
+		MemoryBlockGuard<char*> dataB(dsize+2);
+		char *data = dataB;
+
+		// Fetch rest of data, relying on the Protocol to error on stupidly large sizes for us
+		if(!rStream.ReadFullBuffer(data + 2, dsize - 2, 0 /* not interested in bytes read if this fails */, Timeout))
+		{
+			THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
+		}
+		// Copy in header
+		data[0] = hdr[0]; data[1] = hdr[1];
+
+		// assign to this string, storing the header and the extra data
+		assign(data, dsize);
+	}
+	
+	// Check it
+	CheckValid();
+	
+	// Alert derived classes
+	EncodedFilenameChanged();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::WriteToStream(IOStream &)
+//		Purpose: Writes the filename to a stream
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::WriteToStream(IOStream &rStream) const
+{
+	CheckValid();
+	
+	rStream.Write(c_str(), size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::EncodedFilenameChanged()
+//		Purpose: The encoded filename stored has changed
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::EncodedFilenameChanged()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::IsEncrypted()
+//		Purpose: Returns true if the filename is stored using an encrypting encoding
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreFilename::IsEncrypted() const
+{
+	return BACKUPSTOREFILENAME_GET_ENCODING(*this) != Encoding_Clear;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilename::SetAsClearFilename(const char *)
+//		Purpose: Sets this object to be a valid filename, but with a filename in the clear.
+//				 Used on the server to create filenames when there's no way of encrypting it.
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilename::SetAsClearFilename(const char *Clear)
+{
+	// Make std::string from the clear name
+	std::string toEncode(Clear);
+
+	// Make an encoded string
+	char hdr[2];
+	BACKUPSTOREFILENAME_MAKE_HDR(hdr, toEncode.size()+2, Encoding_Clear);
+	std::string encoded(hdr, 2);
+	encoded += toEncode;
+	ASSERT(encoded.size() == toEncode.size() + 2);
+	
+	// Store the encoded string
+	assign(encoded);
+	
+	// Stuff which must be done
+	EncodedFilenameChanged();
+	CheckValid(false);
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h
new file mode 100755
index 00000000..4c951b6f
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilename.h
@@ -0,0 +1,85 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFilename.h
+//		Purpose: Filename for the backup store
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAME__H
+#define BACKUPSTOREFILENAME__H
+
+#include <string>
+
+class Protocol;
+class IOStream;
+
+// #define BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+// don't define this -- the problem of memory usage still appears without this.
+// It's just that this class really showed up the problem. Instead, malloc allocation
+// is globally defined in BoxPlatform.h, for troublesome libraries.
+
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+	// Use a malloc_allocated string, because the STL default allocators really screw up with
+	// memory allocation, particularly with this class.
+	// Makes a few things a bit messy and inefficient with conversions.
+	// Given up using this, and use global malloc allocation instead, but thought it
+	// worth leaving this code in just in case it's useful for the future.
+	typedef std::basic_string<char, std::string_char_traits<char>, std::malloc_alloc> BackupStoreFilename_base;
+	// If this is changed, change GetClearFilename() back to returning a reference.
+#else
+	typedef std::string BackupStoreFilename_base;
+#endif // PLATFORM_HAVE_STL_MALLOC_ALLOC
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreFilename
+//		Purpose: Filename for the backup store
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilename : public BackupStoreFilename_base
+{
+public:
+	BackupStoreFilename();
+	BackupStoreFilename(const BackupStoreFilename &rToCopy);
+	virtual ~BackupStoreFilename();
+
+	bool CheckValid(bool ExceptionIfInvalid = true) const;
+	
+	void ReadFromProtocol(Protocol &rProtocol);
+	void WriteToProtocol(Protocol &rProtocol) const;
+	
+	void ReadFromStream(IOStream &rStream, int Timeout);
+	void WriteToStream(IOStream &rStream) const;
+
+	void SetAsClearFilename(const char *Clear);
+
+	// Check that it's encrypted
+	bool IsEncrypted() const;
+	
+	// These enumerated types belong in the base class so 
+	// the CheckValid() function can make sure that the encoding
+	// is a valid encoding
+	enum
+	{
+		Encoding_Min = 1,
+		Encoding_Clear = 1,
+		Encoding_Blowfish = 2,
+		Encoding_Max = 2
+	};
+
+protected:
+	virtual void EncodedFilenameChanged();
+};
+
+// On the wire utilities for class and derived class
+#define BACKUPSTOREFILENAME_GET_SIZE(hdr)		(( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2)
+#define BACKUPSTOREFILENAME_GET_ENCODING(hdr)	(((hdr)[0]) & 0x3)
+
+#define BACKUPSTOREFILENAME_MAKE_HDR(hdr, size, encoding)		{uint16_t h = (((uint16_t)size) << 2) | (encoding); ((hdr)[0]) = h & 0xff; ((hdr)[1]) = h >> 8;}
+
+#endif // BACKUPSTOREFILENAME__H
+
diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp
new file mode 100755
index 00000000..3069f612
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilenameClear.cpp
@@ -0,0 +1,335 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFilenameClear.cpp
+//		Purpose: BackupStoreFilenames in the clear
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreFilenameClear.h"
+#include "BackupStoreException.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// Hide private variables from the rest of the world
+namespace
+{
+	int sEncodeMethod = BackupStoreFilename::Encoding_Clear;
+	CipherContext sBlowfishEncrypt;
+	CipherContext sBlowfishDecrypt;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::BackupStoreFilenameClear()
+//		Purpose: Default constructor, creates an invalid filename
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &)
+//		Purpose: Creates a filename, encoding from the given string
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const std::string &rToEncode)
+{
+	SetClearFilename(rToEncode);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &)
+//		Purpose: Copy constructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy)
+	: BackupStoreFilename(rToCopy),
+	  mClearFilename(rToCopy.mClearFilename)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+//		Purpose: Copy from base class
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::BackupStoreFilenameClear(const BackupStoreFilename &rToCopy)
+	: BackupStoreFilename(rToCopy)
+{
+	// Will get a clear filename when it's required
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::~BackupStoreFilenameClear()
+//		Purpose: Destructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+BackupStoreFilenameClear::~BackupStoreFilenameClear()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::GetClearFilename()
+//		Purpose: Get the unencoded filename
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+const std::string BackupStoreFilenameClear::GetClearFilename() const
+{
+	MakeClearAvailable();
+	// When modifying, remember to change back to reference return if at all possible
+	// -- returns an object rather than a reference to allow easy use with other code.
+	return std::string(mClearFilename.c_str(), mClearFilename.size());
+}
+#else
+const std::string &BackupStoreFilenameClear::GetClearFilename() const
+{
+	MakeClearAvailable();
+	return mClearFilename;
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::SetClearFilename(const std::string &)
+//		Purpose: Encode and make available the clear filename
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetClearFilename(const std::string &rToEncode)
+{
+	// Only allow Blowfish encodings
+	if(sEncodeMethod != Encoding_Blowfish)
+	{
+		THROW_EXCEPTION(BackupStoreException, FilenameEncryptionNotSetup)
+	}
+
+	// Make an encoded string with blowfish encryption
+	EncryptClear(rToEncode, sBlowfishEncrypt, Encoding_Blowfish);
+		
+	// Store the clear filename
+	mClearFilename.assign(rToEncode.c_str(), rToEncode.size());
+
+	// Make sure we did the right thing
+	if(!CheckValid(false))
+	{
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::MakeClearAvailable()
+//		Purpose: Private. Make sure the clear filename is available
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::MakeClearAvailable() const
+{
+	if(!mClearFilename.empty())
+		return;		// nothing to do
+
+	// Check valid
+	CheckValid();
+		
+	// Decode the header
+	int size = BACKUPSTOREFILENAME_GET_SIZE(*this);
+	int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this);
+	
+	// Decode based on encoding given in the header
+	switch(encoding)
+	{
+	case Encoding_Clear:
+		TRACE0("**** BackupStoreFilename encoded with Clear encoding ****\n");
+		mClearFilename.assign(c_str() + 2, size - 2);
+		break;
+		
+	case Encoding_Blowfish:
+		DecryptEncoded(sBlowfishDecrypt);
+		break;
+	
+	default:
+		THROW_EXCEPTION(BackupStoreException, UnknownFilenameEncoding)
+		break;	
+	}
+}
+
+
+// Buffer for encoding and decoding -- do this all in one single buffer to
+// avoid lots of string allocation, which stuffs up memory usage.
+// These static memory vars are, of course, not thread safe, but we don't use threads.
+#ifndef NDEBUG
+static int sEncDecBufferSize = 2;	// small size for debug builds
+#else
+static int sEncDecBufferSize = 256;
+#endif
+static MemoryBlockGuard<uint8_t *> spEncDecBuffer(sEncDecBufferSize);
+
+// fudge to stop leak reporting
+#ifdef BOX_MEMORY_LEAK_TESTING
+namespace
+{
+	class leak_off
+	{
+	public:
+		leak_off()
+		{
+			MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer);
+		}
+	};
+	leak_off dont_report_as_leak;
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::EncryptClear(const std::string &, CipherContext &, int)
+//		Purpose: Private. Assigns the encoded filename string, encrypting.
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding)
+{
+	// Work out max size
+	int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rToEncode.size()) + 4;
+	
+	// Make sure encode/decode buffer has enough space
+	if(sEncDecBufferSize < maxOutSize)
+	{
+		TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize);
+		spEncDecBuffer.Resize(maxOutSize);
+		sEncDecBufferSize = maxOutSize;
+	}
+	
+	// Pointer to buffer
+	uint8_t *buffer = spEncDecBuffer;
+	MEMLEAKFINDER_NOT_A_LEAK(buffer);
+	
+	// Encode -- do entire block in one go
+	int encSize = rCipherContext.TransformBlock(buffer + 2, sEncDecBufferSize - 2, rToEncode.c_str(), rToEncode.size());
+	// and add in header size
+	encSize += 2;
+	
+	// Adjust header
+	BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding);
+	
+	// Store the encoded string
+	assign((char*)buffer, encSize);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::DecryptEncoded(CipherContext &)
+//		Purpose: Decrypt the encoded filename using the cipher context
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const
+{
+	// Work out max size
+	int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(size()) + 4;
+	
+	// Make sure encode/decode buffer has enough space
+	if(sEncDecBufferSize < maxOutSize)
+	{
+		TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, maxOutSize);
+		spEncDecBuffer.Resize(maxOutSize);
+		sEncDecBufferSize = maxOutSize;
+	}
+	
+	// Pointer to buffer
+	uint8_t *buffer = spEncDecBuffer;
+	MEMLEAKFINDER_NOT_A_LEAK(buffer);
+	
+	// Decrypt
+	const char *str = c_str() + 2;
+	int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, size() - 2);
+	
+	// Assign to this
+	mClearFilename.assign((char*)buffer, sizeOut);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::EncodedFilenameChanged()
+//		Purpose: The encoded filename stored has changed
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::EncodedFilenameChanged()
+{
+	BackupStoreFilename::EncodedFilenameChanged();
+
+	// Delete stored filename in clear
+	mClearFilename.erase();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::SetBlowfishKey(const void *, int)
+//		Purpose: Set the key used for Blowfish encryption of filenames
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength)
+{
+	// Initialisation vector not used. Can't use a different vector for each filename as
+	// that would stop comparisions on the server working.
+	sBlowfishEncrypt.Reset();
+	sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+	ASSERT(sBlowfishEncrypt.GetIVLength() == IVLength);
+	sBlowfishEncrypt.SetIV(pIV);
+	sBlowfishDecrypt.Reset();
+	sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
+	ASSERT(sBlowfishDecrypt.GetIVLength() == IVLength);
+	sBlowfishDecrypt.SetIV(pIV);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFilenameClear::SetEncodingMethod(int)
+//		Purpose: Set the encoding method used for filenames
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreFilenameClear::SetEncodingMethod(int Method)
+{
+	sEncodeMethod = Method;
+}
+
+
+
diff --git a/lib/backupclient/BackupStoreFilenameClear.h b/lib/backupclient/BackupStoreFilenameClear.h
new file mode 100755
index 00000000..d4c45701
--- /dev/null
+++ b/lib/backupclient/BackupStoreFilenameClear.h
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreFilenameClear.h
+//		Purpose: BackupStoreFilenames in the clear
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREFILENAMECLEAR__H
+#define BACKUPSTOREFILENAMECLEAR__H
+
+#include "BackupStoreFilename.h"
+
+class CipherContext;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreFilenameClear
+//		Purpose: BackupStoreFilenames, handling conversion from and to the in the clear version
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class BackupStoreFilenameClear : public BackupStoreFilename
+{
+public:
+	BackupStoreFilenameClear();
+	BackupStoreFilenameClear(const std::string &rToEncode);
+	BackupStoreFilenameClear(const BackupStoreFilenameClear &rToCopy);
+	BackupStoreFilenameClear(const BackupStoreFilename &rToCopy);
+	virtual ~BackupStoreFilenameClear();
+
+	// Because we need to use a different allocator for this class to avoid
+	// nasty things happening, can't return this as a reference. Which is a
+	// pity. But probably not too bad.
+#ifdef BACKUPSTOREFILEAME_MALLOC_ALLOC_BASE_TYPE
+	const std::string GetClearFilename() const;
+#else
+	const std::string &GetClearFilename() const;
+#endif
+	void SetClearFilename(const std::string &rToEncode);
+
+	// Setup for encryption of filenames	
+	static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength);
+	static void SetEncodingMethod(int Method);
+	
+protected:
+	void MakeClearAvailable() const;
+	virtual void EncodedFilenameChanged();
+	void EncryptClear(const std::string &rToEncode, CipherContext &rCipherContext, int StoreAsEncoding);
+	void DecryptEncoded(CipherContext &rCipherContext) const;
+
+private:
+	mutable BackupStoreFilename_base mClearFilename;
+};
+
+#endif // BACKUPSTOREFILENAMECLEAR__H
+
+
diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupclient/BackupStoreObjectDump.cpp
new file mode 100644
index 00000000..caa0e82f
--- /dev/null
+++ b/lib/backupclient/BackupStoreObjectDump.cpp
@@ -0,0 +1,210 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreObjectDump.cpp
+//		Purpose: Implementations of dumping objects to stdout/TRACE
+//		Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <map>
+
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreFilename.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupStoreObjectMagic.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static void OutputLine(FILE *, bool, const char *, ...)
+//		Purpose: Output a line for the object dumping, to file and/or trace...
+//		Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+static void OutputLine(FILE *file, bool ToTrace, const char *format, ...)
+{
+	char text[512];
+	int r = 0;
+	va_list ap;
+	va_start(ap, format);
+	r = vsnprintf(text, sizeof(text), format, ap);
+	va_end(ap);
+
+	if(file != 0)
+	{
+		::fprintf(file, "%s", text);		
+	}
+	if(ToTrace)
+	{
+		TRACE1("%s", text);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace)
+//		Purpose: (first arg is FILE *, but avoid including stdio.h everywhere)
+//				 Dump the contents to a file, or trace.
+//		Created: 3/5/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace)
+{
+	FILE *file = (FILE*)clibFileHandle;
+
+	OutputLine(file, ToTrace, "Directory object.\nObject ID: %llx\nContainer ID: %llx\nNumber entries: %d\n"\
+		"Attributes mod time: %llx\nAttributes size: %d\n", mObjectID, mContainerID, mEntries.size(),
+		mAttributesModTime, mAttributes.GetSize());
+
+	// So repeated filenames can be illustrated, even though they can't be decoded
+	std::map<BackupStoreFilename, int> nameNum;
+	int nameNumI = 0;
+
+	// Dump items
+	OutputLine(file, ToTrace, "Items:\nID     Size AttrHash         AtSz NSz NIdx Flags\n");
+	for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+	{
+		// Choose file name index number for this file
+		std::map<BackupStoreFilename, int>::iterator nn(nameNum.find((*i)->GetName()));
+		int ni = nameNumI;
+		if(nn != nameNum.end())
+		{
+			ni = nn->second;
+		}
+		else
+		{
+			nameNum[(*i)->GetName()] = nameNumI;
+			++nameNumI;
+		}
+		
+		// Do dependencies
+		char depends[128];
+		depends[0] = '\0';
+		int depends_l = 0;
+		if((*i)->GetDependsNewer() != 0)
+		{
+			depends_l += ::sprintf(depends + depends_l, " depNew(%llx)", (*i)->GetDependsNewer());
+		}
+		if((*i)->GetDependsOlder() != 0)
+		{
+			depends_l += ::sprintf(depends + depends_l, " depOld(%llx)", (*i)->GetDependsOlder());
+		}
+
+		// Output item
+		int16_t f = (*i)->GetFlags();
+		OutputLine(file, ToTrace, "%06llx %4lld %016llx %4d %3d %4d%s%s%s%s%s%s\n",
+			(*i)->GetObjectID(),
+			(*i)->GetSizeInBlocks(),
+			(*i)->GetAttributesHash(),
+			(*i)->GetAttributes().GetSize(),
+			(*i)->GetName().size(),
+			ni,
+			((f & BackupStoreDirectory::Entry::Flags_File)?" file":""),
+			((f & BackupStoreDirectory::Entry::Flags_Dir)?" dir":""),
+			((f & BackupStoreDirectory::Entry::Flags_Deleted)?" del":""),
+			((f & BackupStoreDirectory::Entry::Flags_OldVersion)?" old":""),
+			((f & BackupStoreDirectory::Entry::Flags_RemoveASAP)?" removeASAP":""),
+			depends);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreFile::DumpFile(void *, bool, IOStream &)
+//		Purpose: (first arg is FILE *, but avoid including stdio.h everywhere)
+//				 Dump the contents to a file, or trace.
+//		Created: 4/5/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFile)
+{
+	FILE *file = (FILE*)clibFileHandle;
+
+	// Read header
+	file_StreamFormat hdr;
+	if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr),
+		0 /* not interested in bytes read if this fails */, IOStream::TimeOutInfinite))
+	{
+		// Couldn't read header
+		THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
+	}
+
+	// Check and output header info
+	if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1)
+		&& hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V0))
+	{
+		OutputLine(file, ToTrace, "File header doesn't have the correct magic, aborting dump\n");
+		return;
+	}
+
+	OutputLine(file, ToTrace, "File object.\nContainer ID: %llx\nModification time: %llx\n"\
+		"Max block clear size: %d\nOptions: %08x\nNum blocks: %d\n", ntoh64(hdr.mContainerID),
+			ntoh64(hdr.mModificationTime), ntohl(hdr.mMaxBlockClearSize), ntohl(hdr.mOptions),
+			ntoh64(hdr.mNumBlocks));
+
+	// Read the next two objects
+	BackupStoreFilename fn;
+	fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+	OutputLine(file, ToTrace, "Filename size: %d\n", fn.size());
+	
+	BackupClientFileAttributes attr;
+	attr.ReadFromStream(rFile, IOStream::TimeOutInfinite);
+	OutputLine(file, ToTrace, "Attributes size: %d\n", attr.GetSize());
+	
+	// Dump the blocks
+	rFile.Seek(0, IOStream::SeekType_Absolute);
+	BackupStoreFile::MoveStreamPositionToBlockIndex(rFile);
+
+	// Read in header
+	file_BlockIndexHeader bhdr;
+	rFile.ReadFullBuffer(&bhdr, sizeof(bhdr), 0);
+	if(bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
+		&& bhdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0))
+	{
+		OutputLine(file, ToTrace, "WARNING: Block header doesn't have the correct magic\n");
+	}
+	// number of blocks
+	int64_t nblocks = ntoh64(bhdr.mNumBlocks);
+	OutputLine(file, ToTrace, "Other file ID (for block refs): %llx\nNum blocks (in blk hdr): %lld\n",
+		ntoh64(bhdr.mOtherFileID), nblocks);
+
+	// Dump info about each block
+	OutputLine(file, ToTrace, "======== ===== ==========\n   Index Where  EncSz/Idx\n");
+	int64_t nnew = 0, nold = 0;
+	for(int64_t b = 0; b < nblocks; ++b)
+	{
+		file_BlockIndexEntry en;
+		if(!rFile.ReadFullBuffer(&en, sizeof(en), 0))
+		{
+			OutputLine(file, ToTrace, "Didn't manage to read block %lld from file\n", b);
+			continue;
+		}
+		int64_t s = ntoh64(en.mEncodedSize);
+		if(s > 0)
+		{
+			nnew++;
+			TRACE2("%8lld this  s=%8lld\n", b, s);
+		}
+		else
+		{
+			nold++;
+			TRACE2("%8lld other i=%8lld\n", b, 0 - s);		
+		}
+	}
+	TRACE0("======== ===== ==========\n");
+}
+
diff --git a/lib/backupclient/BackupStoreObjectMagic.h b/lib/backupclient/BackupStoreObjectMagic.h
new file mode 100755
index 00000000..7ee600a2
--- /dev/null
+++ b/lib/backupclient/BackupStoreObjectMagic.h
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreObjectMagic.h
+//		Purpose: Magic values for the start of objects in the backup store
+//		Created: 19/11/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREOBJECTMAGIC__H
+#define BACKUPSTOREOBJECTMAGIC__H
+
+// Each of these values is the first 4 bytes of the object file.
+// Remember to swap from network to host byte order.
+
+// Magic value for file streams
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V1		0x66696C65
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_MAGIC_VALUE_V0		0x46494C45
+
+// Magic for the block index at the file stream -- used to
+// ensure streams are reordered as expected
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 0x62696478
+// Do not use v0 in any new code!
+#define OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 0x46426C6B
+
+// Magic value for directory streams
+#define OBJECTMAGIC_DIR_MAGIC_VALUE 		0x4449525F
+
+#endif // BACKUPSTOREOBJECTMAGIC__H
+
diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra
new file mode 100755
index 00000000..66203e3c
--- /dev/null
+++ b/lib/backupclient/Makefile.extra
@@ -0,0 +1,16 @@
+
+MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
+
+GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt
+
+# AUTOGEN SEEDING
+autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h:	$(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt
+	perl $(GEN_CMD_SRV)
+
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_BackupStoreException.h autogen_BackupStoreException.cpp:	$(MAKEEXCEPTION) BackupStoreException.txt
+	perl $(MAKEEXCEPTION) BackupStoreException.txt
+
diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp
new file mode 100755
index 00000000..91a6b758
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccountDatabase.cpp
@@ -0,0 +1,370 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreAccountDatabase.cpp
+//		Purpose: Database of accounts for the backup store
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <string>
+#include <map>
+#include <stdio.h>
+#include <sys/stat.h>
+
+#include "BackupStoreAccountDatabase.h"
+#include "Guards.h"
+#include "FdGetLine.h"
+#include "BackupStoreException.h"
+#include "CommonException.h"
+#include "FileModificationTime.h"
+
+#include "MemLeakFindOn.h"
+
+class _BackupStoreAccountDatabase
+{
+public:
+	std::string mFilename;
+	std::map<int32_t, BackupStoreAccountDatabase::Entry> mDatabase;
+	box_time_t mModificationTime;
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *)
+//		Purpose: Constructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename)
+	: pImpl(new _BackupStoreAccountDatabase)
+{
+	pImpl->mFilename = Filename;
+	pImpl->mModificationTime = 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::~BackupStoreAccountDatabase()
+//		Purpose: Destructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::~BackupStoreAccountDatabase()
+{
+	delete pImpl;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::Entry::Entry()
+//		Purpose: Default constructor
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry()
+	: mID(-1),
+	  mDiscSet(-1)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::Entry::Entry(int32_t, int)
+//		Purpose: Constructor
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry(int32_t ID, int DiscSet)
+	: mID(ID),
+	  mDiscSet(DiscSet)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::Entry::Entry(const Entry &)
+//		Purpose: Copy constructor
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry(const Entry &rEntry)
+	: mID(rEntry.mID),
+	  mDiscSet(rEntry.mDiscSet)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::Entry::~Entry()
+//		Purpose: Destructor
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::~Entry()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::Read(const char *)
+//		Purpose: Read in a database from disc
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const char *Filename)
+{
+	// Database object to use
+	std::auto_ptr<BackupStoreAccountDatabase> db(new BackupStoreAccountDatabase(Filename));
+	
+	// Read in the file
+	db->ReadFile();
+
+	// Return to called
+	return db;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::ReadFile()
+//		Purpose: Read the file off disc
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::ReadFile() const
+{
+	// Open file
+	FileHandleGuard<> file(pImpl->mFilename.c_str());
+
+	// Clear existing entries
+	pImpl->mDatabase.clear();
+
+	// Read in lines
+	FdGetLine getLine(file);
+	
+	while(!getLine.IsEOF())
+	{
+		// Read and split up line
+		std::string l(getLine.GetLine(true));
+
+		if(!l.empty())
+		{
+			// Check...
+			int32_t id;
+			int discSet;
+			if(::sscanf(l.c_str(), "%x:%d", &id, &discSet) != 2)
+			{
+				THROW_EXCEPTION(BackupStoreException, BadAccountDatabaseFile)
+			}
+
+			// Make a new entry
+			pImpl->mDatabase[id] = Entry(id, discSet);
+		}
+	}
+	
+	// Store the modification time of the file
+	pImpl->mModificationTime = GetDBFileModificationTime();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::CheckUpToDate()
+//		Purpose: Private. Ensure that the in memory database matches the one on disc
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::CheckUpToDate() const
+{
+	if(pImpl->mModificationTime != GetDBFileModificationTime())
+	{
+		// File has changed -- load it in again
+		ReadFile();
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::GetDBFileModificationTime()
+//		Purpose: Get the current modification time of the database
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const
+{
+	struct stat st;
+	if(::stat(pImpl->mFilename.c_str(), &st) == -1)
+	{
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+	
+	return FileModificationTime(st);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::Write()
+//		Purpose: Write the database back to disc after modifying it
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::Write()
+{
+	// Open file for writing
+	// Would use this...
+	//	FileHandleGuard<O_WRONLY | O_TRUNC> file(pImpl->mFilename.c_str());
+	// but gcc fails randomly on it on some platforms. Weird.
+	
+	int file = ::open(pImpl->mFilename.c_str(), O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+	if(file == -1)
+	{
+		THROW_EXCEPTION(CommonException, OSFileOpenError)
+	}
+	
+	try
+	{
+		// Then write each entry
+		for(std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.begin());
+			i != pImpl->mDatabase.end(); ++i)
+		{
+			// Write out the entry
+			char line[256];	// more than enough for a couple of integers in string form
+			int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet());
+			if(::write(file, line, s) != s)
+			{
+				THROW_EXCEPTION(CommonException, OSFileError)
+			}
+		}
+		
+		::close(file);
+	}
+	catch(...)
+	{
+		::close(file);
+		throw;
+	}
+	
+	// Done.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::EntryExists(int32_t)
+//		Purpose: Does an entry exist in the database?
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreAccountDatabase::EntryExists(int32_t ID) const
+{
+	// Check that we're using the latest version of the database
+	CheckUpToDate();
+
+	return pImpl->mDatabase.find(ID) != pImpl->mDatabase.end();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::GetEntry(int32_t)
+//		Purpose: Retrieve an entry
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(int32_t ID) const
+{
+	// Check that we're using the latest version of the database
+	CheckUpToDate();
+
+	std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.find(ID));
+	if(i == pImpl->mDatabase.end())
+	{
+		THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry)
+	}
+	
+	return i->second;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::AddEntry(int32_t, int)
+//		Purpose: Add a new entry to the database
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::AddEntry(int32_t ID, int DiscSet)
+{
+	// Check that we're using the latest version of the database
+	CheckUpToDate();
+
+	pImpl->mDatabase[ID] = Entry(ID, DiscSet);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::DeleteEntry(int32_t)
+//		Purpose: Delete an entry from the database
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::DeleteEntry(int32_t ID)
+{
+	// Check that we're using the latest version of the database
+	CheckUpToDate();
+
+	std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.find(ID));
+	if(i == pImpl->mDatabase.end())
+	{
+		THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry)
+	}
+
+	pImpl->mDatabase.erase(i);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t>)
+//		Purpose: 
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t> &rIDsOut)
+{
+	// Check that we're using the latest version of the database
+	CheckUpToDate();
+
+	// Delete everything in the output list
+	rIDsOut.clear();
+
+	std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.begin());
+	for(; i != pImpl->mDatabase.end(); ++i)
+	{
+		rIDsOut.push_back(i->first);
+	}
+}
+
+
+
+
diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h
new file mode 100755
index 00000000..8d6e7ad8
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccountDatabase.h
@@ -0,0 +1,75 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreAccountDatabase.h
+//		Purpose: Database of accounts for the backup store
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREACCOUNTDATABASE__H
+#define BACKUPSTOREACCOUNTDATABASE__H
+
+#include <memory>
+#include <vector>
+
+#include "BoxTime.h"
+
+class _BackupStoreAccountDatabase;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreAccountDatabase
+//		Purpose: Database of accounts for the backup store
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupStoreAccountDatabase
+{
+public:
+	friend class _BackupStoreAccountDatabase;	// to stop compiler warnings
+	~BackupStoreAccountDatabase();
+private:
+	BackupStoreAccountDatabase(const char *Filename);
+	BackupStoreAccountDatabase(const BackupStoreAccountDatabase &);
+public:
+
+	static std::auto_ptr<BackupStoreAccountDatabase> Read(const char *Filename);
+	void Write();
+
+	class Entry
+	{
+	public:
+		Entry();
+		Entry(int32_t ID, int DiscSet);
+		Entry(const Entry &rEntry);
+		~Entry();
+
+		int32_t GetID() const {return mID;}
+		int GetDiscSet() const {return mDiscSet;}
+		
+	private:
+		int32_t mID;
+		int mDiscSet;
+	};
+
+	bool EntryExists(int32_t ID) const;
+	const Entry &GetEntry(int32_t ID) const;
+	void AddEntry(int32_t ID, int DiscSet);
+	void DeleteEntry(int32_t ID);
+
+	// This interface should change in the future. But for now it'll do.
+	void GetAllAccountIDs(std::vector<int32_t> &rIDsOut);
+
+private:
+	void ReadFile() const;	// const in concept only
+	void CheckUpToDate() const;	// const in concept only
+	box_time_t GetDBFileModificationTime() const;
+
+private:
+	mutable _BackupStoreAccountDatabase *pImpl;
+};
+
+#endif // BACKUPSTOREACCOUNTDATABASE__H
+
diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp
new file mode 100755
index 00000000..36d9cad3
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccounts.cpp
@@ -0,0 +1,162 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreAccounts.cpp
+//		Purpose: Account management for backup store server
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "BoxPortsAndFiles.h"
+#include "BackupStoreAccounts.h"
+#include "BackupStoreAccountDatabase.h"
+#include "RaidFileWrite.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+#include "UnixUser.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &)
+//		Purpose: Constructor
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase)
+	: mrDatabase(rDatabase)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccounts::~BackupStoreAccounts()
+//		Purpose: Destructor
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccounts::~BackupStoreAccounts()
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &)
+//		Purpose: Create a new account on the specified disc set.
+//				 If rAsUsername is not empty, then the account information will be written under the
+//				 username specified.
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername)
+{
+	{
+		// Become the user specified in the config file?
+		std::auto_ptr<UnixUser> user;
+		if(!rAsUsername.empty())
+		{
+			// Username specified, change...
+			user.reset(new UnixUser(rAsUsername.c_str()));
+			user->ChangeProcessUser(true /* temporary */);
+			// Change will be undone at the end of this function
+		}
+
+		// Get directory name
+		std::string dirName(MakeAccountRootDir(ID, DiscSet));
+	
+		// Create a directory on disc
+		RaidFileWrite::CreateDirectory(DiscSet, dirName, true /* recursive */);
+		
+		// Create an info file
+		BackupStoreInfo::CreateNew(ID, dirName, DiscSet, SizeSoftLimit, SizeHardLimit);
+		
+		// And an empty directory
+		BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID);
+		int64_t rootDirSize = 0;
+		// Write it, knowing the directory scheme
+		{
+			RaidFileWrite rf(DiscSet, dirName + "o01");
+			rf.Open();
+			rootDir.WriteToStream(rf);
+			rootDirSize = rf.GetDiscUsageInBlocks();
+			rf.Commit(true);
+		}
+	
+		// Update the store info to reflect the size of the root directory
+		std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */));
+		info->ChangeBlocksUsed(rootDirSize);
+		info->ChangeBlocksInDirectories(rootDirSize);
+		
+		// Save it back
+		info->Save();
+	}
+
+	// As the original user...
+
+	// Create the entry in the database
+	mrDatabase.AddEntry(ID, DiscSet);
+	
+	// Write the database back
+	mrDatabase.Write();	
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccounts::GetAccountRoot(int32_t, std::string &, int &)
+//		Purpose: Gets the root of an account, returning the info via references
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const
+{
+	// Find the account
+	const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID));
+	
+	rRootDirOut = MakeAccountRootDir(ID, en.GetDiscSet());
+	rDiscSetOut = en.GetDiscSet();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccounts::MakeAccountRootDir(int32_t, int)
+//		Purpose: Private. Generates a root directory name for the account
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const
+{
+	char accid[64];	// big enough!
+	::sprintf(accid, "%08x/", ID);
+	return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED DIRECTORY_SEPARATOR) + accid);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreAccounts::AccountExists(int32_t)
+//		Purpose: Does an account exist?
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreAccounts::AccountExists(int32_t ID)
+{
+	return mrDatabase.EntryExists(ID);
+}
+
+
diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h
new file mode 100755
index 00000000..0c3dd103
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccounts.h
@@ -0,0 +1,47 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreAccounts.h
+//		Purpose: Account management for backup store server
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREACCOUNTS__H
+#define BACKUPSTOREACCOUNTS__H
+
+#include <string>
+
+class BackupStoreAccountDatabase;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreAccounts
+//		Purpose: Account management for backup store server
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+class BackupStoreAccounts
+{
+public:
+	BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase);
+	~BackupStoreAccounts();
+private:
+	BackupStoreAccounts(const BackupStoreAccounts &rToCopy);
+
+public:
+	void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername);
+
+	bool AccountExists(int32_t ID);
+	void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const;
+
+private:
+	std::string MakeAccountRootDir(int32_t ID, int DiscSet) const;
+
+private:
+	BackupStoreAccountDatabase &mrDatabase;
+};
+
+#endif // BACKUPSTOREACCOUNTS__H
+
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
new file mode 100644
index 00000000..fb48c3da
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -0,0 +1,745 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreCheck.cpp
+//		Purpose: Check a store for consistency
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "BackupStoreCheck.h"
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::BackupStoreCheck(const std::string &, int, int32_t, bool, bool)
+//		Purpose: Constructor
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet)
+	: mStoreRoot(rStoreRoot),
+	  mDiscSetNumber(DiscSetNumber),
+	  mAccountID(AccountID),
+	  mFixErrors(FixErrors),
+	  mQuiet(Quiet),
+	  mNumberErrorsFound(0),
+	  mLastIDInInfo(0),
+	  mpInfoLastBlock(0),
+	  mInfoLastBlockEntries(0),
+	  mLostDirNameSerial(0),
+	  mLostAndFoundDirectoryID(0),
+	  mBlocksUsed(0),
+	  mBlocksInOldFiles(0),
+	  mBlocksInDeletedFiles(0),
+	  mBlocksInDirectories(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::~BackupStoreCheck()
+//		Purpose: Destructor
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::~BackupStoreCheck()
+{
+	// Clean up
+	FreeInfo();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::Check()
+//		Purpose: Perform the check on the given account
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::Check()
+{
+	// Lock the account
+	{
+		std::string writeLockFilename;
+		StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename);
+
+		bool gotLock = false;
+		int triesLeft = 8;
+		do
+		{
+			gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */);
+			
+			if(!gotLock)
+			{
+				--triesLeft;
+				::sleep(1);
+			}
+		} while(!gotLock && triesLeft > 0);
+	
+		if(!gotLock)
+		{
+			// Couldn't lock the account -- just stop now
+			if(!mQuiet)
+			{
+				::printf("Couldn't lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server.\n");
+			}
+			THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount)
+		}
+	}
+
+	if(!mQuiet && mFixErrors)
+	{
+		::printf("NOTE: Will fix errors encountered during checking.\n");
+	}
+
+	// Phase 1, check objects
+	if(!mQuiet)
+	{
+		::printf("Check store account ID %08x\nPhase 1, check objects...\n", mAccountID);
+	}
+	CheckObjects();
+	
+	// Phase 2, check directories
+	if(!mQuiet)
+	{
+		::printf("Phase 2, check directories...\n");
+	}
+	CheckDirectories();
+	
+	// Phase 3, check root
+	if(!mQuiet)
+	{
+		::printf("Phase 3, check root...\n");
+	}
+	CheckRoot();
+
+	// Phase 4, check unattached objects
+	if(!mQuiet)
+	{
+		::printf("Phase 4, fix unattached objects...\n");
+	}
+	CheckUnattachedObjects();
+
+	// Phase 5, fix bad info
+	if(!mQuiet)
+	{
+		::printf("Phase 5, fix unrecovered inconsistencies...\n");
+	}
+	FixDirsWithWrongContainerID();
+	FixDirsWithLostDirs();
+	
+	// Phase 6, regenerate store info
+	if(!mQuiet)
+	{
+		::printf("Phase 6, regenerate store info...\n");
+	}
+	WriteNewStoreInfo();
+	
+//	DUMP_OBJECT_INFO
+	
+	if(mNumberErrorsFound > 0)
+	{
+		::printf("%lld errors found\n", mNumberErrorsFound);
+		if(!mFixErrors)
+		{
+			::printf("NOTE: No changes to the store account have been made.\n");
+		}
+		if(!mFixErrors && mNumberErrorsFound > 0)
+		{
+			::printf("Run again with fix option to fix these errors\n");
+		}
+		if(mNumberErrorsFound > 0)
+		{
+			::printf("You should now use bbackupquery on the client machine to examine the store.\n");
+			if(mLostAndFoundDirectoryID != 0)
+			{
+				::printf("A lost+found directory was created in the account root.\n"\
+					"This contains files and directories which could not be matched to existing directories.\n"\
+					"bbackupd will delete this directory in a few days time.\n");
+			}
+		}
+	}
+	else
+	{
+		::printf("Store account checked, no errors found.\n");
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    static TwoDigitHexToInt(const char *, int &)
+//		Purpose: Convert a two digit hex string to an int, returning whether it's valid or not
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut)
+{
+	int n = 0;
+	// Char 0
+	if(String[0] >= '0' && String[0] <= '9')
+	{
+		n = (String[0] - '0') << 4;
+	}
+	else if(String[0] >= 'a' && String[0] <= 'f')
+	{
+		n = ((String[0] - 'a') + 0xa) << 4;
+	}
+	else
+	{
+		return false;
+	}
+	// Char 1
+	if(String[1] >= '0' && String[1] <= '9')
+	{
+		n |= String[1] - '0';
+	}
+	else if(String[1] >= 'a' && String[1] <= 'f')
+	{
+		n |= (String[1] - 'a') + 0xa;
+	}
+	else
+	{
+		return false;
+	}
+
+	// Return a valid number
+	rNumberOut = n;
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CheckObjects()
+//		Purpose: Read in the contents of the directory, recurse to other levels,
+//				 checking objects for sanity and readability
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckObjects()
+{
+	// Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything
+	int64_t maxDir = 0;
+
+	// Find the maximum directory starting ID
+	{
+		// Make sure the starting root dir doesn't end with '/'.
+		std::string start(mStoreRoot);
+		if(start.size() > 0 && start[start.size() - 1] == '/')
+		{
+			start.resize(start.size() - 1);
+		}
+	
+		maxDir = CheckObjectsScanDir(0, 1, mStoreRoot);
+		TRACE1("Max dir starting ID is %llx\n", maxDir);
+	}
+	
+	// Then go through and scan all the objects within those directories
+	for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH))
+	{
+		CheckObjectsDir(d);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CheckObjectsScanDir(int64_t, int, int, const std::string &)
+//		Purpose: Read in the contents of the directory, recurse to other levels,
+//				 return the maximum starting ID of any directory found.
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName)
+{
+	//TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID);
+
+	int64_t maxID = StartID;
+
+	// Read in all the directories, and recurse downwards
+	{
+		std::vector<std::string> dirs;
+		RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName,
+			RaidFileRead::DirReadType_DirsOnly, dirs);
+
+		for(std::vector<std::string>::const_iterator i(dirs.begin()); i != dirs.end(); ++i)
+		{
+			// Check to see if it's the right name
+			int n = 0;
+			if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n)
+				&& n < (1<<STORE_ID_SEGMENT_LENGTH))
+			{
+				// Next level down
+				int64_t mi = CheckObjectsScanDir(StartID | (n << (Level * STORE_ID_SEGMENT_LENGTH)), Level + 1,
+					rDirName + DIRECTORY_SEPARATOR + *i);
+				// Found a greater starting ID?
+				if(mi > maxID)
+				{
+					maxID = mi;
+				}
+			}
+			else
+			{
+				::printf("Spurious or invalid directory %s/%s found%s -- delete manually\n", rDirName.c_str(), (*i).c_str(), mFixErrors?", deleting":"");
+				++mNumberErrorsFound;
+			}
+		}
+	}
+
+	return maxID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CheckObjectsDir(int64_t)
+//		Purpose: Check all the files within this directory which has the given starting ID.
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
+{
+	// Make directory name -- first generate the filename of an entry in it
+	std::string dirName;
+	StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */);
+	// Check expectations
+	ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == '/');
+	// Remove the filename from it
+	dirName.resize(dirName.size() - 4); // four chars for "/o00"
+	
+	// Check directory exists
+	if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName))
+	{
+		TRACE1("RaidFile dir %s does not exist\n", dirName.c_str());
+		return;
+	}
+
+	// Read directory contents
+	std::vector<std::string> files;
+	RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName,
+		RaidFileRead::DirReadType_FilesOnly, files);
+	
+	// Array of things present
+	bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)];
+	for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l)
+	{
+		idsPresent[l] = false;
+	}
+	
+	// Parse each entry, building up a list of object IDs which are present in the dir.
+	// This is done so that whatever order is retured from the directory, objects are scanned
+	// in order.
+	// Filename must begin with a 'o' and be three characters long, otherwise it gets deleted.
+	for(std::vector<std::string>::const_iterator i(files.begin()); i != files.end(); ++i)
+	{
+		bool fileOK = true;
+		int n = 0;
+		if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n)
+			&& n < (1<<STORE_ID_SEGMENT_LENGTH))
+		{
+			// Filename is valid, mark as existing
+			idsPresent[n] = true;
+		}
+		else
+		{
+			// info file in root dir is OK!
+			if(StartID != 0 || ::strcmp("info", (*i).c_str()) != 0)
+			{
+				fileOK = false;
+			}
+		}
+		
+		if(!fileOK)
+		{
+			// Unexpected or bad file, delete it
+			::printf("Spurious file %s/%s found%s\n", dirName.c_str(), (*i).c_str(), mFixErrors?", deleting":"");
+			++mNumberErrorsFound;
+			if(mFixErrors)
+			{
+				RaidFileWrite del(mDiscSetNumber, dirName + DIRECTORY_SEPARATOR + *i);
+				del.Delete();
+			}
+		}
+	}
+	
+	// Check all the objects found in this directory
+	for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i)
+	{
+		if(idsPresent[i])
+		{
+			// Check the object is OK, and add entry
+			char leaf[8];
+			::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i);
+			if(!CheckAndAddObject(StartID | i, dirName + leaf))
+			{
+				// File was bad, delete it
+				::printf("Corrupted file %s%s found%s\n", dirName.c_str(), leaf, mFixErrors?", deleting":"");
+				++mNumberErrorsFound;
+				if(mFixErrors)
+				{
+					RaidFileWrite del(mDiscSetNumber, dirName + leaf);
+					del.Delete();
+				}
+			}
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &)
+//		Purpose: Check a specific object and add it to the list if it's OK -- if
+//				 there are any errors with the reading, return false and it'll be deleted.
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename)
+{
+	// Info on object...
+	bool isFile = true;
+	int64_t containerID = -1;
+	int64_t size = -1;
+
+	try
+	{
+		// Open file
+		std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename));
+		size = file->GetDiscUsageInBlocks();
+		
+		// Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile
+		uint32_t signature;
+		if(file->Read(&signature, sizeof(signature)) != sizeof(signature))
+		{
+			// Too short, can't read signature from it
+			return false;
+		}
+		// Seek back to beginning
+		file->Seek(0, IOStream::SeekType_Absolute);
+		
+		// Then... check depending on the type
+		switch(ntohl(signature))
+		{
+		case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+		case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+#endif
+			// File... check
+			containerID = CheckFile(ObjectID, *file);
+			break;
+
+		case OBJECTMAGIC_DIR_MAGIC_VALUE:
+			isFile = false;
+			containerID = CheckDirInitial(ObjectID, *file);
+			break;
+
+		default:
+			// Unknown signature. Bad file. Very bad file.
+			return false;
+			break;
+		}
+		
+		// Add to usage counts
+		int64_t s = file->GetDiscUsageInBlocks();
+		mBlocksUsed += s;
+		if(!isFile)
+		{
+			mBlocksInDirectories += s;
+		}
+	}
+	catch(...)
+	{
+		// Error caught, not a good file then, let it be deleted
+		return false;
+	}
+	
+	// Got a container ID? (ie check was successful)
+	if(containerID == -1)
+	{
+		return false;
+	}
+	
+	// Add to list of IDs known about
+	AddID(ObjectID, containerID, size, isFile);
+
+	// Report success
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CheckFile(int64_t, IOStream &)
+//		Purpose: Do check on file, return original container ID if OK, or -1 on error
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream)
+{
+	// Check that it's not the root directory ID. Having a file as the root directory would be bad.
+	if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+	{
+		// Get that dodgy thing deleted!
+		::printf("Have file as root directory. This is bad.\n");
+		return -1;
+	}
+
+	// Check the format of the file, and obtain the container ID
+	int64_t originalContainerID = -1;
+	if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */,
+		&originalContainerID))
+	{
+		// Didn't verify
+		return -1;
+	}
+
+	return originalContainerID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CheckDirInitial(int64_t, IOStream &)
+//		Purpose: Do initial check on directory, return container ID if OK, or -1 on error
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream)
+{
+	// Simply attempt to read in the directory
+	BackupStoreDirectory dir;
+	dir.ReadFromStream(rStream, IOStream::TimeOutInfinite);
+
+	// Check object ID
+	if(dir.GetObjectID() != ObjectID)
+	{
+		// Wrong object ID
+		return -1;
+	}
+	
+	// Return container ID
+	return dir.GetContainerID();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CheckDirectories()
+//		Purpose: Check the directories
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckDirectories()
+{
+	// Phase 1 did this:
+	//    Checked that all the directories are readable
+	//    Built a list of all directories and files which exist on the store
+	//
+	// This phase will check all the files in the directories, make
+	// a note of all directories which are missing, and do initial fixing.
+
+	// Scan all objects	
+	for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+	{
+		IDBlock *pblock = i->second;
+		int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+		
+		for(int e = 0; e < bentries; ++e)
+		{
+			uint8_t flags = GetFlags(pblock, e);
+			if(flags & Flags_IsDir)
+			{
+				// Found a directory. Read it in.
+				std::string filename;
+				StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */);
+				BackupStoreDirectory dir;
+				{
+					std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+					dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+				}
+				
+				// Flag for modifications
+				bool isModified = false;
+				
+				// Check for validity
+				if(dir.CheckAndFix())
+				{
+					// Wasn't quite right, and has been modified
+					::printf("Directory ID %llx has bad structure\n", pblock->mID[e]);
+					++mNumberErrorsFound;
+					isModified = true;
+				}
+				
+				// Go through, and check that everything in that directory exists and is valid
+				std::vector<int64_t> toDelete;
+				
+				BackupStoreDirectory::Iterator i(dir);
+				BackupStoreDirectory::Entry *en = 0;
+				while((en = i.Next()) != 0)
+				{
+					// Lookup the item
+					int32_t iIndex;
+					IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
+					bool badEntry = false;
+					if(piBlock != 0)
+					{
+						// Found. Get flags
+						uint8_t iflags = GetFlags(piBlock, iIndex);
+						
+						// Is the type the same?
+						if(((iflags & Flags_IsDir) == Flags_IsDir)
+							!= ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir))
+						{
+							// Entry is of wrong type
+							::printf("Directory ID %llx references object %llx which has a different type than expected.\n", pblock->mID[e], en->GetObjectID());
+							badEntry = true;
+						}
+						else
+						{
+							// Check that the entry is not already contained.
+							if(iflags & Flags_IsContained)
+							{
+								badEntry = true;
+								::printf("Directory ID %llx references object %llx which is already contained.\n", pblock->mID[e], en->GetObjectID());
+							}
+							else
+							{
+								// Not already contained -- mark as contained
+								SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
+								
+								// Check that the container ID of the object is correct
+								if(piBlock->mContainer[iIndex] != pblock->mID[e])
+								{
+									// Needs fixing...
+									if(iflags & Flags_IsDir)
+									{
+										// Add to will fix later list
+										::printf("Directory ID %llx has wrong container ID.\n", en->GetObjectID());
+										mDirsWithWrongContainerID.push_back(en->GetObjectID());
+									}
+									else
+									{
+										// This is OK for files, they might move
+										::printf("File ID %llx has different container ID, probably moved\n", en->GetObjectID());
+									}
+									
+									// Fix entry for now
+									piBlock->mContainer[iIndex] = pblock->mID[e];
+								}
+							}
+						}
+						
+						// Check the object size, if it's OK and a file
+						if(!badEntry && !((iflags & Flags_IsDir) == Flags_IsDir))
+						{
+							if(en->GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[iIndex])
+							{
+								// Correct
+								en->SetSizeInBlocks(piBlock->mObjectSizeInBlocks[iIndex]);
+								// Mark as changed
+								isModified = true;
+								// Tell user
+								::printf("Directory ID %llx has wrong size for object %llx\n", pblock->mID[e], en->GetObjectID());
+							}
+						}
+					}
+					else
+					{
+						// Item can't be found. Is it a directory?
+						if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir)
+						{
+							// Store the directory for later attention
+							mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e];
+						}
+						else
+						{
+							// Just remove the entry
+							badEntry = true;
+							::printf("Directory ID %llx references object %llx which does not exist.\n", pblock->mID[e], en->GetObjectID());
+						}
+					}
+					
+					// Is this entry worth keeping?
+					if(badEntry)
+					{
+						toDelete.push_back(en->GetObjectID());
+					}
+					else
+					{
+						// Add to sizes?
+						if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion)
+						{
+							mBlocksInOldFiles += en->GetSizeInBlocks();
+						}
+						if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted)
+						{
+							mBlocksInDeletedFiles += en->GetSizeInBlocks();
+						}
+					}
+				}
+				
+				if(toDelete.size() > 0)
+				{
+					// Delete entries from directory
+					for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
+					{
+						dir.DeleteEntry(*d);
+					}
+					
+					// Mark as modified
+					isModified = true;
+					
+					// Check the directory again, now that entries have been removed
+					dir.CheckAndFix();
+					
+					// Errors found
+					++mNumberErrorsFound;
+				}
+				
+				if(isModified && mFixErrors)
+				{	
+					::printf("Fixing directory ID %llx\n", pblock->mID[e]);
+
+					// Save back to disc
+					RaidFileWrite fixed(mDiscSetNumber, filename);
+					fixed.Open(true /* allow overwriting */);
+					dir.WriteToStream(fixed);
+					// Commit it
+					fixed.Commit(true /* convert to raid representation now */);
+				}
+			}
+		}
+	}
+
+}
+
+
diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h
new file mode 100644
index 00000000..3f48312a
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck.h
@@ -0,0 +1,199 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreCheck.h
+//		Purpose: Check a store for consistency
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECHECK__H
+#define BACKUPSTORECHECK__H
+
+#include <string>
+#include <map>
+#include <vector>
+#include <set>
+
+#include "NamedLock.h"
+class IOStream;
+class BackupStoreFilename;
+
+/*
+
+The following problems can be fixed:
+
+	* Spurious files deleted
+	* Corrupted files deleted
+	* Root ID as file, deleted
+	* Dirs with wrong object id inside, deleted
+	* Direcetory entries pointing to non-existant files, deleted
+	* Doubly references files have second reference deleted
+	* Wrong directory container IDs fixed
+	* Missing root recreated
+	* Reattach files which exist, but aren't referenced
+		- files go into directory original directory, if it still exists
+		- missing directories are inferred, and recreated
+		- or if all else fails, go into lost+found
+		- file dir entries take the original name and mod time
+		- directories go into lost+found
+	* Container IDs on directories corrected
+	* Inside directories,
+		- only one object per name has old version clear
+		- IDs aren't duplicated
+	* Bad store info files regenerated
+	* Bad sizes of files in directories fixed
+
+*/
+
+
+// Size of blocks in the list of IDs
+#ifdef NDEBUG
+	#define BACKUPSTORECHECK_BLOCK_SIZE		(64*1024)
+#else
+	#define BACKUPSTORECHECK_BLOCK_SIZE		8
+#endif
+
+// The object ID type -- can redefine to uint32_t to produce a lower memory version for smaller stores
+typedef int64_t BackupStoreCheck_ID_t;
+// Can redefine the size type for lower memory usage too
+typedef int64_t BackupStoreCheck_Size_t;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreCheck
+//		Purpose: Check a store for consistency
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+class BackupStoreCheck
+{
+public:
+	BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet);
+	~BackupStoreCheck();
+private:
+	// no copying
+	BackupStoreCheck(const BackupStoreCheck &);
+	BackupStoreCheck &operator=(const BackupStoreCheck &);
+public:
+
+	// Do the exciting things
+	void Check();
+	
+	bool ErrorsFound() {return mNumberErrorsFound > 0;}
+
+private:
+	enum
+	{
+		// Bit mask
+		Flags_IsDir = 1,
+		Flags_IsContained = 2,
+		// Mask
+		Flags__MASK = 3,
+		// Number of bits
+		Flags__NumFlags = 2,
+		// Items per uint8_t
+		Flags__NumItemsPerEntry = 4	// ie 8 / 2
+	};
+
+	typedef struct
+	{
+		// Note use arrays within the block, rather than the more obvious array of
+		// objects, to be more memory efficient -- think alignment of the byte values.
+		uint8_t mFlags[BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry];
+		BackupStoreCheck_ID_t mID[BACKUPSTORECHECK_BLOCK_SIZE];
+		BackupStoreCheck_ID_t mContainer[BACKUPSTORECHECK_BLOCK_SIZE];
+		BackupStoreCheck_Size_t mObjectSizeInBlocks[BACKUPSTORECHECK_BLOCK_SIZE];
+	} IDBlock;
+	
+	// Phases of the check
+	void CheckObjects();
+	void CheckDirectories();
+	void CheckRoot();
+	void CheckUnattachedObjects();
+	void FixDirsWithWrongContainerID();
+	void FixDirsWithLostDirs();
+	void WriteNewStoreInfo();
+
+	// Checking functions
+	int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName);
+	void CheckObjectsDir(int64_t StartID);
+	bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename);
+	int64_t CheckFile(int64_t ObjectID, IOStream &rStream);
+	int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream);	
+
+	// Fixing functions
+	bool TryToRecreateDirectory(int64_t MissingDirectoryID);
+	void InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory);
+	int64_t GetLostAndFoundDirID();
+	void CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID);
+
+	// Data handling
+	void FreeInfo();
+	void AddID(BackupStoreCheck_ID_t ID, BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile);
+	IDBlock *LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut);
+	inline void SetFlags(IDBlock *pBlock, int32_t Index, uint8_t Flags)
+	{
+		ASSERT(pBlock != 0);
+		ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE);
+		ASSERT(Flags < (1 << Flags__NumFlags));
+		
+		pBlock->mFlags[Index / Flags__NumItemsPerEntry]
+			|= (Flags << ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags));
+	}
+	inline uint8_t GetFlags(IDBlock *pBlock, int32_t Index)
+	{
+		ASSERT(pBlock != 0);
+		ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE);
+
+		return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK;
+	}
+	
+#ifndef NDEBUG
+	void DumpObjectInfo();
+	#define DUMP_OBJECT_INFO DumpObjectInfo();
+#else
+	#define DUMP_OBJECT_INFO
+#endif
+	
+private:
+	std::string mStoreRoot;
+	int mDiscSetNumber;
+	int32_t mAccountID;
+	bool mFixErrors;
+	bool mQuiet;
+	
+	int64_t mNumberErrorsFound;
+	
+	// Lock for the store account
+	NamedLock mAccountLock;
+	
+	// Storage for ID data
+	typedef std::map<BackupStoreCheck_ID_t, IDBlock*> Info_t;
+	Info_t mInfo;
+	BackupStoreCheck_ID_t mLastIDInInfo;
+	IDBlock *mpInfoLastBlock;
+	int32_t mInfoLastBlockEntries;
+	
+	// List of stuff to fix
+	std::vector<BackupStoreCheck_ID_t> mDirsWithWrongContainerID;
+	// This is a map of lost dir ID -> existing dir ID
+	std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs;
+	
+	// Set of extra directories added
+	std::set<BackupStoreCheck_ID_t> mDirsAdded;
+	
+	// Misc stuff
+	int32_t mLostDirNameSerial;
+	int64_t mLostAndFoundDirectoryID;
+	
+	// Usage
+	int64_t mBlocksUsed;
+	int64_t mBlocksInOldFiles;
+	int64_t mBlocksInDeletedFiles;
+	int64_t mBlocksInDirectories;
+};
+
+#endif // BACKUPSTORECHECK__H
+
diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp
new file mode 100644
index 00000000..fe91d00d
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck2.cpp
@@ -0,0 +1,841 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreCheck2.cpp
+//		Purpose: More backup store checking
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "BackupStoreCheck.h"
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreInfo.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CheckRoot()
+//		Purpose: Check the root directory exists.
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckRoot()
+{
+	int32_t index = 0;
+	IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index);
+	
+	if(pblock != 0)
+	{
+		// Found it. Which is lucky. Mark it as contained.
+		SetFlags(pblock, index, Flags_IsContained);
+	}
+	else
+	{
+		::printf("Root directory doesn't exist\n");
+		
+		++mNumberErrorsFound;
+		
+		if(mFixErrors)
+		{
+			// Create a new root directory
+			CreateBlankDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID);
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CreateBlankDirectory(int64_t, int64_t)
+//		Purpose: Creates a blank directory
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID)
+{
+	if(!mFixErrors)
+	{
+		// Don't do anything if we're not supposed to fix errors
+		return;
+	}
+
+	BackupStoreDirectory dir(DirectoryID, ContainingDirID);
+	
+	// Serialise to disc
+	std::string filename;
+	StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */);
+	RaidFileWrite obj(mDiscSetNumber, filename);
+	obj.Open(false /* don't allow overwriting */);
+	dir.WriteToStream(obj);
+	int64_t size = obj.GetDiscUsageInBlocks();
+	obj.Commit(true /* convert to raid now */);
+	
+	// Record the fact we've done this
+	mDirsAdded.insert(DirectoryID);
+	
+	// Add to sizes
+	mBlocksUsed += size;
+	mBlocksInDirectories += size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::CheckUnattachedObjects()
+//		Purpose: Check for objects which aren't attached to anything
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckUnattachedObjects()
+{
+	// Scan all objects, finding ones which have no container
+	for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+	{
+		IDBlock *pblock = i->second;
+		int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+		
+		for(int e = 0; e < bentries; ++e)
+		{
+			uint8_t flags = GetFlags(pblock, e);
+			if((flags & Flags_IsContained) == 0)
+			{
+				// Unattached object...
+				::printf("Object %llx is unattached.\n", pblock->mID[e]);
+				++mNumberErrorsFound;
+
+				// What's to be done?
+				int64_t putIntoDirectoryID = 0;
+
+				if((flags & Flags_IsDir) == Flags_IsDir)
+				{
+					// Directory. Just put into lost and found.
+					putIntoDirectoryID = GetLostAndFoundDirID();
+				}
+				else
+				{
+					// File. Only attempt to attach it somewhere if it isn't a patch
+					{
+						int64_t diffFromObjectID = 0;
+						std::string filename;
+						StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */);
+						// The easiest way to do this is to verify it again. Not such a bad penalty, because
+						// this really shouldn't be done very often.
+						{
+							std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+							BackupStoreFile::VerifyEncodedFileFormat(*file, &diffFromObjectID);
+						}
+
+						// If not zero, then it depends on another file, which may or may not be available.
+						// Just delete it to be safe.
+						if(diffFromObjectID != 0)
+						{
+							::printf("Object %llx is unattached, and is a patch. Deleting, cannot reliably recover.\n", pblock->mID[e]);
+						
+							// Delete this object instead
+							if(mFixErrors)
+							{
+								RaidFileWrite del(mDiscSetNumber, filename);
+								del.Delete();
+							}
+							
+							// Move on to next item
+							continue;
+						}
+					}
+					
+					// Files contain their original filename, so perhaps the orginal directory still exists,
+					// or we can infer the existance of a directory?
+					// Look for a matching entry in the mDirsWhichContainLostDirs map.
+					// Can't do this with a directory, because the name just wouldn't be known, which is
+					// pretty useless as bbackupd would just delete it. So better to put it in lost+found
+					// where the admin can do something about it.
+					int32_t dirindex;
+					IDBlock *pdirblock = LookupID(pblock->mContainer[e], dirindex);
+					if(pdirblock != 0)
+					{
+						// Something with that ID has been found. Is it a directory?
+						if(GetFlags(pdirblock, dirindex) & Flags_IsDir)
+						{
+							// Directory exists, add to that one
+							putIntoDirectoryID = pblock->mContainer[e];
+						}
+						else
+						{
+							// Not a directory. Use lost and found dir
+							putIntoDirectoryID = GetLostAndFoundDirID();
+						}
+					}
+					else if(mDirsAdded.find(pblock->mContainer[e]) != mDirsAdded.end()
+						|| TryToRecreateDirectory(pblock->mContainer[e]))
+					{
+						// The directory reappeared, or was created somehow elsewhere
+						putIntoDirectoryID = pblock->mContainer[e];
+					}
+					else
+					{
+						putIntoDirectoryID = GetLostAndFoundDirID();
+					}
+				}
+				ASSERT(putIntoDirectoryID != 0);
+
+				// Add it to the directory
+				InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID,
+					((flags & Flags_IsDir) == Flags_IsDir));
+			}
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::TryToRecreateDirectory(int64_t)
+//		Purpose: Recreate a missing directory
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID)
+{
+	// During the directory checking phase, a map of "missing directory" to
+	// containing directory was built. If we can find it here, then it's
+	// something which can be recreated!
+	std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator missing(
+		mDirsWhichContainLostDirs.find(MissingDirectoryID));
+	if(missing == mDirsWhichContainLostDirs.end())
+	{
+		// Not a missing directory, can't recreate.
+		return false;
+	}
+	
+	// Can recreate this! Wooo!
+	if(!mFixErrors)
+	{
+		::printf("Missing directory %llx could be recreated\n", MissingDirectoryID);
+		mDirsAdded.insert(MissingDirectoryID);
+		return true;
+	}
+	::printf("Recreating missing directory %llx\n", MissingDirectoryID);
+	
+	// Create a blank directory
+	BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */);
+	// Note that this directory already contains a directory entry pointing to
+	// this dir, so it doesn't have to be added.
+	
+	// Serialise to disc
+	std::string filename;
+	StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */);
+	RaidFileWrite root(mDiscSetNumber, filename);
+	root.Open(false /* don't allow overwriting */);
+	dir.WriteToStream(root);
+	root.Commit(true /* convert to raid now */);
+	
+	// Record the fact we've done this
+	mDirsAdded.insert(MissingDirectoryID);
+	
+	// Remove the entry from the map, so this doesn't happen again
+	mDirsWhichContainLostDirs.erase(missing);
+
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::GetLostAndFoundDirID()
+//		Purpose: Returns the ID of the lost and found directory, creating it if necessary
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::GetLostAndFoundDirID()
+{
+	// Already allocated it?
+	if(mLostAndFoundDirectoryID != 0)
+	{
+		return mLostAndFoundDirectoryID;
+	}
+
+	if(!mFixErrors)
+	{
+		// The result will never be used anyway if errors aren't being fixed
+		return 1;
+	}
+
+	// Load up the root directory
+	BackupStoreDirectory dir;
+	std::string filename;
+	StoreStructure::MakeObjectFilename(BACKUPSTORE_ROOT_DIRECTORY_ID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+	{
+		std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+		dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+	}
+
+	// Find a suitable name
+	BackupStoreFilename lostAndFound;
+	int n = 0;
+	while(true)
+	{
+		char name[32];
+		::sprintf(name, "lost+found%d", n++);
+		lostAndFound.SetAsClearFilename(name);
+		if(!dir.NameInUse(lostAndFound))
+		{
+			// Found a name which can be used
+			::printf("Lost and found dir has name %s\n", name);
+			break;
+		}
+	}
+
+	// Allocate an ID
+	int64_t id = mLastIDInInfo + 1;
+
+	// Create a blank directory
+	CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID);
+	
+	// Add an entry for it
+	dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0);
+
+	// Write out root dir
+	RaidFileWrite root(mDiscSetNumber, filename);
+	root.Open(true /* allow overwriting */);
+	dir.WriteToStream(root);
+	root.Commit(true /* convert to raid now */);
+	
+	// Store
+	mLostAndFoundDirectoryID = id;
+
+	// Tell caller
+	return mLostAndFoundDirectoryID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool)
+//		Purpose: 
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory)
+{
+	if(!mFixErrors)
+	{
+		// Don't do anything if we're not supposed to fix errors
+		return;
+	}
+
+	// Data for the object
+	BackupStoreFilename objectStoreFilename;
+	int64_t modTime = 100;	// something which isn't zero or a special time
+	int32_t sizeInBlocks = 0; // suitable for directories
+
+	if(IsDirectory)
+	{
+		// Directory -- simply generate a name for it.
+		char name[32];
+		::sprintf(name, "dir%08x", mLostDirNameSerial++);
+		objectStoreFilename.SetAsClearFilename(name);
+	}
+	else
+	{
+		// Files require a little more work...
+		// Open file
+		std::string fileFilename;
+		StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */);
+		std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, fileFilename));
+		// Fill in size information
+		sizeInBlocks = file->GetDiscUsageInBlocks();
+		// Read in header
+		file_StreamFormat hdr;
+		if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+			&& ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif		
+			))
+		{
+			// This should never happen, everything has been checked before.
+			THROW_EXCEPTION(BackupStoreException, Internal)
+		}
+		// This tells us nice things
+		modTime = ntoh64(hdr.mModificationTime);
+		// And the filename comes next
+		objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite);
+	}
+
+	// Directory object
+	BackupStoreDirectory dir;
+
+	// Generate filename
+	std::string filename;
+	StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+	
+	// Read it in
+	{
+		std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+		dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+	}
+	
+	// Add a new entry in an appropraite place
+	dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks,
+		IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File));
+
+	// Fix any flags which have been broken, which there's a good change of going
+	dir.CheckAndFix();
+	
+	// Write it out
+	if(mFixErrors)
+	{
+		RaidFileWrite root(mDiscSetNumber, filename);
+		root.Open(true /* allow overwriting */);
+		dir.WriteToStream(root);
+		root.Commit(true /* convert to raid now */);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::FixDirsWithWrongContainerID()
+//		Purpose: Rewrites container IDs where required
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FixDirsWithWrongContainerID()
+{
+	if(!mFixErrors)
+	{
+		// Don't do anything if we're not supposed to fix errors
+		return;
+	}
+
+	// Run through things which need fixing
+	for(std::vector<BackupStoreCheck_ID_t>::iterator i(mDirsWithWrongContainerID.begin());
+			i != mDirsWithWrongContainerID.end(); ++i)
+	{
+		int32_t index = 0;
+		IDBlock *pblock = LookupID(*i, index);
+		if(pblock == 0) continue;
+		
+		// Load in
+		BackupStoreDirectory dir;
+		std::string filename;
+		StoreStructure::MakeObjectFilename(*i, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+		{
+			std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+			dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+		}
+
+		// Adjust container ID
+		dir.SetContainerID(pblock->mContainer[index]);
+		
+		// Write it out
+		RaidFileWrite root(mDiscSetNumber, filename);
+		root.Open(true /* allow overwriting */);
+		dir.WriteToStream(root);
+		root.Commit(true /* convert to raid now */);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::FixDirsWithLostDirs()
+//		Purpose: Fix directories
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FixDirsWithLostDirs()
+{
+	if(!mFixErrors)
+	{
+		// Don't do anything if we're not supposed to fix errors
+		return;
+	}
+
+	// Run through things which need fixing
+	for(std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator i(mDirsWhichContainLostDirs.begin());
+			i != mDirsWhichContainLostDirs.end(); ++i)
+	{
+		int32_t index = 0;
+		IDBlock *pblock = LookupID(i->second, index);
+		if(pblock == 0) continue;
+		
+		// Load in
+		BackupStoreDirectory dir;
+		std::string filename;
+		StoreStructure::MakeObjectFilename(i->second, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+		{
+			std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+			dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+		}
+
+		// Delete the dodgy entry
+		dir.DeleteEntry(i->first);
+		
+		// Fix it up
+		dir.CheckAndFix();
+		
+		// Write it out
+		RaidFileWrite root(mDiscSetNumber, filename);
+		root.Open(true /* allow overwriting */);
+		dir.WriteToStream(root);
+		root.Commit(true /* convert to raid now */);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::WriteNewStoreInfo()
+//		Purpose: Regenerate store info
+//		Created: 23/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::WriteNewStoreInfo()
+{
+	// Attempt to load the existing store info file
+	std::auto_ptr<BackupStoreInfo> poldInfo;
+	try
+	{
+		poldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release());
+	}
+	catch(...)
+	{
+		::printf("Load of existing store info failed, regenerating.\n");
+		++mNumberErrorsFound;
+	}
+
+	// Minimum soft and hard limits
+	int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024;
+	int64_t minHard = ((minSoft * 11) / 10) + 1024;
+
+	// Need to do anything?
+	if(poldInfo.get() != 0 && mNumberErrorsFound == 0 && poldInfo->GetAccountID() == mAccountID)
+	{
+		// Leave the store info as it is, no need to alter it because nothing really changed,
+		// and the only essential thing was that the account ID was correct, which is was.
+		return;
+	}
+	
+	// NOTE: We will always build a new store info, so the client store marker gets changed.
+
+	// Work out the new limits
+	int64_t softLimit = minSoft;
+	int64_t hardLimit = minHard;
+	if(poldInfo.get() != 0 && poldInfo->GetBlocksSoftLimit() > minSoft)
+	{
+		softLimit = poldInfo->GetBlocksSoftLimit();
+	}
+	else
+	{
+		::printf("NOTE: Soft limit for account changed to ensure housekeeping doesn't delete files on next run\n");
+	}
+	if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard)
+	{
+		hardLimit = poldInfo->GetBlocksHardLimit();
+	}
+	else
+	{
+		::printf("NOTE: Hard limit for account changed to ensure housekeeping doesn't delete files on next run\n");
+	}
+	
+	// Object ID
+	int64_t lastObjID = mLastIDInInfo;
+	if(mLostAndFoundDirectoryID != 0)
+	{
+		mLastIDInInfo++;
+	}
+
+	// Build a new store info
+	std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration(
+		mAccountID,
+		mStoreRoot,
+		mDiscSetNumber,
+		lastObjID,
+		mBlocksUsed,
+		mBlocksInOldFiles,
+		mBlocksInDeletedFiles,
+		mBlocksInDirectories,
+		softLimit,
+		hardLimit));
+
+	// Save to disc?
+	if(mFixErrors)
+	{
+		info->Save();
+		::printf("New store info file written successfully.\n");
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::CheckAndFix()
+//		Purpose: Check the directory for obvious logical problems, and fix them.
+//				 Return true if the directory was changed.
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDirectory::CheckAndFix()
+{
+	bool changed = false;
+	
+	// Check that if a file depends on a new version, that version is in this directory
+	{
+		std::vector<Entry*>::iterator i(mEntries.begin());
+		for(; i != mEntries.end(); ++i)
+		{
+			int64_t dependsNewer = (*i)->GetDependsNewer();
+			if(dependsNewer != 0)
+			{
+				BackupStoreDirectory::Entry *newerEn = FindEntryByID(dependsNewer);
+				if(newerEn == 0)
+				{
+					// Depends on something, but it isn't there.
+					TRACE2("Entry id %llx removed because depends on newer version %llx which doesn't exist\n", (*i)->GetObjectID(), dependsNewer);
+					
+					// Remove
+					delete *i;
+					mEntries.erase(i);
+					
+					// Start again at the beginning of the vector, the iterator is now invalid
+					i = mEntries.begin();
+					
+					// Mark as changed
+					changed = true;
+				}
+				else
+				{
+					// Check that newerEn has it marked
+					if(newerEn->GetDependsOlder() != (*i)->GetObjectID())
+					{
+						// Wrong entry
+						TRACE3("Entry id %llx, correcting DependsOlder to %llx, was %llx\n", dependsNewer, (*i)->GetObjectID(), newerEn->GetDependsOlder());
+						newerEn->SetDependsOlder((*i)->GetObjectID());
+						// Mark as changed
+						changed = true;
+					}
+				}
+			}
+		}
+	}
+	
+	// Check that if a file has a dependency marked, it exists, and remove it if it doesn't
+	{
+		std::vector<Entry*>::iterator i(mEntries.begin());
+		for(; i != mEntries.end(); ++i)
+		{
+			int64_t dependsOlder = (*i)->GetDependsOlder();
+			if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0)
+			{
+				// Has an older version marked, but this doesn't exist. Remove this mark
+				TRACE2("Entry id %llx was marked that %llx depended on it, which doesn't exist, dependency info cleaered\n", (*i)->GetObjectID(), dependsOlder);
+
+				(*i)->SetDependsOlder(0);
+				
+				// Mark as changed
+				changed = true;
+			}
+		}
+	}
+
+	bool ch = false;
+	do
+	{
+		// Reset change marker
+		ch = false;
+		
+		// Search backwards -- so see newer versions first
+		std::vector<Entry*>::iterator i(mEntries.end());
+		if(i == mEntries.begin())
+		{
+			// Directory is empty, stop now
+			return changed; // changed flag
+		}
+
+		// Records of things seen
+		std::set<int64_t> idsEncountered;
+		std::set<BackupStoreFilename> filenamesEncountered;
+
+		do
+		{
+			// Look at previous
+			--i;
+
+			bool removeEntry = false;
+			if((*i) == 0)
+			{
+				TRACE0("Remove because null pointer found\n");
+				removeEntry = true;
+			}
+			else
+			{
+				bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir);
+				
+				// Check mutually exclusive flags
+				if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File))
+				{
+					// Bad! Unset the file flag
+					TRACE1("Entry %llx: File flag set when dir flag set\n", (*i)->GetObjectID());
+					(*i)->RemoveFlags(Entry::Flags_File);
+					changed = true;
+				}
+			
+				// Check...
+				if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end())
+				{
+					// ID already seen, or type doesn't match
+					TRACE1("Entry %llx: Remove because ID already seen\n", (*i)->GetObjectID());
+					removeEntry = true;
+				}
+				else
+				{
+					// Haven't already seen this ID, remember it
+					idsEncountered.insert((*i)->GetObjectID());
+					
+					// Check to see if the name has already been encountered -- if not, then it
+					// needs to have the old version flag set
+					if(filenamesEncountered.find((*i)->GetName()) != filenamesEncountered.end())
+					{
+						// Seen before -- check old version flag set
+						if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion
+							&& ((*i)->GetFlags() & Entry::Flags_Deleted) == 0)
+						{
+							// Not set, set it
+							TRACE1("Entry %llx: Set old flag\n", (*i)->GetObjectID());
+							(*i)->AddFlags(Entry::Flags_OldVersion);
+							changed = true;
+						}
+					}
+					else
+					{
+						// Check old version flag NOT set
+						if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion)
+						{
+							// Set, unset it
+							TRACE1("Entry %llx: Old flag unset\n", (*i)->GetObjectID());
+							(*i)->RemoveFlags(Entry::Flags_OldVersion);
+							changed = true;
+						}
+						
+						// Remember filename
+						filenamesEncountered.insert((*i)->GetName());
+					}
+				}
+			}
+			
+			if(removeEntry)
+			{
+				// Mark something as changed, in loop
+				ch = true;
+				
+				// Mark something as globally changed
+				changed = true;
+				
+				// erase the thing from the list
+				Entry *pentry = (*i);
+				mEntries.erase(i);
+				
+				// And delete the entry object
+				delete pentry;
+				
+				// Stop going around this loop, as the iterator is now invalid
+				break;
+			}
+		} while(i != mEntries.begin());
+
+	} while(ch != false);
+	
+	return changed;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::AddUnattactedObject(...)
+//		Purpose: Adds an object which is currently unattached. Assume that CheckAndFix() will be called afterwards.
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::AddUnattactedObject(const BackupStoreFilename &rName,
+	box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags)
+{
+	Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags,
+			ModificationTime /* use as attr mod time too */);
+	try
+	{
+		// Want to order this just before the first object which has a higher ID,
+		// which is the place it's most likely to be correct.
+		std::vector<Entry*>::iterator i(mEntries.begin());
+		for(; i != mEntries.end(); ++i)
+		{
+			if((*i)->GetObjectID() > ObjectID)
+			{
+				// Found a good place to insert it
+				break;
+			}
+		}
+		if(i == mEntries.end())
+		{
+			mEntries.push_back(pnew);
+		}
+		else
+		{
+			mEntries.insert(i, 1 /* just the one copy */, pnew);
+		}
+	}
+	catch(...)
+	{
+		delete pnew;
+		throw;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDirectory::NameInUse(const BackupStoreFilename &)
+//		Purpose: Returns true if the name is currently in use in the directory
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDirectory::NameInUse(const BackupStoreFilename &rName)
+{
+	for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+	{
+		if((*i)->GetName() == rName)
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
+
diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp
new file mode 100644
index 00000000..f22c8339
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheckData.cpp
@@ -0,0 +1,205 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreCheckData.cpp
+//		Purpose: Data handling for store checking
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <memory>
+
+#include "BackupStoreCheck.h"
+#include "autogen_BackupStoreException.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::FreeInfo()
+//		Purpose: Free all the data stored
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FreeInfo()
+{
+	// Free all the blocks
+	for(Info_t::iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+	{
+		::free(i->second);
+	}
+	
+	// Clear the contents of the map
+	mInfo.clear();
+	
+	// Reset the last ID, just in case
+	mpInfoLastBlock = 0;
+	mInfoLastBlockEntries = 0;
+	mLastIDInInfo = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::AddID(BackupStoreCheck_ID_t, BackupStoreCheck_ID_t, bool)
+//		Purpose: Add an ID to the list
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::AddID(BackupStoreCheck_ID_t ID,
+	BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile)
+{
+	// Check ID is OK.
+	if(ID <= mLastIDInInfo)
+	{
+		THROW_EXCEPTION(BackupStoreException, InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing)
+	}
+	
+	// Can this go in the current block?
+	if(mpInfoLastBlock == 0 || mInfoLastBlockEntries >= BACKUPSTORECHECK_BLOCK_SIZE)
+	{
+		// No. Allocate a new one
+		IDBlock *pblk = (IDBlock*)::malloc(sizeof(IDBlock));
+		if(pblk == 0)
+		{
+			throw std::bad_alloc();
+		}
+		// Zero all the flags entries
+		for(int z = 0; z < (BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry); ++z)
+		{
+			pblk->mFlags[z] = 0;
+		} 
+		// Store in map
+		mInfo[ID] = pblk;
+		// Allocated and stored OK, setup for use
+		mpInfoLastBlock = pblk;
+		mInfoLastBlockEntries = 0;
+	}
+	ASSERT(mpInfoLastBlock != 0 && mInfoLastBlockEntries < BACKUPSTORECHECK_BLOCK_SIZE);
+	
+	// Add to block
+	mpInfoLastBlock->mID[mInfoLastBlockEntries] = ID;
+	mpInfoLastBlock->mContainer[mInfoLastBlockEntries] = Container;
+	mpInfoLastBlock->mObjectSizeInBlocks[mInfoLastBlockEntries] = ObjectSize;
+	SetFlags(mpInfoLastBlock, mInfoLastBlockEntries, IsFile?(0):(Flags_IsDir));
+	
+	// Increment size
+	++mInfoLastBlockEntries;
+	
+	// Store last ID
+	mLastIDInInfo = ID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::LookupID(BackupStoreCheck_ID_t, int32_t
+//		Purpose: Look up an ID. Return the block it's in, or zero if not found, and the 
+//				 index within that block if the thing is found.
+//		Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut)
+{
+	IDBlock *pblock = 0;
+
+	// Find the lower matching block who's first entry is not less than ID
+	Info_t::const_iterator ib(mInfo.lower_bound(ID));
+	
+	// Was there a block
+	if(ib == mInfo.end())
+	{
+		// Block wasn't found... could be in last block
+		pblock = mpInfoLastBlock;
+	}
+	else
+	{
+		// Found it as first entry?
+		if(ib->first == ID)
+		{
+			rIndexOut = 0;
+			return ib->second;
+		}
+		
+		// Go back one block as it's not the first entry in this one
+		if(ib == mInfo.begin())
+		{
+			// Was first block, can't go back
+			return 0;
+		}
+		// Go back...
+		--ib;
+
+		// So, the ID will be in this block, if it's in anything
+		pblock = ib->second;
+	}
+
+	ASSERT(pblock != 0);
+	if(pblock == 0) return 0;
+	
+	// How many entries are there in the block
+	int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+	
+	// Do binary search within block
+	int high = bentries;
+	int low = -1;
+	while(high - low > 1)
+	{
+		int i = (high + low) / 2;
+		if(ID <= pblock->mID[i])
+		{
+			high = i;
+		}
+		else
+		{
+			low = i;
+		}
+	}
+	if(ID == pblock->mID[high])
+	{
+		// Found
+		rIndexOut = high;
+		return pblock;
+	}
+
+	// Not found
+	return 0;
+}
+
+
+#ifndef NDEBUG
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreCheck::DumpObjectInfo()
+//		Purpose: Debug only. Trace out all object info.
+//		Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::DumpObjectInfo()
+{
+	for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+	{
+		IDBlock *pblock = i->second;
+		int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+		TRACE2("BLOCK @ 0x%08x, %d entries\n", pblock, bentries);
+		
+		for(int e = 0; e < bentries; ++e)
+		{
+			uint8_t flags = GetFlags(pblock, e);
+			TRACE4("id %llx, c %llx, %s, %s\n",
+				pblock->mID[e], pblock->mContainer[e],
+				(flags & Flags_IsDir)?"dir":"file",
+				(flags & Flags_IsContained)?"contained":"unattached");
+		}
+	}
+}
+#endif
+
diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp
new file mode 100755
index 00000000..6fa05d06
--- /dev/null
+++ b/lib/backupstore/BackupStoreConfigVerify.cpp
@@ -0,0 +1,48 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreConfigVerify.h
+//		Purpose: Configuration definition for the backup store server
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreConfigVerify.h"
+#include "ServerTLS.h"
+#include "BoxPortsAndFiles.h"
+
+#include "MemLeakFindOn.h"
+
+static const ConfigurationVerifyKey verifyserverkeys[] = 
+{
+	SERVERTLS_VERIFY_SERVER_KEYS(0)	// no default listen addresses
+};
+
+static const ConfigurationVerify verifyserver[] = 
+{
+	{
+		"Server",
+		0,
+		verifyserverkeys,
+		ConfigTest_Exists | ConfigTest_LastEntry,
+		0
+	}
+};
+
+static const ConfigurationVerifyKey verifyrootkeys[] = 
+{
+	{"AccountDatabase",	0, ConfigTest_Exists, 0},
+	{"TimeBetweenHousekeeping",	0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"ExtendedLogging",	"no", ConfigTest_IsBool, 0},			// make value "yes" to enable in config file
+	{"RaidFileConf",	BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 	0}
+};
+
+const ConfigurationVerify BackupConfigFileVerify =
+{
+	"root",
+	verifyserver,
+	verifyrootkeys,
+	ConfigTest_Exists | ConfigTest_LastEntry,
+	0
+};
diff --git a/lib/backupstore/BackupStoreConfigVerify.h b/lib/backupstore/BackupStoreConfigVerify.h
new file mode 100755
index 00000000..815cfaed
--- /dev/null
+++ b/lib/backupstore/BackupStoreConfigVerify.h
@@ -0,0 +1,18 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreConfigVerify.h
+//		Purpose: Configuration definition for the backup store server
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECONFIGVERIFY__H
+#define BACKUPSTORECONFIGVERIFY__H
+
+#include "Configuration.h"
+
+extern const ConfigurationVerify BackupConfigFileVerify;
+
+#endif // BACKUPSTORECONFIGVERIFY__H
+
diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp
new file mode 100755
index 00000000..a9effe00
--- /dev/null
+++ b/lib/backupstore/BackupStoreInfo.cpp
@@ -0,0 +1,592 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreInfo.cpp
+//		Purpose: Main backup store information storage
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <algorithm>
+
+#include "BackupStoreInfo.h"
+#include "BackupStoreException.h"
+#include "RaidFileWrite.h"
+#include "RaidFileRead.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// ******************
+// make sure the defaults in CreateNew are modified!
+// ******************
+typedef struct
+{
+	int32_t mMagicValue;	// also the version number
+	int32_t mAccountID;
+	int64_t mClientStoreMarker;
+	int64_t mLastObjectIDUsed;
+	int64_t mBlocksUsed;
+	int64_t mBlocksInOldFiles;
+	int64_t mBlocksInDeletedFiles;
+	int64_t mBlocksInDirectories;
+	int64_t mBlocksSoftLimit;
+	int64_t mBlocksHardLimit;
+	uint32_t mCurrentMarkNumber;
+	uint32_t mOptionsPresent;		// bit mask of optional elements present
+	int64_t mNumberDeletedDirectories;
+	// Then loads of int64_t IDs for the deleted directories
+} info_StreamFormat;
+
+#define INFO_MAGIC_VALUE	0x34832476
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#ifdef NDEBUG
+	#define 	NUM_DELETED_DIRS_BLOCK	256
+#else
+	#define 	NUM_DELETED_DIRS_BLOCK	2
+#endif
+
+#define INFO_FILENAME	"info"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::BackupStoreInfo()
+//		Purpose: Default constructor
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreInfo::BackupStoreInfo()
+	: mAccountID(-1),
+	  mDiscSet(-1),
+	  mReadOnly(true),
+	  mIsModified(false),
+	  mClientStoreMarker(0),
+	  mLastObjectIDUsed(-1),
+	  mBlocksUsed(0),
+	  mBlocksInOldFiles(0),
+	  mBlocksInDeletedFiles(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::~BackupStoreInfo
+//		Purpose: Destructor
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreInfo::~BackupStoreInfo()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::CreateNew(int32_t, const std::string &, int)
+//		Purpose: Create a new info file on disc
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+	// Initial header (is entire file)
+	info_StreamFormat hdr = {
+		htonl(INFO_MAGIC_VALUE), // mMagicValue
+		htonl(AccountID), // mAccountID
+		0, // mClientStoreMarker
+		hton64(1), // mLastObjectIDUsed (which is the root directory)
+		0, // mBlocksUsed
+		0, // mBlocksInOldFiles
+		0, // mBlocksInDeletedFiles
+		0, // mBlocksInDirectories
+		hton64(BlockSoftLimit), // mBlocksSoftLimit
+		hton64(BlockHardLimit), // mBlocksHardLimit
+		0, // mCurrentMarkNumber
+		0, // mOptionsPresent
+		0 // mNumberDeletedDirectories
+	};
+	
+	// Generate the filename
+	ASSERT(rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR);
+	std::string fn(rRootDir + INFO_FILENAME);
+	
+	// Open the file for writing
+	RaidFileWrite rf(DiscSet, fn);
+	rf.Open(false);		// no overwriting, as this is a new file
+	
+	// Write header
+	rf.Write(&hdr, sizeof(hdr));
+	
+	// Commit it to disc, converting it to RAID now
+	rf.Commit(true);
+	
+	// Done.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::Load(int32_t, const std::string &, int, bool)
+//		Purpose: Loads the info from disc, given the root information. Can be marked as read only.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID)
+{
+	// Generate the filename
+	std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME);
+	
+	// Open the file for reading (passing on optional request for revision ID)
+	std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID));
+	
+	// Read in a header
+	info_StreamFormat hdr;
+	if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+	{
+		THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+	}
+	
+	// Check it
+	if(ntohl(hdr.mMagicValue) != INFO_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != AccountID)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+	}
+	
+	// Make new object
+	std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
+	
+	// Put in basic location info
+	info->mAccountID = AccountID;
+	info->mDiscSet = DiscSet;
+	info->mFilename = fn;
+	info->mReadOnly = ReadOnly;
+	
+	// Insert info from file
+	info->mClientStoreMarker	= ntoh64(hdr.mClientStoreMarker);
+	info->mLastObjectIDUsed		= ntoh64(hdr.mLastObjectIDUsed);
+	info->mBlocksUsed 			= ntoh64(hdr.mBlocksUsed);
+	info->mBlocksInOldFiles 	= ntoh64(hdr.mBlocksInOldFiles);
+	info->mBlocksInDeletedFiles	= ntoh64(hdr.mBlocksInDeletedFiles);
+	info->mBlocksInDirectories	= ntoh64(hdr.mBlocksInDirectories);
+	info->mBlocksSoftLimit		= ntoh64(hdr.mBlocksSoftLimit);
+	info->mBlocksHardLimit		= ntoh64(hdr.mBlocksHardLimit);
+	
+	// Load up array of deleted objects
+	int64_t numDelObj = ntoh64(hdr.mNumberDeletedDirectories);
+	
+	// Then load them in
+	if(numDelObj > 0)
+	{
+		int64_t objs[NUM_DELETED_DIRS_BLOCK];
+		
+		int64_t toload = numDelObj;
+		while(toload > 0)
+		{
+			// How many in this one?
+			int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload));
+			
+			if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */))
+			{
+				THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+			}
+			
+			// Add them
+			for(int t = 0; t < b; ++t)
+			{
+				info->mDeletedDirectories.push_back(ntoh64(objs[t]));
+			}
+			
+			// Number loaded
+			toload -= b;
+		}
+	}
+
+	// Final check
+	if(static_cast<int64_t>(info->mDeletedDirectories.size()) != numDelObj)
+	{
+		THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+	}
+	
+	// return it to caller
+	return info;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::CreateForRegeneration(...)
+//		Purpose: Return an object which can be used to save for regeneration.
+//		Created: 23/4/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration(int32_t AccountID, const std::string &rRootDir,
+	int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles,
+	int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+	// Generate the filename
+	std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME);
+	
+	// Make new object
+	std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
+	
+	// Put in basic info
+	info->mAccountID = AccountID;
+	info->mDiscSet = DiscSet;
+	info->mFilename = fn;
+	info->mReadOnly = false;
+	
+	// Insert info starting info
+	info->mClientStoreMarker	= 0;
+	info->mLastObjectIDUsed		= LastObjectID;
+	info->mBlocksUsed 			= BlocksUsed;
+	info->mBlocksInOldFiles 	= BlocksInOldFiles;
+	info->mBlocksInDeletedFiles	= BlocksInDeletedFiles;
+	info->mBlocksInDirectories	= BlocksInDirectories;
+	info->mBlocksSoftLimit		= BlockSoftLimit;
+	info->mBlocksHardLimit		= BlockHardLimit;
+	
+	// return it to caller
+	return info;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::Save()
+//		Purpose: Save modified info back to disc
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::Save()
+{
+	// Make sure we're initialised (although should never come to this)
+	if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1)
+	{
+		THROW_EXCEPTION(BackupStoreException, Internal)
+	}
+
+	// Can we do this?
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	
+	// Then... open a write file
+	RaidFileWrite rf(mDiscSet, mFilename);
+	rf.Open(true);		// allow overwriting
+	
+	// Make header
+	info_StreamFormat hdr;
+	hdr.mMagicValue 				= htonl(INFO_MAGIC_VALUE);
+	hdr.mAccountID 					= htonl(mAccountID);
+	hdr.mClientStoreMarker			= hton64(mClientStoreMarker);
+	hdr.mLastObjectIDUsed			= hton64(mLastObjectIDUsed);
+	hdr.mBlocksUsed 				= hton64(mBlocksUsed);
+	hdr.mBlocksInOldFiles 			= hton64(mBlocksInOldFiles);
+	hdr.mBlocksInDeletedFiles 		= hton64(mBlocksInDeletedFiles);
+	hdr.mBlocksInDirectories		= hton64(mBlocksInDirectories);
+	hdr.mBlocksSoftLimit			= hton64(mBlocksSoftLimit);
+	hdr.mBlocksHardLimit			= hton64(mBlocksHardLimit);
+	hdr.mCurrentMarkNumber			= 0;
+	hdr.mOptionsPresent				= 0;
+	hdr.mNumberDeletedDirectories	= hton64(mDeletedDirectories.size());
+	
+	// Write header
+	rf.Write(&hdr, sizeof(hdr));
+	
+	// Write the deleted object list
+	if(mDeletedDirectories.size() > 0)
+	{
+		int64_t objs[NUM_DELETED_DIRS_BLOCK];
+		
+		int tosave = mDeletedDirectories.size();
+		std::vector<int64_t>::iterator i(mDeletedDirectories.begin());
+		while(tosave > 0)
+		{
+			// How many in this one?
+			int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave));
+			
+			// Add them
+			for(int t = 0; t < b; ++t)
+			{
+				ASSERT(i != mDeletedDirectories.end());
+				objs[t] = hton64((*i));
+				i++;
+			}
+
+			// Write			
+			rf.Write(objs, b * sizeof(int64_t));
+			
+			// Number saved
+			tosave -= b;
+		}
+	}
+
+	// Commit it to disc, converting it to RAID now
+	rf.Commit(true);
+	
+	// Mark is as not modified
+	mIsModified = false;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::ChangeBlocksUsed(int32_t)
+//		Purpose: Change number of blocks used, by a delta amount
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	if((mBlocksUsed + Delta) < 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+	}
+	
+	mBlocksUsed += Delta;
+	
+	mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::ChangeBlocksInOldFiles(int32_t)
+//		Purpose: Change number of blocks in old files, by a delta amount
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	if((mBlocksInOldFiles + Delta) < 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+	}
+	
+	mBlocksInOldFiles += Delta;
+	
+	mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::ChangeBlocksInDeletedFiles(int32_t)
+//		Purpose: Change number of blocks in deleted files, by a delta amount
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	if((mBlocksInDeletedFiles + Delta) < 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+	}
+	
+	mBlocksInDeletedFiles += Delta;
+	
+	mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::ChangeBlocksInDirectories(int32_t)
+//		Purpose: Change number of blocks in directories, by a delta amount
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	if((mBlocksInDirectories + Delta) < 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+	}
+	
+	mBlocksInDirectories += Delta;
+	
+	mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::CorrectAllUsedValues(int64_t, int64_t, int64_t, int64_t)
+//		Purpose: Set all the usage counts to specific values -- use for correcting in housekeeping
+//				 if something when wrong during the backup connection, and the store info wasn't
+//				 saved back to disc.
+//		Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	
+	// Set the values
+	mBlocksUsed = Used;
+	mBlocksInOldFiles = InOldFiles;
+	mBlocksInDeletedFiles = InDeletedFiles;
+	mBlocksInDirectories = InDirectories;
+	
+	mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::AddDeletedDirectory(int64_t)
+//		Purpose: Add a directory ID to the deleted list
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::AddDeletedDirectory(int64_t DirID)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	
+	mDeletedDirectories.push_back(DirID);
+	
+	mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::RemovedDeletedDirectory(int64_t)
+//		Purpose: Remove a directory from the deleted list
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	
+	std::vector<int64_t>::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID));
+	if(i == mDeletedDirectories.end())
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList)
+	}
+	mDeletedDirectories.erase(i);
+	
+	mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::ChangeLimits(int64_t, int64_t)
+//		Purpose: Change the soft and hard limits
+//		Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+
+	mBlocksSoftLimit = BlockSoftLimit;
+	mBlocksHardLimit = BlockHardLimit;
+	
+	mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::AllocateObjectID()
+//		Purpose: Allocate an ID for a new object in the store.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreInfo::AllocateObjectID()
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	if(mLastObjectIDUsed < 0)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised)
+	}
+	
+	// Return the next object ID
+	return ++mLastObjectIDUsed;
+	
+	mIsModified = true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreInfo::SetClientStoreMarker(int64_t)
+//		Purpose: Sets the client store marker
+//		Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker)
+{
+	if(mReadOnly)
+	{
+		THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+	}
+	
+	mClientStoreMarker = ClientStoreMarker;
+	
+	mIsModified = true;
+}
+
+
+
diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h
new file mode 100755
index 00000000..1aba5edb
--- /dev/null
+++ b/lib/backupstore/BackupStoreInfo.h
@@ -0,0 +1,111 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreInfo.h
+//		Purpose: Main backup store information storage
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREINFO__H
+#define BACKUPSTOREINFO__H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+class BackupStoreCheck;
+
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreInfo
+//		Purpose: Main backup store information storage
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class BackupStoreInfo
+{
+	friend class BackupStoreCheck;
+public:
+	~BackupStoreInfo();
+private:
+	// Creation through static functions only
+	BackupStoreInfo();
+	// No copying allowed
+	BackupStoreInfo(const BackupStoreInfo &);
+	
+public:
+	// Create a New account, saving a blank info object to the disc
+	static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit);
+	
+	// Load it from the store
+	static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0);
+	
+	// Has info been modified?
+	bool IsModified() const {return mIsModified;}
+	
+	// Save modified infomation back to store
+	void Save();
+	
+	// Data access functions
+	int32_t GetAccountID() const {return mAccountID;}
+	int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;}
+	int64_t GetBlocksUsed() const {return mBlocksUsed;}
+	int64_t GetBlocksInOldFiles() const {return mBlocksInOldFiles;}
+	int64_t GetBlocksInDeletedFiles() const {return mBlocksInDeletedFiles;}
+	int64_t GetBlocksInDirectories() const {return mBlocksInDirectories;}
+	const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;}
+	int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;}
+	int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;}
+	bool IsReadOnly() const {return mReadOnly;}
+	int GetDiscSetNumber() const {return mDiscSet;}
+	
+	// Data modification functions
+	void ChangeBlocksUsed(int64_t Delta);
+	void ChangeBlocksInOldFiles(int64_t Delta);
+	void ChangeBlocksInDeletedFiles(int64_t Delta);
+	void ChangeBlocksInDirectories(int64_t Delta);
+	void CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories);
+	void AddDeletedDirectory(int64_t DirID);
+	void RemovedDeletedDirectory(int64_t DirID);
+	void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit);
+	
+	// Object IDs
+	int64_t AllocateObjectID();
+		
+	// Client marker set and get
+	int64_t GetClientStoreMarker() {return mClientStoreMarker;}
+	void SetClientStoreMarker(int64_t ClientStoreMarker);
+
+private:
+	static std::auto_ptr<BackupStoreInfo> CreateForRegeneration(int32_t AccountID, const std::string &rRootDir,
+		int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles,
+		int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+private:
+	// Location information
+	int32_t mAccountID;
+	int mDiscSet;
+	std::string mFilename;
+	bool mReadOnly;
+	bool mIsModified;
+	
+	// Client infomation
+	int64_t mClientStoreMarker;
+	
+	// Account information
+	int64_t mLastObjectIDUsed;
+	int64_t mBlocksUsed;
+	int64_t mBlocksInOldFiles;
+	int64_t mBlocksInDeletedFiles;
+	int64_t mBlocksInDirectories;
+	int64_t mBlocksSoftLimit;
+	int64_t mBlocksHardLimit;
+	std::vector<int64_t> mDeletedDirectories;
+};
+
+
+#endif // BACKUPSTOREINFO__H
+
+
diff --git a/lib/backupstore/StoreStructure.cpp b/lib/backupstore/StoreStructure.cpp
new file mode 100755
index 00000000..45a1ce91
--- /dev/null
+++ b/lib/backupstore/StoreStructure.cpp
@@ -0,0 +1,95 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    StoreStructure.cpp
+//		Purpose: 
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "RaidFileController.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StoreStructure::MakeObjectFilename(int64_t, const std::string &, int, std::string &, bool)
+//		Purpose: Builds the object filename for a given object, given a root. Optionally ensure that the
+//				 directory exists.
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void StoreStructure::MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists)
+{
+	const static char *hex = "0123456789abcdef";
+
+	// Set output to root string
+	rFilenameOut = rStoreRoot;
+
+	// get the id value from the stored object ID so we can do
+	// bitwise operations on it.
+	uint64_t id = (uint64_t)ObjectID;
+
+	// get leafname, shift the bits which make up the leafname off
+	unsigned int leafname(id & STORE_ID_SEGMENT_MASK);
+	id >>= STORE_ID_SEGMENT_LENGTH;
+
+	// build pathname
+	while(id != 0)
+	{
+		// assumes that the segments are no bigger than 8 bits
+		int v = id & STORE_ID_SEGMENT_MASK;
+		rFilenameOut += hex[(v & 0xf0) >> 4];
+		rFilenameOut += hex[v & 0xf];
+		rFilenameOut += DIRECTORY_SEPARATOR_ASCHAR;
+
+		// shift the bits we used off the pathname
+		id >>= STORE_ID_SEGMENT_LENGTH;
+	}
+	
+	// Want to make sure this exists?
+	if(EnsureDirectoryExists)
+	{
+		if(!RaidFileRead::DirectoryExists(DiscSet, rFilenameOut))
+		{
+			// Create it
+			RaidFileWrite::CreateDirectory(DiscSet, rFilenameOut, true /* recusive */);
+		}
+	}
+
+	// append the filename
+	rFilenameOut += 'o';
+	rFilenameOut += hex[(leafname & 0xf0) >> 4];
+	rFilenameOut += hex[leafname & 0xf];
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StoreStructure::MakeWriteLockFilename(const std::string &, int, std::string &)
+//		Purpose: Generate the on disc filename of the write lock file
+//		Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void StoreStructure::MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut)
+{
+	// Find the disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(DiscSet));
+	
+	// Make the filename
+	std::string writeLockFile(rdiscSet[0] + DIRECTORY_SEPARATOR + rStoreRoot + "write.lock");
+
+	// Return it to the caller
+	rFilenameOut = writeLockFile;
+}
+
+
diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h
new file mode 100755
index 00000000..094c0deb
--- /dev/null
+++ b/lib/backupstore/StoreStructure.h
@@ -0,0 +1,32 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    StoreStructure.h
+//		Purpose: Functions for placing files in the store
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef STORESTRUCTURE__H
+#define STORESTRUCTURE__H
+
+#include <string>
+
+#ifdef NDEBUG
+	#define STORE_ID_SEGMENT_LENGTH		8
+	#define STORE_ID_SEGMENT_MASK		0xff
+#else
+	// Debug we'll use lots and lots of directories to stress things
+	#define STORE_ID_SEGMENT_LENGTH		2
+	#define STORE_ID_SEGMENT_MASK		0x03
+#endif
+
+
+namespace StoreStructure
+{
+	void MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists);
+	void MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut);
+};
+
+#endif // STORESTRUCTURE__H
+
diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h
new file mode 100755
index 00000000..8c471171
--- /dev/null
+++ b/lib/common/BannerText.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BannerText.h
+//		Purpose: Banner text for daemons and utilities
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BANNERTEXT__H
+#define BANNERTEXT__H
+
+#define BANNER_TEXT(UtilityName) \
+	"Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers 2003, 2004\n"
+
+#endif // BANNERTEXT__H
+
diff --git a/lib/common/BeginStructPackForWire.h b/lib/common/BeginStructPackForWire.h
new file mode 100644
index 00000000..f9f8f616
--- /dev/null
+++ b/lib/common/BeginStructPackForWire.h
@@ -0,0 +1,23 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BeginStructPackForWire.h
+//		Purpose: Begin structure packing for wire
+//		Created: 25/11/03
+//
+// --------------------------------------------------------------------------
+
+// No header guard -- this is intentional
+
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+
+#pragma pack(1)
+
+#else
+
+	logical error -- check BoxPlatform.h and including file
+
+#endif
+
+
+
diff --git a/lib/common/Box.h b/lib/common/Box.h
new file mode 100755
index 00000000..19e78ada
--- /dev/null
+++ b/lib/common/Box.h
@@ -0,0 +1,180 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Box.h
+//		Purpose: Main header file for the Box project
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOX__H
+#define BOX__H
+
+// Use the same changes as gcc3 for gcc4
+#ifdef PLATFORM_GCC4
+	#define PLATFORM_GCC3
+#endif
+
+#include "BoxPlatform.h"
+
+// uncomment this line to enable full memory leak finding on all malloc-ed blocks (at least, ones used by the STL)
+//#define MEMLEAKFINDER_FULL_MALLOC_MONITORING
+
+#ifndef NDEBUG
+	// not available on OpenBSD... oh well.
+	//#define SHOW_BACKTRACE_ON_EXCEPTION
+#endif
+
+#ifdef SHOW_BACKTRACE_ON_EXCEPTION
+	// include "Utils.h"
+	#define OPTIONAL_DO_BACKTRACE DumpStackBacktrace();
+#else
+	#define OPTIONAL_DO_BACKTRACE
+#endif
+
+#include "CommonException.h"
+
+#ifndef NDEBUG
+	
+	extern bool AssertFailuresToSyslog;
+	#define ASSERT_FAILS_TO_SYSLOG_ON {AssertFailuresToSyslog = true;}
+	void BoxDebugAssertFailed(char *cond, char *file, int line);
+	#define ASSERT(cond) {if(!(cond)) {BoxDebugAssertFailed(#cond, __FILE__, __LINE__); THROW_EXCEPTION(CommonException, AssertFailed)}}
+
+	// Note that syslog tracing is independent of BoxDebugTraceOn, but stdout tracing is not
+	extern bool BoxDebugTraceToSyslog;
+	#define TRACE_TO_SYSLOG(x) {BoxDebugTraceToSyslog = x;}
+	extern bool BoxDebugTraceToStdout;
+	#define TRACE_TO_STDOUT(x) {BoxDebugTraceToStdout = x;}
+
+	extern bool BoxDebugTraceOn;
+	int BoxDebug_printf(const char *format, ...);
+	int BoxDebugTrace(const char *format, ...);
+	#define	TRACE0(msg) {BoxDebugTrace("%s", msg);}
+	#define	TRACE1(msg, a0) {BoxDebugTrace(msg, a0);}
+	#define	TRACE2(msg, a0, a1) {BoxDebugTrace(msg, a0, a1);}
+	#define	TRACE3(msg, a0, a1, a2) {BoxDebugTrace(msg, a0, a1, a2);}
+	#define	TRACE4(msg, a0, a1, a2, a3) {BoxDebugTrace(msg, a0, a1, a2, a3);}
+	#define	TRACE5(msg, a0, a1, a2, a3, a4) {BoxDebugTrace(msg, a0, a1, a2, a3, a4);}
+	#define	TRACE6(msg, a0, a1, a2, a3, a4, a5) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5);}
+	#define	TRACE7(msg, a0, a1, a2, a3, a4, a5, a6) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6);}
+	#define	TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6, a7);}
+	
+	#ifndef PLATFORM_DISABLE_MEM_LEAK_TESTING
+		#define BOX_MEMORY_LEAK_TESTING
+	#endif
+	
+	// Exception names
+	#define EXCEPTION_CODENAMES_EXTENDED
+	
+#else
+	#define ASSERT_FAILS_TO_SYSLOG_ON
+	#define ASSERT(cond)
+
+	#define TRACE_TO_SYSLOG(x) {}
+	#define TRACE_TO_STDOUT(x) {}
+
+	#define TRACE0(msg)
+	#define	TRACE1(msg, a0)
+	#define	TRACE2(msg, a0, a1)
+	#define	TRACE3(msg, a0, a1, a2)
+	#define	TRACE4(msg, a0, a1, a2, a3)
+	#define	TRACE5(msg, a0, a1, a2, a3, a4)
+	#define	TRACE6(msg, a0, a1, a2, a3, a4, a5)
+	#define	TRACE7(msg, a0, a1, a2, a3, a4, a5, a6)
+	#define	TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7)
+	
+	// Box Backup builds release get extra information for exception logging
+	#define EXCEPTION_CODENAMES_EXTENDED
+	#define EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION
+	
+#endif
+
+#ifdef BOX_MEMORY_LEAK_TESTING
+	// Memory leak testing
+	#include "MemLeakFinder.h"
+	#define MEMLEAKFINDER_NOT_A_LEAK(x)	memleakfinder_notaleak(x);
+	#define MEMLEAKFINDER_START {memleakfinder_global_enable = true;}
+	#define MEMLEAKFINDER_STOP {memleakfinder_global_enable = false;}
+#else
+	#define DEBUG_NEW new
+	#define MEMLEAKFINDER_NOT_A_LEAK(x)
+	#define MEMLEAKFINDER_START
+	#define MEMLEAKFINDER_STOP
+#endif
+
+
+#define THROW_EXCEPTION(type, subtype)														\
+	{																						\
+		OPTIONAL_DO_BACKTRACE																\
+		TRACE1("Exception thrown: " #type "(" #subtype ") at " __FILE__ "(%d)\n", __LINE__)	\
+		throw type(type::subtype);															\
+	}
+
+// extra macros for converting to network byte order
+
+// Always define a swap64 function, as it's useful.
+inline uint64_t box_swap64(uint64_t x)
+{
+	return ((x & 0xff) << 56 |
+		(x & 0xff00LL) << 40 |
+		(x & 0xff0000LL) << 24 |
+		(x & 0xff000000LL) << 8 |
+		(x & 0xff00000000LL) >> 8 |
+		(x & 0xff0000000000LL) >> 24 |
+		(x & 0xff000000000000LL) >> 40 |
+		(x & 0xff00000000000000LL) >> 56);
+}
+
+// Does the platform provide a built in SWAP64 we can use?
+#ifdef PLATFORM_NO_BUILT_IN_SWAP64
+
+	#define hton64(x) box_swap64(x)
+	#define ntoh64(x) box_swap64(x)
+
+#else
+
+	#if BYTE_ORDER == BIG_ENDIAN
+	
+		// Less hassle than trying to find some working things
+		// on Darwin PPC
+		#define hton64(x) (x)
+		#define ntoh64(x) (x)
+	
+	#else
+	
+		#ifdef PLATFORM_LINUX
+			// On Linux, use some internal kernal stuff to do this
+			#include <asm/byteorder.h>
+			#define hton64 __cpu_to_be64
+			#define ntoh64 __be64_to_cpu
+		#else
+			#define hton64 htobe64
+			#define ntoh64 betoh64
+		#endif
+	
+		// hack to make some of these work
+		// version in  /usr/include/sys/endian.h  doesn't include the 'LL' at the end of the constants
+		// provoking complaints from the compiler
+		#ifdef __GNUC__
+		#undef __swap64gen
+		#define __swap64gen(x) __extension__({                                  \
+				u_int64_t __swap64gen_x = (x);                                  \
+																				\
+				(u_int64_t)((__swap64gen_x & 0xff) << 56 |                      \
+					(__swap64gen_x & 0xff00LL) << 40 |                            \
+					(__swap64gen_x & 0xff0000LL) << 24 |                          \
+					(__swap64gen_x & 0xff000000LL) << 8 |                         \
+					(__swap64gen_x & 0xff00000000LL) >> 8 |                       \
+					(__swap64gen_x & 0xff0000000000LL) >> 24 |                    \
+					(__swap64gen_x & 0xff000000000000LL) >> 40 |                  \
+					(__swap64gen_x & 0xff00000000000000LL) >> 56);                \
+		})
+		#endif // __GNUC__
+	
+	#endif // n BYTE_ORDER == BIG_ENDIAN
+
+#endif // PLATFORM_NO_BUILT_IN_SWAP64
+
+#endif // BOX__H
+
diff --git a/lib/common/BoxException.cpp b/lib/common/BoxException.cpp
new file mode 100755
index 00000000..2503ca63
--- /dev/null
+++ b/lib/common/BoxException.cpp
@@ -0,0 +1,21 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BoxException.cpp
+//		Purpose: Exception
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BoxException.h"
+
+#include "MemLeakFindOn.h"
+
+BoxException::BoxException()
+{
+}
+
+BoxException::~BoxException() throw ()
+{
+}
diff --git a/lib/common/BoxException.h b/lib/common/BoxException.h
new file mode 100755
index 00000000..eb992f57
--- /dev/null
+++ b/lib/common/BoxException.h
@@ -0,0 +1,37 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BoxException.h
+//		Purpose: Exception
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXEXCEPTION__H
+#define BOXEXCEPTION__H
+
+#include <exception>
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BoxException
+//		Purpose: Exception
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+class BoxException : public std::exception
+{
+public:
+	BoxException();
+	~BoxException() throw ();
+	
+	virtual unsigned int GetType() const throw() = 0;
+	virtual unsigned int GetSubType() const throw() = 0;
+
+private:
+};
+
+
+#endif // BOXEXCEPTION__H
+
diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h
new file mode 100755
index 00000000..716366d2
--- /dev/null
+++ b/lib/common/BoxPlatform.h
@@ -0,0 +1,236 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BoxPlatform.h
+//		Purpose: Specifies what each platform supports in more detail, and includes
+//				 extra files to get basic support for types.
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXPLATFORM__H
+#define BOXPLATFORM__H
+
+#define DIRECTORY_SEPARATOR			"/"
+#define DIRECTORY_SEPARATOR_ASCHAR	'/'
+
+#define PLATFORM_DEV_NULL			"/dev/null"
+
+
+// Other flags which might be useful...
+//
+//	#define PLATFORM_BERKELEY_DB_NOT_SUPPORTED
+//    -- dbopen etc not on this platform
+//
+//  #define PLATFORM_REGEX_NOT_SUPPORTED
+//	  -- regex support not available on this platform
+
+
+#ifdef PLATFORM_OPENBSD
+
+	#include <sys/types.h>
+
+	#define PLATFORM_HAVE_setproctitle
+
+	#define PLATFORM_STATIC_TEMP_DIRECTORY_NAME	"/tmp"
+	
+	#define PLATFORM_HAVE_getpeereid
+
+	#define PLATFORM_RANDOM_DEVICE	"/dev/arandom"
+
+#endif // PLATFORM_OPENBSD
+
+#ifdef PLATFORM_NETBSD
+
+	#include <sys/types.h>
+
+	#define PLATFORM_HAVE_setproctitle
+
+	#define PLATFORM_NO_BUILT_IN_SWAP64
+
+	#define PLATFORM_STATIC_TEMP_DIRECTORY_NAME	"/tmp"
+	
+	#define PLATFORM_KQUEUE_NOT_SUPPORTED
+
+	#define PLATFORM_RANDOM_DEVICE	"/dev/urandom"
+
+#endif
+
+#ifdef PLATFORM_FREEBSD
+
+	#include <sys/types.h>
+	#include <netinet/in.h>
+
+	#define PLATFORM_HAVE_setproctitle
+
+	#define PLATFORM_NO_BUILT_IN_SWAP64
+
+	#define PLATFORM_STATIC_TEMP_DIRECTORY_NAME	"/tmp"
+
+	#define PLATFORM_HAVE_getpeereid
+
+	#define PLATFORM_RANDOM_DEVICE  "/dev/urandom"
+
+#endif // PLATFORM_FREEBSD
+
+#ifdef PLATFORM_DARWIN
+
+	#include <sys/types.h>
+	#include <netdb.h>
+	
+	// types 'missing'
+	#ifndef _SOCKLEN_T
+		typedef int socklen_t;
+	#endif
+	typedef u_int8_t uint8_t;
+	typedef signed char int8_t;
+	typedef u_int64_t uint64_t;
+	typedef u_int32_t uint32_t;
+	typedef u_int16_t uint16_t;
+	
+	// poll() emulator on Darwin misses this declaration
+	#define INFTIM		-1
+
+	#define PLATFORM_STATIC_TEMP_DIRECTORY_NAME	"/tmp"
+	
+	#define PLATFORM_LCHOWN_NOT_SUPPORTED
+	
+	#define PLATFORM_READLINE_NOT_SUPPORTED
+
+	#define PLATFORM_RANDOM_DEVICE	"/dev/random"
+	
+#endif // PLATFORM_DARWIN
+
+#ifdef PLATFORM_LINUX
+	
+	#include <sys/types.h>
+
+	// for ntohl etc...
+	#include <netinet/in.h>
+
+	// types 'missing'
+	typedef u_int8_t uint8_t;
+	typedef signed char int8_t;
+	typedef u_int32_t uint32_t;
+	typedef u_int16_t uint16_t;
+	typedef u_int64_t uint64_t;
+
+	// not defined in Linux, a BSD thing
+	#define INFTIM -1
+
+	#define LLONG_MAX    9223372036854775807LL
+	#define LLONG_MIN    (-LLONG_MAX - 1LL)
+
+	#define PLATFORM_STATIC_TEMP_DIRECTORY_NAME	"/tmp"
+
+	#define PLATFORM_HAVE_getsockopt_SO_PEERCRED
+
+	// load in installation specific linux configuration
+	#include "../../local/_linux_platform.h"
+
+	#define PLATFORM_KQUEUE_NOT_SUPPORTED
+	#define PLATFORM_dirent_BROKEN_d_type
+	#define PLATFORM_stat_SHORT_mtime
+	#define PLATFORM_stat_NO_st_flags
+	#define PLATFORM_USES_MTAB_FILE_FOR_MOUNTS
+	#define PLATFORM_open_NO_O_EXLOCK
+	#define PLATFORM_sockaddr_NO_len
+
+	#define PLATFORM_RANDOM_DEVICE	"/dev/urandom"
+
+	// If large file support is on, can't do the intercepts in the test/raidfile
+	#if _FILE_OFFSET_BITS == 64
+		#define PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE
+	#endif
+
+#endif // PLATFORM_LINUX
+
+#ifdef PLATFORM_CYGWIN
+
+	#define PLATFORM_BERKELEY_DB_NOT_SUPPORTED
+
+	#define PLATFORM_KQUEUE_NOT_SUPPORTED
+	#define PLATFORM_dirent_BROKEN_d_type
+	#define PLATFORM_stat_SHORT_mtime
+	#define PLATFORM_stat_NO_st_flags
+	#define PLATFORM_USES_MTAB_FILE_FOR_MOUNTS
+	#define PLATFORM_open_NO_O_EXLOCK
+	#define PLATFORM_sockaddr_NO_len
+	#define PLATFORM_NO_BUILT_IN_SWAP64
+
+	#define PLATFORM_STATIC_TEMP_DIRECTORY_NAME	"/tmp"
+
+	#define PLATFORM_READLINE_NOT_SUPPORTED
+
+	#define LLONG_MAX    9223372036854775807LL
+	#define LLONG_MIN    (-LLONG_MAX - 1LL)
+
+	#define INFTIM -1
+
+	// File listing canonical interesting mount points.  
+	#define MNTTAB          _PATH_MNTTAB   
+
+	// File listing currently active mount points.  
+	#define MOUNTED         _PATH_MOUNTED   
+
+	#define __need_FILE
+
+	// Extra includes
+	#include <stdint.h>
+	#include <stdlib.h>
+	#include <netinet/in.h>
+	#include <sys/socket.h>
+	#include <sys/stat.h>
+	#include <sys/types.h>
+	#include <dirent.h>
+	#include <stdio.h>
+	#include <paths.h>
+
+	// No easy random entropy source
+	#define PLATFORM_RANDOM_DEVICE_NONE
+
+#endif // PLATFORM_CYGWIN
+
+
+// Find out if credentials on UNIX sockets can be obtained
+#ifndef PLATFORM_HAVE_getpeereid
+	#ifndef PLATFORM_HAVE_getsockopt_SO_PEERCRED
+		#define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET
+	#endif
+#endif
+
+
+// Compiler issues
+#ifdef __GNUC__
+
+	#ifdef PLATFORM_GCC3
+
+		// GCC v3 doesn't like pragmas in #defines
+		#define STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+		
+		// But fortunately, the STL allocations are much better behaved.
+	
+	#else
+	
+		// Force STL to use malloc() for memory allocation
+		// -- slower, but doesn't gradually use more and more memory
+		// HOWEVER -- this 'fix' is broken on some platforms. Lots of fun!
+		#ifndef PLATFORM_STL_USE_MALLOC_BROKEN
+			#define __USE_MALLOC
+		#endif
+		
+		// set packing to one bytes (can't use push/pop on gcc)
+		#define BEGIN_STRUCTURE_PACKING_FOR_WIRE	#pragma pack(1)
+		
+		// Use default packing
+		#define END_STRUCTURE_PACKING_FOR_WIRE		#pragma pack()
+
+	#endif
+
+#else
+	compiler not supported!
+#endif
+
+
+#endif // BOXPLATFORM__H
+
diff --git a/lib/common/BoxPortsAndFiles.h b/lib/common/BoxPortsAndFiles.h
new file mode 100755
index 00000000..7788537e
--- /dev/null
+++ b/lib/common/BoxPortsAndFiles.h
@@ -0,0 +1,31 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BoxPortsAndFiles.h
+//		Purpose: Central list of which tcp/ip ports and hardcoded file locations
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXPORTSANDFILES__H
+#define BOXPORTSANDFILES__H
+
+#define BOX_PORT_BASE		2200
+
+
+// Backup store daemon
+#define BOX_PORT_BBSTORED					(BOX_PORT_BASE+1)
+#define BOX_FILE_BBSTORED_DEFAULT_CONFIG	"/etc/box/bbstored.conf"
+// directory within the RAIDFILE root for the backup store daemon
+#define BOX_RAIDFILE_ROOT_BBSTORED			"backup"
+
+// Backup client daemon
+#define BOX_FILE_BBACKUPD_DEFAULT_CONFIG	"/etc/box/bbackupd.conf"
+
+
+// RaidFile conf location efault
+#define BOX_FILE_RAIDFILE_DEFAULT_CONFIG	"/etc/box/raidfile.conf"
+
+
+#endif // BOXPORTSANDFILES__H
+
diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp
new file mode 100755
index 00000000..feada309
--- /dev/null
+++ b/lib/common/BoxTime.cpp
@@ -0,0 +1,32 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BoxTime.cpp
+//		Purpose: Time for the box
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <time.h>
+
+#include "BoxTime.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    GetCurrentBoxTime()
+//		Purpose: Returns the current time as a box time. (1 sec precision)
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+box_time_t GetCurrentBoxTime()
+{
+	ASSERT(sizeof(uint32_t) == sizeof(time_t));
+	return SecondsToBoxTime((uint32_t)time(0));
+}
+
+
diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h
new file mode 100755
index 00000000..a7a25ac6
--- /dev/null
+++ b/lib/common/BoxTime.h
@@ -0,0 +1,44 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BoxTime.h
+//		Purpose: How time is represented
+//		Created: 2003/10/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXTIME__H
+#define BOXTIME__H
+
+// Time is presented as an unsigned 64 bit integer, in microseconds
+typedef uint64_t	box_time_t;
+
+#define NANO_SEC_IN_SEC		(1000000000LL)
+#define NANO_SEC_IN_USEC 	(1000)
+#define NANO_SEC_IN_USEC_LL (1000LL)
+#define MICRO_SEC_IN_SEC 	(1000000)
+#define MICRO_SEC_IN_SEC_LL	(1000000LL)
+#define MILLI_SEC_IN_NANO_SEC		(1000)
+#define MILLI_SEC_IN_NANO_SEC_LL	(1000LL)
+
+box_time_t GetCurrentBoxTime();
+
+inline box_time_t SecondsToBoxTime(uint32_t Seconds)
+{
+	return ((box_time_t)Seconds * MICRO_SEC_IN_SEC_LL);
+}
+inline box_time_t SecondsToBoxTime(uint64_t Seconds)
+{
+	return ((box_time_t)Seconds * MICRO_SEC_IN_SEC_LL);
+}
+inline int64_t BoxTimeToSeconds(box_time_t Time)
+{
+	return Time / MICRO_SEC_IN_SEC_LL;
+}
+inline int64_t BoxTimeToMilliSeconds(box_time_t Time)
+{
+	return Time / MILLI_SEC_IN_NANO_SEC_LL;
+}
+
+#endif // BOXTIME__H
+
diff --git a/lib/common/BoxTimeToText.cpp b/lib/common/BoxTimeToText.cpp
new file mode 100755
index 00000000..94b0d152
--- /dev/null
+++ b/lib/common/BoxTimeToText.cpp
@@ -0,0 +1,41 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BoxTimeToText.cpp
+//		Purpose: Convert box time to text
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <time.h>
+#include <stdio.h>
+
+#include "BoxTimeToText.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BoxTimeToISO8601String(box_time_t)
+//		Purpose: Convert a 64 bit box time to a ISO 8601 complient string
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+std::string BoxTimeToISO8601String(box_time_t Time)
+{
+	time_t timeInSecs = (time_t)BoxTimeToSeconds(Time);
+	struct tm time;
+	gmtime_r(&timeInSecs, &time);
+	
+	char str[128];	// more than enough space
+	sprintf(str, "%04d-%02d-%02dT%02d:%02d:%02d", time.tm_year + 1900,
+		time.tm_mon + 1, time.tm_mday, time.tm_hour, time.tm_min, time.tm_sec);
+		
+	return std::string(str);
+}
+
+
diff --git a/lib/common/BoxTimeToText.h b/lib/common/BoxTimeToText.h
new file mode 100755
index 00000000..a25c70b6
--- /dev/null
+++ b/lib/common/BoxTimeToText.h
@@ -0,0 +1,19 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BoxTimeToText.h
+//		Purpose: Convert box time to text
+//		Created: 2003/10/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef BOXTIMETOTEXT__H
+#define BOXTIMETOTEXT__H
+
+#include <string>
+#include "BoxTime.h"
+
+std::string BoxTimeToISO8601String(box_time_t Time);
+
+#endif // BOXTIMETOTEXT__H
+
diff --git a/lib/common/BoxTimeToUnix.h b/lib/common/BoxTimeToUnix.h
new file mode 100755
index 00000000..17e57e27
--- /dev/null
+++ b/lib/common/BoxTimeToUnix.h
@@ -0,0 +1,30 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BoxTimeToUnix.h
+//		Purpose: Convert times in 64 bit values to UNIX structures
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#ifndef FILEMODIFICATIONTIMETOTIMEVAL__H
+#define FILEMODIFICATIONTIMETOTIMEVAL__H
+
+#include <sys/time.h>
+
+#include "BoxTime.h"
+
+inline void BoxTimeToTimeval(box_time_t Time, struct timeval &tv)
+{
+	tv.tv_sec = (long)(Time / MICRO_SEC_IN_SEC_LL);
+	tv.tv_usec = (long)(Time % MICRO_SEC_IN_SEC_LL);
+}
+
+inline void BoxTimeToTimespec(box_time_t Time, struct timespec &tv)
+{
+	tv.tv_sec = (time_t)(Time / MICRO_SEC_IN_SEC_LL);
+	tv.tv_nsec = ((long)(Time % MICRO_SEC_IN_SEC_LL)) * NANO_SEC_IN_USEC;
+}
+
+#endif // FILEMODIFICATIONTIMETOTIMEVAL__H
+
diff --git a/lib/common/CollectInBufferStream.cpp b/lib/common/CollectInBufferStream.cpp
new file mode 100755
index 00000000..90e2e7bc
--- /dev/null
+++ b/lib/common/CollectInBufferStream.cpp
@@ -0,0 +1,274 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CollectInBufferStream.cpp
+//		Purpose: Collect data in a buffer, and then read it out.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "CollectInBufferStream.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+#define INITIAL_BUFFER_SIZE	1024
+#define MAX_BUFFER_ADDITION	(1024*64)
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::CollectInBufferStream()
+//		Purpose: Constructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+CollectInBufferStream::CollectInBufferStream()
+	: mBuffer(INITIAL_BUFFER_SIZE),
+	  mBufferSize(INITIAL_BUFFER_SIZE),
+	  mBytesInBuffer(0),
+	  mReadPosition(0),
+	  mInWritePhase(true)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::~CollectInBufferStream()
+//		Purpose: Destructor
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+CollectInBufferStream::~CollectInBufferStream()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::Read(void *, int, int)
+//		Purpose: As interface. But only works in read phase
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+int CollectInBufferStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+	
+	// Adjust to number of bytes left
+	if(NBytes > (mBytesInBuffer - mReadPosition))
+	{
+		NBytes = (mBytesInBuffer - mReadPosition);
+	}
+	ASSERT(NBytes >= 0);
+	if(NBytes <= 0) return 0;	// careful now
+	
+	// Copy in the requested number of bytes and adjust the read pointer
+	::memcpy(pBuffer, ((char*)mBuffer) + mReadPosition, NBytes);
+	mReadPosition += NBytes;
+	
+	return NBytes;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::BytesLeftToRead()
+//		Purpose: As interface. But only works in read phase
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type CollectInBufferStream::BytesLeftToRead()
+{
+	if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+	
+	return (mBytesInBuffer - mReadPosition);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::Write(void *, int)
+//		Purpose: As interface. But only works in write phase
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::Write(const void *pBuffer, int NBytes)
+{
+	if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+	
+	// Enough space in the buffer
+	if((mBytesInBuffer + NBytes) > mBufferSize)
+	{
+		// Need to reallocate... what's the block size we'll use?
+		int allocateBlockSize = mBufferSize;
+		if(allocateBlockSize > MAX_BUFFER_ADDITION)
+		{
+			allocateBlockSize = MAX_BUFFER_ADDITION;
+		}
+		
+		// Write it the easy way. Although it's not the most efficient...
+		int newSize = mBufferSize;
+		while(newSize < (mBytesInBuffer + NBytes))
+		{
+			newSize += allocateBlockSize;
+		}
+		
+		// Reallocate buffer
+		mBuffer.Resize(newSize);
+		
+		// Store new size
+		mBufferSize = newSize;
+	}
+	
+	// Copy in data and adjust counter
+	::memcpy(((char*)mBuffer) + mBytesInBuffer, pBuffer, NBytes);
+	mBytesInBuffer += NBytes;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::GetPosition()
+//		Purpose: In write phase, returns the number of bytes written, in read
+//				 phase, the number of bytes to go
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type CollectInBufferStream::GetPosition() const
+{
+	return mInWritePhase?mBytesInBuffer:mReadPosition;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::Seek(pos_type, int)
+//		Purpose: As interface. But read phase only. 
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::Seek(pos_type Offset, int SeekType)
+{
+	if(mInWritePhase != false) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+	
+	int newPos = 0;
+	switch(SeekType)
+	{
+	case IOStream::SeekType_Absolute:
+		newPos = Offset;
+		break;
+	case IOStream::SeekType_Relative:
+		newPos = mReadPosition + Offset;
+		break;
+	case IOStream::SeekType_End:
+		newPos = mBytesInBuffer + Offset;
+		break;
+	default:
+		THROW_EXCEPTION(CommonException, IOStreamBadSeekType)
+		break;
+	}
+	
+	// Make sure it doesn't go over
+	if(newPos > mBytesInBuffer)
+	{
+		newPos = mBytesInBuffer;
+	}
+	// or under
+	if(newPos < 0)
+	{
+		newPos = 0;
+	}
+	
+	// Set the new read position
+	mReadPosition = newPos;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::StreamDataLeft()
+//		Purpose: As interface
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool CollectInBufferStream::StreamDataLeft()
+{
+	return mInWritePhase?(false):(mReadPosition < mBytesInBuffer);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::StreamClosed()
+//		Purpose: As interface
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool CollectInBufferStream::StreamClosed()
+{
+	return !mInWritePhase;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::SetForReading()
+//		Purpose: Switch to read phase, after all data written
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::SetForReading()
+{
+	if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) }
+	
+	// Move to read phase
+	mInWritePhase = false;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::GetBuffer()
+//		Purpose: Returns the buffer
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void *CollectInBufferStream::GetBuffer() const
+{
+	return mBuffer.GetPtr();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::GetSize()
+//		Purpose: Returns the buffer size
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+int CollectInBufferStream::GetSize() const
+{
+	return mBytesInBuffer;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CollectInBufferStream::Reset()
+//		Purpose: Reset the stream, so it is empty and ready to be written to.
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+void CollectInBufferStream::Reset()
+{
+	mInWritePhase = true;
+	mBytesInBuffer = 0;
+	mReadPosition = 0;
+}
+
diff --git a/lib/common/CollectInBufferStream.h b/lib/common/CollectInBufferStream.h
new file mode 100755
index 00000000..d73af8db
--- /dev/null
+++ b/lib/common/CollectInBufferStream.h
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CollectInBufferStream.h
+//		Purpose: Collect data in a buffer, and then read it out.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef COLLECTINBUFFERSTREAM__H
+#define COLLECTINBUFFERSTREAM__H
+
+#include "IOStream.h"
+#include "Guards.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    CollectInBufferStream
+//		Purpose: Collect data in a buffer, and then read it out.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class CollectInBufferStream : public IOStream
+{
+public:
+	CollectInBufferStream();
+	~CollectInBufferStream();
+private:
+	// No copying
+	CollectInBufferStream(const CollectInBufferStream &);
+	CollectInBufferStream(const IOStream &);
+public:
+
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type BytesLeftToRead();
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual pos_type GetPosition() const;
+	virtual void Seek(pos_type Offset, int SeekType);
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+	void SetForReading();
+	
+	void Reset();
+	
+	void *GetBuffer() const;
+	int GetSize() const;
+	bool IsSetForReading() const {return !mInWritePhase;}
+
+private:
+	MemoryBlockGuard<char*> mBuffer;
+	int mBufferSize;
+	int mBytesInBuffer;
+	int mReadPosition;
+	bool mInWritePhase;
+};
+
+#endif // COLLECTINBUFFERSTREAM__H
+
diff --git a/lib/common/CommonException.h b/lib/common/CommonException.h
new file mode 100755
index 00000000..a0eb3bf5
--- /dev/null
+++ b/lib/common/CommonException.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CommonException.h
+//		Purpose: Exception
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef COMMONEXCEPTION__H
+#define COMMONEXCEPTION__H
+
+// Compatibility header with old non-autogen exception scheme
+#include "autogen_CommonException.h"
+
+#endif // COMMONEXCEPTION__H
+
diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt
new file mode 100644
index 00000000..3875ed71
--- /dev/null
+++ b/lib/common/CommonException.txt
@@ -0,0 +1,44 @@
+
+# NOTE: Exception descriptions are for public distributions of Box Backup only -- do not rely for other applications.
+
+
+EXCEPTION Common 1
+
+Internal			0
+AssertFailed		1
+OSFileOpenError		2	Can't open a file -- attempted to load a non-existant config file or bad file referenced within?
+OSFileCloseError	3
+FileAlreadyClosed	4
+BadArguments		5
+ConfigNoKey			6
+ConfigNoSubConfig	7
+GetLineNoHandle		8
+OSFileError			9	Error accessing a file. Check permissions.
+GetLineEOF			10
+ConfigBadIntValue	11
+GetLineTooLarge		12	Protects against very large lines using up lots of memory.
+NotSupported		13
+OSFileReadError		14
+OSFileWriteError	15
+FileClosed			16
+IOStreamBadSeekType	17
+CantWriteToPartialReadStream			18
+CollectInBufferStreamNotInCorrectPhase	19
+NamedLockAlreadyLockingSomething		20
+NamedLockNotHeld						21
+StreamableMemBlockIncompleteRead		22
+MemBlockStreamNotSupported				23
+StreamDoesntHaveRequiredProperty		24
+CannotWriteToReadGatherStream			25
+ReadGatherStreamAddingBadBlock			26
+CouldNotLookUpUsername					27
+CouldNotRestoreProcessUser				28
+CouldNotChangeProcessUser				29
+RegexNotSupportedOnThisPlatform			30	Your platform does not have built in regular expression libraries.
+BadRegularExpression					31
+CouldNotCreateKQueue					32
+KEventErrorAdd							33
+KEventErrorWait							34
+KEventErrorRemove						35
+KQueueNotSupportedOnThisPlatform		36
+IOStreamGetLineNotEnoughDataToIgnore	37	Bad value passed to IOStreamGetLine::IgnoreBufferedData()
diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp
new file mode 100755
index 00000000..def93571
--- /dev/null
+++ b/lib/common/Configuration.cpp
@@ -0,0 +1,741 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Configuration.cpp
+//		Purpose: Reading configuration files
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <limits.h>
+
+#include "Configuration.h"
+#include "CommonException.h"
+#include "Guards.h"
+#include "FdGetLine.h"
+
+#include "MemLeakFindOn.h"
+
+// utility whitespace function
+inline bool iw(int c)
+{
+	return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded
+}
+
+// boolean values
+static const char *sValueBooleanStrings[] = {"yes", "true", "no", "false", 0};
+static const bool sValueBooleanValue[] = {true, true, false, false};
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::Configuration(const std::string &)
+//		Purpose: Constructor
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+Configuration::Configuration(const std::string &rName)
+	: mName(rName)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::Configuration(const Configuration &)
+//		Purpose: Copy constructor
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+Configuration::Configuration(const Configuration &rToCopy)
+	: mName(rToCopy.mName),
+	  mSubConfigurations(rToCopy.mSubConfigurations),
+	  mKeys(rToCopy.mKeys)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::~Configuration()
+//		Purpose: Destructor
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+Configuration::~Configuration()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::LoadAndVerify(const std::string &, const ConfigurationVerify *, std::string &)
+//		Purpose: Loads a configuration file from disc, checks it. Returns NULL if it was faulting, in which
+//				 case they'll be an error message.
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<Configuration> Configuration::LoadAndVerify(const char *Filename, const ConfigurationVerify *pVerify, std::string &rErrorMsg)
+{
+	// Check arguments
+	if(Filename == 0)
+	{
+		THROW_EXCEPTION(CommonException, BadArguments)
+	}
+	
+	// Just to make sure
+	rErrorMsg.erase();
+	
+	// Open the file
+	FileHandleGuard<O_RDONLY> file(Filename);
+	
+	// GetLine object
+	FdGetLine getline(file);
+	
+	// Object to create
+	Configuration *pconfig = new Configuration(std::string("<root>"));
+	
+	try
+	{
+		// Load
+		LoadInto(*pconfig, getline, rErrorMsg, true);
+
+		if(!rErrorMsg.empty())
+		{
+			// An error occured, return now
+			//TRACE1("Error message from LoadInto: %s", rErrorMsg.c_str());
+			TRACE0("Error at Configuration::LoadInfo\n");
+			delete pconfig;
+			pconfig = 0;
+			return std::auto_ptr<Configuration>(0);
+		}
+
+		// Verify?
+		if(pVerify)
+		{
+			if(!Verify(*pconfig, *pVerify, std::string(), rErrorMsg))
+			{
+				//TRACE1("Error message from Verify: %s", rErrorMsg.c_str());
+				TRACE0("Error at Configuration::Verify\n");
+				delete pconfig;
+				pconfig = 0;
+				return std::auto_ptr<Configuration>(0);
+			}
+		}
+	}
+	catch(...)
+	{
+		// Clean up
+		delete pconfig;
+		pconfig = 0;
+		throw;
+	}
+	
+	// Success. Return result.
+	return std::auto_ptr<Configuration>(pconfig);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    LoadInto(Configuration &, FdGetLine &, std::string &, bool)
+//		Purpose: Private. Load configuration information from the file into the config object.
+//				 Returns 'abort' flag, if error, will be appended to rErrorMsg.
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel)
+{
+	bool startBlockExpected = false;
+	std::string blockName;
+
+	//TRACE1("BLOCK: |%s|\n", rConfig.mName.c_str());
+
+	while(!rGetLine.IsEOF())
+	{
+		std::string line(rGetLine.GetLine(true));	/* preprocess out whitespace and comments */
+		
+		if(line.empty())
+		{
+			// Ignore blank lines
+			continue;
+		}
+		
+		// Line an open block string?
+		if(line == "{")
+		{
+			if(startBlockExpected)
+			{
+				// New config object
+				Configuration config(blockName);
+				
+				// Continue processing into this block
+				if(!LoadInto(config, rGetLine, rErrorMsg, false))
+				{
+					// Abort error
+					return false;
+				}
+
+				startBlockExpected = false;
+
+				// Store...
+				rConfig.mSubConfigurations.push_back(std::pair<std::string, Configuration>(blockName, config));
+			}
+			else
+			{
+				rErrorMsg += "Unexpected start block in " + rConfig.mName + "\n";
+			}
+		}
+		else
+		{
+			// Close block?
+			if(line == "}")
+			{
+				if(RootLevel)
+				{
+					// error -- root level doesn't have a close
+					rErrorMsg += "Root level has close block -- forget to terminate subblock?\n";
+					// but otherwise ignore
+				}
+				else
+				{
+					//TRACE0("ENDBLOCK\n");
+					return true;		// All very good and nice
+				}
+			}
+			// Either a key, or a sub block beginning
+			else
+			{
+				// Can't be a start block
+				if(startBlockExpected)
+				{
+					rErrorMsg += "Block " + blockName + " wasn't started correctly (no '{' on line of it's own)\n";
+					startBlockExpected = false;
+				}
+
+				// Has the line got an = in it?
+				unsigned int equals = 0;
+				for(; equals < line.size(); ++equals)
+				{
+					if(line[equals] == '=')
+					{
+						// found!
+						break;
+					}
+				}
+				if(equals < line.size())
+				{
+					// Make key value pair
+					unsigned int keyend = equals;
+					while(keyend > 0 && iw(line[keyend-1]))
+					{
+						keyend--;
+					}
+					unsigned int valuestart = equals+1;
+					while(valuestart < line.size() && iw(line[valuestart]))
+					{
+						valuestart++;
+					}
+					if(keyend > 0 && valuestart <= line.size())
+					{
+						std::string key(line.substr(0, keyend));
+						std::string value(line.substr(valuestart));
+						//TRACE2("KEY: |%s|=|%s|\n", key.c_str(), value.c_str());
+
+						// Check for duplicate values
+						if(rConfig.mKeys.find(key) != rConfig.mKeys.end())
+						{
+							// Multi-values allowed here, but checked later on
+							rConfig.mKeys[key] += MultiValueSeparator;
+							rConfig.mKeys[key] += value;
+						}
+						else
+						{
+							// Store
+							rConfig.mKeys[key] = value;
+						}
+					}
+					else
+					{
+						rErrorMsg += "Invalid key in block "+rConfig.mName+"\n";
+					}
+				}
+				else
+				{
+					// Start of sub block
+					blockName = line;
+					startBlockExpected = true;
+				}
+			}
+		}	
+	}
+
+	// End of file?
+	if(!RootLevel && rGetLine.IsEOF())
+	{
+		// Error if EOF and this isn't the root level
+		rErrorMsg += "File ended without terminating all subblocks\n";
+	}
+	
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::KeyExists(const char *)
+//		Purpose: Checks to see if a key exists
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+bool Configuration::KeyExists(const char *pKeyName) const
+{
+	if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+	return mKeys.find(pKeyName) != mKeys.end();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::GetKeyValue(const char *)
+//		Purpose: Returns the value of a configuration variable
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+const std::string &Configuration::GetKeyValue(const char *pKeyName) const
+{
+	if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+	std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName));
+	
+	if(i == mKeys.end())
+	{
+		THROW_EXCEPTION(CommonException, ConfigNoKey)
+	}
+	else
+	{
+		return i->second;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::GetKeyValueInt(const char *)
+//		Purpose: Gets a key value as an integer
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+int Configuration::GetKeyValueInt(const char *pKeyName) const
+{
+	if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+	std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName));
+	
+	if(i == mKeys.end())
+	{
+		THROW_EXCEPTION(CommonException, ConfigNoKey)
+	}
+	else
+	{
+		long value = ::strtol((i->second).c_str(), NULL, 0 /* C style handling */);
+		if(value == LONG_MAX || value == LONG_MIN)
+		{
+			THROW_EXCEPTION(CommonException, ConfigBadIntValue)
+		}
+		return (int)value;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::GetKeyValueBool(const char *) const
+//		Purpose: Gets a key value as a boolean
+//		Created: 17/2/04
+//
+// --------------------------------------------------------------------------
+bool Configuration::GetKeyValueBool(const char *pKeyName) const
+{
+	if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+	std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName));
+	
+	if(i == mKeys.end())
+	{
+		THROW_EXCEPTION(CommonException, ConfigNoKey)
+	}
+	else
+	{
+		bool value = false;
+		
+		// Anything this is called for should have been verified as having a correct
+		// string in the verification section. However, this does default to false
+		// if it isn't in the string table.
+		
+		for(int l = 0; sValueBooleanStrings[l] != 0; ++l)
+		{
+			if(::strcasecmp((i->second).c_str(), sValueBooleanStrings[l]) == 0)
+			{
+				// Found.
+				value = sValueBooleanValue[l];
+				break;
+			}
+		}
+		
+		return value;
+	}
+
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::GetKeyNames()
+//		Purpose: Returns list of key names
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+std::vector<std::string> Configuration::GetKeyNames() const
+{
+	std::map<std::string, std::string>::const_iterator i(mKeys.begin());
+	
+	std::vector<std::string> r;
+	
+	for(; i != mKeys.end(); ++i)
+	{
+		r.push_back(i->first);
+	}
+	
+	return r;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::SubConfigurationExists(const char *)
+//		Purpose: Checks to see if a sub configuration exists
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+bool Configuration::SubConfigurationExists(const char *pSubName) const
+{
+	if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+	// Attempt to find it...
+	std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin());
+	
+	for(; i != mSubConfigurations.end(); ++i)
+	{
+		// This the one?
+		if(i->first == pSubName)
+		{
+			// Yes.
+			return true;
+		}
+	}
+
+	// didn't find it.
+	return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::GetSubConfiguration(const char *)
+//		Purpose: Gets a sub configuration
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+const Configuration &Configuration::GetSubConfiguration(const char *pSubName) const
+{
+	if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+	// Attempt to find it...
+	std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin());
+	
+	for(; i != mSubConfigurations.end(); ++i)
+	{
+		// This the one?
+		if(i->first == pSubName)
+		{
+			// Yes.
+			return i->second;
+		}
+	}
+
+	THROW_EXCEPTION(CommonException, ConfigNoSubConfig)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::GetSubConfigurationNames()
+//		Purpose: Return list of sub configuration names
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+std::vector<std::string> Configuration::GetSubConfigurationNames() const
+{
+	std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin());
+	
+	std::vector<std::string> r;
+	
+	for(; i != mSubConfigurations.end(); ++i)
+	{
+		r.push_back(i->first);
+	}
+	
+	return r;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Configuration::Verify(const Configuration &, const ConfigurationVerify &, const std::string &, std::string &)
+//		Purpose: Return list of sub configuration names
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg)
+{
+	bool ok = true;
+
+	// First... check the keys
+	if(rVerify.mpKeys != 0)
+	{
+		const ConfigurationVerifyKey *pvkey = rVerify.mpKeys;
+		
+		bool todo = true;
+		do
+		{
+			// Can the key be found?
+			ASSERT(pvkey->mpName);
+			if(rConfig.KeyExists(pvkey->mpName))
+			{
+				// Get value
+				const std::string &rval = rConfig.GetKeyValue(pvkey->mpName);
+				const char *val = rval.c_str();
+
+				// Check it's a number?
+				if((pvkey->Tests & ConfigTest_IsInt) == ConfigTest_IsInt)
+				{					
+					// Test it...
+					char *end;
+					long r = ::strtol(val, &end, 0);
+					if(r == LONG_MIN || r == LONG_MAX || end != (val + rval.size()))
+					{
+						// not a good value
+						ok = false;
+						rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid integer.\n";
+					}
+				}
+				
+				// Check it's a bool?
+				if((pvkey->Tests & ConfigTest_IsBool) == ConfigTest_IsBool)
+				{				
+					// See if it's one of the allowed strings.
+					bool found = false;
+					for(int l = 0; sValueBooleanStrings[l] != 0; ++l)
+					{
+						if(::strcasecmp(val, sValueBooleanStrings[l]) == 0)
+						{
+							// Found.
+							found = true;
+							break;
+						}
+					}
+					
+					// Error if it's not one of them.
+					if(!found)
+					{
+						ok = false;
+						rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid boolean value.\n";
+					}
+				}
+				
+				// Check for multi valued statments where they're not allowed
+				if((pvkey->Tests & ConfigTest_MultiValueAllowed) == 0)
+				{
+					// Check to see if this key is a multi-value -- it shouldn't be
+					if(rval.find(MultiValueSeparator) != rval.npos)
+					{
+						ok = false;
+						rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) multi value not allowed (duplicated key?).\n";
+					}
+				}				
+			}
+			else
+			{
+				// Is it required to exist?
+				if((pvkey->Tests & ConfigTest_Exists) == ConfigTest_Exists)
+				{
+					// Should exist, but doesn't.
+					ok = false;
+					rErrorMsg += rLevel + rConfig.mName + "." + pvkey->mpName + " (key) is missing.\n";
+				}
+				else if(pvkey->mpDefaultValue)
+				{
+					rConfig.mKeys[std::string(pvkey->mpName)] = std::string(pvkey->mpDefaultValue);
+				}
+			}
+		
+			if((pvkey->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry)
+			{
+				// No more!
+				todo = false;
+			}
+			
+			// next
+			pvkey++;
+			
+		} while(todo);
+
+		// Check for additional keys
+		for(std::map<std::string, std::string>::const_iterator i = rConfig.mKeys.begin();
+			i != rConfig.mKeys.end(); ++i)
+		{
+			// Is the name in the list?
+			const ConfigurationVerifyKey *scan = rVerify.mpKeys;
+			bool found = false;
+			while(scan)
+			{
+				if(scan->mpName == i->first)
+				{
+					found = true;
+					break;
+				}
+				
+				// Next?
+				if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry)
+				{
+					break;
+				}
+				scan++;
+			}
+			
+			if(!found)
+			{
+				// Shouldn't exist, but does.
+				ok = false;
+				rErrorMsg += rLevel + rConfig.mName + "." + i->first + " (key) is not a known key. Check spelling and placement.\n";
+			}
+		}
+	}
+	
+	// Then the sub configurations
+	if(rVerify.mpSubConfigurations)
+	{
+		// Find the wildcard entry, if it exists, and check that required subconfigs are there
+		const ConfigurationVerify *wildcardverify = 0;
+	
+		const ConfigurationVerify *scan = rVerify.mpSubConfigurations;
+		while(scan)
+		{
+			ASSERT(scan->mpName);
+			if(scan->mpName[0] == '*')
+			{
+				wildcardverify = scan;
+			}
+			
+			// Required?
+			if((scan->Tests & ConfigTest_Exists) == ConfigTest_Exists)
+			{
+				if(scan->mpName[0] == '*')
+				{
+					// Check something exists
+					if(rConfig.mSubConfigurations.size() < 1)
+					{
+						// A sub config should exist, but doesn't.
+						ok = false;
+						rErrorMsg += rLevel + rConfig.mName + ".* (block) is missing (a block must be present).\n";
+					}
+				}
+				else
+				{
+					// Check real thing exists
+					if(!rConfig.SubConfigurationExists(scan->mpName))
+					{
+						// Should exist, but doesn't.
+						ok = false;
+						rErrorMsg += rLevel + rConfig.mName + "." + scan->mpName + " (block) is missing.\n";
+					}
+				}
+			}
+
+			// Next?
+			if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry)
+			{
+				break;
+			}
+			scan++;
+		}
+		
+		// Go through the sub configurations, one by one
+		for(std::list<std::pair<std::string, Configuration> >::const_iterator i(rConfig.mSubConfigurations.begin());
+			i != rConfig.mSubConfigurations.end(); ++i)
+		{
+			// Can this be found?
+			const ConfigurationVerify *subverify = 0;
+			
+			const ConfigurationVerify *scan = rVerify.mpSubConfigurations;
+			const char *name = i->first.c_str();
+			ASSERT(name);
+			while(scan)
+			{
+				if(strcmp(scan->mpName, name) == 0)
+				{
+					// found it!
+					subverify = scan;
+				}
+			
+				// Next?
+				if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry)
+				{
+					break;
+				}
+				scan++;
+			}
+			
+			// Use wildcard?
+			if(subverify == 0)
+			{
+				subverify = wildcardverify;
+			}
+			
+			// Verify
+			if(subverify)
+			{
+				// override const-ness here...
+				if(!Verify((Configuration&)i->second, *subverify, rConfig.mName + '.', rErrorMsg))
+				{
+					ok = false;
+				}
+			}
+		}
+	}
+	
+	return ok;
+}
+
+
diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h
new file mode 100755
index 00000000..4c455b0f
--- /dev/null
+++ b/lib/common/Configuration.h
@@ -0,0 +1,98 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Configuration
+//		Purpose: Reading configuration files
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+
+#ifndef CONFIGURATION__H
+#define CONFIGURATION__H
+
+#include <map>
+#include <list>
+#include <vector>
+#include <string>
+#include <memory>
+
+// For defining tests
+enum
+{
+	ConfigTest_LastEntry = 1,
+	ConfigTest_Exists = 2,
+	ConfigTest_IsInt = 4,
+	ConfigTest_MultiValueAllowed = 8,
+	ConfigTest_IsBool = 16
+};
+
+class ConfigurationVerifyKey
+{
+public:
+	const char *mpName;			// "*" for all other keys (not implemented yet)
+	const char *mpDefaultValue;	// default for when it's not present
+	int Tests;
+	void *TestFunction;			// set to zero for now, will implement later
+};
+
+class ConfigurationVerify
+{
+public:
+	const char *mpName;			// "*" for all other sub config names
+	const ConfigurationVerify *mpSubConfigurations;
+	const ConfigurationVerifyKey *mpKeys;
+	int Tests;	
+	void *TestFunction;			// set to zero for now, will implement later
+};
+
+class FdGetLine;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    Configuration
+//		Purpose: Loading, checking, and representing configuration files
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+class Configuration
+{
+private:
+	Configuration(const std::string &rName);
+public:
+	Configuration(const Configuration &rToCopy);
+	~Configuration();
+	
+	enum
+	{
+		// The character to separate multi-values
+		MultiValueSeparator = '\x01'
+	};
+	
+	static std::auto_ptr<Configuration> LoadAndVerify(const char *Filename, const ConfigurationVerify *pVerify, std::string &rErrorMsg);
+	static std::auto_ptr<Configuration> Load(const char *Filename, std::string &rErrorMsg) { return LoadAndVerify(Filename, 0, rErrorMsg); }
+	
+	bool KeyExists(const char *pKeyName) const;
+	const std::string &GetKeyValue(const char *pKeyName) const;
+	int GetKeyValueInt(const char *pKeyName) const;
+	bool GetKeyValueBool(const char *pKeyName) const;
+	std::vector<std::string> GetKeyNames() const;
+	
+	bool SubConfigurationExists(const char *pSubName) const;
+	const Configuration &GetSubConfiguration(const char *pSubName) const;
+	std::vector<std::string> GetSubConfigurationNames() const;
+	
+	std::string mName;
+	// Order of sub blocks preserved
+	typedef std::list<std::pair<std::string, Configuration> > SubConfigListType;
+	SubConfigListType mSubConfigurations;
+	// Order of keys, not preserved
+	std::map<std::string, std::string> mKeys;
+	
+private:
+	static bool LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel);
+	static bool Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg);
+};
+
+#endif // CONFIGURATION__H
+
diff --git a/lib/common/Conversion.h b/lib/common/Conversion.h
new file mode 100644
index 00000000..cba5bb08
--- /dev/null
+++ b/lib/common/Conversion.h
@@ -0,0 +1,98 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Conversion.h
+//		Purpose: Convert between various types
+//		Created: 9/4/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef CONVERSION__H
+#define CONVERSION__H
+
+#include <string>
+
+namespace BoxConvert
+{
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    BoxConvert::Convert<to_type, from_type>(to_type &, from_type)
+	//		Purpose: Convert from types to types
+	//		Created: 9/4/04
+	//
+	// --------------------------------------------------------------------------
+	template<typename to_type, typename from_type>
+	inline to_type Convert(from_type From)
+	{
+		// Default conversion, simply use C++ conversion
+		return From;
+	}
+
+	// Specialise for string -> integer
+	int32_t _ConvertStringToInt(const char *pString, int Size);
+	template<>
+	inline int32_t Convert<int32_t, const std::string &>(const std::string &rFrom)
+	{
+		return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 32);
+	}
+	template<>
+	inline int16_t Convert<int16_t, const std::string &>(const std::string &rFrom)
+	{
+		return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 16);
+	}
+	template<>
+	inline int8_t Convert<int8_t, const std::string &>(const std::string &rFrom)
+	{
+		return BoxConvert::_ConvertStringToInt(rFrom.c_str(), 8);
+	}
+	template<>
+	inline int32_t Convert<int32_t, const char *>(const char *pFrom)
+	{
+		return BoxConvert::_ConvertStringToInt(pFrom, 32);
+	}
+	template<>
+	inline int16_t Convert<int16_t, const char *>(const char *pFrom)
+	{
+		return BoxConvert::_ConvertStringToInt(pFrom, 16);
+	}
+	template<>
+	inline int8_t Convert<int8_t, const char *>(const char *pFrom)
+	{
+		return BoxConvert::_ConvertStringToInt(pFrom, 8);
+	}
+	
+	// Specialise for integer -> string
+	void _ConvertIntToString(std::string &rTo, int32_t From);
+	template<>
+	inline std::string Convert<std::string, int32_t>(int32_t From)
+	{
+		std::string r;
+		BoxConvert::_ConvertIntToString(r, From);
+		return r;
+	}
+	template<>
+	inline std::string Convert<std::string, int16_t>(int16_t From)
+	{
+		std::string r;
+		BoxConvert::_ConvertIntToString(r, From);
+		return r;
+	}
+	template<>
+	inline std::string Convert<std::string, int8_t>(int8_t From)
+	{
+		std::string r;
+		BoxConvert::_ConvertIntToString(r, From);
+		return r;
+	}
+	
+	// Specialise for bool -> string
+	template<>
+	inline std::string Convert<std::string, bool>(bool From)
+	{
+		return std::string(From?"true":"false");
+	}
+};
+
+#endif // CONVERSION__H
+
diff --git a/lib/common/ConversionException.txt b/lib/common/ConversionException.txt
new file mode 100644
index 00000000..91b5fa9a
--- /dev/null
+++ b/lib/common/ConversionException.txt
@@ -0,0 +1,8 @@
+
+EXCEPTION Conversion 12
+
+Internal							0
+CannotConvertEmptyStringToInt		1
+BadStringRepresentationOfInt		2
+IntOverflowInConvertFromString		3
+BadIntSize							4
diff --git a/lib/common/ConversionString.cpp b/lib/common/ConversionString.cpp
new file mode 100644
index 00000000..9e237a77
--- /dev/null
+++ b/lib/common/ConversionString.cpp
@@ -0,0 +1,123 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ConversionString.cpp
+//		Purpose: Conversions to and from strings
+//		Created: 9/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+
+#include "Conversion.h"
+#include "autogen_ConversionException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BoxConvert::_ConvertStringToInt(const char *, int)
+//		Purpose: Convert from string to integer, with range checking.
+//				 Always does signed -- no point in unsigned as C++ type checking
+//				 isn't up to handling it properly.
+//				 If a null pointer is passed in, then returns 0.
+//		Created: 9/4/04
+//
+// --------------------------------------------------------------------------
+int32_t BoxConvert::_ConvertStringToInt(const char *pString, int Size)
+{
+	// Handle null strings gracefully.
+	if(pString == 0)
+	{
+		return 0;
+	}
+
+	// Check for initial validity
+	if(*pString == '\0')
+	{
+		THROW_EXCEPTION(ConversionException, CannotConvertEmptyStringToInt)
+	}
+	
+	// Convert.
+	char *numEnd = 0;
+	errno = 0;			// Some platforms don't reset it.
+	long r = ::strtol(pString, &numEnd, 0);
+
+	// Check that all the characters were used
+	if(*numEnd != '\0')
+	{
+		THROW_EXCEPTION(ConversionException, BadStringRepresentationOfInt)
+	}
+	
+	// Error check
+	if(r == 0 && errno == EINVAL)
+	{
+		THROW_EXCEPTION(ConversionException, BadStringRepresentationOfInt)
+	}
+
+	// Range check from strtol
+	if((r == LONG_MIN || r == LONG_MAX) && errno == ERANGE)
+	{
+		THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString)
+	}
+	
+	// Check range for size of integer
+	switch(Size)
+	{
+	case 32:
+		{
+			// No extra checking needed, if this assert holds true
+			ASSERT(sizeof(long) == sizeof(int32_t));
+		}
+		break;
+		
+	case 16:
+		{
+			if(r <= (0 - 0x7fff) || r > 0x7fff)
+			{
+				THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString)
+			}
+			break;
+		}
+	
+	case 8:
+		{
+			if(r <= (0 - 0x7f) || r > 0x7f)
+			{
+				THROW_EXCEPTION(ConversionException, IntOverflowInConvertFromString)
+			}
+			break;
+		}
+		
+	default:
+		{
+			THROW_EXCEPTION(ConversionException, BadIntSize)
+			break;
+		}
+	}
+	
+	// Return number
+	return r;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BoxConvert::_ConvertIntToString(std::string &, int32_t)
+//		Purpose: Convert signed interger to a string
+//		Created: 9/4/04
+//
+// --------------------------------------------------------------------------
+void BoxConvert::_ConvertIntToString(std::string &rTo, int32_t From)
+{
+	char text[64];	// size more than enough
+	::sprintf(text, "%d", From);
+	rTo = text;
+}
+
diff --git a/lib/common/DebugAssertFailed.cpp b/lib/common/DebugAssertFailed.cpp
new file mode 100755
index 00000000..2acf4602
--- /dev/null
+++ b/lib/common/DebugAssertFailed.cpp
@@ -0,0 +1,32 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    AssertFailed.cpp
+//		Purpose: Assert failure code
+//		Created: 2003/09/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef NDEBUG
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <syslog.h>
+
+#include "MemLeakFindOn.h"
+
+bool AssertFailuresToSyslog = false;
+
+void BoxDebugAssertFailed(char *cond, char *file, int line)
+{
+	printf("ASSERT FAILED: [%s] at %s(%d)\n", cond, file, line);
+	if(AssertFailuresToSyslog)
+	{
+		::syslog(LOG_ERR, "ASSERT FAILED: [%s] at %s(%d)\n", cond, file, line);
+	}
+}
+
+
+#endif // NDEBUG
+
diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp
new file mode 100755
index 00000000..e1877a13
--- /dev/null
+++ b/lib/common/DebugMemLeakFinder.cpp
@@ -0,0 +1,377 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    MemLeakFinder.cpp
+//		Purpose: Memory leak finder
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+
+#ifndef NDEBUG
+
+#include "Box.h"
+
+#undef malloc
+#undef realloc
+#undef free
+
+#include <map>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <set>
+
+bool memleakfinder_global_enable = false;
+
+typedef struct
+{
+	size_t size;
+	const char *file;
+	int line;
+} MallocBlockInfo;
+
+typedef struct
+{
+	size_t size;
+	const char *file;
+	int line;
+	bool array;
+} ObjectInfo;
+
+namespace
+{
+	static std::map<void *, MallocBlockInfo> sMallocBlocks;
+	static std::map<void *, ObjectInfo> sObjectBlocks;
+	
+	static bool sTrackMallocInSection = false;
+	static std::set<void *> sSectionMallocBlocks;
+	static bool sTrackObjectsInSection = false;
+	static std::map<void *, ObjectInfo> sSectionObjectBlocks;
+	
+	static std::set<void *> sNotLeaks;
+
+	void *sNotLeaksPre[1024];
+	int sNotLeaksPreNum = 0;
+}
+
+void memleakfinder_malloc_add_block(void *b, size_t size, const char *file, int line)
+{
+	if(b != 0)
+	{
+		MallocBlockInfo i;
+		i.size = size;
+		i.file = file;
+		i.line = line;
+		sMallocBlocks[b] = i;
+		
+		if(sTrackMallocInSection)
+		{
+			sSectionMallocBlocks.insert(b);
+		}
+	}
+}
+
+
+void *memleakfinder_malloc(size_t size, const char *file, int line)
+{
+	void *b = ::malloc(size);
+	if(!memleakfinder_global_enable) return b;
+
+	memleakfinder_malloc_add_block(b, size, file, line);
+
+	//TRACE4("malloc(), %d, %s, %d, %08x\n", size, file, line, b);
+	return b;
+}
+
+void *memleakfinder_realloc(void *ptr, size_t size)
+{
+	if(!memleakfinder_global_enable)
+	{
+		return ::realloc(ptr, size);
+	}
+
+	// Check it's been allocated
+	std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr));
+	if(i == sMallocBlocks.end())
+	{
+		TRACE1("Block %x realloc(), but not in list. Error? Or allocated in startup static objects?\n", ptr);
+		void *b = ::realloc(ptr, size);
+		memleakfinder_malloc_add_block(b, size, "FOUND-IN-REALLOC", 0);
+		return b;
+	}
+
+	void *b = ::realloc(ptr, size);
+	
+	// Worked?
+	if(b != 0)
+	{
+		// Update map
+		MallocBlockInfo inf = i->second;
+		inf.size = size;
+		sMallocBlocks.erase(i);
+		sMallocBlocks[b] = inf;
+		
+		if(sTrackMallocInSection)
+		{
+			std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr));
+			if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si);
+			sSectionMallocBlocks.insert(b);
+		}
+	}
+	
+	//TRACE3("realloc(), %d, %08x->%08x\n", size, ptr, b);
+	return b;
+}
+
+void memleakfinder_free(void *ptr)
+{
+	if(memleakfinder_global_enable)
+	{
+		// Check it's been allocated
+		std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr));
+		if(i != sMallocBlocks.end())
+		{
+			sMallocBlocks.erase(i);
+		}
+		else
+		{
+			TRACE1("Block %x freed, but not known. Error? Or allocated in startup static allocation?\n", ptr);
+		}
+
+		if(sTrackMallocInSection)
+		{
+			std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr));
+			if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si);
+		}
+	}
+
+	//TRACE1("free(), %08x\n", ptr);
+	::free(ptr);
+}
+
+
+void memleakfinder_notaleak_insert_pre()
+{
+	if(!memleakfinder_global_enable) return;
+	for(int l = 0; l < sNotLeaksPreNum; l++)
+	{
+		sNotLeaks.insert(sNotLeaksPre[l]);
+	}
+	sNotLeaksPreNum = 0;
+}
+
+bool is_leak(void *ptr)
+{
+	memleakfinder_notaleak_insert_pre();
+	return sNotLeaks.find(ptr) == sNotLeaks.end();
+}
+
+void memleakfinder_notaleak(void *ptr)
+{
+	memleakfinder_notaleak_insert_pre();
+	if(memleakfinder_global_enable)
+	{
+		sNotLeaks.insert(ptr);
+	}
+	else
+	{
+		sNotLeaksPre[sNotLeaksPreNum++] = ptr;
+	}
+/*	{
+		std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr));
+		if(i != sMallocBlocks.end()) sMallocBlocks.erase(i);
+	}
+	{
+		std::set<void *>::iterator si(sSectionMallocBlocks.find(ptr));
+		if(si != sSectionMallocBlocks.end()) sSectionMallocBlocks.erase(si);
+	}
+	{
+		std::map<void *, ObjectInfo>::iterator i(sObjectBlocks.find(ptr));
+		if(i != sObjectBlocks.end()) sObjectBlocks.erase(i);
+	}*/
+}
+
+
+
+// start monitoring a section of code
+void memleakfinder_startsectionmonitor()
+{
+	sTrackMallocInSection = true;
+	sSectionMallocBlocks.clear();
+	sTrackObjectsInSection = true;
+	sSectionObjectBlocks.clear();
+}
+
+// trace all blocks allocated and still allocated since memleakfinder_startsectionmonitor() called
+void memleakfinder_traceblocksinsection()
+{
+	std::set<void *>::iterator s(sSectionMallocBlocks.begin());
+	for(; s != sSectionMallocBlocks.end(); ++s)
+	{
+		std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.find(*s));
+		if(i == sMallocBlocks.end())
+		{
+			TRACE0("Logical error in section block finding\n");
+		}
+		else
+		{
+			TRACE4("Block 0x%08x size %d allocated at %s:%d\n", (int)i->first, i->second.size, i->second.file, i->second.line);
+		}
+	}
+	for(std::map<void *, ObjectInfo>::const_iterator i(sSectionObjectBlocks.begin()); i != sSectionObjectBlocks.end(); ++i)
+	{
+		TRACE5("Object%s 0x%08x size %d allocated at %s:%d\n", i->second.array?" []":"", (int)i->first, i->second.size, i->second.file, i->second.line);
+	}
+}
+
+int memleakfinder_numleaks()
+{
+	int n = 0;
+	
+	for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i)
+	{
+		if(is_leak(i->first)) ++n;
+	}
+	
+	for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i)
+	{
+		if(is_leak(i->first)) ++n;
+	}
+
+	return n;
+}
+
+void memleakfinder_reportleaks_file(FILE *file)
+{
+	for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i)
+	{
+		if(is_leak(i->first)) ::fprintf(file, "Block 0x%08x size %d allocated at %s:%d\n", (int)i->first, i->second.size, i->second.file, i->second.line);
+	}
+	for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i)
+	{
+		if(is_leak(i->first)) ::fprintf(file, "Object%s 0x%08x size %d allocated at %s:%d\n", i->second.array?" []":"", (int)i->first, i->second.size, i->second.file, i->second.line);
+	}
+}
+
+void memleakfinder_reportleaks()
+{
+	// report to stdout
+	memleakfinder_reportleaks_file(stdout);
+}
+
+void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext)
+{
+	FILE *file = ::fopen(filename, "a");
+	if(file != 0)
+	{
+		if(memleakfinder_numleaks() > 0)
+		{
+			fprintf(file, "MEMORY LEAKS FROM PROCESS %d (%s)\n", getpid(), markertext);
+			memleakfinder_reportleaks_file(file);
+		}
+	
+		::fclose(file);
+	}
+	else
+	{
+		printf("WARNING: Couldn't open memory leak results file %s for appending\n", filename);
+	}
+}
+
+static char atexit_filename[512];
+static char atexit_markertext[512];
+static bool atexit_registered = false;
+
+void memleakfinder_atexit()
+{
+	memleakfinder_reportleaks_appendfile(atexit_filename, atexit_markertext);
+}
+
+void memleakfinder_setup_exit_report(const char *filename, const char *markertext)
+{
+	::strcpy(atexit_filename, filename);
+	::strcpy(atexit_markertext, markertext);
+	if(!atexit_registered)
+	{
+		atexit(memleakfinder_atexit);
+		atexit_registered = true;
+	}
+}
+
+
+
+
+void add_object_block(void *block, size_t size, const char *file, int line, bool array)
+{
+	if(!memleakfinder_global_enable) return;
+
+	if(block != 0)
+	{
+		ObjectInfo i;
+		i.size = size;
+		i.file = file;
+		i.line = line;
+		i.array = array;
+		sObjectBlocks[block] = i;
+		
+		if(sTrackObjectsInSection)
+		{
+			sSectionObjectBlocks[block] = i;
+		}
+	}
+}
+
+void remove_object_block(void *block)
+{
+	if(!memleakfinder_global_enable) return;
+
+	std::map<void *, ObjectInfo>::iterator i(sObjectBlocks.find(block));
+	if(i != sObjectBlocks.end())
+	{
+		sObjectBlocks.erase(i);
+	}
+
+	if(sTrackObjectsInSection)
+	{
+		std::map<void *, ObjectInfo>::iterator i(sSectionObjectBlocks.find(block));
+		if(i != sSectionObjectBlocks.end())
+		{
+			sSectionObjectBlocks.erase(i);
+		}
+	}
+
+	// If it's not in the list, just ignore it, as lots of stuff goes this way...
+}
+
+void *operator new(size_t size, const char *file, int line)
+{
+	void *r = ::malloc(size);
+	add_object_block(r, size, file, line, false);
+	//TRACE4("new(), %d, %s, %d, %08x\n", size, file, line, r);
+	return r;
+}
+
+void *operator new[](size_t size, const char *file, int line)
+{
+	void *r = ::malloc(size);
+	add_object_block(r, size, file, line, true);
+	//TRACE4("new[](), %d, %s, %d, %08x\n", size, file, line, r);
+	return r;
+}
+
+void operator delete[](void *ptr) throw ()
+{
+	::free(ptr);
+	remove_object_block(ptr);
+	//TRACE1("delete[]() called, %08x\n", ptr);
+}
+
+void operator delete(void *ptr) throw ()
+{
+	::free(ptr);
+	remove_object_block(ptr);
+	//TRACE1("delete() called, %08x\n", ptr);
+}
+
+#endif // NDEBUG
diff --git a/lib/common/DebugPrintf.cpp b/lib/common/DebugPrintf.cpp
new file mode 100755
index 00000000..02c25496
--- /dev/null
+++ b/lib/common/DebugPrintf.cpp
@@ -0,0 +1,72 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    DebugPrintf.cpp
+//		Purpose: Implementation of a printf function, to avoid a stdio.h include in Box.h
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+
+#ifndef NDEBUG
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <syslog.h>
+
+#include "MemLeakFindOn.h"
+
+// Use this apparently superflous printf function to avoid having to 
+// include stdio.h in every file in the project.
+
+int BoxDebug_printf(const char *format, ...)
+{
+	va_list ap;
+	va_start(ap, format);
+	int r = vprintf(format, ap);
+	va_end(ap);
+	return r;
+}
+
+
+bool BoxDebugTraceOn = true;
+bool BoxDebugTraceToStdout = true;
+bool BoxDebugTraceToSyslog = false;
+
+int BoxDebugTrace(const char *format, ...)
+{
+	char text[512];
+	int r = 0;
+	if(BoxDebugTraceOn || BoxDebugTraceToSyslog)
+	{
+		va_list ap;
+		va_start(ap, format);
+		r = vsnprintf(text, sizeof(text), format, ap);
+		va_end(ap);
+	}
+
+	// Send to stdout if trace is on and std out is enabled
+	if(BoxDebugTraceOn && BoxDebugTraceToStdout)
+	{
+		printf("%s", text);		
+	}
+
+	// But tracing to syslog is independent of tracing being on or not
+	if(BoxDebugTraceToSyslog)
+	{
+		// Remove trailing '\n', if it's there
+		if(r > 0 && text[r] == '\n')
+		{
+			text[r] = '\0';
+			--r;
+		}
+		// Log it
+		::syslog(LOG_INFO, "TRACE: %s", text);
+	}
+
+	return r;
+}
+
+
+#endif // NDEBUG
diff --git a/lib/common/EndStructPackForWire.h b/lib/common/EndStructPackForWire.h
new file mode 100644
index 00000000..c9655cd7
--- /dev/null
+++ b/lib/common/EndStructPackForWire.h
@@ -0,0 +1,23 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    EndStructPackForWire.h
+//		Purpose: End structure packing for wire
+//		Created: 25/11/03
+//
+// --------------------------------------------------------------------------
+
+// No header guard -- this is intentional
+
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+
+#pragma pack()
+
+#else
+
+	logical error -- check BoxPlatform.h and including file
+
+#endif
+
+
+
diff --git a/lib/common/EventWatchFilesystemObject.cpp b/lib/common/EventWatchFilesystemObject.cpp
new file mode 100644
index 00000000..1611d5bd
--- /dev/null
+++ b/lib/common/EventWatchFilesystemObject.cpp
@@ -0,0 +1,99 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    EventWatchFilesystemObject.cpp
+//		Purpose: WaitForEvent compatible object for watching directories
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "EventWatchFilesystemObject.h"
+#include "autogen_CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    EventWatchFilesystemObject::EventWatchFilesystemObject(const char *)
+//		Purpose: Constructor -- opens the file object
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename)
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	: mDescriptor(::open(Filename, O_RDONLY /*O_EVTONLY*/, 0))
+#endif
+{
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	if(mDescriptor == -1)
+	{
+		THROW_EXCEPTION(CommonException, OSFileOpenError)
+	}
+#else
+	THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform)
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    EventWatchFilesystemObject::~EventWatchFilesystemObject()
+//		Purpose: Destructor
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+EventWatchFilesystemObject::~EventWatchFilesystemObject()
+{
+	if(mDescriptor != -1)
+	{
+		::close(mDescriptor);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &)
+//		Purpose: Copy constructor
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy)
+	: mDescriptor(::dup(rToCopy.mDescriptor))
+{
+	if(mDescriptor == -1)
+	{
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+}
+
+
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    EventWatchFilesystemObject::FillInKEvent(struct kevent &, int)
+//		Purpose: For WaitForEvent
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, int Flags) const
+{
+	EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, NOTE_DELETE | NOTE_WRITE, 0, (void*)this);
+}
+#else
+void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, int Flags) const
+{
+	THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform)
+}
+#endif
+
diff --git a/lib/common/EventWatchFilesystemObject.h b/lib/common/EventWatchFilesystemObject.h
new file mode 100644
index 00000000..d8a7245b
--- /dev/null
+++ b/lib/common/EventWatchFilesystemObject.h
@@ -0,0 +1,48 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    EventWatchFilesystemObject.h
+//		Purpose: WaitForEvent compatible object for watching directories
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef EVENTWATCHFILESYSTEMOBJECT__H
+#define EVENTWATCHFILESYSTEMOBJECT__H
+
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	#include <sys/event.h>
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    EventWatchFilesystemObject
+//		Purpose: WaitForEvent compatible object for watching files and directories
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+class EventWatchFilesystemObject
+{
+public:
+	EventWatchFilesystemObject(const char *Filename);
+	~EventWatchFilesystemObject();
+	EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy);
+private:
+	// Assignment not allowed
+	EventWatchFilesystemObject &operator=(const EventWatchFilesystemObject &);
+public:
+
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	void FillInKEvent(struct kevent &rEvent, int Flags = 0) const;
+#else
+	void FillInPoll(int &fd, short &events, int Flags = 0) const;
+#endif
+
+private:
+	int mDescriptor;
+};
+
+#endif // EventWatchFilesystemObject__H
+
diff --git a/lib/common/ExcludeList.cpp b/lib/common/ExcludeList.cpp
new file mode 100755
index 00000000..6350869d
--- /dev/null
+++ b/lib/common/ExcludeList.cpp
@@ -0,0 +1,219 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ExcludeList.cpp
+//		Purpose: General purpose exclusion list
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+	#include <regex.h>
+	#define EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED
+#endif
+
+#include "ExcludeList.h"
+#include "Utils.h"
+#include "Configuration.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ExcludeList::ExcludeList()
+//		Purpose: Constructor. Generates an exclude list which will allow everything
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+ExcludeList::ExcludeList()
+	: mpAlwaysInclude(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ExcludeList::~ExcludeList()
+//		Purpose: Destructor
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+ExcludeList::~ExcludeList()
+{
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+	// free regex memory
+	while(mRegex.size() > 0)
+	{
+		regex_t *pregex = mRegex.back();
+		mRegex.pop_back();
+		// Free regex storage, and the structure itself
+		::regfree(pregex);
+		delete pregex;
+	}
+#endif
+
+	// Clean up exceptions list
+	if(mpAlwaysInclude != 0)
+	{
+		delete mpAlwaysInclude;
+		mpAlwaysInclude = 0;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ExcludeList::AddDefiniteEntries(const std::string &)
+//		Purpose: Adds a number of definite entries to the exclude list -- ones which
+//				 will be excluded if and only if the test string matches exactly.
+//				 Uses the Configuration classes' multi-value conventions, with
+//				 multiple entires in one string separated by Configuration::MultiValueSeparator
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+void ExcludeList::AddDefiniteEntries(const std::string &rEntries)
+{
+	// Split strings up
+	std::vector<std::string> ens;
+	SplitString(rEntries, Configuration::MultiValueSeparator, ens);
+	
+	// Add to set of excluded strings
+	for(std::vector<std::string>::const_iterator i(ens.begin()); i != ens.end(); ++i)
+	{
+		if(i->size() > 0)
+		{
+			mDefinite.insert(*i);
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ExcludeList::AddRegexEntries(const std::string &)
+//		Purpose: Adds a number of regular expression entries to the exclude list -- 
+//				 if the test expression matches any of these regex, it will be excluded.
+//				 Uses the Configuration classes' multi-value conventions, with
+//				 multiple entires in one string separated by Configuration::MultiValueSeparator
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+void ExcludeList::AddRegexEntries(const std::string &rEntries)
+{
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+
+	// Split strings up
+	std::vector<std::string> ens;
+	SplitString(rEntries, Configuration::MultiValueSeparator, ens);
+	
+	// Create and add new regular expressions
+	for(std::vector<std::string>::const_iterator i(ens.begin()); i != ens.end(); ++i)
+	{
+		if(i->size() > 0)
+		{
+			// Allocate memory
+			regex_t *pregex = new regex_t;
+			
+			try
+			{
+				// Compile
+				if(::regcomp(pregex, i->c_str(), REG_EXTENDED | REG_NOSUB) != 0)
+				{
+					THROW_EXCEPTION(CommonException, BadRegularExpression)
+				}
+				
+				// Store in list of regular expressions
+				mRegex.push_back(pregex);
+			}
+			catch(...)
+			{
+				delete pregex;
+				throw;
+			}
+		}
+	}
+
+#else
+	THROW_EXCEPTION(CommonException, RegexNotSupportedOnThisPlatform)
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ExcludeList::IsExcluded(const std::string &)
+//		Purpose: Returns true if the entry should be excluded
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+bool ExcludeList::IsExcluded(const std::string &rTest) const
+{
+	// Check against the always include list
+	if(mpAlwaysInclude != 0)
+	{
+		if(mpAlwaysInclude->IsExcluded(rTest))
+		{
+			// Because the "always include" list says it's 'excluded'
+			// this means it should actually be included.
+			return false;
+		}
+	}
+
+	// Is it in the set of definite entries?
+	if(mDefinite.find(rTest) != mDefinite.end())
+	{
+		return true;
+	}
+	
+	// Check against regular expressions
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+	for(std::vector<regex_t *>::const_iterator i(mRegex.begin()); i != mRegex.end(); ++i)
+	{
+		// Test against this expression
+		if(regexec(*i, rTest.c_str(), 0, 0 /* no match information required */, 0 /* no flags */) == 0)
+		{
+			// match happened
+			return true;
+		}
+		// In all other cases, including an error, just continue to the next expression
+	}
+#endif
+
+	return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ExcludeList::SetAlwaysIncludeList(ExcludeList *)
+//		Purpose: Takes ownership of the list, deletes any pre-existing list.
+//				 NULL is acceptable to delete the list.
+//				 The AlwaysInclude list is a list of exceptions to the exclusions.
+//		Created: 19/2/04
+//
+// --------------------------------------------------------------------------
+void ExcludeList::SetAlwaysIncludeList(ExcludeList *pAlwaysInclude)
+{
+	// Delete old list
+	if(mpAlwaysInclude != 0)
+	{
+		delete mpAlwaysInclude;
+		mpAlwaysInclude = 0;
+	}
+	
+	// Store the pointer
+	mpAlwaysInclude = pAlwaysInclude;
+}
+
+
+	
+
+
diff --git a/lib/common/ExcludeList.h b/lib/common/ExcludeList.h
new file mode 100755
index 00000000..a1954044
--- /dev/null
+++ b/lib/common/ExcludeList.h
@@ -0,0 +1,65 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ExcludeList.h
+//		Purpose: General purpose exclusion list
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef EXCLUDELIST__H
+#define EXCLUDELIST__H
+
+#include <string>
+#include <set>
+#include <vector>
+
+// avoid including regex.h in lots of places
+#ifndef EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED
+	typedef int regex_t;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    ExcludeList
+//		Purpose: General purpose exclusion list
+//		Created: 28/1/04
+//
+// --------------------------------------------------------------------------
+class ExcludeList
+{
+public:
+	ExcludeList();
+	~ExcludeList();
+
+	void AddDefiniteEntries(const std::string &rEntries);
+	void AddRegexEntries(const std::string &rEntries);
+
+	// Add exceptions to the exclusions (takes ownership)
+	void SetAlwaysIncludeList(ExcludeList *pAlwaysInclude);
+	
+	// Test function
+	bool IsExcluded(const std::string &rTest) const;
+	
+	// Mainly for tests
+	unsigned int SizeOfDefiniteList() const {return mDefinite.size();}
+	unsigned int SizeOfRegexList() const
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+		{return mRegex.size();}
+#else
+		{return 0;}
+#endif
+
+private:
+	std::set<std::string> mDefinite;
+#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+	std::vector<regex_t *> mRegex;
+#endif
+
+	// For exceptions to the excludes
+	ExcludeList *mpAlwaysInclude;
+};
+
+#endif // EXCLUDELIST__H
+
diff --git a/lib/common/FdGetLine.cpp b/lib/common/FdGetLine.cpp
new file mode 100755
index 00000000..dee02604
--- /dev/null
+++ b/lib/common/FdGetLine.cpp
@@ -0,0 +1,211 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    FdGetLine.cpp
+//		Purpose: Line based file descriptor reading
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "FdGetLine.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// utility whitespace function
+inline bool iw(int c)
+{
+	return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FdGetLine::FdGetLine(int)
+//		Purpose: Constructor, taking file descriptor
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+FdGetLine::FdGetLine(int fd)
+	: mFileHandle(fd),
+	  mLineNumber(0),
+	  mBufferBegin(0),
+	  mBytesInBuffer(0),
+	  mPendingEOF(false),
+	  mEOF(false)
+{
+	if(mFileHandle < 0) {THROW_EXCEPTION(CommonException, BadArguments)}
+	//printf("FdGetLine buffer size = %d\n", sizeof(mBuffer));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FdGetLine::~FdGetLine()
+//		Purpose: Destructor
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+FdGetLine::~FdGetLine()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FdGetLine::GetLine(bool)
+//		Purpose: Returns a file from the file. If Preprocess is true, leading
+//				 and trailing whitespace is removed, and comments (after #)
+//				 are deleted.
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+std::string FdGetLine::GetLine(bool Preprocess)
+{
+	if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)}
+
+	// EOF?
+	if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)}
+	
+	std::string r;
+
+	bool foundLineEnd = false;
+
+	while(!foundLineEnd && !mEOF)
+	{
+		// Use any bytes left in the buffer
+		while(mBufferBegin < mBytesInBuffer)
+		{
+			int c = mBuffer[mBufferBegin++];
+			if(c == '\r')
+			{
+				// Ignore nasty Windows line ending extra chars
+			}
+			else if(c == '\n')
+			{
+				// Line end!
+				foundLineEnd = true;
+				break;
+			}
+			else
+			{
+				// Add to string
+				r += c;
+			}
+			
+			// Implicit line ending at EOF
+			if(mBufferBegin >= mBytesInBuffer && mPendingEOF)
+			{
+				foundLineEnd = true;
+			}
+		}
+		
+		// Check size
+		if(r.size() > FDGETLINE_MAX_LINE_SIZE)
+		{
+			THROW_EXCEPTION(CommonException, GetLineTooLarge)
+		}
+		
+		// Read more in?
+		if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF)
+		{
+			int bytes = ::read(mFileHandle, mBuffer, sizeof(mBuffer));
+			
+			// Error?
+			if(bytes == -1)
+			{
+				THROW_EXCEPTION(CommonException, OSFileError)
+			}
+			
+			// Adjust buffer info
+			mBytesInBuffer = bytes;
+			mBufferBegin = 0;
+			
+			// EOF / closed?
+			if(bytes == 0)
+			{
+				mPendingEOF = true;
+			}
+		}
+		
+		// EOF?
+		if(mPendingEOF && mBufferBegin >= mBytesInBuffer)
+		{
+			// File is EOF, and now we've depleted the buffer completely, so tell caller as well.
+			mEOF = true;
+		}
+	}
+
+	if(!Preprocess)
+	{
+		return r;
+	}
+	else
+	{
+		// Check for comment char, but char before must be whitespace
+		int end = 0;
+		int size = r.size();
+		while(end < size)
+		{
+			if(r[end] == '#' && (end == 0 || (iw(r[end-1]))))
+			{
+				break;
+			}
+			end++;
+		}
+		
+		// Remove whitespace
+		int begin = 0;
+		while(begin < size && iw(r[begin]))
+		{
+			begin++;
+		}
+		if(!iw(r[end])) end--;
+		while(end > begin && iw(r[end]))
+		{
+			end--;
+		}
+		
+		// Return a sub string
+		return r.substr(begin, end - begin + 1);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FdGetLine::DetachFile()
+//		Purpose: Detaches the file handle, setting the file pointer correctly.
+//				 Probably not good for sockets...
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+void FdGetLine::DetachFile()
+{
+	if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)}
+
+	// Adjust file pointer
+	int bytesOver = mBufferBegin - mBufferBegin;
+	ASSERT(bytesOver >= 0);
+	if(bytesOver > 0)
+	{
+		if(::lseek(mFileHandle, 0 - bytesOver, SEEK_CUR) == -1)
+		{
+			THROW_EXCEPTION(CommonException, OSFileError)
+		}
+	}
+
+	// Unset file pointer
+	mFileHandle = -1;
+}
+
+
diff --git a/lib/common/FdGetLine.h b/lib/common/FdGetLine.h
new file mode 100755
index 00000000..fecb0371
--- /dev/null
+++ b/lib/common/FdGetLine.h
@@ -0,0 +1,61 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    FdGetLine.h
+//		Purpose: Line based file descriptor reading
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+
+#ifndef FDGETLINE__H
+#define FDGETLINE__H
+
+#include <string>
+
+#ifdef NDEBUG
+	#define FDGETLINE_BUFFER_SIZE		1024
+#else
+	#define FDGETLINE_BUFFER_SIZE		4
+#endif
+
+// Just a very large upper bound for line size to avoid
+// people sending lots of data over sockets and causing memory problems.
+#define FDGETLINE_MAX_LINE_SIZE			(1024*256)
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    FdGetLine
+//		Purpose: Line based file descriptor reading
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+class FdGetLine
+{
+public:
+	FdGetLine(int fd);
+	~FdGetLine();
+private:
+	FdGetLine(const FdGetLine &rToCopy);
+
+public:
+	std::string GetLine(bool Preprocess = false);
+	bool IsEOF() {return mEOF;}
+	int GetLineNumber() {return mLineNumber;}
+	
+	// Call to detach, setting file pointer correctly to last bit read.
+	// Only works for lseek-able file descriptors.
+	void DetachFile();
+	
+private:
+	char mBuffer[FDGETLINE_BUFFER_SIZE];
+	int mFileHandle;
+	int mLineNumber;
+	int mBufferBegin;
+	int mBytesInBuffer;
+	bool mPendingEOF;
+	bool mEOF;
+};
+
+#endif // FDGETLINE__H
+
diff --git a/lib/common/FileModificationTime.h b/lib/common/FileModificationTime.h
new file mode 100755
index 00000000..78f5c115
--- /dev/null
+++ b/lib/common/FileModificationTime.h
@@ -0,0 +1,57 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    FileModificationTime.h
+//		Purpose: Function for getting file modification time.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef FILEMODIFICATIONTIME__H
+#define FILEMODIFICATIONTIME__H
+
+#include <sys/stat.h>
+
+#include "BoxTime.h"
+
+inline box_time_t FileModificationTime(struct stat &st)
+{
+#ifdef PLATFORM_stat_SHORT_mtime
+	box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL);
+#else
+	box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
+			+ (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
+#endif
+	
+	return datamodified;
+}
+
+inline box_time_t FileAttrModificationTime(struct stat &st)
+{
+#ifdef PLATFORM_stat_SHORT_mtime
+	box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL);
+#else
+	box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
+			+ (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
+#endif
+	
+	return statusmodified;
+}
+
+inline box_time_t FileModificationTimeMaxModAndAttr(struct stat &st)
+{
+#ifdef PLATFORM_stat_SHORT_mtime
+	box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL);
+	box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL);
+#else
+	box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
+			+ (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
+	box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL)
+			+ (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL));
+#endif
+	
+	return (datamodified > statusmodified)?datamodified:statusmodified;
+}
+
+#endif // FILEMODIFICATIONTIME__H
+
diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp
new file mode 100755
index 00000000..ca6894ae
--- /dev/null
+++ b/lib/common/FileStream.cpp
@@ -0,0 +1,245 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    FileStream.cpp
+//		Purpose: IOStream interface to files
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "FileStream.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::FileStream(const char *, int, int)
+//		Purpose: Constructor, opens file
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+FileStream::FileStream(const char *Filename, int flags, int mode)
+	: mOSFileHandle(::open(Filename, flags, mode)),
+	  mIsEOF(false)
+{
+	if(mOSFileHandle < 0)
+	{
+		MEMLEAKFINDER_NOT_A_LEAK(this);
+		THROW_EXCEPTION(CommonException, OSFileOpenError)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::FileStream(int)
+//		Purpose: Constructor, using existing file descriptor
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+FileStream::FileStream(int FileDescriptor)
+	: mOSFileHandle(FileDescriptor),
+	  mIsEOF(false)
+{
+	if(mOSFileHandle < 0)
+	{
+		MEMLEAKFINDER_NOT_A_LEAK(this);
+		THROW_EXCEPTION(CommonException, OSFileOpenError)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::FileStream(const FileStream &)
+//		Purpose: Copy constructor, creates a duplicate of the file handle
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+FileStream::FileStream(const FileStream &rToCopy)
+	: mOSFileHandle(::dup(rToCopy.mOSFileHandle)),
+	  mIsEOF(rToCopy.mIsEOF)
+{
+	if(mOSFileHandle < 0)
+	{
+		MEMLEAKFINDER_NOT_A_LEAK(this);
+		THROW_EXCEPTION(CommonException, OSFileOpenError)
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::~FileStream()
+//		Purpose: Destructor, closes file
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+FileStream::~FileStream()
+{
+	if(mOSFileHandle >= 0)
+	{
+		Close();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::Read(void *, int)
+//		Purpose: Reads bytes from the file
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+int FileStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)}
+	int r = ::read(mOSFileHandle, pBuffer, NBytes);
+	if(r == -1)
+	{
+		THROW_EXCEPTION(CommonException, OSFileReadError)
+	}
+	if(r == 0)
+	{
+		mIsEOF = true;
+	}
+	
+	return r;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::BytesLeftToRead()
+//		Purpose: Returns number of bytes to read (may not be most efficient function ever)
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type FileStream::BytesLeftToRead()
+{
+	struct stat st;
+	if(::fstat(mOSFileHandle, &st) != 0)
+	{
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+	
+	return st.st_size - GetPosition();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::Write(void *, int)
+//		Purpose: Writes bytes to the file
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void FileStream::Write(const void *pBuffer, int NBytes)
+{
+	if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)}
+	if(::write(mOSFileHandle, pBuffer, NBytes) != NBytes)
+	{
+		THROW_EXCEPTION(CommonException, OSFileWriteError)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::GetPosition()
+//		Purpose: Get position in stream
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type FileStream::GetPosition() const
+{
+	if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)}
+	off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR);
+	if(p == -1)
+	{
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+	
+	return (IOStream::pos_type)p;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::Seek(pos_type, int)
+//		Purpose: Seeks within file, as lseek
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void FileStream::Seek(IOStream::pos_type Offset, int SeekType)
+{
+	if(mOSFileHandle == -1) {THROW_EXCEPTION(CommonException, FileClosed)}
+	if(::lseek(mOSFileHandle, Offset, ConvertSeekTypeToOSWhence(SeekType)) == -1)
+	{
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+	
+	// Not end of file any more!
+	mIsEOF = false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::Close()
+//		Purpose: Closes the underlying file
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void FileStream::Close()
+{
+	if(mOSFileHandle < 0)
+	{
+		THROW_EXCEPTION(CommonException, FileAlreadyClosed)
+	}
+	if(::close(mOSFileHandle) != 0)
+	{
+		THROW_EXCEPTION(CommonException, OSFileCloseError)
+	}
+	mOSFileHandle = -1;
+	mIsEOF = true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::StreamDataLeft()
+//		Purpose: Any data left to write?
+//		Created: 2003/08/02
+//
+// --------------------------------------------------------------------------
+bool FileStream::StreamDataLeft()
+{
+	return !mIsEOF;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileStream::StreamClosed()
+//		Purpose: Is the stream closed?
+//		Created: 2003/08/02
+//
+// --------------------------------------------------------------------------
+bool FileStream::StreamClosed()
+{
+	return mIsEOF;
+}
+
diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h
new file mode 100755
index 00000000..bb3cc459
--- /dev/null
+++ b/lib/common/FileStream.h
@@ -0,0 +1,46 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    FileStream.h
+//		Purpose: FileStream interface to files
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef FILESTREAM__H
+#define FILESTREAM__H
+
+#include "IOStream.h"
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+class FileStream : public IOStream
+{
+public:
+	FileStream(const char *Filename, int flags = O_RDONLY, int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH));
+	FileStream(int FileDescriptor);
+	FileStream(const FileStream &rToCopy);
+	virtual ~FileStream();
+	
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type BytesLeftToRead();
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual pos_type GetPosition() const;
+	virtual void Seek(IOStream::pos_type Offset, int SeekType);
+	virtual void Close();
+	
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+private:
+	int mOSFileHandle;
+	bool mIsEOF;
+};
+
+
+#endif // FILESTREAM__H
+
+
diff --git a/lib/common/Guards.h b/lib/common/Guards.h
new file mode 100755
index 00000000..6efc5614
--- /dev/null
+++ b/lib/common/Guards.h
@@ -0,0 +1,112 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Guards.h
+//		Purpose: Classes which ensure things are closed/deleted properly when
+//				 going out of scope. Easy exception proof code, etc
+//		Created: 2003/07/12
+//
+// --------------------------------------------------------------------------
+
+#ifndef GUARDS__H
+#define GUARDS__H
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <new>
+
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+template <int flags = O_RDONLY, int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)>
+class FileHandleGuard
+{
+public:
+	FileHandleGuard(const char *filename)
+		: mOSFileHandle(::open(filename, flags, mode))
+	{
+		if(mOSFileHandle < 0)
+		{
+			THROW_EXCEPTION(CommonException, OSFileOpenError)
+		}
+	}
+	
+	~FileHandleGuard()
+	{
+		if(mOSFileHandle >= 0)
+		{
+			Close();
+		}
+	}
+	
+	void Close()
+	{
+		if(mOSFileHandle < 0)
+		{
+			THROW_EXCEPTION(CommonException, FileAlreadyClosed)
+		}
+		if(::close(mOSFileHandle) != 0)
+		{
+			THROW_EXCEPTION(CommonException, OSFileCloseError)
+		}
+		mOSFileHandle = -1;
+	}
+	
+	operator int() const
+	{
+		return mOSFileHandle;
+	}
+
+private:
+	int mOSFileHandle;
+};
+
+template<typename type>
+class MemoryBlockGuard
+{
+public:
+	MemoryBlockGuard(int BlockSize)
+		: mpBlock(::malloc(BlockSize))
+	{
+		if(mpBlock == 0)
+		{
+			throw std::bad_alloc();
+		}
+	}
+	
+	~MemoryBlockGuard()
+	{
+		free(mpBlock);
+	}
+	
+	operator type() const
+	{
+		return (type)mpBlock;
+	}
+	
+	type GetPtr() const
+	{
+		return (type)mpBlock;
+	}
+	
+	void Resize(int NewSize)
+	{
+		void *ptrn = ::realloc(mpBlock, NewSize);
+		if(ptrn == 0)
+		{
+			throw std::bad_alloc();
+		}
+		mpBlock = ptrn;
+	}
+	
+private:
+	void *mpBlock;
+};
+
+#include "MemLeakFindOff.h"
+
+#endif // GUARDS__H
+
diff --git a/lib/common/IOStream.cpp b/lib/common/IOStream.cpp
new file mode 100755
index 00000000..024eefcc
--- /dev/null
+++ b/lib/common/IOStream.cpp
@@ -0,0 +1,229 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    IOStream.cpp
+//		Purpose: I/O Stream abstraction
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "IOStream.h"
+#include "CommonException.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::IOStream()
+//		Purpose: Constructor
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+IOStream::IOStream()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::IOStream(const IOStream &)
+//		Purpose: Copy constructor (exceptions)
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+IOStream::IOStream(const IOStream &rToCopy)
+{
+	THROW_EXCEPTION(CommonException, NotSupported)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::~IOStream()
+//		Purpose: Destructor
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+IOStream::~IOStream()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::Close()
+//		Purpose: Close the stream
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void IOStream::Close()
+{
+	// Do nothing by default -- let the destructor clear everything up.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::Seek(int, int)
+//		Purpose: Seek in stream (if supported)
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void IOStream::Seek(IOStream::pos_type Offset, int SeekType)
+{
+	THROW_EXCEPTION(CommonException, NotSupported)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::GetPosition()
+//		Purpose: Returns current position in stream (if supported)
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type IOStream::GetPosition() const
+{
+	THROW_EXCEPTION(CommonException, NotSupported)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::ConvertSeekTypeToOSWhence(int)
+//		Purpose: Return an whence arg for lseek given a IOStream seek type
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+int IOStream::ConvertSeekTypeToOSWhence(int SeekType)
+{
+	// Should be nicely optimised out as values are choosen in header file to match OS values.
+	int ostype = SEEK_SET;
+	switch(SeekType)
+	{
+	case SeekType_Absolute:
+		ostype = SEEK_SET;
+		break;
+	case SeekType_Relative:
+		ostype = SEEK_CUR;
+		break;
+	case SeekType_End:
+		ostype = SEEK_END;
+		break;
+	
+	default:
+		THROW_EXCEPTION(CommonException, IOStreamBadSeekType)
+	}
+	
+	return ostype;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::ReadFullBuffer(void *, int, int)
+//		Purpose: Reads bytes into buffer, returning whether or not it managed to
+//				 get all the bytes required. Exception and abort use of stream
+//				 if this returns false.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool IOStream::ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout)
+{
+	int bytesToGo = NBytes;
+	char *buffer = (char*)pBuffer;
+	if(pNBytesRead) (*pNBytesRead) = 0;
+	
+	while(bytesToGo > 0)
+	{
+		int bytesRead = Read(buffer, bytesToGo, Timeout);
+		if(bytesRead == 0)
+		{
+			// Timeout or something
+			return false;
+		}
+		// Increment things
+		bytesToGo -= bytesRead;
+		buffer += bytesRead;
+		if(pNBytesRead) (*pNBytesRead) += bytesRead;
+	}
+	
+	// Got everything
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::WriteAllBuffered()
+//		Purpose: Ensures that any data which has been buffered is written to the stream
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void IOStream::WriteAllBuffered()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::BytesLeftToRead()
+//		Purpose: Numbers of bytes left to read in the stream, or
+//				 IOStream::SizeOfStreamUnknown if this isn't known.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type IOStream::BytesLeftToRead()
+{
+	return IOStream::SizeOfStreamUnknown;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStream::CopyStreamTo(IOStream &, int Timeout)
+//		Purpose: Copies the entire stream to another stream (reading from this,
+//				 writing to rCopyTo). Returns whether the copy completed (ie
+//				 StreamDataLeft() returns false)
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool IOStream::CopyStreamTo(IOStream &rCopyTo, int Timeout, int BufferSize)
+{
+	// Make sure there's something to do before allocating that buffer
+	if(!StreamDataLeft())
+	{
+		return true;	// complete, even though nothing happened
+	}
+
+	// Buffer
+	MemoryBlockGuard<char*> buffer(BufferSize);
+	
+	// Get copying!
+	while(StreamDataLeft())
+	{
+		// Read some data
+		int bytes = Read(buffer, BufferSize, Timeout);
+		if(bytes == 0 && StreamDataLeft())
+		{
+			return false;	// incomplete, timed out
+		}
+		
+		// Write some data
+		if(bytes != 0)
+		{
+			rCopyTo.Write(buffer, bytes);
+		}
+	}
+	
+	return true;	// completed
+}
+
+
diff --git a/lib/common/IOStream.h b/lib/common/IOStream.h
new file mode 100755
index 00000000..042ccca4
--- /dev/null
+++ b/lib/common/IOStream.h
@@ -0,0 +1,67 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    IOStream.h
+//		Purpose: I/O Stream abstraction
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef IOSTREAM__H
+#define IOSTREAM__H
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    IOStream
+//		Purpose: Abstract interface to streams of data
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+class IOStream
+{
+public:
+	IOStream();
+	IOStream(const IOStream &rToCopy);
+	virtual ~IOStream();
+	
+	enum
+	{
+		TimeOutInfinite = -1,
+		SizeOfStreamUnknown = -1
+	};
+	
+	enum
+	{
+		SeekType_Absolute = 0,
+		SeekType_Relative = 1,
+		SeekType_End = 2
+	};
+	
+	// Timeout in milliseconds
+	// Read may return 0 -- does not mean end of stream.
+	typedef int64_t pos_type;
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite) = 0;
+	virtual pos_type BytesLeftToRead();	// may return IOStream::SizeOfStreamUnknown (and will for most stream types)
+	virtual void Write(const void *pBuffer, int NBytes) = 0;
+	virtual void WriteAllBuffered();
+	virtual pos_type GetPosition() const;
+	virtual void Seek(pos_type Offset, int SeekType);
+	virtual void Close();
+	
+	// Has all data that can be read been read?
+	virtual bool StreamDataLeft() = 0;
+	// Has the stream been closed (writing not possible)
+	virtual bool StreamClosed() = 0;
+	
+	// Utility functions
+	bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout = IOStream::TimeOutInfinite);
+	bool CopyStreamTo(IOStream &rCopyTo, int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024);
+	
+	static int ConvertSeekTypeToOSWhence(int SeekType);
+};
+
+
+#endif // IOSTREAM__H
+
+
diff --git a/lib/common/IOStreamGetLine.cpp b/lib/common/IOStreamGetLine.cpp
new file mode 100755
index 00000000..27a77c29
--- /dev/null
+++ b/lib/common/IOStreamGetLine.cpp
@@ -0,0 +1,227 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    IOStreamGetLine.cpp
+//		Purpose: Line based file descriptor reading
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "IOStreamGetLine.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// utility whitespace function
+inline bool iw(int c)
+{
+	return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStreamGetLine::IOStreamGetLine(int)
+//		Purpose: Constructor, taking file descriptor
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+IOStreamGetLine::IOStreamGetLine(IOStream &Stream)
+	: mrStream(Stream),
+	  mLineNumber(0),
+	  mBufferBegin(0),
+	  mBytesInBuffer(0),
+	  mPendingEOF(false),
+	  mEOF(false)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStreamGetLine::~IOStreamGetLine()
+//		Purpose: Destructor
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+IOStreamGetLine::~IOStreamGetLine()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStreamGetLine::GetLine(std::string &, bool, int)
+//		Purpose: Gets a line from the file, returning it in rOutput. If Preprocess is true, leading
+//				 and trailing whitespace is removed, and comments (after #)
+//				 are deleted.
+//				 Returns true if a line is available now, false if retrying may get a line (eg timeout, signal),
+//				 and exceptions if it's EOF.
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+bool IOStreamGetLine::GetLine(std::string &rOutput, bool Preprocess, int Timeout)
+{
+	// EOF?
+	if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)}
+	
+	// Initialise string to stored into
+	std::string r(mPendingString);
+	mPendingString.erase();
+
+	bool foundLineEnd = false;
+
+	while(!foundLineEnd && !mEOF)
+	{
+		// Use any bytes left in the buffer
+		while(mBufferBegin < mBytesInBuffer)
+		{
+			int c = mBuffer[mBufferBegin++];
+			if(c == '\r')
+			{
+				// Ignore nasty Windows line ending extra chars
+			}
+			else if(c == '\n')
+			{
+				// Line end!
+				foundLineEnd = true;
+				break;
+			}
+			else
+			{
+				// Add to string
+				r += c;
+			}
+			
+			// Implicit line ending at EOF
+			if(mBufferBegin >= mBytesInBuffer && mPendingEOF)
+			{
+				foundLineEnd = true;
+			}
+		}
+		
+		// Check size
+		if(r.size() > IOSTREAMGETLINE_MAX_LINE_SIZE)
+		{
+			THROW_EXCEPTION(CommonException, GetLineTooLarge)
+		}
+		
+		// Read more in?
+		if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF)
+		{
+			int bytes = mrStream.Read(mBuffer, sizeof(mBuffer), Timeout);
+			
+			// Adjust buffer info
+			mBytesInBuffer = bytes;
+			mBufferBegin = 0;
+			
+			// EOF / closed?
+			if(!mrStream.StreamDataLeft())
+			{
+				mPendingEOF = true;
+			}
+			
+			// No data returned?
+			if(bytes == 0 && mrStream.StreamDataLeft())
+			{
+				// store string away
+				mPendingString = r;
+				// Return false;
+				return false;
+			}
+		}
+		
+		// EOF?
+		if(mPendingEOF && mBufferBegin >= mBytesInBuffer)
+		{
+			// File is EOF, and now we've depleted the buffer completely, so tell caller as well.
+			mEOF = true;
+		}
+	}
+
+	if(!Preprocess)
+	{
+		rOutput = r;
+		return true;
+	}
+	else
+	{
+		// Check for comment char, but char before must be whitespace
+		int end = 0;
+		int size = r.size();
+		while(end < size)
+		{
+			if(r[end] == '#' && (end == 0 || (iw(r[end-1]))))
+			{
+				break;
+			}
+			end++;
+		}
+		
+		// Remove whitespace
+		int begin = 0;
+		while(begin < size && iw(r[begin]))
+		{
+			begin++;
+		}
+		if(!iw(r[end])) end--;
+		while(end > begin && iw(r[end]))
+		{
+			end--;
+		}
+		
+		// Return a sub string
+		rOutput = r.substr(begin, end - begin + 1);
+		return true;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStreamGetLine::DetachFile()
+//		Purpose: Detaches the file handle, setting the file pointer correctly.
+//				 Probably not good for sockets...
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+void IOStreamGetLine::DetachFile()
+{
+	// Adjust file pointer
+	int bytesOver = mBytesInBuffer - mBufferBegin;
+	ASSERT(bytesOver >= 0);
+	if(bytesOver > 0)
+	{
+		mrStream.Seek(0 - bytesOver, IOStream::SeekType_Relative);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStreamGetLine::IgnoreBufferedData(int)
+//		Purpose: Ignore buffered bytes (effectively removing them from the
+//				 beginning of the buffered data.)
+//				 Cannot remove more bytes than are currently in the buffer.
+//				 Be careful when this is used!
+//		Created: 22/12/04
+//
+// --------------------------------------------------------------------------
+void IOStreamGetLine::IgnoreBufferedData(int BytesToIgnore)
+{
+	int bytesInBuffer = mBytesInBuffer - mBufferBegin;
+	if(BytesToIgnore < 0 || BytesToIgnore > bytesInBuffer)
+	{
+		THROW_EXCEPTION(CommonException, IOStreamGetLineNotEnoughDataToIgnore)
+	}
+	mBufferBegin += BytesToIgnore;
+}
+
+
+
diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h
new file mode 100755
index 00000000..cf152e5a
--- /dev/null
+++ b/lib/common/IOStreamGetLine.h
@@ -0,0 +1,71 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    IOStreamGetLine.h
+//		Purpose: Line based file descriptor reading
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+
+#ifndef IOSTREAMGETLINE__H
+#define IOSTREAMGETLINE__H
+
+#include <string>
+
+#include "IOStream.h"
+
+#ifdef NDEBUG
+	#define IOSTREAMGETLINE_BUFFER_SIZE		1024
+#else
+	#define IOSTREAMGETLINE_BUFFER_SIZE		4
+#endif
+
+// Just a very large upper bound for line size to avoid
+// people sending lots of data over sockets and causing memory problems.
+#define IOSTREAMGETLINE_MAX_LINE_SIZE			(1024*256)
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    IOStreamGetLine
+//		Purpose: Line based stream reading
+//		Created: 2003/07/24
+//
+// --------------------------------------------------------------------------
+class IOStreamGetLine
+{
+public:
+	IOStreamGetLine(IOStream &Stream);
+	~IOStreamGetLine();
+private:
+	IOStreamGetLine(const IOStreamGetLine &rToCopy);
+
+public:
+	bool GetLine(std::string &rOutput, bool Preprocess = false, int Timeout = IOStream::TimeOutInfinite);
+	bool IsEOF() {return mEOF;}
+	int GetLineNumber() {return mLineNumber;}
+	
+	// Call to detach, setting file pointer correctly to last bit read.
+	// Only works for lseek-able file descriptors.
+	void DetachFile();
+	
+	// For doing interesting stuff with the remaining data...
+	// Be careful with this!
+	const void *GetBufferedData() const {return mBuffer + mBufferBegin;}
+	int GetSizeOfBufferedData() const {return mBytesInBuffer - mBufferBegin;}
+	void IgnoreBufferedData(int BytesToIgnore);
+	IOStream &GetUnderlyingStream() {return mrStream;}
+	
+private:
+	char mBuffer[IOSTREAMGETLINE_BUFFER_SIZE];
+	IOStream &mrStream;
+	int mLineNumber;
+	int mBufferBegin;
+	int mBytesInBuffer;
+	bool mPendingEOF;
+	bool mEOF;
+	std::string mPendingString;
+};
+
+#endif // IOSTREAMGETLINE__H
+
diff --git a/lib/common/LinuxWorkaround.cpp b/lib/common/LinuxWorkaround.cpp
new file mode 100755
index 00000000..7900fa6e
--- /dev/null
+++ b/lib/common/LinuxWorkaround.cpp
@@ -0,0 +1,73 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    LinuxWorkaround.cpp
+//		Purpose: Workarounds for Linux
+//		Created: 2003/10/31
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/stat.h>
+#include <unistd.h>
+#include <dirent.h>
+
+#include <string>
+
+#include "LinuxWorkaround.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+#ifdef PLATFORM_LINUX
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    LinuxWorkaround_FinishDirentStruct(struct dirent *, const char *)
+//		Purpose: Finishes off filling in a dirent structure, which Linux leaves incomplete.
+//		Created: 2003/10/31
+//
+// --------------------------------------------------------------------------
+void LinuxWorkaround_FinishDirentStruct(struct dirent *entry, const char *DirectoryName)
+{
+	// From man readdir under Linux:
+	//
+	// BUGS
+    //   Field d_type is not implemented as  of  libc6  2.1  and  will  always  return
+    //   DT_UNKNOWN (0).
+	//
+	// What kind of an OS is this?
+	
+	
+	// Build filename of this entry
+	std::string fn(DirectoryName);
+	fn += '/';
+	fn += entry->d_name;
+	
+	// Do a stat on it
+	struct stat st;
+	if(::lstat(fn.c_str(), &st) != 0)
+	{
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+	
+	// Fill in the d_type field.
+	if(S_ISREG(st.st_mode))
+	{
+		entry->d_type = DT_REG;	
+	}
+	else if(S_ISDIR(st.st_mode))
+	{
+		entry->d_type = DT_DIR;
+	}
+	else if(S_ISLNK(st.st_mode))
+	{
+		entry->d_type = DT_LNK;
+	}
+	// otherwise leave it as we found it
+}
+
+#endif // PLATFORM_LINUX
+
diff --git a/lib/common/LinuxWorkaround.h b/lib/common/LinuxWorkaround.h
new file mode 100755
index 00000000..bcd27495
--- /dev/null
+++ b/lib/common/LinuxWorkaround.h
@@ -0,0 +1,20 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    LinuxWorkaround.h
+//		Purpose: Workarounds for Linux
+//		Created: 2003/10/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef LINUXWORKAROUND__H
+#define LINUXWORKAROUND__H
+
+#ifdef PLATFORM_LINUX
+
+void LinuxWorkaround_FinishDirentStruct(struct dirent *entry, const char *DirectoryName);
+
+#endif // PLATFORM_LINUX
+
+#endif // LINUXWORKAROUND__H
+
diff --git a/lib/common/MainHelper.h b/lib/common/MainHelper.h
new file mode 100755
index 00000000..ab75af96
--- /dev/null
+++ b/lib/common/MainHelper.h
@@ -0,0 +1,42 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    MainHelper.h
+//		Purpose: Helper stuff for main() programs
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#ifndef MAINHELPER__H
+#define MAINHELPER__H
+
+#include <stdio.h>
+
+#include "BoxException.h"
+
+#define MAINHELPER_START									\
+	if(argc == 2 && ::strcmp(argv[1], "--version") == 0)	\
+	{ printf(BOX_VERSION "\n"); return 0; }					\
+	MEMLEAKFINDER_START										\
+	try {
+#define MAINHELPER_END																\
+	} catch(BoxException &e) {														\
+	printf("Exception: %s (%d/%d)\n", e.what(), e.GetType(), e.GetSubType());		\
+	return 1;																		\
+	} catch(std::exception &e) {													\
+	printf("Exception: %s\n", e.what());											\
+	return 1;																		\
+	} catch(...) {																	\
+	printf("Exception: <UNKNOWN>\n");												\
+	return 1; }
+
+#ifdef BOX_MEMORY_LEAK_TESTING
+	#define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker)				\
+		memleakfinder_setup_exit_report(file, marker);
+#else
+	#define MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, marker)
+#endif // BOX_MEMORY_LEAK_TESTING
+
+
+#endif // MAINHELPER__H
+
diff --git a/lib/common/Makefile.extra b/lib/common/Makefile.extra
new file mode 100755
index 00000000..92e6fbb3
--- /dev/null
+++ b/lib/common/Makefile.extra
@@ -0,0 +1,11 @@
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_CommonException.h autogen_CommonException.cpp:	$(MAKEEXCEPTION) CommonException.txt
+	perl $(MAKEEXCEPTION) CommonException.txt
+
+# AUTOGEN SEEDING
+autogen_ConversionException.h autogen_ConversionException.cpp:	$(MAKEEXCEPTION) ConversionException.txt
+	perl $(MAKEEXCEPTION) ConversionException.txt
+
diff --git a/lib/common/MemBlockStream.cpp b/lib/common/MemBlockStream.cpp
new file mode 100755
index 00000000..538a7ef8
--- /dev/null
+++ b/lib/common/MemBlockStream.cpp
@@ -0,0 +1,235 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    MemBlockStream.cpp
+//		Purpose: Stream out data from any memory block
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "MemBlockStream.h"
+#include "CommonException.h"
+#include "StreamableMemBlock.h"
+#include "CollectInBufferStream.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::MemBlockStream()
+//		Purpose: Constructor (doesn't copy block, careful with lifetimes)
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::MemBlockStream(const void *pBuffer, int Size)
+	: mpBuffer((char*)pBuffer),
+	  mBytesInBuffer(Size),
+	  mReadPosition(0)
+{
+	ASSERT(pBuffer != 0);
+	ASSERT(Size >= 0);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::MemBlockStream(const StreamableMemBlock &)
+//		Purpose: Constructor (doesn't copy block, careful with lifetimes)
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::MemBlockStream(const StreamableMemBlock &rBlock)
+	: mpBuffer((char*)rBlock.GetBuffer()),
+	  mBytesInBuffer(rBlock.GetSize()),
+	  mReadPosition(0)
+{
+	ASSERT(mpBuffer != 0);
+	ASSERT(mBytesInBuffer >= 0);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::MemBlockStream(const StreamableMemBlock &)
+//		Purpose: Constructor (doesn't copy block, careful with lifetimes)
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::MemBlockStream(const CollectInBufferStream &rBuffer)
+	: mpBuffer((char*)rBuffer.GetBuffer()),
+	  mBytesInBuffer(rBuffer.GetSize()),
+	  mReadPosition(0)
+{
+	ASSERT(mpBuffer != 0);
+	ASSERT(mBytesInBuffer >= 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::MemBlockStream(const MemBlockStream &)
+//		Purpose: Copy constructor
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::MemBlockStream(const MemBlockStream &rToCopy)
+	: mpBuffer(rToCopy.mpBuffer),
+	  mBytesInBuffer(rToCopy.mBytesInBuffer),
+	  mReadPosition(0)
+{
+	ASSERT(mpBuffer != 0);
+	ASSERT(mBytesInBuffer >= 0);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::~MemBlockStream()
+//		Purpose: Destructor
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+MemBlockStream::~MemBlockStream()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::Read(void *, int, int)
+//		Purpose: As interface. But only works in read phase
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+int MemBlockStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// Adjust to number of bytes left
+	if(NBytes > (mBytesInBuffer - mReadPosition))
+	{
+		NBytes = (mBytesInBuffer - mReadPosition);
+	}
+	ASSERT(NBytes >= 0);
+	if(NBytes <= 0) return 0;	// careful now
+	
+	// Copy in the requested number of bytes and adjust the read pointer
+	::memcpy(pBuffer, mpBuffer + mReadPosition, NBytes);
+	mReadPosition += NBytes;
+	
+	return NBytes;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::BytesLeftToRead()
+//		Purpose: As interface. But only works in read phase
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type MemBlockStream::BytesLeftToRead()
+{
+	return (mBytesInBuffer - mReadPosition);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::Write(void *, int)
+//		Purpose: As interface. But only works in write phase
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void MemBlockStream::Write(const void *pBuffer, int NBytes)
+{
+	THROW_EXCEPTION(CommonException, MemBlockStreamNotSupported)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::GetPosition()
+//		Purpose: In write phase, returns the number of bytes written, in read
+//				 phase, the number of bytes to go
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type MemBlockStream::GetPosition() const
+{
+	return mReadPosition;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::Seek(pos_type, int)
+//		Purpose: As interface.
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void MemBlockStream::Seek(pos_type Offset, int SeekType)
+{
+	int newPos = 0;
+	switch(SeekType)
+	{
+	case IOStream::SeekType_Absolute:
+		newPos = Offset;
+		break;
+	case IOStream::SeekType_Relative:
+		newPos = mReadPosition + Offset;
+		break;
+	case IOStream::SeekType_End:
+		newPos = mBytesInBuffer + Offset;
+		break;
+	default:
+		THROW_EXCEPTION(CommonException, IOStreamBadSeekType)
+		break;
+	}
+	
+	// Make sure it doesn't go over
+	if(newPos > mBytesInBuffer)
+	{
+		newPos = mBytesInBuffer;
+	}
+	// or under
+	if(newPos < 0)
+	{
+		newPos = 0;
+	}
+	
+	// Set the new read position
+	mReadPosition = newPos;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::StreamDataLeft()
+//		Purpose: As interface
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+bool MemBlockStream::StreamDataLeft()
+{
+	return mReadPosition < mBytesInBuffer;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MemBlockStream::StreamClosed()
+//		Purpose: As interface
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+bool MemBlockStream::StreamClosed()
+{
+	return true;
+}
+
diff --git a/lib/common/MemBlockStream.h b/lib/common/MemBlockStream.h
new file mode 100755
index 00000000..f78ff8e6
--- /dev/null
+++ b/lib/common/MemBlockStream.h
@@ -0,0 +1,52 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    MemBlockStream.h
+//		Purpose: Stream out data from any memory block
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+
+#ifndef MEMBLOCKSTREAM__H
+#define MEMBLOCKSTREAM__H
+
+#include "IOStream.h"
+
+class StreamableMemBlock;
+class CollectInBufferStream;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    MemBlockStream
+//		Purpose: Stream out data from any memory block -- be careful the lifetime
+//				 of the block is greater than the lifetime of this stream.
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+class MemBlockStream : public IOStream
+{
+public:
+	MemBlockStream(const void *pBuffer, int Size);
+	MemBlockStream(const StreamableMemBlock &rBlock);
+	MemBlockStream(const CollectInBufferStream &rBuffer);
+	MemBlockStream(const MemBlockStream &rToCopy);
+	~MemBlockStream();
+public:
+
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type BytesLeftToRead();
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual pos_type GetPosition() const;
+	virtual void Seek(pos_type Offset, int SeekType);
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+private:
+	const char *mpBuffer;
+	int mBytesInBuffer;
+	int mReadPosition;
+};
+
+#endif // MEMBLOCKSTREAM__H
+
diff --git a/lib/common/MemLeakFindOff.h b/lib/common/MemLeakFindOff.h
new file mode 100755
index 00000000..1cc98bac
--- /dev/null
+++ b/lib/common/MemLeakFindOff.h
@@ -0,0 +1,27 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    MemLeakFindOff.h
+//		Purpose: Switch memory leak finding off
+//		Created: 13/1/04
+//
+// --------------------------------------------------------------------------
+
+// no header guard
+
+#ifdef BOX_MEMORY_LEAK_TESTING
+
+#undef new
+
+#ifndef MEMLEAKFINDER_FULL_MALLOC_MONITORING
+	#ifdef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+		#undef malloc
+		#undef realloc
+		#undef free
+		#undef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+	#endif
+#endif
+
+#undef MEMLEAKFINDER_ENABLED
+
+#endif
diff --git a/lib/common/MemLeakFindOn.h b/lib/common/MemLeakFindOn.h
new file mode 100755
index 00000000..c20fe25a
--- /dev/null
+++ b/lib/common/MemLeakFindOn.h
@@ -0,0 +1,25 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    MemLeakFindOn.h
+//		Purpose: Switch memory leak finding on
+//		Created: 13/1/04
+//
+// --------------------------------------------------------------------------
+
+// no header guard
+
+#ifdef BOX_MEMORY_LEAK_TESTING
+
+#define new DEBUG_NEW
+
+#ifndef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+	#define malloc(X)	memleakfinder_malloc(X, __FILE__, __LINE__)
+	#define realloc		memleakfinder_realloc
+	#define free		memleakfinder_free
+	#define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+#endif
+
+#define MEMLEAKFINDER_ENABLED
+
+#endif
diff --git a/lib/common/MemLeakFinder.h b/lib/common/MemLeakFinder.h
new file mode 100755
index 00000000..f5887dac
--- /dev/null
+++ b/lib/common/MemLeakFinder.h
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    MemLeakFinder.h
+//		Purpose: Memory leak finder
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef MEMLEAKFINDER__H
+#define MEMLEAKFINDER__H
+
+#define DEBUG_NEW new(__FILE__,__LINE__)
+
+#ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING
+	// include stdlib now, to avoid problems with having the macros defined already
+	#include <stdlib.h>
+#endif
+
+// global enable flag
+extern bool memleakfinder_global_enable;
+
+extern "C"
+{
+	void *memleakfinder_malloc(size_t size, const char *file, int line);
+	void *memleakfinder_realloc(void *ptr, size_t size);
+	void memleakfinder_free(void *ptr);
+}
+
+int memleakfinder_numleaks();
+
+void memleakfinder_reportleaks();
+
+void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext);
+
+void memleakfinder_setup_exit_report(const char *filename, const char *markertext);
+
+void memleakfinder_startsectionmonitor();
+
+void memleakfinder_traceblocksinsection();
+
+void memleakfinder_notaleak(void *ptr);
+
+void *operator new(size_t size, const char *file, int line);
+void *operator new[](size_t size, const char *file, int line);
+
+void operator delete(void *ptr) throw ();
+void operator delete[](void *ptr) throw ();
+
+// define the malloc functions now, if required
+#ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING
+	#define malloc(X)	memleakfinder_malloc(X, __FILE__, __LINE__)
+	#define realloc		memleakfinder_realloc
+	#define free		memleakfinder_free
+	#define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED
+#endif
+
+
+#endif // MEMLEAKFINDER__H
+
diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp
new file mode 100755
index 00000000..1f6e038a
--- /dev/null
+++ b/lib/common/NamedLock.cpp
@@ -0,0 +1,144 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    NamedLock.cpp
+//		Purpose: A global named lock, implemented as a lock file in file system
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <fcntl.h>
+#include <errno.h>
+#include <unistd.h>
+#ifdef PLATFORM_LINUX
+	#include <sys/file.h>
+#endif // PLATFORM_LINUX
+#ifdef PLATFORM_CYGWIN
+	#include <sys/file.h>
+#endif // PLATFORM_CYGWIN
+
+#include "NamedLock.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    NamedLock::NamedLock()
+//		Purpose: Constructor
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+NamedLock::NamedLock()
+	: mFileDescriptor(-1)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    NamedLock::~NamedLock()
+//		Purpose: Destructor (automatically unlocks if locked)
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+NamedLock::~NamedLock()
+{
+	if(mFileDescriptor != -1)
+	{
+		ReleaseLock();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    NamedLock::TryAndGetLock(const char *, int)
+//		Purpose: Trys to get a lock on the name in the file system.
+//				 IMPORTANT NOTE: If a file exists with this name, it will be deleted.
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+bool NamedLock::TryAndGetLock(const char *Filename, int mode)
+{
+	// Check
+	if(mFileDescriptor != -1)
+	{
+		THROW_EXCEPTION(CommonException, NamedLockAlreadyLockingSomething)
+	}
+
+	// See if the lock can be got
+#ifdef PLATFORM_open_NO_O_EXLOCK
+	int fd = ::open(Filename, O_WRONLY | O_CREAT | O_TRUNC, mode);
+	if(fd == -1)
+	{
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+	if(::flock(fd, LOCK_EX | LOCK_NB) != 0)
+	{
+		::close(fd);
+		if(errno == EWOULDBLOCK)
+		{
+			return false;
+		}
+		else
+		{
+			THROW_EXCEPTION(CommonException, OSFileError)
+		}
+	}
+
+	// Success
+	mFileDescriptor = fd;
+
+	return true;
+#else
+	int fd = ::open(Filename, O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode);
+	if(fd != -1)
+	{
+		// Got a lock, lovely
+		mFileDescriptor = fd;
+		return true;
+	}
+	
+	// Failed. Why?
+	if(errno != EWOULDBLOCK)
+	{
+		// Not the expected error
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+
+	return false;
+#endif
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    NamedLock::ReleaseLock()
+//		Purpose: Release the lock. Exceptions if the lock is not held
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void NamedLock::ReleaseLock()
+{
+	// Got a lock?
+	if(mFileDescriptor == -1)
+	{
+		THROW_EXCEPTION(CommonException, NamedLockNotHeld)
+	}
+	
+	// Close the file
+	if(::close(mFileDescriptor) != 0)
+	{
+		THROW_EXCEPTION(CommonException, OSFileError)
+	}
+	// Mark as unlocked
+	mFileDescriptor = -1;
+}
+
+
+
+
diff --git a/lib/common/NamedLock.h b/lib/common/NamedLock.h
new file mode 100755
index 00000000..5eac712a
--- /dev/null
+++ b/lib/common/NamedLock.h
@@ -0,0 +1,41 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    NamedLock.h
+//		Purpose: A global named lock, implemented as a lock file in file system
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef NAMEDLOCK__H
+#define NAMEDLOCK__H
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    NamedLock
+//		Purpose: A global named lock, implemented as a lock file in file system
+//		Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class NamedLock
+{
+public:
+	NamedLock();
+	~NamedLock();
+private:
+	// No copying allowed
+	NamedLock(const NamedLock &);
+
+public:
+	bool TryAndGetLock(const char *Filename, int mode = 0755);
+	bool GotLock() {return mFileDescriptor != -1;}
+	void ReleaseLock();
+	
+	
+private:
+	int mFileDescriptor;
+};
+
+#endif // NAMEDLOCK__H
+
diff --git a/lib/common/PartialReadStream.cpp b/lib/common/PartialReadStream.cpp
new file mode 100755
index 00000000..0b5c4cf6
--- /dev/null
+++ b/lib/common/PartialReadStream.cpp
@@ -0,0 +1,135 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    PartialReadStream.h
+//		Purpose: Read part of another stream
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "PartialReadStream.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    PartialReadStream::PartialReadStream(IOStream &, int)
+//		Purpose: Constructor, taking another stream and the number of bytes
+//				 to be read from it.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+PartialReadStream::PartialReadStream(IOStream &rSource, int BytesToRead)
+	: mrSource(rSource),
+	  mBytesLeft(BytesToRead)
+{
+	ASSERT(BytesToRead > 0);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    PartialReadStream::~PartialReadStream()
+//		Purpose: Destructor. Won't absorb any unread bytes.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+PartialReadStream::~PartialReadStream()
+{
+	// Warn in debug mode
+	if(mBytesLeft != 0)
+	{
+		TRACE1("PartialReadStream::~PartialReadStream when mBytesLeft = %d\n", mBytesLeft);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    PartialReadStream::Read(void *, int, int)
+//		Purpose: As interface.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+int PartialReadStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// Finished?
+	if(mBytesLeft <= 0)
+	{
+		return 0;
+	}
+
+	// Asking for more than is allowed?
+	if(NBytes > mBytesLeft)
+	{
+		// Adjust downwards
+		NBytes = mBytesLeft;
+	}
+	
+	// Route the request to the source
+	int read = mrSource.Read(pBuffer, NBytes, Timeout);
+	ASSERT(read <= mBytesLeft);
+	
+	// Adjust the count
+	mBytesLeft -= read;
+	
+	// Return the number read
+	return read;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    PartialReadStream::BytesLeftToRead()
+//		Purpose: As interface.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type PartialReadStream::BytesLeftToRead()
+{
+	return mBytesLeft;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    PartialReadStream::Write(const void *, int)
+//		Purpose: As interface. But will exception.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void PartialReadStream::Write(const void *pBuffer, int NBytes)
+{
+	THROW_EXCEPTION(CommonException, CantWriteToPartialReadStream)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    PartialReadStream::StreamDataLeft()
+//		Purpose: As interface.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool PartialReadStream::StreamDataLeft()
+{
+	return mBytesLeft != 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    PartialReadStream::StreamClosed()
+//		Purpose: As interface.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+bool PartialReadStream::StreamClosed()
+{
+	// always closed
+	return true;
+}
+
diff --git a/lib/common/PartialReadStream.h b/lib/common/PartialReadStream.h
new file mode 100755
index 00000000..42cb7aeb
--- /dev/null
+++ b/lib/common/PartialReadStream.h
@@ -0,0 +1,46 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    PartialReadStream.h
+//		Purpose: Read part of another stream
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+
+#ifndef PARTIALREADSTREAM__H
+#define PARTIALREADSTREAM__H
+
+#include "IOStream.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    PartialReadStream
+//		Purpose: Read part of another stream
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+class PartialReadStream : public IOStream
+{
+public:
+	PartialReadStream(IOStream &rSource, int BytesToRead);
+	~PartialReadStream();
+private:
+	// no copying allowed
+	PartialReadStream(const IOStream &);
+	PartialReadStream(const PartialReadStream &);
+	
+public:
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type BytesLeftToRead();
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+private:
+	IOStream &mrSource;
+	int mBytesLeft;
+};
+
+#endif // PARTIALREADSTREAM__H
+
diff --git a/lib/common/ReadGatherStream.cpp b/lib/common/ReadGatherStream.cpp
new file mode 100755
index 00000000..9ccc3a54
--- /dev/null
+++ b/lib/common/ReadGatherStream.cpp
@@ -0,0 +1,262 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ReadGatherStream.cpp
+//		Purpose: Build a stream (for reading only) out of a number of other streams.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "ReadGatherStream.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::ReadGatherStream(bool)
+//		Purpose: Constructor. Args says whether or not all the component streams will be deleted when this
+//				 object is deleted.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+ReadGatherStream::ReadGatherStream(bool DeleteComponentStreamsOnDestruction)
+	: mDeleteComponentStreamsOnDestruction(DeleteComponentStreamsOnDestruction),
+	  mCurrentPosition(0),
+	  mTotalSize(0),
+	  mCurrentBlock(0),
+	  mPositionInCurrentBlock(0),
+	  mSeekDoneForCurrent(false)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::~ReadGatherStream()
+//		Purpose: Destructor. Will delete all the stream objects, if required.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+ReadGatherStream::~ReadGatherStream()
+{
+	// Delete compoenent streams?
+	if(mDeleteComponentStreamsOnDestruction)
+	{
+		for(unsigned int l = 0; l < mComponents.size(); ++l)
+		{
+			delete mComponents[l];
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::AddComponent(IOStream *)
+//		Purpose: Add a component to this stream, returning the index of this component
+//				 in the internal list. Use this with AddBlock()
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+int ReadGatherStream::AddComponent(IOStream *pStream)
+{
+	ASSERT(pStream != 0);
+
+	// Just add the component to the list, returning it's index.
+	int index = mComponents.size();
+	mComponents.push_back(pStream);
+	return index;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::AddBlock(int, pos_type, bool, pos_type)
+//		Purpose: Add a block to the list of blocks being gathered into one stream.
+//				 Length is length of block to read from this component, Seek == true
+//				 if a seek is required, and if true, SeekTo is the position (absolute)
+//				 in the stream to be seeked to when this block is required.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+void ReadGatherStream::AddBlock(int Component, pos_type Length, bool Seek, pos_type SeekTo)
+{
+	// Check block
+	if(Component < 0 || Component >= (int)mComponents.size() || Length < 0 || SeekTo < 0)
+	{
+		THROW_EXCEPTION(CommonException, ReadGatherStreamAddingBadBlock);
+	}
+	
+	// Add to list
+	Block b;
+	b.mLength = Length;
+	b.mSeekTo = SeekTo;
+	b.mComponent = Component;
+	b.mSeek = Seek;
+	
+	mBlocks.push_back(b);
+	
+	// And update the total size
+	mTotalSize += Length;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::Read(void *, int, int)
+//		Purpose: As interface.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+int ReadGatherStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	int bytesToRead = NBytes;
+	uint8_t *buffer = (uint8_t*)pBuffer;
+	
+	while(bytesToRead > 0)
+	{
+		// Done?
+		if(mCurrentBlock >= mBlocks.size())
+		{
+			// Stop now, as have finished the last block
+			return NBytes - bytesToRead;
+		}
+			
+		// Seek?
+		if(mPositionInCurrentBlock == 0 && mBlocks[mCurrentBlock].mSeek && !mSeekDoneForCurrent)
+		{
+			// Do seeks in this manner so that seeks are done regardless of whether the block
+			// has length > 0, and it will only be done once, and at as late a stage as possible.
+			
+			mComponents[mBlocks[mCurrentBlock].mComponent]->Seek(mBlocks[mCurrentBlock].mSeekTo, IOStream::SeekType_Absolute);
+		
+			mSeekDoneForCurrent = true;
+		}
+
+		// Anything in the current block?
+		if(mPositionInCurrentBlock < mBlocks[mCurrentBlock].mLength)
+		{
+			// Read!
+			int s = mBlocks[mCurrentBlock].mLength - mPositionInCurrentBlock;
+			if(s > bytesToRead) s = bytesToRead;
+			
+			int r = mComponents[mBlocks[mCurrentBlock].mComponent]->Read(buffer, s, Timeout);
+			
+			// update variables
+			mPositionInCurrentBlock += r;
+			buffer += r;
+			bytesToRead -= r;
+			mCurrentPosition += r;
+			
+			if(r != s)
+			{
+				// Stream returned less than requested. To avoid blocking when not necessary,
+				// return now.
+				return NBytes - bytesToRead;
+			}
+		}
+		else
+		{
+			// Move to next block
+			++mCurrentBlock;
+			mPositionInCurrentBlock = 0;
+			mSeekDoneForCurrent = false;
+		}
+	}
+
+	return NBytes - bytesToRead;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::GetPosition()
+//		Purpose: As interface
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type ReadGatherStream::GetPosition() const
+{
+	return mCurrentPosition;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::BytesLeftToRead()
+//		Purpose: As interface
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type ReadGatherStream::BytesLeftToRead()
+{
+	return mTotalSize - mCurrentPosition;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::Write(const void *, int)
+//		Purpose: As interface.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+void ReadGatherStream::Write(const void *pBuffer, int NBytes)
+{
+	THROW_EXCEPTION(CommonException, CannotWriteToReadGatherStream);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::StreamDataLeft()
+//		Purpose: As interface.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool ReadGatherStream::StreamDataLeft()
+{
+	if(mCurrentBlock >= mBlocks.size())
+	{
+		// Done all the blocks
+		return false;
+	}
+	
+	if(mCurrentBlock == (mBlocks.size() - 1)
+		&& mPositionInCurrentBlock >= mBlocks[mCurrentBlock].mLength)
+	{
+		// Are on the last block, and have got all the data from it.
+		return false;
+	}
+
+	// Otherwise, there's more data to be read
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadGatherStream::StreamClosed()
+//		Purpose: As interface. But the stream is always closed.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+bool ReadGatherStream::StreamClosed()
+{
+	return true;
+}
+
+
diff --git a/lib/common/ReadGatherStream.h b/lib/common/ReadGatherStream.h
new file mode 100755
index 00000000..613ede3e
--- /dev/null
+++ b/lib/common/ReadGatherStream.h
@@ -0,0 +1,67 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ReadGatherStream.h
+//		Purpose: Build a stream (for reading only) out of a number of other streams.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef READGATHERSTREAM_H
+#define READGATHERSTREAM_H
+
+#include "IOStream.h"
+
+#include <vector>
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    ReadGatherStream
+//		Purpose: Build a stream (for reading only) out of a number of other streams.
+//		Created: 10/12/03
+//
+// --------------------------------------------------------------------------
+class ReadGatherStream : public IOStream
+{
+public:
+	ReadGatherStream(bool DeleteComponentStreamsOnDestruction);
+	~ReadGatherStream();
+private:
+	ReadGatherStream(const ReadGatherStream &);
+	ReadGatherStream &operator=(const ReadGatherStream &);
+public:
+
+	int AddComponent(IOStream *pStream);
+	void AddBlock(int Component, pos_type Length, bool Seek = false, pos_type SeekTo = 0);
+
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type BytesLeftToRead();
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+	virtual pos_type GetPosition() const;
+
+private:
+	bool mDeleteComponentStreamsOnDestruction;
+	std::vector<IOStream *> mComponents;
+	
+	typedef struct
+	{
+		pos_type mLength;
+		pos_type mSeekTo;
+		int mComponent;
+		bool mSeek;
+	} Block;
+	
+	std::vector<Block> mBlocks;
+	
+	pos_type mCurrentPosition;
+	pos_type mTotalSize;
+	unsigned int mCurrentBlock;
+	pos_type mPositionInCurrentBlock;
+	bool mSeekDoneForCurrent;
+};
+
+
+#endif // READGATHERSTREAM_H
diff --git a/lib/common/StreamableMemBlock.cpp b/lib/common/StreamableMemBlock.cpp
new file mode 100755
index 00000000..4ea9e398
--- /dev/null
+++ b/lib/common/StreamableMemBlock.cpp
@@ -0,0 +1,365 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    StreamableMemBlock.cpp
+//		Purpose: Memory blocks which can be loaded and saved from streams
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <new>
+#include <stdlib.h>
+#include <string.h>
+
+#include "StreamableMemBlock.h"
+#include "IOStream.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::StreamableMemBlock()
+//		Purpose: Constructor, making empty block
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock::StreamableMemBlock()
+	: mpBuffer(0),
+	  mSize(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::StreamableMemBlock(void *, int)
+//		Purpose: Create block, copying data from another bit of memory
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock::StreamableMemBlock(void *pBuffer, int Size)
+	: mpBuffer(0),
+	  mSize(0)
+{
+	AllocateBlock(Size);
+	::memcpy(mpBuffer, pBuffer, Size);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::StreamableMemBlock(int)
+//		Purpose: Create block, initialising it to all zeros
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock::StreamableMemBlock(int Size)
+	: mpBuffer(0),
+	  mSize(0)
+{
+	AllocateBlock(Size);
+	::memset(mpBuffer, 0, Size);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::StreamableMemBlock(const StreamableMemBlock &)
+//		Purpose: Copy constructor
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock::StreamableMemBlock(const StreamableMemBlock &rToCopy)
+	: mpBuffer(0),
+	  mSize(0)
+{
+	AllocateBlock(rToCopy.mSize);
+	::memcpy(mpBuffer, rToCopy.mpBuffer, mSize);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::Set(void *, int)
+//		Purpose: Set the contents of the block
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void StreamableMemBlock::Set(void *pBuffer, int Size)
+{
+	FreeBlock();
+	AllocateBlock(Size);
+	::memcpy(mpBuffer, pBuffer, Size);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::Set(IOStream &)
+//		Purpose: Set from stream. Stream must support BytesLeftToRead()
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void StreamableMemBlock::Set(IOStream &rStream, int Timeout)
+{
+	// Get size
+	IOStream::pos_type size = rStream.BytesLeftToRead();
+	if(size == IOStream::SizeOfStreamUnknown)
+	{
+		THROW_EXCEPTION(CommonException, StreamDoesntHaveRequiredProperty)
+	}
+	
+	// Allocate a new block (this way to be exception safe)
+	char *pblock = (char*)malloc(size);
+	if(pblock == 0)
+	{
+		throw std::bad_alloc();
+	}
+	
+	try
+	{
+		// Read in
+		if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */))
+		{
+			THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+		}
+	
+		// Free the block ready for replacement
+		FreeBlock();
+	}
+	catch(...)
+	{
+		::free(pblock);
+		throw;
+	}
+	
+	// store...
+	ASSERT(mpBuffer == 0);
+	mpBuffer = pblock;
+	mSize = size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::Set(const StreamableMemBlock &)
+//		Purpose: Set from other block.
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+void StreamableMemBlock::Set(const StreamableMemBlock &rBlock)
+{
+	Set(rBlock.mpBuffer, rBlock.mSize);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::~StreamableMemBlock()
+//		Purpose: Destructor
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+StreamableMemBlock::~StreamableMemBlock()
+{
+	FreeBlock();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::FreeBlock()
+//		Purpose: Protected. Frees block of memory
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void StreamableMemBlock::FreeBlock()
+{
+	if(mpBuffer != 0)
+	{
+		::free(mpBuffer);
+	}
+	mpBuffer = 0;
+	mSize = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::AllocateBlock(int)
+//		Purpose: Protected. Allocate the block of memory
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void StreamableMemBlock::AllocateBlock(int Size)
+{
+	ASSERT(mpBuffer == 0);
+	if(Size > 0)
+	{
+		mpBuffer = ::malloc(Size);
+		if(mpBuffer == 0)
+		{
+			throw std::bad_alloc();
+		}
+	}
+	mSize = Size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::ResizeBlock(int)
+//		Purpose: Protected. Resizes the allocated block.
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void StreamableMemBlock::ResizeBlock(int Size)
+{
+	ASSERT(mpBuffer != 0);
+	ASSERT(Size > 0);
+	if(Size > 0)
+	{
+		void *pnewBuffer = ::realloc(mpBuffer, Size);
+		if(pnewBuffer == 0)
+		{
+			throw std::bad_alloc();
+		}
+		mpBuffer = pnewBuffer;
+	}
+	mSize = Size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::ReadFromStream(IOStream &, int)
+//		Purpose: Read the block in from a stream
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void StreamableMemBlock::ReadFromStream(IOStream &rStream, int Timeout)
+{
+	// Get the size of the block
+	int32_t size_s;
+	if(!rStream.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
+	{
+		THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+	}
+	
+	int size = ntohl(size_s);
+	
+	
+	// Allocate a new block (this way to be exception safe)
+	char *pblock = (char*)malloc(size);
+	if(pblock == 0)
+	{
+		throw std::bad_alloc();
+	}
+	
+	try
+	{
+		// Read in
+		if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */))
+		{
+			THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
+		}
+	
+		// Free the block ready for replacement
+		FreeBlock();
+	}
+	catch(...)
+	{
+		::free(pblock);
+		throw;
+	}
+	
+	// store...
+	ASSERT(mpBuffer == 0);
+	mpBuffer = pblock;
+	mSize = size;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::WriteToStream(IOStream &)
+//		Purpose: Write the block to a stream
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void StreamableMemBlock::WriteToStream(IOStream &rStream) const
+{
+	int32_t sizenbo = htonl(mSize);
+	// Size
+	rStream.Write(&sizenbo, sizeof(sizenbo));
+	// Buffer
+	if(mSize > 0)
+	{
+		rStream.Write(mpBuffer, mSize);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::WriteEmptyBlockToStream(IOStream &)
+//		Purpose: Writes an empty block to a stream.
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void StreamableMemBlock::WriteEmptyBlockToStream(IOStream &rStream)
+{
+	int32_t sizenbo = htonl(0);
+	rStream.Write(&sizenbo, sizeof(sizenbo));
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::GetBuffer()
+//		Purpose: Get pointer to buffer
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+void *StreamableMemBlock::GetBuffer() const
+{
+	if(mSize == 0)
+	{
+		// Return something which isn't a null pointer
+		static const int validptr = 0;
+		return (void*)&validptr;
+	}
+	
+	// return the buffer
+	ASSERT(mpBuffer != 0);
+	return mpBuffer;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    StreamableMemBlock::operator==(const StreamableMemBlock &)
+//		Purpose: Test for equality of memory blocks
+//		Created: 2003/09/06
+//
+// --------------------------------------------------------------------------
+bool StreamableMemBlock::operator==(const StreamableMemBlock &rCompare) const
+{
+	if(mSize != rCompare.mSize) return false;
+	if(mSize == 0 && rCompare.mSize == 0) return true;	// without memory comparison!
+	return ::memcmp(mpBuffer, rCompare.mpBuffer, mSize) == 0;
+}
+
+
diff --git a/lib/common/StreamableMemBlock.h b/lib/common/StreamableMemBlock.h
new file mode 100755
index 00000000..250c0aea
--- /dev/null
+++ b/lib/common/StreamableMemBlock.h
@@ -0,0 +1,71 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    StreamableMemBlock.h
+//		Purpose: Memory blocks which can be loaded and saved from streams
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+
+#ifndef STREAMABLEMEMBLOCK__H
+#define STREAMABLEMEMBLOCK__H
+
+class IOStream;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    StreamableMemBlock
+//		Purpose: Memory blocks which can be loaded and saved from streams
+//		Created: 2003/09/05
+//
+// --------------------------------------------------------------------------
+class StreamableMemBlock
+{
+public:
+	StreamableMemBlock();
+	StreamableMemBlock(int Size);
+	StreamableMemBlock(void *pBuffer, int Size);
+	StreamableMemBlock(const StreamableMemBlock &rToCopy);
+	~StreamableMemBlock();
+	
+	void Set(const StreamableMemBlock &rBlock);
+	void Set(void *pBuffer, int Size);
+	void Set(IOStream &rStream, int Timeout);
+	StreamableMemBlock &operator=(const StreamableMemBlock &rBlock)
+	{
+		Set(rBlock);
+		return *this;
+	}
+
+	void ReadFromStream(IOStream &rStream, int Timeout);
+	void WriteToStream(IOStream &rStream) const;
+	
+	static void WriteEmptyBlockToStream(IOStream &rStream);
+	
+	void *GetBuffer() const;
+	
+	// Size of block
+	int GetSize() const {return mSize;}
+
+	// Buffer empty?
+	bool IsEmpty() const {return mSize == 0;}
+
+	// Clear the contents of the block
+	void Clear() {FreeBlock();}
+	
+	bool operator==(const StreamableMemBlock &rCompare) const;
+
+	void ResizeBlock(int Size);
+
+protected:	// be careful with these!
+	void AllocateBlock(int Size);
+	void FreeBlock();
+
+private:
+	void *mpBuffer;
+	int mSize;
+};
+
+#endif // STREAMABLEMEMBLOCK__H
+
diff --git a/lib/common/TemporaryDirectory.h b/lib/common/TemporaryDirectory.h
new file mode 100755
index 00000000..62010f79
--- /dev/null
+++ b/lib/common/TemporaryDirectory.h
@@ -0,0 +1,26 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    TemporaryDirectory.h
+//		Purpose: Location of temporary directory
+//		Created: 2003/10/13
+//
+// --------------------------------------------------------------------------
+
+#ifndef TEMPORARYDIRECTORY__H
+#define TEMPORARYDIRECTORY__H
+
+#include <string>
+
+#ifdef PLATFORM_STATIC_TEMP_DIRECTORY_NAME
+	// Prefix name with Box to avoid clashing with OS API names
+	inline std::string BoxGetTemporaryDirectoryName()
+	{
+		return std::string(PLATFORM_STATIC_TEMP_DIRECTORY_NAME);
+	}
+#else
+	non-static temporary directory names not supported yet
+#endif
+
+#endif // TEMPORARYDIRECTORY__H
+
diff --git a/lib/common/Test.h b/lib/common/Test.h
new file mode 100755
index 00000000..bd69cd5a
--- /dev/null
+++ b/lib/common/Test.h
@@ -0,0 +1,165 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Test.h
+//		Purpose: Useful stuff for tests
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+
+#ifndef TEST__H
+#define TEST__H
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include <stdio.h>
+ 
+extern int failures;
+
+#define TEST_FAIL_WITH_MESSAGE(msg) {failures++; printf("FAILURE: " msg " at " __FILE__ "(%d)\n", __LINE__);}
+#define TEST_ABORT_WITH_MESSAGE(msg) {failures++; printf("FAILURE: " msg " at " __FILE__ "(%d)\n", __LINE__); return 1;}
+
+#define TEST_THAT(condition) {if(!(condition)) TEST_FAIL_WITH_MESSAGE("Condition [" #condition "] failed")}
+#define TEST_THAT_ABORTONFAIL(condition) {if(!(condition)) TEST_ABORT_WITH_MESSAGE("Condition [" #condition "] failed")}
+
+// NOTE: The 0- bit it to allow this to work with stuff which has negative constants for flags (eg ConnectionException)
+#define TEST_CHECK_THROWS(statement, excepttype, subtype)									\
+	{																						\
+		bool didthrow = false;																\
+		try																					\
+		{																					\
+			statement;																		\
+		}																					\
+		catch(excepttype &e)																\
+		{																					\
+			if(e.GetSubType() != ((unsigned int)excepttype::subtype)						\
+					&& e.GetSubType() != (unsigned int)(0-excepttype::subtype)) 			\
+			{																				\
+				throw;																		\
+			}																				\
+			didthrow = true;																\
+		}																					\
+		catch(...)																			\
+		{																					\
+			throw;																			\
+		}																					\
+		if(!didthrow)																		\
+		{																					\
+			TEST_FAIL_WITH_MESSAGE("Didn't throw exception " #excepttype "(" #subtype ")")	\
+		}																					\
+	}
+
+inline bool TestFileExists(const char *Filename)
+{
+	struct stat st;
+	return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0;
+}
+
+inline bool TestDirExists(const char *Filename)
+{
+	struct stat st;
+	return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == S_IFDIR;
+}
+
+// -1 if doesn't exist
+inline int TestGetFileSize(const char *Filename)
+{
+	struct stat st;
+	if(::stat(Filename, &st) == 0)
+	{
+		return st.st_size;
+	}
+	return -1;
+}
+
+inline int LaunchServer(const char *CommandLine, const char *pidFile)
+{
+	if(::system(CommandLine) != 0)
+	{
+		printf("Server: %s\n", CommandLine);
+		TEST_FAIL_WITH_MESSAGE("Couldn't start server");
+		return -1;
+	}
+	// time for it to start up
+	::sleep(1);
+	
+	// read pid file
+	if(!TestFileExists(pidFile))
+	{
+		printf("Server: %s\n", CommandLine);
+		TEST_FAIL_WITH_MESSAGE("Server didn't save PID file");	
+		return -1;
+	}
+	
+	FILE *f = fopen(pidFile, "r");
+	int pid = -1;
+	if(f == NULL || fscanf(f, "%d", &pid) != 1)
+	{
+		printf("Server: %s (pidfile %s)\n", CommandLine, pidFile);
+		TEST_FAIL_WITH_MESSAGE("Couldn't read PID file");	
+		return -1;
+	}
+	fclose(f);
+	
+	return pid;
+}
+
+inline bool ServerIsAlive(int pid)
+{
+	if(pid == 0) return false;
+	return ::kill(pid, 0) != -1;
+}
+
+inline bool HUPServer(int pid)
+{
+	if(pid == 0) return false;
+	return ::kill(pid, SIGHUP) != -1;
+}
+
+inline bool KillServer(int pid)
+{
+	if(pid == 0 || pid == -1) return false;
+	bool KilledOK = ::kill(pid, SIGTERM) != -1;
+	TEST_THAT(KilledOK);
+	::sleep(1);
+	return !ServerIsAlive(pid);
+}
+
+inline void TestRemoteProcessMemLeaks(const char *filename)
+{
+#ifdef BOX_MEMORY_LEAK_TESTING
+	// Does the file exist?
+	if(!TestFileExists(filename))
+	{
+		++failures;
+		printf("FAILURE: MemLeak report not available (file %s)\n", filename);
+	}
+	else
+	{
+		// Is it empty?
+		if(TestGetFileSize(filename) > 0)
+		{
+			++failures;
+			printf("FAILURE: Memory leaks found in other process (file %s)\n==========\n", filename);
+			FILE *f = fopen(filename, "r");
+			char line[512];
+			while(::fgets(line, sizeof(line), f) != 0)
+			{
+				printf("%s", line);
+			}
+			fclose(f);
+			printf("==========\n");
+		}
+		
+		// Delete it
+		::unlink(filename);
+	}
+#endif
+}
+
+#endif // TEST__H
+
diff --git a/lib/common/UnixUser.cpp b/lib/common/UnixUser.cpp
new file mode 100755
index 00000000..7a60b263
--- /dev/null
+++ b/lib/common/UnixUser.cpp
@@ -0,0 +1,121 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    UnixUser.cpp
+//		Purpose: Interface for managing the UNIX user of the current process
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <pwd.h>
+#include <unistd.h>
+
+#include "UnixUser.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    UnixUser::UnixUser(const char *)
+//		Purpose: Constructor, initialises to info of given username
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+UnixUser::UnixUser(const char *Username)
+	: mUID(0),
+	  mGID(0),
+	  mRevertOnDestruction(false)
+{
+	// Get password info
+	struct passwd *pwd = ::getpwnam(Username);
+	if(pwd == 0)
+	{
+		THROW_EXCEPTION(CommonException, CouldNotLookUpUsername)
+	}
+	
+	// Store UID and GID
+	mUID = pwd->pw_uid;
+	mGID = pwd->pw_gid;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    UnixUser::UnixUser(uid_t, gid_t)
+//		Purpose: Construct from given UNIX user ID and group ID
+//		Created: 15/3/04
+//
+// --------------------------------------------------------------------------
+UnixUser::UnixUser(uid_t UID, gid_t GID)
+	: mUID(UID),
+	  mGID(GID),
+	  mRevertOnDestruction(false)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    UnixUser::~UnixUser()
+//		Purpose: Destructor -- reverts to previous user if the change wasn't perminant
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+UnixUser::~UnixUser()
+{
+	if(mRevertOnDestruction)
+	{
+		// Revert to "real" user and group id of the process
+		if(::setegid(::getgid()) != 0
+			|| ::seteuid(::getuid()) != 0)
+		{
+			THROW_EXCEPTION(CommonException, CouldNotRestoreProcessUser)
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    UnixUser::ChangeProcessUser(bool)
+//		Purpose: Change the process user and group ID to the user. If Temporary == true
+//				 the process username will be changed back when the object is destructed.
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+void UnixUser::ChangeProcessUser(bool Temporary)
+{
+	if(Temporary)
+	{
+		// Change temporarily (change effective only)
+		if(::setegid(mGID) != 0
+			|| ::seteuid(mUID) != 0)
+		{
+			THROW_EXCEPTION(CommonException, CouldNotChangeProcessUser)
+		}
+		
+		// Mark for change on destruction
+		mRevertOnDestruction = true;
+	}
+	else
+	{
+		// Change perminantely (change all UIDs and GIDs)
+		if(::setgid(mGID) != 0
+			|| ::setuid(mUID) != 0)
+		{
+			THROW_EXCEPTION(CommonException, CouldNotChangeProcessUser)
+		}
+	}
+}
+
+
+
+
diff --git a/lib/common/UnixUser.h b/lib/common/UnixUser.h
new file mode 100755
index 00000000..c895eb2a
--- /dev/null
+++ b/lib/common/UnixUser.h
@@ -0,0 +1,37 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    UnixUser.h
+//		Purpose: Interface for managing the UNIX user of the current process
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef UNIXUSER__H
+#define UNIXUSER__H
+
+class UnixUser
+{
+public:
+	UnixUser(const char *Username);
+	UnixUser(uid_t UID, gid_t GID);
+	~UnixUser();
+private:
+	// no copying allowed
+	UnixUser(const UnixUser &);
+	UnixUser &operator=(const UnixUser &);
+public:
+
+	void ChangeProcessUser(bool Temporary = false);
+
+	uid_t GetUID() {return mUID;}
+	gid_t GetGID() {return mGID;}
+
+private:
+	uid_t mUID;
+	gid_t mGID;
+	bool mRevertOnDestruction;
+};
+
+#endif // UNIXUSER__H
+
diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp
new file mode 100755
index 00000000..b4d3144b
--- /dev/null
+++ b/lib/common/Utils.cpp
@@ -0,0 +1,159 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Utils.cpp
+//		Purpose: Utility function
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+
+#ifdef SHOW_BACKTRACE_ON_EXCEPTION
+	#include <execinfo.h>
+	#include <stdlib.h>
+#endif
+
+#include "Utils.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SplitString(const std::string &, char, std::vector<std::string> &)
+//		Purpose: Splits a string at a given character
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput)
+{
+	// Split it up.
+	std::string::size_type b = 0;
+	std::string::size_type e = 0;
+	while(e = String.find_first_of(SplitOn, b), e != String.npos)
+	{
+		// Get this string
+		unsigned int len = e - b;
+		if(len >= 1)
+		{
+			rOutput.push_back(String.substr(b, len));
+		}
+		b = e + 1;
+	}
+	// Last string
+	if(b < String.size())
+	{
+		rOutput.push_back(String.substr(b));
+	}
+/*#ifndef NDEBUG
+	TRACE2("Splitting string '%s' on %c\n", String.c_str(), SplitOn);
+	for(unsigned int l = 0; l < rOutput.size(); ++l)
+	{
+		TRACE2("%d = '%s'\n", l, rOutput[l].c_str());
+	}
+#endif*/
+}
+
+#ifdef SHOW_BACKTRACE_ON_EXCEPTION
+void DumpStackBacktrace()
+{
+	void *array[10];
+	size_t size;
+	char **strings;
+	size_t i;
+
+	size = backtrace (array, 10);
+	strings = backtrace_symbols (array, size);
+
+	printf ("Obtained %zd stack frames.\n", size);
+
+	for(i = 0; i < size; i++)
+		printf("%s\n", strings[i]);
+
+	free (strings);
+}
+#endif
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FileExists(const char *)
+//		Purpose: Does a file exist?
+//		Created: 20/11/03
+//
+// --------------------------------------------------------------------------
+bool FileExists(const char *Filename, int64_t *pFileSize, bool TreatLinksAsNotExisting)
+{
+	struct stat st;
+	if(::stat(Filename, &st) != 0)
+	{
+		if(errno == ENOENT)
+		{
+			return false;
+		}
+		else
+		{
+			THROW_EXCEPTION(CommonException, OSFileError);
+		}
+	}
+
+	// is it a file?	
+	if((st.st_mode & S_IFDIR) == 0)
+	{
+		if(TreatLinksAsNotExisting && ((st.st_mode & S_IFLNK) != 0))
+		{
+			return false;
+		}
+	
+		// Yes. Tell caller the size?
+		if(pFileSize != 0)
+		{
+			*pFileSize = st.st_size;
+		}
+	
+		return true;
+	}
+	else
+	{
+		return false;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ObjectExists(const char *)
+//		Purpose: Does a object exist, and if so, is it a file or a directory?
+//		Created: 23/11/03
+//
+// --------------------------------------------------------------------------
+int ObjectExists(const char *Filename)
+{
+	struct stat st;
+	if(::stat(Filename, &st) != 0)
+	{
+		if(errno == ENOENT)
+		{
+			return ObjectExists_NoObject;
+		}
+		else
+		{
+			THROW_EXCEPTION(CommonException, OSFileError);
+		}
+	}
+
+	// is it a file or a dir?
+	return ((st.st_mode & S_IFDIR) == 0)?ObjectExists_File:ObjectExists_Dir;
+}
+
+
+
+
diff --git a/lib/common/Utils.h b/lib/common/Utils.h
new file mode 100755
index 00000000..5da84d9a
--- /dev/null
+++ b/lib/common/Utils.h
@@ -0,0 +1,36 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Utils.h
+//		Purpose: Utility function
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef UTILS__H
+#define UTILS__H
+
+#include <string>
+#include <vector>
+
+#include "MemLeakFindOn.h"
+
+void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput);
+
+#ifdef SHOW_BACKTRACE_ON_EXCEPTION
+	void DumpStackBacktrace();
+#endif
+
+bool FileExists(const char *Filename, int64_t *pFileSize = 0, bool TreatLinksAsNotExisting = false);
+
+enum
+{
+	ObjectExists_NoObject = 0,
+	ObjectExists_File = 1,
+	ObjectExists_Dir = 2
+};
+int ObjectExists(const char *Filename);
+
+#include "MemLeakFindOff.h"
+
+#endif // UTILS__H
diff --git a/lib/common/WaitForEvent.cpp b/lib/common/WaitForEvent.cpp
new file mode 100644
index 00000000..e9ee58d2
--- /dev/null
+++ b/lib/common/WaitForEvent.cpp
@@ -0,0 +1,194 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    WaitForEvent.cpp
+//		Purpose: Generic waiting for events, using an efficient method (platform dependent)
+//		Created: 9/3/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+
+#include "WaitForEvent.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    WaitForEvent::WaitForEvent()
+//		Purpose: Constructor
+//		Created: 9/3/04
+//
+// --------------------------------------------------------------------------
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+WaitForEvent::WaitForEvent(int Timeout)
+	: mKQueue(::kqueue()),
+	  mpTimeout(0)
+{
+	if(mKQueue == -1)
+	{
+		THROW_EXCEPTION(CommonException, CouldNotCreateKQueue)
+	}
+
+	// Set the choosen timeout
+	SetTimeout(Timeout);
+}
+#else
+WaitForEvent::WaitForEvent(int Timeout)
+	: mTimeout(Timeout),
+	  mpPollInfo(0)
+{
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    WaitForEvent::~WaitForEvent()
+//		Purpose: Destructor
+//		Created: 9/3/04
+//
+// --------------------------------------------------------------------------
+WaitForEvent::~WaitForEvent()
+{
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	::close(mKQueue);
+	mKQueue = -1;
+#else
+	if(mpPollInfo != 0)
+	{
+		::free(mpPollInfo);
+		mpPollInfo = 0;
+	}
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    WaitForEvent::SetTimeout
+//		Purpose: Sets the timeout for future wait calls
+//		Created: 9/3/04
+//
+// --------------------------------------------------------------------------
+void WaitForEvent::SetTimeout(int Timeout)
+{
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	// Generate timeout
+	if(Timeout != TimeoutInfinite)
+	{
+		mTimeout.tv_sec = Timeout / 1000;
+		mTimeout.tv_nsec = (Timeout % 1000) * 1000000;
+	}
+	
+	// Infinite or not?
+	mpTimeout = (Timeout != TimeoutInfinite)?(&mTimeout):(NULL);
+#else
+	mTimeout = Timeout;
+#endif
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    WaitForEvent::Wait(int)
+//		Purpose: Wait for an event to take place. Returns a pointer to the object
+//				 which has been signalled, or returns 0 for the timeout condition.
+//				 Timeout specified in milliseconds.
+//		Created: 9/3/04
+//
+// --------------------------------------------------------------------------
+void *WaitForEvent::Wait()
+{
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	// Event return structure
+	struct kevent e;
+	::memset(&e, 0, sizeof(e));
+	
+	switch(::kevent(mKQueue, NULL, 0, &e, 1, mpTimeout))
+	{
+	case 0:
+		// Timeout
+		return 0;
+		break;
+
+	case 1:
+		// Event happened!
+		return e.udata;
+		break;
+		
+	default:
+		// Interrupted system calls aren't an error, just equivalent to a timeout
+		if(errno != EINTR)
+		{
+			THROW_EXCEPTION(CommonException, KEventErrorWait)
+		}
+		return 0;
+		break;
+	}
+#else
+	// Use poll() instead.
+	// Need to build the structures?
+	if(mpPollInfo == 0)
+	{
+		// Yes...
+		mpPollInfo = (struct pollfd *)::malloc((sizeof(struct pollfd) * mItems.size()) + 4);
+		if(mpPollInfo == 0)
+		{
+			throw std::bad_alloc();
+		}
+		
+		// Build...
+		for(unsigned int l = 0; l < mItems.size(); ++l)
+		{
+			mpPollInfo[l].fd = mItems[l].fd;
+			mpPollInfo[l].events = mItems[l].events;
+			mpPollInfo[l].revents = 0;
+		}
+	}
+	
+	// Make sure everything is reset (don't really have to do this, but don't trust the OS)
+	for(unsigned int l = 0; l < mItems.size(); ++l)
+	{
+		mpPollInfo[l].revents = 0;
+	}
+	
+	// Poll!
+	switch(::poll(mpPollInfo, mItems.size(), mTimeout))
+	{
+	case -1:
+		// Interrupted system calls aren't an error, just equivalent to a timeout
+		if(errno != EINTR)
+		{
+			THROW_EXCEPTION(CommonException, KEventErrorWait)
+		}
+		return 0;
+		break;
+	case 0:	// timed out
+		return 0;
+		break;
+	default:	// got some thing...
+		// control flows on...
+		break;
+	}
+	
+	// Find the item which was ready
+	for(unsigned int s = 0; s < mItems.size(); ++s)
+	{
+		if(mpPollInfo[s].revents & POLLIN)
+		{
+			return mItems[s].item;
+			break;
+		}
+	}
+#endif
+
+	return 0;
+}
+
diff --git a/lib/common/WaitForEvent.h b/lib/common/WaitForEvent.h
new file mode 100644
index 00000000..b8f79da6
--- /dev/null
+++ b/lib/common/WaitForEvent.h
@@ -0,0 +1,146 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    WaitForEvent.h
+//		Purpose: Generic waiting for events, using an efficient method (platform dependent)
+//		Created: 9/3/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef WAITFOREVENT__H
+#define WAITFOREVENT__H
+
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	#include <sys/event.h>
+	#include <sys/time.h>
+#else
+	#include <vector>
+	#include <poll.h>
+#endif
+
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+class WaitForEvent
+{
+public:
+	WaitForEvent(int Timeout = TimeoutInfinite);
+	~WaitForEvent();
+private:
+	// No copying.
+	WaitForEvent(const WaitForEvent &);
+	WaitForEvent &operator=(const WaitForEvent &);
+public:
+
+	enum
+	{
+		TimeoutInfinite = -1
+	};
+
+	void SetTimeout(int Timeout = TimeoutInfinite);
+
+	void *Wait();
+
+#ifdef PLATFORM_KQUEUE_NOT_SUPPORTED
+	typedef struct
+	{
+		int fd;
+		short events;
+		void *item;
+	} ItemInfo;
+#endif
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    WaitForEvent::Add(const Type &, int)
+	//		Purpose: Adds an event to the list of items to wait on. The flags are passed to the object.
+	//		Created: 9/3/04
+	//
+	// --------------------------------------------------------------------------
+	template<typename T>
+	void Add(const T *pItem, int Flags = 0)
+	{
+		ASSERT(pItem != 0);
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+		struct kevent e;
+		pItem->FillInKEvent(e, Flags);
+		// Fill in extra flags to say what to do
+		e.flags |= EV_ADD;
+		e.udata = (void*)pItem;
+		if(::kevent(mKQueue, &e, 1, NULL, 0, NULL) == -1)
+		{
+			THROW_EXCEPTION(CommonException, KEventErrorAdd)
+		}
+#else
+		// Add item
+		ItemInfo i;
+		pItem->FillInPoll(i.fd, i.events, Flags);
+		i.item = (void*)pItem;
+		mItems.push_back(i);
+		// Delete any pre-prepared poll info, as it's now out of date
+		if(mpPollInfo != 0)
+		{
+			::free(mpPollInfo);
+			mpPollInfo = 0;
+		}
+#endif
+	}
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    WaitForEvent::Remove(const Type &, int)
+	//		Purpose: Removes an event from the list of items to wait on. The flags are passed to the object.
+	//		Created: 9/3/04
+	//
+	// --------------------------------------------------------------------------
+	template<typename T>
+	void Remove(const T *pItem, int Flags = 0)
+	{
+		ASSERT(pItem != 0);
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+		struct kevent e;
+		pItem->FillInKEvent(e, Flags);
+		// Fill in extra flags to say what to do
+		e.flags |= EV_DELETE;
+		e.udata = (void*)pItem;
+		if(::kevent(mKQueue, &e, 1, NULL, 0, NULL) == -1)
+		{
+			THROW_EXCEPTION(CommonException, KEventErrorRemove)
+		}
+#else
+		if(mpPollInfo != 0)
+		{
+			::free(mpPollInfo);
+			mpPollInfo = 0;
+		}
+		for(std::vector<ItemInfo>::iterator i(mItems.begin()); i != mItems.end(); ++i)
+		{
+			if(i->item == pItem)
+			{
+				mItems.erase(i);
+				return;
+			}
+		}
+#endif
+	}
+
+private:
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	int mKQueue;
+	struct timespec mTimeout;
+	struct timespec *mpTimeout;
+#else
+	int mTimeout;
+	std::vector<ItemInfo> mItems;
+	struct pollfd *mpPollInfo;
+#endif
+};
+
+#include "MemLeakFindOff.h"
+
+#endif // WAITFOREVENT__H
+
+
diff --git a/lib/common/makeexception.pl b/lib/common/makeexception.pl
new file mode 100755
index 00000000..c0388286
--- /dev/null
+++ b/lib/common/makeexception.pl
@@ -0,0 +1,277 @@
+#!/usr/bin/perl
+
+# global exception list file
+my $global_list = '../../ExceptionCodes.txt';
+
+
+my @exception;
+my @exception_desc;
+my $class;
+my $class_number;
+
+# read the description!
+
+open EXCEPTION_DESC,$ARGV[0] or die "Can't open $ARGV[0]";
+
+while(<EXCEPTION_DESC>)
+{
+	chomp; s/\A\s+//; s/#.+\Z//; s/\s+\Z//; s/\s+/ /g;
+	next unless m/\S/;
+
+	if(m/\AEXCEPTION\s+(.+)\s+(\d+)\Z/)
+	{
+		$class = $1;
+		$class_number = $2;
+	}
+	else
+	{
+		my ($name,$number,$description) = split /\s+/,$_,3;
+		if($name eq '' || $number =~ m/\D/)
+		{
+			die "Bad line '$_'";
+		}
+		if($exception[$number] ne '')
+		{
+			die "Duplicate exception number $number";
+		}
+		$exception[$number] = $name;
+		$exception_desc[$number] = $description;
+	}
+}
+
+die "Exception class and number not specified" unless $class ne '' && $class_number ne '';
+
+close EXCEPTION_DESC;
+
+# write the code
+print "Generating $class exception...\n";
+
+open CPP,">autogen_${class}Exception.cpp" or die "Can't open cpp file for writing";
+open H,">autogen_${class}Exception.h" or die "Can't open h file for writing";
+
+# write header file
+my $guardname = uc 'AUTOGEN_'.$class.'EXCEPTION_H';
+print H <<__E;
+
+// Auto-generated file -- do not edit
+
+#ifndef $guardname
+#define $guardname
+
+#include "BoxException.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    ${class}Exception
+//		Purpose: Exception
+//		Created: autogen
+//
+// --------------------------------------------------------------------------
+class ${class}Exception : public BoxException
+{
+public:
+	${class}Exception(unsigned int SubType)
+		: mSubType(SubType)
+	{
+	}
+	
+	${class}Exception(const ${class}Exception &rToCopy)
+		: mSubType(rToCopy.mSubType)
+	{
+	}
+	
+	~${class}Exception() throw ()
+	{
+	}
+	
+	enum
+	{
+		ExceptionType = $class_number
+	};
+
+	enum
+	{
+__E
+
+for(my $e = 0; $e <= $#exception; $e++)
+{
+	if($exception[$e] ne '')
+	{
+		print H "\t\t".$exception[$e].' = '.$e.(($e==$#exception)?'':',')."\n"
+	}
+}
+
+print H <<__E;
+	};
+	
+	virtual unsigned int GetType() const throw();
+	virtual unsigned int GetSubType() const throw();
+	virtual const char *what() const throw();
+
+private:
+	unsigned int mSubType;
+};
+
+#endif // $guardname
+__E
+
+# -----------------------------------------------------------------------------------------------------------
+
+print CPP <<__E;
+
+// Auto-generated file -- do not edit
+
+#include "Box.h"
+#include "autogen_${class}Exception.h"
+
+#include "MemLeakFindOn.h"
+
+#ifdef EXCEPTION_CODENAMES_EXTENDED
+	#ifdef EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION
+static const char *whats[] = {
+__E
+
+my $last_seen = -1;
+for(my $e = 0; $e <= $#exception; $e++)
+{
+	if($exception[$e] ne '')
+	{
+		for(my $s = $last_seen + 1; $s < $e; $s++)
+		{
+			print CPP "\t\"UNUSED\",\n"
+		}
+		my $ext = ($exception_desc[$e] ne '')?" ($exception_desc[$e])":'';
+		print CPP "\t\"${class} ".$exception[$e].$ext.'"'.(($e==$#exception)?'':',')."\n";
+		$last_seen = $e;
+	}
+}
+
+print CPP <<__E;
+};
+	#else
+static const char *whats[] = {
+__E
+
+$last_seen = -1;
+for(my $e = 0; $e <= $#exception; $e++)
+{
+	if($exception[$e] ne '')
+	{
+		for(my $s = $last_seen + 1; $s < $e; $s++)
+		{
+			print CPP "\t\"UNUSED\",\n"
+		}
+		print CPP "\t\"${class} ".$exception[$e].'"'.(($e==$#exception)?'':',')."\n";
+		$last_seen = $e;
+	}
+}
+
+print CPP <<__E;
+};
+	#endif
+#endif
+
+unsigned int ${class}Exception::GetType() const throw()
+{
+	return ${class}Exception::ExceptionType;
+}
+
+unsigned int ${class}Exception::GetSubType() const throw()
+{
+	return mSubType;
+}
+
+const char *${class}Exception::what() const throw()
+{
+#ifdef EXCEPTION_CODENAMES_EXTENDED
+	if(mSubType < 0 || mSubType > (sizeof(whats) / sizeof(whats[0])))
+	{
+		return "${class}";
+	}
+	return whats[mSubType];
+#else
+	return "${class}";
+#endif
+}
+
+__E
+
+close H;
+close CPP;
+
+# update the global exception list
+my $list_before;
+my $list_after;
+my $is_after = 0;
+if(open CURRENT,$global_list)
+{
+	while(<CURRENT>)
+	{
+		next if m/\A#/;
+		
+		if(m/\AEXCEPTION TYPE (\w+) (\d+)/)
+		{
+			# check that the number isn't being reused
+			if($2 == $class_number && $1 ne $class)
+			{
+				die "Class number $class_number is being used by $class and $1 -- correct this.\n";
+			}
+			if($2 > $class_number)
+			{
+				# This class comes after the current one (ensures numerical ordering)
+				$is_after = 1;
+			}
+			if($1 eq $class)
+			{
+				# skip this entry
+				while(<CURRENT>)
+				{
+					last if m/\AEND TYPE/;
+				}
+				$_ = '';
+			}
+		}
+		
+		if($is_after)
+		{
+			$list_after .= $_;
+		}
+		else
+		{
+			$list_before .= $_;
+		}
+	}
+
+	close CURRENT;
+}
+
+open GLOBAL,">$global_list" or die "Can't open global exception code listing for writing";
+
+print GLOBAL <<__E;
+#
+# automatically generated file, do not edit.
+#
+# This file lists all the exception codes used by the system.
+# Use to look up more detailed descriptions of meanings of errors.
+#
+__E
+
+print GLOBAL $list_before;
+
+print GLOBAL "EXCEPTION TYPE $class $class_number\n";
+for(my $e = 0; $e <= $#exception; $e++)
+{
+	if($exception[$e] ne '')
+	{
+		my $ext = ($exception_desc[$e] ne '')?" - $exception_desc[$e]":'';
+		print GLOBAL "($class_number/$e) - ${class} ".$exception[$e].$ext."\n";
+	}
+}
+print GLOBAL "END TYPE\n";
+
+print GLOBAL $list_after;
+
+close GLOBAL;
+
+
diff --git a/lib/compress/Compress.h b/lib/compress/Compress.h
new file mode 100755
index 00000000..9b609adb
--- /dev/null
+++ b/lib/compress/Compress.h
@@ -0,0 +1,195 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CompressContext.h
+//		Purpose: Interface to zlib compression
+//		Created: 5/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef COMPRESSCONTEXT__H
+#define COMPRESSCONTEXT__H
+
+#include <zlib.h>
+
+#include "CompressException.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    CompressContext
+//		Purpose: Interface to zlib compression, only very slight wrapper.
+//				 (Use CompressStream for a more friendly interface.)
+//		Created: 5/12/03
+//
+// --------------------------------------------------------------------------
+template<bool Compressing>
+class Compress
+{
+public:
+	Compress()
+		: mFinished(false),
+		  mFlush(Z_NO_FLUSH)
+	{	
+		// initialise stream
+		mStream.zalloc = Z_NULL;
+		mStream.zfree = Z_NULL;
+		mStream.opaque = Z_NULL;
+		mStream.data_type = Z_BINARY;
+
+		if((Compressing)?(deflateInit(&mStream, Z_DEFAULT_COMPRESSION))
+			:(inflateInit(&mStream)) != Z_OK)
+		{
+			THROW_EXCEPTION(CompressException, InitFailed)
+		}
+		
+		mStream.avail_in = 0;
+	}
+	
+	~Compress()
+	{
+		int r = 0;
+		if((r = ((Compressing)?(deflateEnd(&mStream))
+			:(inflateEnd(&mStream)))) != Z_OK)
+		{
+			TRACE1("zlib error code = %d\n", r);
+			if(r == Z_DATA_ERROR)
+			{
+				TRACE0("WARNING: End of compress/decompress without all input being consumed -- possible corruption?\n");
+			}
+			else
+			{
+				THROW_EXCEPTION(CompressException, EndFailed)
+			}
+		}
+	}
+		
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Compress<Function>::InputRequired()
+	//		Purpose: Input required yet?
+	//		Created: 5/12/03
+	//
+	// --------------------------------------------------------------------------
+	bool InputRequired()
+	{
+		return mStream.avail_in <= 0;
+	}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Compress<Function>::Input(const void *, int)
+	//		Purpose: Set the input buffer ready for next output call.
+	//		Created: 5/12/03
+	//
+	// --------------------------------------------------------------------------
+	void Input(const void *pInBuffer, int InLength)
+	{
+		// Check usage
+		if(mStream.avail_in != 0)
+		{
+			THROW_EXCEPTION(CompressException, BadUsageInputNotRequired)
+		}
+		
+		// Store info
+		mStream.next_in = (unsigned char *)pInBuffer;
+		mStream.avail_in = InLength;
+	}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Compress<Function>::FinishInput()
+	//		Purpose: When compressing, no more input will be given.
+	//		Created: 5/12/03
+	//
+	// --------------------------------------------------------------------------
+	void FinishInput()
+	{
+		mFlush = Z_FINISH;
+	}
+		
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Compress<Function>::Output(void *, int)
+	//		Purpose: Get some output data
+	//		Created: 5/12/03
+	//
+	// --------------------------------------------------------------------------
+	int Output(void *pOutBuffer, int OutLength, bool SyncFlush = false)
+	{
+		// need more input?
+		if(mStream.avail_in == 0 && mFlush != Z_FINISH && !SyncFlush)
+		{
+			return 0;
+		}
+	
+		// Buffers
+		mStream.next_out = (unsigned char *)pOutBuffer;
+		mStream.avail_out = OutLength;
+		
+		// Call one of the functions
+		int flush = mFlush;
+		if(SyncFlush && mFlush != Z_FINISH)
+		{
+			flush = Z_SYNC_FLUSH;
+		}
+		int ret = (Compressing)?(deflate(&mStream, flush)):(inflate(&mStream, flush));
+		
+		if(SyncFlush && ret == Z_BUF_ERROR)
+		{
+			// No progress possible. Just return 0.
+			return 0;
+		}
+		
+		// Check errors
+		if(ret < 0)
+		{
+			TRACE1("zlib error code = %d\n", ret);			
+			THROW_EXCEPTION(CompressException, TransformFailed)
+		}
+		
+		// Parse result
+		if(ret == Z_STREAM_END)
+		{
+			mFinished = true;
+		}
+		
+		// Return how much data was output
+		return OutLength - mStream.avail_out;
+	}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Compress<Function>::OutputHasFinished()
+	//		Purpose: No more output to be recieved 
+	//		Created: 5/12/03
+	//
+	// --------------------------------------------------------------------------
+	bool OutputHasFinished()
+	{
+		return mFinished;
+	}
+	
+	
+private:
+	z_stream mStream;
+	bool mFinished;
+	int mFlush;
+};
+
+template<typename Integer>
+Integer Compress_MaxSizeForCompressedData(Integer InLen)
+{
+	// Conservative rendition of the info found here: http://www.gzip.org/zlib/zlib_tech.html
+	int blocks = (InLen + 32*1024 - 1) / (32*1024);
+	return InLen + (blocks * 6) + 8;
+}
+
+
+#endif // COMPRESSCONTEXT__H
+
diff --git a/lib/compress/CompressException.h b/lib/compress/CompressException.h
new file mode 100755
index 00000000..7e8c9ca2
--- /dev/null
+++ b/lib/compress/CompressException.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherException.h
+//		Purpose: Exception
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef COMPRESSEXCEPTION__H
+#define COMPRESSEXCEPTION__H
+
+// Compatibility
+#include "autogen_CompressException.h"
+
+#endif // COMPRESSEXCEPTION__H
+
diff --git a/lib/compress/CompressException.txt b/lib/compress/CompressException.txt
new file mode 100644
index 00000000..278b76d3
--- /dev/null
+++ b/lib/compress/CompressException.txt
@@ -0,0 +1,12 @@
+EXCEPTION Compress 6
+
+Internal					0
+InitFailed					1
+EndFailed					2
+BadUsageInputNotRequired	3
+TransformFailed				4
+CopyCompressStreamNotAllowed				5
+NullPointerPassedToCompressStream			6
+CompressStreamReadSupportNotRequested		7	Specify read in the constructor
+CompressStreamWriteSupportNotRequested		8	Specify write in the constructor
+CannotWriteToClosedCompressStream			9
diff --git a/lib/compress/CompressStream.cpp b/lib/compress/CompressStream.cpp
new file mode 100644
index 00000000..2839b647
--- /dev/null
+++ b/lib/compress/CompressStream.cpp
@@ -0,0 +1,425 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CompressStream.h
+//		Purpose: Compressing stream
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <memory>
+
+#include "CompressStream.h"
+#include "Compress.h"
+#include "autogen_CompressException.h"
+
+#include "MemLeakFindOn.h"
+
+// How big a buffer to use
+#ifndef NDEBUG
+	// debug!
+	#define BUFFER_SIZE	256
+#else
+	#define BUFFER_SIZE	(32*1024)
+#endif
+
+#define USE_READ_COMPRESSOR		\
+	CheckRead();				\
+	Compress<false> *pDecompress = (Compress<false> *)mpReadCompressor;
+
+#define USE_WRITE_COMPRESSOR	\
+	CheckWrite();				\
+	Compress<true> *pCompress = (Compress<true> *)mpWriteCompressor;
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::CompressStream(IOStream *, bool, bool, bool, bool)
+//		Purpose: Constructor
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+CompressStream::CompressStream(IOStream *pStream, bool TakeOwnership,
+		bool DecompressRead, bool CompressWrite, bool PassThroughWhenNotCompressed)
+	: mpStream(pStream),
+	  mHaveOwnership(TakeOwnership),
+	  mDecompressRead(DecompressRead),
+	  mCompressWrite(CompressWrite),
+	  mPassThroughWhenNotCompressed(PassThroughWhenNotCompressed),
+	  mpReadCompressor(0),
+	  mpWriteCompressor(0),
+	  mpBuffer(0),
+	  mIsClosed(false)
+{
+	if(mpStream == 0)
+	{
+		THROW_EXCEPTION(CompressException, NullPointerPassedToCompressStream)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::~CompressStream()
+//		Purpose: Destructor
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+CompressStream::~CompressStream()
+{
+	// Clean up compressors
+	if(mpReadCompressor)
+	{
+		delete ((Compress<false>*)mpReadCompressor);
+		mpReadCompressor = 0;
+	}
+	if(mpWriteCompressor)
+	{
+		delete ((Compress<true>*)mpWriteCompressor);
+		mpWriteCompressor = 0;
+	}
+
+	// Delete the stream, if we have ownership
+	if(mHaveOwnership)
+	{
+		delete mpStream;
+		mpStream = 0;
+	}
+	
+	// Free any buffer
+	if(mpBuffer != 0)
+	{
+		::free(mpBuffer);
+		mpBuffer = 0;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::CompressStream(const CompressStream &)
+//		Purpose: Copy constructor, will exception
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+CompressStream::CompressStream(const CompressStream &)
+{
+	THROW_EXCEPTION(CompressException, CopyCompressStreamNotAllowed)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::operator=(const CompressStream &)
+//		Purpose: Assignment operator, will exception
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+CompressStream &CompressStream::operator=(const CompressStream &)
+{
+	THROW_EXCEPTION(CompressException, CopyCompressStreamNotAllowed)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::Read(void *, int, int)
+//		Purpose: As interface
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+int CompressStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	USE_READ_COMPRESSOR
+	if(pDecompress == 0)
+	{
+		return mpStream->Read(pBuffer, NBytes, Timeout);
+	}
+
+	// Where is the buffer? (note if writing as well, read buffer is second in a block of two buffer sizes)
+	void *pbuf = (mDecompressRead && mCompressWrite)?(((uint8_t*)mpBuffer) + BUFFER_SIZE):mpBuffer;
+
+	// Any data left to go?
+	if(!pDecompress->InputRequired())
+	{
+		// Output some data from the existing data read
+		return pDecompress->Output(pBuffer, NBytes, true /* write as much as possible */);
+	}
+
+	// Read data into the buffer -- read as much as possible in one go
+	int s = mpStream->Read(pbuf, BUFFER_SIZE, Timeout);
+	if(s == 0)
+	{
+		return 0;
+	}
+	
+	// Give input to the compressor
+	pDecompress->Input(pbuf, s);
+
+	// Output as much as possible
+	return pDecompress->Output(pBuffer, NBytes, true /* write as much as possible */);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::Write(const void *, int)
+//		Purpose: As interface
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+void CompressStream::Write(const void *pBuffer, int NBytes)
+{
+	USE_WRITE_COMPRESSOR
+	if(pCompress == 0)
+	{
+		mpStream->Write(pBuffer, NBytes);
+		return;
+	}
+	
+	if(mIsClosed)
+	{
+		THROW_EXCEPTION(CompressException, CannotWriteToClosedCompressStream)
+	}
+
+	// Give the data to the compressor
+	pCompress->Input(pBuffer, NBytes);
+	
+	// Write the data to the stream
+	WriteCompressedData();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::WriteAllBuffered()
+//		Purpose: As interface
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+void CompressStream::WriteAllBuffered()
+{
+	if(mIsClosed)
+	{
+		THROW_EXCEPTION(CompressException, CannotWriteToClosedCompressStream)
+	}
+
+	// Just ask compressed data to be written out, but with the sync flag set
+	WriteCompressedData(true);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::Close()
+//		Purpose: As interface
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+void CompressStream::Close()
+{
+	if(mCompressWrite)
+	{
+		USE_WRITE_COMPRESSOR
+		if(pCompress != 0)
+		{
+			// Flush anything from the write buffer
+			pCompress->FinishInput();
+			WriteCompressedData();
+			
+			// Mark as definately closed
+			mIsClosed = true;
+		}
+	}
+
+	// Close
+	mpStream->Close();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::WriteCompressedData(bool)
+//		Purpose: Private. Writes the output of the compressor to the stream,
+//				 optionally doing a sync flush.
+//		Created: 28/5/04
+//
+// --------------------------------------------------------------------------
+void CompressStream::WriteCompressedData(bool SyncFlush)
+{
+	USE_WRITE_COMPRESSOR
+	if(pCompress == 0) {THROW_EXCEPTION(CompressException, Internal)}
+	
+	int s = 0;
+	do
+	{
+		s = pCompress->Output(mpBuffer, BUFFER_SIZE, SyncFlush);
+		if(s > 0)
+		{
+			mpStream->Write(mpBuffer, s);
+		}
+	} while(s > 0);
+	// Check assumption -- all input has been consumed
+	if(!pCompress->InputRequired()) {THROW_EXCEPTION(CompressException, Internal)}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::StreamDataLeft()
+//		Purpose: As interface
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+bool CompressStream::StreamDataLeft()
+{
+	USE_READ_COMPRESSOR
+	if(pDecompress == 0)
+	{
+		return mpStream->StreamDataLeft();
+	}
+
+	// Any bytes left in our buffer?
+	if(!pDecompress->InputRequired())
+	{
+		// Still buffered data to decompress
+		return true;
+	}
+
+	// Otherwise ask the stream
+	return mpStream->StreamDataLeft();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::StreamClosed()
+//		Purpose: As interface
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+bool CompressStream::StreamClosed()
+{
+	if(!mIsClosed && mpStream->StreamClosed())
+	{
+		mIsClosed = true;
+	}
+	return mIsClosed;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::CheckRead()
+//		Purpose: Checks that everything is set up to read
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+void CompressStream::CheckRead()
+{
+	// Has the read compressor already been created?
+	if(mpReadCompressor != 0)
+	{
+		return;
+	}
+	
+	// Need to create one?
+	if(mDecompressRead)
+	{
+		mpReadCompressor = new Compress<false>();
+		// And make sure there's a buffer
+		CheckBuffer();
+	}
+	else
+	{
+		// Not decompressing. Should be passing through data?
+		if(!mPassThroughWhenNotCompressed)
+		{
+			THROW_EXCEPTION(CompressException, CompressStreamReadSupportNotRequested)
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::CheckWrite()
+//		Purpose: Checks that everything is set up to write
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+void CompressStream::CheckWrite()
+{
+	// Has the read compressor already been created?
+	if(mpWriteCompressor != 0)
+	{
+		return;
+	}
+	
+	// Need to create one?
+	if(mCompressWrite)
+	{
+		mpWriteCompressor = new Compress<true>();
+		// And make sure there's a buffer
+		CheckBuffer();
+	}
+	else
+	{
+		// Not decompressing. Should be passing through data?
+		if(!mPassThroughWhenNotCompressed)
+		{
+			THROW_EXCEPTION(CompressException, CompressStreamWriteSupportNotRequested)
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CompressStream::CheckBuffer()
+//		Purpose: Allocates the buffer for (de)compression operations
+//		Created: 28/5/04
+//
+// --------------------------------------------------------------------------
+void CompressStream::CheckBuffer()
+{
+	// Already done
+	if(mpBuffer != 0)
+	{
+		return;
+	}
+	
+	// Allocate the buffer -- which may actually be two buffers in one
+	// The temporary use buffer is first (used by write only, so only present if writing)
+	// and the read buffer follows.
+	int size = BUFFER_SIZE;
+	if(mDecompressRead && mCompressWrite)
+	{
+		size *= 2;
+	}
+	TRACE1("Allocating CompressStream buffer, size %d\n", size);
+	mpBuffer = ::malloc(size);
+	if(mpBuffer == 0)
+	{
+		throw std::bad_alloc();
+	}
+}
+
+
diff --git a/lib/compress/CompressStream.h b/lib/compress/CompressStream.h
new file mode 100644
index 00000000..7959e3dc
--- /dev/null
+++ b/lib/compress/CompressStream.h
@@ -0,0 +1,62 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CompressStream.h
+//		Purpose: Compressing stream
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef COMPRESSSTREAM__H
+#define COMPRESSSTREAM__H
+
+#include "IOStream.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    CompressStream
+//		Purpose: Compressing stream
+//		Created: 27/5/04
+//
+// --------------------------------------------------------------------------
+class CompressStream : public IOStream
+{
+public:
+	CompressStream(IOStream *pStream, bool TakeOwnership,
+		bool DecompressRead, bool CompressWrite, bool PassThroughWhenNotCompressed = false);
+	~CompressStream();
+private:
+	// No copying (have implementations which exception)
+	CompressStream(const CompressStream &);
+	CompressStream &operator=(const CompressStream &);
+public:
+
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual void WriteAllBuffered();
+	virtual void Close();
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+protected:
+	void CheckRead();
+	void CheckWrite();
+	void CheckBuffer();
+	void WriteCompressedData(bool SyncFlush = false);
+
+private:
+	IOStream *mpStream;
+	bool mHaveOwnership;
+	bool mDecompressRead;
+	bool mCompressWrite;
+	bool mPassThroughWhenNotCompressed;
+	// Avoid having to include Compress.h
+	void *mpReadCompressor;
+	void *mpWriteCompressor;
+	void *mpBuffer;
+	bool mIsClosed;
+};
+
+#endif // COMPRESSSTREAM__H
+
diff --git a/lib/compress/Makefile.extra b/lib/compress/Makefile.extra
new file mode 100755
index 00000000..7ef9930c
--- /dev/null
+++ b/lib/compress/Makefile.extra
@@ -0,0 +1,7 @@
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_CompressException.h autogen_CompressException.cpp:	$(MAKEEXCEPTION) CompressException.txt
+	perl $(MAKEEXCEPTION) CompressException.txt
+
diff --git a/lib/crypto/CipherAES.cpp b/lib/crypto/CipherAES.cpp
new file mode 100644
index 00000000..b4de6048
--- /dev/null
+++ b/lib/crypto/CipherAES.cpp
@@ -0,0 +1,163 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherAES.cpp
+//		Purpose: AES cipher description
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+// Only available in new versions of openssl
+#ifndef PLATFORM_OLD_OPENSSL
+
+#include <openssl/evp.h>
+
+#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE
+
+#include "CipherAES.h"
+#include "CipherException.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherAES::CipherAES(CipherDescription::CipherMode, const void *, unsigned int, const void *)
+//		Purpose: Constructor -- note key material and IV are not copied. KeyLength in bytes.
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+CipherAES::CipherAES(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector)
+	: CipherDescription(),
+	  mMode(Mode),
+	  mpKey(pKey),
+	  mKeyLength(KeyLength),
+	  mpInitialisationVector(pInitialisationVector)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherAES::CipherAES(const CipherAES &)
+//		Purpose: Copy constructor
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+CipherAES::CipherAES(const CipherAES &rToCopy)
+	: CipherDescription(rToCopy),
+	  mMode(rToCopy.mMode),
+	  mpKey(rToCopy.mpKey),
+	  mKeyLength(rToCopy.mKeyLength),
+	  mpInitialisationVector(rToCopy.mpInitialisationVector)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ~CipherAES::CipherAES()
+//		Purpose: Destructor
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+CipherAES::~CipherAES()
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherAES::operator=(const CipherAES &)
+//		Purpose: Assignment operator
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+CipherAES &CipherAES::operator=(const CipherAES &rToCopy)
+{
+	CipherDescription::operator=(rToCopy);
+
+	mMode = rToCopy.mMode;
+	mpKey = rToCopy.mpKey;
+	mKeyLength = rToCopy.mKeyLength;
+	mpInitialisationVector = rToCopy.mpInitialisationVector;
+
+	return *this;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherAES::GetCipher()
+//		Purpose: Returns cipher object
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+const EVP_CIPHER *CipherAES::GetCipher() const
+{
+	switch(mMode)
+	{
+	case CipherDescription::Mode_ECB:
+		switch(mKeyLength)
+		{
+			case (128/8): return EVP_aes_128_ecb(); break;
+			case (192/8): return EVP_aes_192_ecb(); break;
+			case (256/8): return EVP_aes_256_ecb(); break;
+		default:
+			THROW_EXCEPTION(CipherException, EVPBadKeyLength)
+			break;
+		}
+		break;
+	
+	case CipherDescription::Mode_CBC:
+		switch(mKeyLength)
+		{
+			case (128/8): return EVP_aes_128_cbc(); break;
+			case (192/8): return EVP_aes_192_cbc(); break;
+			case (256/8): return EVP_aes_256_cbc(); break;
+		default:
+			THROW_EXCEPTION(CipherException, EVPBadKeyLength)
+			break;
+		}
+		break;
+	
+	default:
+		break;
+	}
+
+	// Unknown!
+	THROW_EXCEPTION(CipherException, UnknownCipherMode)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherAES::SetupParameters(EVP_CIPHER_CTX *)
+//		Purpose: Set up various parameters for cipher
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+void CipherAES::SetupParameters(EVP_CIPHER_CTX *pCipherContext) const
+{
+	ASSERT(pCipherContext != 0);
+	
+	// Set key (key length is implied)
+	if(EVP_CipherInit_ex(pCipherContext, NULL, NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1)
+	{
+		THROW_EXCEPTION(CipherException, EVPInitFailure)
+	}
+	
+}
+
+
+
+#endif // n PLATFORM_OLD_OPENSSL
+
diff --git a/lib/crypto/CipherAES.h b/lib/crypto/CipherAES.h
new file mode 100644
index 00000000..51fb146a
--- /dev/null
+++ b/lib/crypto/CipherAES.h
@@ -0,0 +1,50 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherAES.h
+//		Purpose: AES cipher description
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef CIPHERAES__H
+#define CIPHERAES__H
+
+// Only available in new versions of openssl
+#ifndef PLATFORM_OLD_OPENSSL
+
+#include "CipherDescription.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    CipherAES
+//		Purpose: AES cipher description
+//		Created: 27/4/04
+//
+// --------------------------------------------------------------------------
+class CipherAES : public CipherDescription
+{
+public:
+	CipherAES(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector = 0);
+	CipherAES(const CipherAES &rToCopy);
+	virtual ~CipherAES();
+	CipherAES &operator=(const CipherAES &rToCopy);
+	
+	// Return OpenSSL cipher object
+	virtual const EVP_CIPHER *GetCipher() const;
+	
+	// Setup any other parameters
+	virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const;
+
+private:
+	CipherDescription::CipherMode mMode;
+	const void *mpKey;
+	unsigned int mKeyLength;
+	const void *mpInitialisationVector;
+};
+
+#endif // n PLATFORM_OLD_OPENSSL
+
+#endif // CIPHERAES__H
+
diff --git a/lib/crypto/CipherBlowfish.cpp b/lib/crypto/CipherBlowfish.cpp
new file mode 100755
index 00000000..e27e3b0a
--- /dev/null
+++ b/lib/crypto/CipherBlowfish.cpp
@@ -0,0 +1,220 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherBlowfish.cpp
+//		Purpose: Blowfish cipher description
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <openssl/evp.h>
+
+#ifdef PLATFORM_OLD_OPENSSL
+	#include <string.h>
+	#include <strings.h>
+#endif
+
+#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE
+
+#include "CipherBlowfish.h"
+#include "CipherException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherBlowfish::CipherBlowfish(CipherDescription::CipherMode, const void *, unsigned int, const void *)
+//		Purpose: Constructor -- note key material and IV are not copied. KeyLength in bytes.
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherBlowfish::CipherBlowfish(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector)
+	: CipherDescription(),
+	  mMode(Mode)
+#ifndef PLATFORM_OLD_OPENSSL
+	, mpKey(pKey),
+	  mKeyLength(KeyLength),
+	  mpInitialisationVector(pInitialisationVector)
+{
+}
+#else
+{
+	mKey.assign((const char *)pKey, KeyLength);
+	if(pInitialisationVector == 0)
+	{
+		bzero(mInitialisationVector, sizeof(mInitialisationVector));
+	}
+	else
+	{
+		::memcpy(mInitialisationVector, pInitialisationVector, sizeof(mInitialisationVector));
+	}
+}
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherBlowfish::CipherBlowfish(const CipherBlowfish &)
+//		Purpose: Copy constructor
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherBlowfish::CipherBlowfish(const CipherBlowfish &rToCopy)
+	: CipherDescription(rToCopy),
+	  mMode(rToCopy.mMode),
+#ifndef PLATFORM_OLD_OPENSSL
+	  mpKey(rToCopy.mpKey),
+	  mKeyLength(rToCopy.mKeyLength),
+	  mpInitialisationVector(rToCopy.mpInitialisationVector)
+{
+}
+#else
+	  mKey(rToCopy.mKey)
+{
+	::memcpy(mInitialisationVector, rToCopy.mInitialisationVector, sizeof(mInitialisationVector));
+}
+#endif
+
+
+#ifdef PLATFORM_OLD_OPENSSL
+// Hack functions to support old OpenSSL API
+CipherDescription *CipherBlowfish::Clone() const
+{
+	return new CipherBlowfish(*this);
+}
+void CipherBlowfish::SetIV(const void *pIV)
+{
+	if(pIV == 0)
+	{
+		bzero(mInitialisationVector, sizeof(mInitialisationVector));
+	}
+	else
+	{
+		::memcpy(mInitialisationVector, pIV, sizeof(mInitialisationVector));
+	}
+}
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ~CipherBlowfish::CipherBlowfish()
+//		Purpose: Destructor
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherBlowfish::~CipherBlowfish()
+{
+#ifdef PLATFORM_OLD_OPENSSL
+	// Zero copy of key
+	for(unsigned int l = 0; l < mKey.size(); ++l)
+	{
+		mKey[l] = '\0';
+	}
+#endif
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherBlowfish::operator=(const CipherBlowfish &)
+//		Purpose: Assignment operator
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherBlowfish &CipherBlowfish::operator=(const CipherBlowfish &rToCopy)
+{
+	CipherDescription::operator=(rToCopy);
+
+	mMode = rToCopy.mMode;
+#ifndef PLATFORM_OLD_OPENSSL
+	mpKey = rToCopy.mpKey;
+	mKeyLength = rToCopy.mKeyLength;
+	mpInitialisationVector = rToCopy.mpInitialisationVector;
+#else
+	mKey = rToCopy.mKey;
+	::memcpy(mInitialisationVector, rToCopy.mInitialisationVector, sizeof(mInitialisationVector));
+#endif
+
+	return *this;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherBlowfish::GetCipher()
+//		Purpose: Returns cipher object
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+const EVP_CIPHER *CipherBlowfish::GetCipher() const
+{
+	switch(mMode)
+	{
+	case CipherDescription::Mode_ECB:
+		return EVP_bf_ecb();
+		break;
+	
+	case CipherDescription::Mode_CBC:
+		return EVP_bf_cbc();
+		break;
+	
+	case CipherDescription::Mode_CFB:
+		return EVP_bf_cfb();
+		break;
+	
+	case CipherDescription::Mode_OFB:
+		return EVP_bf_ofb();
+		break;
+	
+	default:
+		break;
+	}
+
+	// Unknown!
+	THROW_EXCEPTION(CipherException, UnknownCipherMode)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherBlowfish::SetupParameters(EVP_CIPHER_CTX *)
+//		Purpose: Set up various parameters for cipher
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void CipherBlowfish::SetupParameters(EVP_CIPHER_CTX *pCipherContext) const
+{
+	ASSERT(pCipherContext != 0);
+	
+	// Set key length
+#ifndef PLATFORM_OLD_OPENSSL
+	if(EVP_CIPHER_CTX_set_key_length(pCipherContext, mKeyLength) != 1)
+#else
+	if(EVP_CIPHER_CTX_set_key_length(pCipherContext, mKey.size()) != 1)
+#endif
+	{
+		THROW_EXCEPTION(CipherException, EVPBadKeyLength)
+	}
+	// Set key
+#ifndef PLATFORM_OLD_OPENSSL
+	if(EVP_CipherInit_ex(pCipherContext, NULL, NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1)
+#else
+	if(EVP_CipherInit(pCipherContext, NULL, (unsigned char*)mKey.c_str(), (unsigned char*)mInitialisationVector, -1) != 1)
+#endif
+	{
+		THROW_EXCEPTION(CipherException, EVPInitFailure)
+	}
+	
+}
+
+
+
diff --git a/lib/crypto/CipherBlowfish.h b/lib/crypto/CipherBlowfish.h
new file mode 100755
index 00000000..92db1d85
--- /dev/null
+++ b/lib/crypto/CipherBlowfish.h
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherBlowfish.h
+//		Purpose: Blowfish cipher description
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef CIPHERBLOWFISH__H
+#define CIPHERBLOWFISH__H
+
+#ifdef PLATFORM_OLD_OPENSSL
+	#include <string>
+#endif
+
+#include "CipherDescription.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    CipherBlowfish
+//		Purpose: Description of Blowfish cipher parameters -- note that copies are not made of key material and IV, careful with object lifetimes.
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+class CipherBlowfish : public CipherDescription
+{
+public:
+	CipherBlowfish(CipherDescription::CipherMode Mode, const void *pKey, unsigned int KeyLength, const void *pInitialisationVector = 0);
+	CipherBlowfish(const CipherBlowfish &rToCopy);
+	virtual ~CipherBlowfish();
+	CipherBlowfish &operator=(const CipherBlowfish &rToCopy);
+	
+	// Return OpenSSL cipher object
+	virtual const EVP_CIPHER *GetCipher() const;
+	
+	// Setup any other parameters
+	virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const;
+
+#ifdef PLATFORM_OLD_OPENSSL
+	CipherDescription *Clone() const;
+	void SetIV(const void *pIV);
+#endif
+
+private:
+	CipherDescription::CipherMode mMode;
+#ifndef PLATFORM_OLD_OPENSSL
+	const void *mpKey;
+	unsigned int mKeyLength;
+	const void *mpInitialisationVector;
+#else
+	std::string mKey;
+	uint8_t mInitialisationVector[8];
+#endif
+};
+
+
+#endif // CIPHERBLOWFISH__H
+
diff --git a/lib/crypto/CipherContext.cpp b/lib/crypto/CipherContext.cpp
new file mode 100755
index 00000000..42707497
--- /dev/null
+++ b/lib/crypto/CipherContext.cpp
@@ -0,0 +1,615 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherContext.cpp
+//		Purpose: Context for symmetric encryption / descryption
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE
+#include "CipherContext.h"
+#include "CipherDescription.h"
+#include "CipherException.h"
+#include "Random.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::CipherContext()
+//		Purpose: Constructor
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherContext::CipherContext()
+	: mInitialised(false),
+	  mWithinTransform(false),
+	  mPaddingOn(true)
+#ifdef PLATFORM_OLD_OPENSSL
+	, mFunction(Decrypt),
+	  mpDescription(0)
+#endif
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::~CipherContext()
+//		Purpose: Destructor
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherContext::~CipherContext()
+{
+	if(mInitialised)
+	{
+		// Clean up
+		EVP_CIPHER_CTX_cleanup(&ctx);
+		mInitialised = false;
+	}
+#ifdef PLATFORM_OLD_OPENSSL
+	if(mpDescription != 0)
+	{
+		delete mpDescription;
+		mpDescription = 0;
+	}
+#endif
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::Init(CipherContext::CipherFunction, const CipherDescription &)
+//		Purpose: Initialises the context, specifying the direction for the encryption, and a
+//				 description of the cipher to use, it's keys, etc
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDescription &rDescription)
+{
+	// Check for bad usage
+	if(mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, AlreadyInitialised)
+	}
+	if(Function != Decrypt && Function != Encrypt)
+	{
+		THROW_EXCEPTION(CipherException, BadArguments)
+	}
+	
+	// Initialise the cipher
+#ifndef PLATFORM_OLD_OPENSSL
+	EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does
+
+	if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, Function) != 1)
+#else
+	// Store function for later
+	mFunction = Function;
+
+	// Use old version of init call
+	if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL, Function) != 1)
+#endif
+	{
+		THROW_EXCEPTION(CipherException, EVPInitFailure)
+	}
+	
+	try
+	{
+#ifndef PLATFORM_OLD_OPENSSL
+		// Let the description set up everything else
+		rDescription.SetupParameters(&ctx);
+#else
+		// With the old version, a copy needs to be taken first.
+		mpDescription = rDescription.Clone();
+		// Mark it as not a leak, otherwise static cipher contexts cause supriously memory leaks to be reported
+		MEMLEAKFINDER_NOT_A_LEAK(mpDescription);
+		mpDescription->SetupParameters(&ctx);
+#endif
+	}
+	catch(...)
+	{
+		EVP_CIPHER_CTX_cleanup(&ctx);
+		throw;
+	}
+
+	// mark as initialised
+	mInitialised = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::Reset()
+//		Purpose: Reset the context, so it can be initialised again with a different key
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void CipherContext::Reset()
+{
+	if(mInitialised)
+	{
+		// Clean up
+		EVP_CIPHER_CTX_cleanup(&ctx);
+		mInitialised = false;
+	}
+#ifdef PLATFORM_OLD_OPENSSL
+	if(mpDescription != 0)
+	{
+		delete mpDescription;
+		mpDescription = 0;
+	}
+#endif
+	mWithinTransform = false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::Begin()
+//		Purpose: Begin a transformation
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+void CipherContext::Begin()
+{
+	if(!mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, NotInitialised)
+	}
+
+	// Warn if in a transformation (not an error, because a context might not have been finalised if an exception occured)
+	if(mWithinTransform)
+	{
+		TRACE0("CipherContext::Begin called when context flagged as within a transform\n");
+	}
+
+	// Initialise the cipher context again
+	if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1)
+	{
+		THROW_EXCEPTION(CipherException, EVPInitFailure)
+	}
+	
+	// Mark as being within a transform
+	mWithinTransform = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::Transform(void *, int, const void *, int)
+//		Purpose: Transforms the data in the in buffer to the out buffer. If pInBuffer == 0 && InLength == 0
+//				 then Final() is called instead.
+//				 Returns the number of bytes placed in the out buffer.
+//				 There must be room in the out buffer for all the data in the in buffer.
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+int CipherContext::Transform(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength)
+{
+	if(!mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, NotInitialised)
+	}
+	
+	if(!mWithinTransform)
+	{
+		THROW_EXCEPTION(CipherException, BeginNotCalled)
+	}
+
+	// Check parameters
+	if(pOutBuffer == 0 || OutLength < 0 || (pInBuffer != 0 && InLength <= 0) || (pInBuffer == 0 && InLength != 0))
+	{
+		THROW_EXCEPTION(CipherException, BadArguments)
+	}
+	
+	// Is this the final call?
+	if(pInBuffer == 0)
+	{
+		return Final(pOutBuffer, OutLength);
+	}
+	
+	// Check output buffer size
+	if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx)))
+	{
+		THROW_EXCEPTION(CipherException, OutputBufferTooSmall);
+	}
+	
+	// Do the transform
+	int outLength = OutLength;
+	if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1)
+	{
+		THROW_EXCEPTION(CipherException, EVPUpdateFailure)
+	}
+
+	return outLength;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::Final(void *, int)
+//		Purpose: Transforms the data as per Transform, and returns the final data in the out buffer.
+//				 Returns the number of bytes written in the out buffer.
+//				 Two main causes of exceptions being thrown: 1) Data is corrupt, and so the end isn't
+//				 padded properly. 2) Padding is off, and the data to be encrypted isn't a multiple
+//				 of a block long.
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+int CipherContext::Final(void *pOutBuffer, int OutLength)
+{
+	if(!mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, NotInitialised)
+	}
+
+	if(!mWithinTransform)
+	{
+		THROW_EXCEPTION(CipherException, BeginNotCalled)
+	}
+
+	// Check parameters
+	if(pOutBuffer == 0 || OutLength < 0)
+	{
+		THROW_EXCEPTION(CipherException, BadArguments)
+	}
+
+	// Check output buffer size
+	if(OutLength < (2 * EVP_CIPHER_CTX_block_size(&ctx)))
+	{
+		THROW_EXCEPTION(CipherException, OutputBufferTooSmall);
+	}
+	
+	// Do the transform
+	int outLength = OutLength;
+#ifndef PLATFORM_OLD_OPENSSL
+	if(EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1)
+	{
+		THROW_EXCEPTION(CipherException, EVPFinalFailure)
+	}
+#else
+	OldOpenSSLFinal((unsigned char*)pOutBuffer, outLength);
+#endif
+	
+	mWithinTransform = false;
+
+	return outLength;
+}
+
+
+#ifdef PLATFORM_OLD_OPENSSL
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::OldOpenSSLFinal(unsigned char *, int &)
+//		Purpose: The old version of OpenSSL needs more work doing to finalise the cipher,
+//				 and reset it so that it's ready for another go.
+//		Created: 27/3/04
+//
+// --------------------------------------------------------------------------
+void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut)
+{
+	// Old version needs to use a different form, and then set up the cipher again for next time around
+	int outLength = rOutLengthOut;
+	// Have to emulate padding off...
+	int blockSize = EVP_CIPHER_CTX_block_size(&ctx);
+	if(mPaddingOn)
+	{
+		// Just use normal final call
+		if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1)
+		{
+			THROW_EXCEPTION(CipherException, EVPFinalFailure)
+		}
+	}
+	else
+	{
+		// Padding is off. OpenSSL < 0.9.7 doesn't support this, so it has to be
+		// bodged in there. Which isn't nice.
+		if(mFunction == Decrypt)
+		{
+			// NASTY -- fiddling around with internals like this is bad.
+			// But only way to get this working on old versions of OpenSSL.
+			if(!EVP_EncryptUpdate(&ctx,Buffer,&outLength,ctx.buf,0)
+				|| outLength != blockSize)
+			{
+				THROW_EXCEPTION(CipherException, EVPFinalFailure)
+			}
+			// Clean up
+			EVP_CIPHER_CTX_cleanup(&ctx);
+		}
+		else
+		{
+			// Check that the length is correct
+			if((ctx.buf_len % blockSize) != 0)
+			{
+				THROW_EXCEPTION(CipherException, EVPFinalFailure)
+			}
+			// For encryption, assume that the last block entirely is
+			// padding, and remove it.
+			char temp[1024];
+			outLength = sizeof(temp);
+			if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1)
+			{
+				THROW_EXCEPTION(CipherException, EVPFinalFailure)
+			}
+			// Remove last block, assuming it's full of padded bytes only.
+			outLength -= blockSize;
+			// Copy anything to the main buffer
+			// (can't just use main buffer, because it might overwrite something important)
+			if(outLength > 0)
+			{
+				::memcpy(Buffer, temp, outLength);
+			}
+		}
+	}
+	// Reinitialise the cipher for the next time around
+	if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, mFunction) != 1)
+	{
+		THROW_EXCEPTION(CipherException, EVPInitFailure)
+	}
+	mpDescription->SetupParameters(&ctx);
+
+	// Update length for caller
+	rOutLengthOut = outLength;
+}
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::InSizeForOutBufferSize(int)
+//		Purpose: Returns the maximum amount of data that can be sent in
+//				 given a output buffer size.
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+int CipherContext::InSizeForOutBufferSize(int OutLength)
+{
+	if(!mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, NotInitialised)
+	}
+
+	// Strictly speaking, the *2 is unnecessary. However... 
+	// Final() is paranoid, and requires two input blocks of space to work.
+	return OutLength - (EVP_CIPHER_CTX_block_size(&ctx) * 2);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::MaxOutSizeForInBufferSize(int)
+//		Purpose: Returns the maximum output size for an input of a given length.
+//				 Will tend to over estimate, as it needs to allow space for Final() to be called.
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+int CipherContext::MaxOutSizeForInBufferSize(int InLength)
+{
+	if(!mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, NotInitialised)
+	}
+
+	// Final() is paranoid, and requires two input blocks of space to work, and so we need to add
+	// three blocks on to be absolutely sure.
+	return InLength + (EVP_CIPHER_CTX_block_size(&ctx) * 3);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::TransformBlock(void *, int, const void *, int)
+//		Purpose: Transform one block to another all in one go, no Final required.
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength)
+{
+	if(!mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, NotInitialised)
+	}
+	
+	// Warn if in a transformation
+	if(mWithinTransform)
+	{
+		TRACE0("CipherContext::TransformBlock called when context flagged as within a transform\n");
+	}
+
+	// Check output buffer size
+	if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx)))
+	{
+		// Check if padding is off, in which case the buffer can be smaller
+		if(!mPaddingOn && OutLength <= InLength)
+		{
+			// This is OK.
+		}
+		else
+		{
+			THROW_EXCEPTION(CipherException, OutputBufferTooSmall);
+		}
+	}
+	
+	// Initialise the cipher context again
+	if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1)
+	{
+		THROW_EXCEPTION(CipherException, EVPInitFailure)
+	}
+	
+	// Do the entire block
+	int outLength = 0;
+	try
+	{
+		// Update
+		outLength = OutLength;
+		if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1)
+		{
+			THROW_EXCEPTION(CipherException, EVPUpdateFailure)
+		}
+		// Finalise
+		int outLength2 = OutLength - outLength;
+#ifndef PLATFORM_OLD_OPENSSL
+		if(EVP_CipherFinal_ex(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1)
+		{
+			THROW_EXCEPTION(CipherException, EVPFinalFailure)
+		}
+#else
+		OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2);
+#endif
+		outLength += outLength2;
+	}
+	catch(...)
+	{
+		// Finalise the context, so definately ready for the next caller
+		int outs = OutLength;
+#ifndef PLATFORM_OLD_OPENSSL
+		EVP_CipherFinal_ex(&ctx, (unsigned char*)pOutBuffer, &outs);
+#else
+		OldOpenSSLFinal((unsigned char*)pOutBuffer, outs);
+#endif
+		throw;
+	}
+
+	return outLength;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::GetIVLength()
+//		Purpose: Returns the size of the IV for this context
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+int CipherContext::GetIVLength()
+{
+	if(!mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, NotInitialised)
+	}
+	
+	return EVP_CIPHER_CTX_iv_length(&ctx);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::SetIV(const void *)
+//		Purpose: Sets the IV for this context (must be correctly sized, use GetIVLength)
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+void CipherContext::SetIV(const void *pIV)
+{
+	if(!mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, NotInitialised)
+	}
+	
+	// Warn if in a transformation
+	if(mWithinTransform)
+	{
+		TRACE0("CipherContext::SetIV called when context flagged as within a transform\n");
+	}
+
+	// Set IV
+	if(EVP_CipherInit(&ctx, NULL, NULL, (unsigned char *)pIV, -1) != 1)
+	{
+		THROW_EXCEPTION(CipherException, EVPInitFailure)
+	}
+
+#ifdef PLATFORM_OLD_OPENSSL
+	// Update description
+	if(mpDescription != 0)
+	{
+		mpDescription->SetIV(pIV);
+	}
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::SetRandomIV(int &)
+//		Purpose: Set a random IV for the context, and return a pointer to the IV used,
+//				 and the length of this IV in the rLengthOut arg.
+//		Created: 3/12/03
+//
+// --------------------------------------------------------------------------
+const void *CipherContext::SetRandomIV(int &rLengthOut)
+{
+	if(!mInitialised)
+	{
+		THROW_EXCEPTION(CipherException, NotInitialised)
+	}
+	
+	// Warn if in a transformation
+	if(mWithinTransform)
+	{
+		TRACE0("CipherContext::SetRandomIV called when context flagged as within a transform\n");
+	}
+
+	// Get length of IV
+	unsigned int ivLen = EVP_CIPHER_CTX_iv_length(&ctx);
+	if(ivLen > sizeof(mGeneratedIV))
+	{
+		THROW_EXCEPTION(CipherException, IVSizeImplementationLimitExceeded)
+	}
+	
+	// Generate some random data
+	Random::Generate(mGeneratedIV, ivLen);
+
+	// Set IV
+	if(EVP_CipherInit(&ctx, NULL, NULL, mGeneratedIV, -1) != 1)
+	{
+		THROW_EXCEPTION(CipherException, EVPInitFailure)
+	}	
+
+#ifdef PLATFORM_OLD_OPENSSL
+	// Update description
+	if(mpDescription != 0)
+	{
+		mpDescription->SetIV(mGeneratedIV);
+	}
+#endif
+
+	// Return the IV and it's length
+	rLengthOut = ivLen;
+	return mGeneratedIV;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherContext::UsePadding(bool)
+//		Purpose: Set whether or not the context uses padding. 
+//		Created: 12/12/03
+//
+// --------------------------------------------------------------------------
+void CipherContext::UsePadding(bool Padding)
+{
+#ifndef PLATFORM_OLD_OPENSSL
+	if(EVP_CIPHER_CTX_set_padding(&ctx, Padding) != 1)
+	{
+		THROW_EXCEPTION(CipherException, EVPSetPaddingFailure)
+	}
+#endif
+	mPaddingOn = Padding;
+}
+
+
+
diff --git a/lib/crypto/CipherContext.h b/lib/crypto/CipherContext.h
new file mode 100755
index 00000000..f9dcd022
--- /dev/null
+++ b/lib/crypto/CipherContext.h
@@ -0,0 +1,83 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherContext.h
+//		Purpose: Context for symmetric encryption / descryption
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef CIPHERCONTEXT__H
+#define CIPHERCONTEXT__H
+
+#ifdef BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_FALSE
+	always include CipherContext.h first in any .cpp file
+#endif
+#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE
+#include <openssl/evp.h>
+class CipherDescription;
+
+#define CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH		32
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    CipherContext
+//		Purpose: Context for symmetric encryption / descryption
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+class CipherContext
+{
+public:
+	CipherContext();
+	~CipherContext();
+private:
+	CipherContext(const CipherContext &);	// no copying
+	CipherContext &operator=(const CipherContext &);	// no assignment
+public:
+
+	typedef enum
+	{
+		Decrypt = 0,
+		Encrypt = 1
+	} CipherFunction;
+
+	void Init(CipherContext::CipherFunction Function, const CipherDescription &rDescription);
+	void Reset();
+	
+	void Begin();
+	int Transform(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength);
+	int Final(void *pOutBuffer, int OutLength);
+	int InSizeForOutBufferSize(int OutLength);
+	int MaxOutSizeForInBufferSize(int InLength);
+	
+	int TransformBlock(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength);
+
+	bool IsInitialised() {return mInitialised;}
+	
+	int GetIVLength();
+	void SetIV(const void *pIV);
+	const void *SetRandomIV(int &rLengthOut);
+	
+	void UsePadding(bool Padding = true);
+
+#ifdef PLATFORM_OLD_OPENSSL
+	void OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut);
+#endif
+	
+private:
+	EVP_CIPHER_CTX ctx;
+	bool mInitialised;
+	bool mWithinTransform;
+	bool mPaddingOn;
+	uint8_t mGeneratedIV[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH];
+#ifdef PLATFORM_OLD_OPENSSL
+	CipherFunction mFunction;
+	CipherDescription *mpDescription;
+#endif
+};
+
+
+#endif // CIPHERCONTEXT__H
+
diff --git a/lib/crypto/CipherDescription.cpp b/lib/crypto/CipherDescription.cpp
new file mode 100755
index 00000000..f0ba6ec8
--- /dev/null
+++ b/lib/crypto/CipherDescription.cpp
@@ -0,0 +1,73 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherDescription.cpp
+//		Purpose: Pure virtual base class for describing ciphers
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <openssl/evp.h>
+
+#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE
+
+#include "CipherDescription.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherDescription::CipherDescription()
+//		Purpose: Constructor
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherDescription::CipherDescription()
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherDescription::CipherDescription(const CipherDescription &)
+//		Purpose: Copy constructor
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherDescription::CipherDescription(const CipherDescription &rToCopy)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ~CipherDescription::CipherDescription()
+//		Purpose: Destructor
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherDescription::~CipherDescription()
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    CipherDescription::operator=(const CipherDescription &)
+//		Purpose: Assignment operator
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+CipherDescription &CipherDescription::operator=(const CipherDescription &rToCopy)
+{
+	return *this;
+}
+
+
diff --git a/lib/crypto/CipherDescription.h b/lib/crypto/CipherDescription.h
new file mode 100755
index 00000000..2977f7da
--- /dev/null
+++ b/lib/crypto/CipherDescription.h
@@ -0,0 +1,59 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherDescription.h
+//		Purpose: Pure virtual base class for describing ciphers
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef CIPHERDESCRIPTION__H
+#define CIPHERDESCRIPTION__H
+
+#ifndef BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_TRUE
+	#define BOX_LIB_CRYPTO_OPENSSL_HEADERS_INCLUDED_FALSE
+	class EVP_CIPHER;
+	class EVP_CIPHER_CTX;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    CipherDescription
+//		Purpose: Describes a cipher
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+class CipherDescription
+{
+public:
+	CipherDescription();
+	CipherDescription(const CipherDescription &rToCopy);
+	virtual ~CipherDescription();
+	CipherDescription &operator=(const CipherDescription &rToCopy);
+	
+	// Return OpenSSL cipher object
+	virtual const EVP_CIPHER *GetCipher() const = 0;
+	
+	// Setup any other parameters
+	virtual void SetupParameters(EVP_CIPHER_CTX *pCipherContext) const = 0;
+	
+	// Mode parameter for cipher -- used in derived classes
+	typedef enum
+	{
+		Mode_ECB = 0,
+		Mode_CBC = 1,
+		Mode_CFB = 2,
+		Mode_OFB = 3
+	} CipherMode;
+
+#ifdef PLATFORM_OLD_OPENSSL
+	// For the old version of OpenSSL, we need to be able to store cipher descriptions.
+	virtual CipherDescription *Clone() const = 0;
+	// And to be able to store new IVs
+	virtual void SetIV(const void *pIV) = 0;
+#endif
+};
+
+#endif // CIPHERDESCRIPTION__H
+
diff --git a/lib/crypto/CipherException.h b/lib/crypto/CipherException.h
new file mode 100755
index 00000000..b0c9f8cd
--- /dev/null
+++ b/lib/crypto/CipherException.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    CipherException.h
+//		Purpose: Exception
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef CIPHEREXCEPTION__H
+#define CIPHEREXCEPTION__H
+
+// Compatibility
+#include "autogen_CipherException.h"
+
+#endif // CIPHEREXCEPTION__H
+
diff --git a/lib/crypto/CipherException.txt b/lib/crypto/CipherException.txt
new file mode 100644
index 00000000..abdbac87
--- /dev/null
+++ b/lib/crypto/CipherException.txt
@@ -0,0 +1,18 @@
+EXCEPTION Cipher 5
+
+Internal							0
+UnknownCipherMode					1
+AlreadyInitialised					2
+BadArguments						3
+EVPInitFailure						4
+EVPUpdateFailure					5
+EVPFinalFailure						6
+NotInitialised						7
+OutputBufferTooSmall				8
+EVPBadKeyLength						9
+BeginNotCalled						10
+IVSizeImplementationLimitExceeded	11
+PseudoRandNotAvailable				12
+EVPSetPaddingFailure				13
+RandomInitFailed					14	Failed to read from random device
+LengthRequestedTooLongForRandomHex	15
diff --git a/lib/crypto/MD5Digest.cpp b/lib/crypto/MD5Digest.cpp
new file mode 100755
index 00000000..58cc90ee
--- /dev/null
+++ b/lib/crypto/MD5Digest.cpp
@@ -0,0 +1,82 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    MD5Digest.cpp
+//		Purpose: Simple interface for creating MD5 digests
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+
+
+#include "Box.h"
+
+#include "MD5Digest.h"
+
+#include "MemLeakFindOn.h"
+
+
+MD5Digest::MD5Digest()
+{
+	MD5_Init(&md5);
+	for(unsigned int l = 0; l < sizeof(mDigest); ++l)
+	{
+		mDigest[l] = 0;
+	}
+}
+
+MD5Digest::~MD5Digest()
+{
+}
+
+void MD5Digest::Add(const std::string &rString)
+{
+	MD5_Update(&md5, rString.c_str(), rString.size());
+}
+
+void MD5Digest::Add(const void *pData, int Length)
+{
+	MD5_Update(&md5, pData, Length);
+}
+
+void MD5Digest::Finish()
+{
+	MD5_Final(mDigest, &md5);
+}
+
+std::string MD5Digest::DigestAsString()
+{
+	std::string r;
+
+	static const char *hex = "0123456789abcdef";
+
+	for(unsigned int l = 0; l < sizeof(mDigest); ++l)
+	{
+		r += hex[(mDigest[l] & 0xf0) >> 4];
+		r += hex[(mDigest[l] & 0x0f)];
+	}
+
+	return r;
+}
+
+int MD5Digest::CopyDigestTo(uint8_t *to)
+{
+	for(int l = 0; l < MD5_DIGEST_LENGTH; ++l)
+	{
+		to[l] = mDigest[l];
+	}
+
+	return MD5_DIGEST_LENGTH;
+}
+
+
+bool MD5Digest::DigestMatches(uint8_t *pCompareWith) const
+{
+	for(int l = 0; l < MD5_DIGEST_LENGTH; ++l)
+	{
+		if(pCompareWith[l] != mDigest[l])
+			return false;
+	}
+
+	return true;
+}
+
diff --git a/lib/crypto/MD5Digest.h b/lib/crypto/MD5Digest.h
new file mode 100755
index 00000000..1be01ea9
--- /dev/null
+++ b/lib/crypto/MD5Digest.h
@@ -0,0 +1,57 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    MD5Digest.h
+//		Purpose: Simple interface for creating MD5 digests
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef MD5DIGEST_H
+#define MD5DIGEST_H
+
+#include <openssl/md5.h>
+#include <string>
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    MD5Digest
+//		Purpose: Simple interface for creating MD5 digests
+//		Created: 8/12/03
+//
+// --------------------------------------------------------------------------
+class MD5Digest  
+{
+public:
+	MD5Digest();
+	virtual ~MD5Digest();
+
+	void Add(const std::string &rString);
+	void Add(const void *pData, int Length);
+
+	void Finish();
+
+	std::string DigestAsString();
+	uint8_t *DigestAsData(int *pLength = 0)
+	{
+		if(pLength) *pLength = sizeof(mDigest);
+		return mDigest;
+	}
+
+	enum
+	{
+		DigestLength = MD5_DIGEST_LENGTH
+	};
+
+	int CopyDigestTo(uint8_t *to);
+
+	bool DigestMatches(uint8_t *pCompareWith) const;
+
+private:
+	MD5_CTX	md5;
+	uint8_t mDigest[MD5_DIGEST_LENGTH];
+};
+
+#endif // MD5DIGEST_H
+
diff --git a/lib/crypto/Makefile.extra b/lib/crypto/Makefile.extra
new file mode 100755
index 00000000..a7e42000
--- /dev/null
+++ b/lib/crypto/Makefile.extra
@@ -0,0 +1,7 @@
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_CipherException.cpp autogen_CipherException.h:	$(MAKEEXCEPTION) CipherException.txt
+	perl $(MAKEEXCEPTION) CipherException.txt
+
diff --git a/lib/crypto/Random.cpp b/lib/crypto/Random.cpp
new file mode 100755
index 00000000..3df5fc5e
--- /dev/null
+++ b/lib/crypto/Random.cpp
@@ -0,0 +1,127 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Random.cpp
+//		Purpose: Random numbers
+//		Created: 31/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <openssl/rand.h>
+#include <stdio.h>
+
+#include "Random.h"
+#include "CipherException.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Random::Initialise()
+//		Purpose: Add additional randomness to the standard library initialisation
+//		Created: 18/6/04
+//
+// --------------------------------------------------------------------------
+void Random::Initialise()
+{
+#ifndef PLATFORM_RANDOM_DEVICE_NONE
+	if(::RAND_load_file(PLATFORM_RANDOM_DEVICE, 1024) != 1024)
+	{
+		THROW_EXCEPTION(CipherException, RandomInitFailed)
+	}
+#else
+	::fprintf(stderr, "No random device -- additional seeding of random number generator not performed.\n");
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Random::Generate(void *, int)
+//		Purpose: Generate Length bytes of random data
+//		Created: 31/12/03
+//
+// --------------------------------------------------------------------------
+void Random::Generate(void *pOutput, int Length)
+{
+	if(RAND_pseudo_bytes((uint8_t*)pOutput, Length) == -1)
+	{
+		THROW_EXCEPTION(CipherException, PseudoRandNotAvailable)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Random::GenerateHex(int)
+//		Purpose: Generate Length bytes of hex encoded data. Note that the
+//				 maximum length requested is limited. (Returns a string
+//				 2 x Length characters long.)
+//		Created: 1/11/04
+//
+// --------------------------------------------------------------------------
+std::string Random::GenerateHex(int Length)
+{
+	uint8_t r[256];
+	if(Length > sizeof(r))
+	{
+		THROW_EXCEPTION(CipherException, LengthRequestedTooLongForRandomHex)
+	}
+	Random::Generate(r, Length);
+	
+	std::string o;
+	static const char *h = "0123456789abcdef";
+	for(int l = 0; l < Length; ++l)
+	{
+		o += h[r[l] >> 4];
+		o += h[r[l] & 0xf];
+	}
+	
+	return o;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Random::RandomInt(int)
+//		Purpose: Return a random integer between 0 and MaxValue inclusive.
+//		Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+uint32_t Random::RandomInt(uint32_t MaxValue)
+{
+	uint32_t v = 0;
+
+	// Generate a mask
+	uint32_t mask = 0;
+	while(mask < MaxValue)
+	{
+		mask = (mask << 1) | 1;
+	}
+
+	do
+	{
+		// Generate a random number
+		uint32_t r = 0;
+		Random::Generate(&r, sizeof(r));
+		
+		// Mask off relevant bits
+		v = r & mask;
+		
+		// Check that it's in the right range.
+	} while(v > MaxValue);
+	
+	// NOTE: don't do a mod, because this doesn't give a correct random distribution
+	
+	return v;
+}
+
+
+
diff --git a/lib/crypto/Random.h b/lib/crypto/Random.h
new file mode 100755
index 00000000..da3e4335
--- /dev/null
+++ b/lib/crypto/Random.h
@@ -0,0 +1,25 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Random.h
+//		Purpose: Random numbers
+//		Created: 31/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef RANDOM__H
+#define RANDOM__H
+
+#include <string>
+
+namespace Random
+{
+	void Initialise();
+	void Generate(void *pOutput, int Length);
+	std::string GenerateHex(int Length);
+	uint32_t RandomInt(uint32_t MaxValue);
+};
+
+
+#endif // RANDOM__H
+
diff --git a/lib/crypto/RollingChecksum.cpp b/lib/crypto/RollingChecksum.cpp
new file mode 100755
index 00000000..75bad7df
--- /dev/null
+++ b/lib/crypto/RollingChecksum.cpp
@@ -0,0 +1,38 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RollingChecksum.cpp
+//		Purpose: A simple rolling checksum over a block of data
+//		Created: 6/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "RollingChecksum.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RollingChecksum::RollingChecksum(const void *, unsigned int)
+//		Purpose: Constructor -- does initial computation of the checksum.
+//		Created: 6/12/03
+//
+// --------------------------------------------------------------------------
+RollingChecksum::RollingChecksum(const void *data, unsigned int Length)
+	: a(0),
+	  b(0)
+{
+	uint8_t *block = (uint8_t *)data;
+	for(unsigned int x = Length; x >= 1; --x)
+	{
+		a += (*block);
+		b += x * (*block);
+		
+		++block;
+	}
+}
+
+
+
diff --git a/lib/crypto/RollingChecksum.h b/lib/crypto/RollingChecksum.h
new file mode 100755
index 00000000..99d116b9
--- /dev/null
+++ b/lib/crypto/RollingChecksum.h
@@ -0,0 +1,96 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RollingChecksum.h
+//		Purpose: A simple rolling checksum over a block of data
+//		Created: 6/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef ROLLINGCHECKSUM__H
+#define ROLLINGCHECKSUM__H
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RollingChecksum
+//		Purpose: A simple rolling checksum over a block of data -- can move the block
+//				 "forwards" in memory and get the next checksum efficiently.
+//
+//				 Implementation of http://rsync.samba.org/tech_report/node3.html
+//		Created: 6/12/03
+//
+// --------------------------------------------------------------------------
+class RollingChecksum
+{
+public:
+	RollingChecksum(const void *data, unsigned int Length);
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    RollingChecksum::RollForward(uint8_t, uint8_t, unsigned int)
+	//		Purpose: Move the checksum forward a block, given the first byte of the current block,
+	//				 last byte of the next block (it's rolling forward to) and the length of the block.
+	//		Created: 6/12/03
+	//
+	// --------------------------------------------------------------------------
+	inline void RollForward(uint8_t StartOfThisBlock, uint8_t LastOfNextBlock, unsigned int Length)
+	{
+		// IMPLEMENTATION NOTE: Everything is implicitly mod 2^16 -- uint16_t's will overflow nicely.
+		a -= StartOfThisBlock;
+		a += LastOfNextBlock;
+		b -= Length * StartOfThisBlock;
+		b += a;
+	}
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    RollingChecksum::GetChecksum()
+	//		Purpose: Returns the checksum
+	//		Created: 6/12/03
+	//
+	// --------------------------------------------------------------------------	
+	inline uint32_t GetChecksum()
+	{
+		return ((uint32_t)a) | (((uint32_t)b) << 16);
+	}
+	
+	// Components, just in case they're handy
+	inline uint16_t GetComponent1() {return a;}
+	inline uint16_t GetComponent2() {return b;}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    RollingChecksum::GetComponentForHashing()
+	//		Purpose: Return the 16 bit component used for hashing and/or quick checks
+	//		Created: 6/12/03
+	//
+	// --------------------------------------------------------------------------
+	inline uint16_t GetComponentForHashing()
+	{
+		return b;
+	}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    RollingChecksum::ExtractHashingComponent(uint32_t)
+	//		Purpose: Static. Given a full checksum, extract the component used in the hashing table.
+	//		Created: 14/1/04
+	//
+	// --------------------------------------------------------------------------
+	static inline uint16_t ExtractHashingComponent(uint32_t Checksum)
+	{
+		return Checksum >> 16;
+	}
+	
+private:
+	uint16_t a;
+	uint16_t b;
+};
+
+#endif // ROLLINGCHECKSUM__H
+
diff --git a/lib/raidfile/Makefile.extra b/lib/raidfile/Makefile.extra
new file mode 100755
index 00000000..8d036633
--- /dev/null
+++ b/lib/raidfile/Makefile.extra
@@ -0,0 +1,7 @@
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_RaidFileException.h autogen_RaidFileException.cpp:	$(MAKEEXCEPTION) RaidFileException.txt
+	perl $(MAKEEXCEPTION) RaidFileException.txt
+
diff --git a/lib/raidfile/RaidFileController.cpp b/lib/raidfile/RaidFileController.cpp
new file mode 100755
index 00000000..89ae6791
--- /dev/null
+++ b/lib/raidfile/RaidFileController.cpp
@@ -0,0 +1,217 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileController.cpp
+//		Purpose: Controls config and daemon comms for RaidFile classes
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "RaidFileController.h"
+#include "RaidFileException.h"
+#include "Configuration.h"
+
+#include "MemLeakFindOn.h"
+
+RaidFileController RaidFileController::mController;
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::RaidFileController()
+//		Purpose: Constructor
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+RaidFileController::RaidFileController()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::~RaidFileController()
+//		Purpose: Destructor
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+RaidFileController::~RaidFileController()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::RaidFileController()
+//		Purpose: Copy constructor
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+RaidFileController::RaidFileController(const RaidFileController &rController)
+{
+	THROW_EXCEPTION(RaidFileException, Internal)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::Initialise(const char *)
+//		Purpose: Initialises the system, loading the configuration file.
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+void RaidFileController::Initialise(const char *ConfigFilename)
+{
+	static const ConfigurationVerifyKey verifykeys[] =
+	{
+		{"SetNumber",	0,	ConfigTest_Exists | ConfigTest_IsInt, 0},
+		{"BlockSize",	0,	ConfigTest_Exists | ConfigTest_IsInt, 0},
+		{"Dir0", 		0,	ConfigTest_Exists, 0},
+		{"Dir1", 		0,	ConfigTest_Exists, 0},
+		{"Dir2", 		0,	ConfigTest_Exists | ConfigTest_LastEntry, 0}
+	};
+	
+	static const ConfigurationVerify subverify = 
+	{
+		"*",
+		0,
+		verifykeys,
+		ConfigTest_LastEntry,
+		0
+	};
+	
+	static const ConfigurationVerify verify = 
+	{
+		"RAID FILE CONFIG",
+		&subverify,
+		0,
+		ConfigTest_LastEntry,
+		0
+	};
+	
+	// Load the configuration
+	std::string err;
+	std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(ConfigFilename, &verify, err);
+	
+	if(pconfig.get() == 0 || !err.empty())
+	{
+		fprintf(stderr, "RaidFile configuation file errors:\n%s", err.c_str());
+		THROW_EXCEPTION(RaidFileException, BadConfigFile)
+	}
+	
+	// Use the values
+	int expectedSetNum = 0;
+	std::vector<std::string> confdiscs(pconfig->GetSubConfigurationNames());
+	for(std::vector<std::string>::const_iterator i(confdiscs.begin()); i != confdiscs.end(); ++i)
+	{
+		const Configuration &disc(pconfig->GetSubConfiguration((*i).c_str()));
+		
+		int setNum = disc.GetKeyValueInt("SetNumber");
+		if(setNum != expectedSetNum)
+		{
+			THROW_EXCEPTION(RaidFileException, BadConfigFile)			
+		}
+		RaidFileDiscSet set(setNum, (unsigned int)disc.GetKeyValueInt("BlockSize"));
+		// Get the values of the directory keys
+		std::string d0(disc.GetKeyValue("Dir0"));
+		std::string d1(disc.GetKeyValue("Dir1"));
+		std::string d2(disc.GetKeyValue("Dir2"));
+		// Are they all different (using RAID) or all the same (not using RAID)
+		if(d0 != d1 && d1 != d2 && d0 != d2)
+		{
+			set.push_back(d0);
+			set.push_back(d1);
+			set.push_back(d2);
+		}
+		else if(d0 == d1 && d0 == d2)
+		{
+			// Just push the first one, which is the non-RAID place to store files
+			set.push_back(d0);
+		}
+		else
+		{
+			// One must be the same as another! Which is bad.
+			THROW_EXCEPTION(RaidFileException, BadConfigFile)			
+		}
+		mSetList.push_back(set);
+		expectedSetNum++;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::GetDiscSet(int)
+//		Purpose: Returns the numbered disc set
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+RaidFileDiscSet &RaidFileController::GetDiscSet(unsigned int DiscSetNum)
+{
+	if(DiscSetNum < 0 || DiscSetNum >= mSetList.size())
+	{
+		THROW_EXCEPTION(RaidFileException, NoSuchDiscSet)
+	}
+
+	return mSetList[DiscSetNum];
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileDiscSet::GetSetNumForWriteFiles(const std::string &)
+//		Purpose: Returns the set number the 'temporary' written files should
+//				 be stored on, given a filename.
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+int RaidFileDiscSet::GetSetNumForWriteFiles(const std::string &rFilename) const
+{
+	// Simple hash function, add up the ASCII values of all the characters,
+	// and get modulo number of partitions in the set.
+	std::string::const_iterator i(rFilename.begin());
+	int h = 0;
+	for(; i != rFilename.end(); ++i)
+	{
+		h += (*i);
+	}
+	return h % size();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::DiscSetPathToFileSystemPath(unsigned int, const std::string &, int)
+//		Purpose: Given a Raid File style file name, return a filename for the physical filing system.
+//				 DiscOffset is effectively the disc number (but remember files are rotated around the
+//				 discs in a disc set)
+//		Created: 19/1/04
+//
+// --------------------------------------------------------------------------
+std::string RaidFileController::DiscSetPathToFileSystemPath(unsigned int DiscSetNum, const std::string &rFilename, int DiscOffset)
+{
+	if(DiscSetNum < 0 || DiscSetNum >= mController.mSetList.size())
+	{
+		THROW_EXCEPTION(RaidFileException, NoSuchDiscSet)
+	}
+
+	// Work out which disc it's to be on
+	int disc = (mController.mSetList[DiscSetNum].GetSetNumForWriteFiles(rFilename) + DiscOffset)
+						% mController.mSetList[DiscSetNum].size();
+	
+	// Make the string
+	std::string r((mController.mSetList[DiscSetNum])[disc]);
+	r += DIRECTORY_SEPARATOR_ASCHAR;
+	r += rFilename;
+	return r;
+}
+
+
+
diff --git a/lib/raidfile/RaidFileController.h b/lib/raidfile/RaidFileController.h
new file mode 100755
index 00000000..4962d236
--- /dev/null
+++ b/lib/raidfile/RaidFileController.h
@@ -0,0 +1,107 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileController.h
+//		Purpose: Controls config and daemon comms for RaidFile classes
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+/*  NOTE: will log to local5: include a line like
+	local5.info                                             /var/log/raidfile
+	in /etc/syslog.conf
+*/
+
+#ifndef RAIDFILECONTROLLER__H
+#define RAIDFILECONTROLLER__H
+
+#include <string>
+#include <vector>
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileDiscSet
+//		Purpose: Describes a set of paritions for RAID like files.
+//				 Use as list of directories containing the files.
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+class RaidFileDiscSet : public std::vector<std::string>
+{
+public:
+	RaidFileDiscSet(int SetID, unsigned int BlockSize)
+		: mSetID(SetID),
+		  mBlockSize(BlockSize)
+	{
+	}
+	RaidFileDiscSet(const RaidFileDiscSet &rToCopy)
+		: std::vector<std::string>(rToCopy),
+		  mSetID(rToCopy.mSetID),
+		  mBlockSize(rToCopy.mBlockSize)
+	{
+	}
+	
+	~RaidFileDiscSet()
+	{
+	}
+
+	int GetSetID() const {return mSetID;}
+	
+	int GetSetNumForWriteFiles(const std::string &rFilename) const;
+	
+	unsigned int GetBlockSize() const {return mBlockSize;}
+
+	// Is this disc set a non-RAID disc set? (ie files never get transformed to raid storage)
+	bool IsNonRaidSet() const {return 1 == size();}
+
+private:
+	int mSetID;
+	unsigned int mBlockSize;
+};
+
+class _RaidFileController;	// compiler warning avoidance
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileController
+//		Purpose: Manages the configuration of the RaidFile system, handles
+//				 communication with the daemon.
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+class RaidFileController
+{
+	friend class _RaidFileController;	// to avoid compiler warning
+private:
+	RaidFileController();
+	RaidFileController(const RaidFileController &rController);
+public:
+	~RaidFileController();
+	
+public:
+	void Initialise(const char *ConfigFilename = "/etc/box/raidfile.conf");
+	int GetNumDiscSets() {return mSetList.size();}
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    RaidFileController::GetController()
+	//		Purpose: Gets the one and only controller object.
+	//		Created: 2003/07/08
+	//
+	// --------------------------------------------------------------------------	
+	static RaidFileController &GetController() {return mController;}
+	RaidFileDiscSet &GetDiscSet(unsigned int DiscSetNum);
+
+	static std::string DiscSetPathToFileSystemPath(unsigned int DiscSetNum, const std::string &rFilename, int DiscOffset);
+	
+private:
+	std::vector<RaidFileDiscSet> mSetList;
+	
+	static RaidFileController mController;
+};
+
+#endif // RAIDFILECONTROLLER__H
+
diff --git a/lib/raidfile/RaidFileException.h b/lib/raidfile/RaidFileException.h
new file mode 100755
index 00000000..809d4d5a
--- /dev/null
+++ b/lib/raidfile/RaidFileException.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileException.h
+//		Purpose: Exception
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef RAIDFILEEXCEPTION__H
+#define RAIDFILEEXCEPTION__H
+
+// Compatibility
+#include "autogen_RaidFileException.h"
+
+#endif // RAIDFILEEXCEPTION__H
+
diff --git a/lib/raidfile/RaidFileException.txt b/lib/raidfile/RaidFileException.txt
new file mode 100644
index 00000000..6ad74563
--- /dev/null
+++ b/lib/raidfile/RaidFileException.txt
@@ -0,0 +1,25 @@
+EXCEPTION RaidFile 2
+
+Internal						0
+CantOpenConfigFile				1	The raidfile.conf file is not accessible. Check that it is present in the default location or daemon configuration files point to the correct location.
+BadConfigFile					2
+NoSuchDiscSet					3
+CannotOverwriteExistingFile		4
+AlreadyOpen						5
+ErrorOpeningWriteFile			6
+NotOpen							7
+OSError							8	Error when accessing an underlying file. Check file permissions allow files to be read and written in the configured raid directories.
+WriteFileOpenOnTransform		9
+WrongNumberOfDiscsInSet			10	There should be three directories in each disc set.
+RaidFileDoesntExist				11
+ErrorOpeningFileForRead			12
+FileIsDamagedNotRecoverable		13
+InvalidRaidFile					14
+DirectoryIncomplete				15
+UnexpectedFileInDirPlace		16
+FileExistsInDirectoryCreation	17
+UnsupportedReadWriteOrClose		18
+CanOnlyGetUsageBeforeCommit		19
+CanOnlyGetFileSizeBeforeCommit	20
+ErrorOpeningWriteFileOnTruncate	21
+FileIsCurrentlyOpenForWriting	22
diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp
new file mode 100755
index 00000000..fed22ac0
--- /dev/null
+++ b/lib/raidfile/RaidFileRead.cpp
@@ -0,0 +1,1701 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileRead.cpp
+//		Purpose: Read Raid like Files
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/uio.h>
+#include <syslog.h>
+#include <stdarg.h>
+#include <dirent.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <memory>
+#include <map>
+
+#include "RaidFileRead.h"
+#include "RaidFileException.h"
+#include "RaidFileController.h"
+#include "RaidFileUtil.h"
+
+#ifdef PLATFORM_LINUX
+	#include "LinuxWorkaround.h"
+#endif
+
+#include "MemLeakFindOn.h"
+
+#define READ_NUMBER_DISCS_REQUIRED	3
+#define READV_MAX_BLOCKS			64
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileRead_NonRaid
+//		Purpose: Internal class for reading RaidFiles which haven't been transformed
+//				 into the RAID like form yet.
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+class RaidFileRead_NonRaid : public RaidFileRead
+{
+public:
+	RaidFileRead_NonRaid(int SetNumber, const std::string &Filename, int OSFileHandle);
+	virtual ~RaidFileRead_NonRaid();
+private:
+	RaidFileRead_NonRaid(const RaidFileRead_NonRaid &rToCopy);
+
+public:
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type GetPosition() const;
+	virtual void Seek(IOStream::pos_type Offset, int SeekType);
+	virtual void Close();
+	virtual pos_type GetFileSize() const;
+	virtual bool StreamDataLeft();
+
+private:
+	int mOSFileHandle;
+	bool mEOF;
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid(int, const std::string &, const std::string &)
+//		Purpose: Constructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead_NonRaid::RaidFileRead_NonRaid(int SetNumber, const std::string &Filename, int OSFileHandle)
+	: RaidFileRead(SetNumber, Filename),
+	  mOSFileHandle(OSFileHandle),
+	  mEOF(false)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::~RaidFileRead_NonRaid()
+//		Purpose: Destructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead_NonRaid::~RaidFileRead_NonRaid()
+{
+	if(mOSFileHandle != -1)
+	{
+		Close();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::Read(const void *, int)
+//		Purpose: Reads bytes from the file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+int RaidFileRead_NonRaid::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Read data
+	int bytesRead = ::read(mOSFileHandle, pBuffer, NBytes);
+	if(bytesRead == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	// Check for EOF
+	if(bytesRead == 0)
+	{
+		mEOF = true;
+	}
+
+	return bytesRead;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::GetPosition()
+//		Purpose: Returns current position
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead::pos_type RaidFileRead_NonRaid::GetPosition() const
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Use lseek to find the current file position
+	off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR);
+	if(p == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	return p;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::Seek(pos_type, int)
+//		Purpose: Seek within the file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_NonRaid::Seek(IOStream::pos_type Offset, int SeekType)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Seek...
+	if(::lseek(mOSFileHandle, Offset, ConvertSeekTypeToOSWhence(SeekType)) == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	// Not EOF any more
+	mEOF = false;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::Close()
+//		Purpose: Close the file (automatically done by destructor)
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_NonRaid::Close()
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Close file...
+	if(::close(mOSFileHandle) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	mOSFileHandle = -1;
+	mEOF = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::GetFileSize()
+//		Purpose: Returns file size.
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+RaidFileRead::pos_type RaidFileRead_NonRaid::GetFileSize() const
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// stat the file
+	struct stat st;
+	if(::fstat(mOSFileHandle, &st) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	return st.st_size;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::StreamDataLeft()
+//		Purpose: Any data left?
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead_NonRaid::StreamDataLeft()
+{
+	return !mEOF;
+}
+
+// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileRead_Raid
+//		Purpose: Internal class for reading RaidFiles have been transformed.
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+class RaidFileRead_Raid : public RaidFileRead
+{
+public:
+	friend class RaidFileRead;
+	RaidFileRead_Raid(int SetNumber, const std::string &Filename, int Stripe1Handle,
+		int Stripe2Handle, int ParityHandle, pos_type FileSize, unsigned int BlockSize,
+		bool LastBlockHasSize);
+	virtual ~RaidFileRead_Raid();
+private:
+	RaidFileRead_Raid(const RaidFileRead_Raid &rToCopy);
+
+public:
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type GetPosition() const;
+	virtual void Seek(IOStream::pos_type Offset, int SeekType);
+	virtual void Close();
+	virtual pos_type GetFileSize() const;
+	virtual bool StreamDataLeft();
+
+private:
+	int ReadRecovered(void *pBuffer, int NBytes);
+	void AttemptToRecoverFromIOError(bool Stripe1);
+	void SetPosition(pos_type FilePosition);
+	static void MoveDamagedFileAlertDaemon(int SetNumber, const std::string &Filename, bool Stripe1);
+
+private:
+	int mStripe1Handle;
+	int mStripe2Handle;
+	int mParityHandle;
+	pos_type mFileSize;
+	unsigned int mBlockSize;
+	pos_type mCurrentPosition;
+	char *mRecoveryBuffer;
+	pos_type mRecoveryBufferStart;
+	bool mLastBlockHasSize;
+	bool mEOF;
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid(int, const std::string &, const std::string &)
+//		Purpose: Constructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead_Raid::RaidFileRead_Raid(int SetNumber, const std::string &Filename, int Stripe1Handle, int Stripe2Handle, int ParityHandle, pos_type FileSize, unsigned int BlockSize, bool LastBlockHasSize)
+	: RaidFileRead(SetNumber, Filename),
+	  mStripe1Handle(Stripe1Handle),
+	  mStripe2Handle(Stripe2Handle),
+	  mParityHandle(ParityHandle),
+	  mFileSize(FileSize),
+	  mBlockSize(BlockSize),
+	  mCurrentPosition(0),
+	  mRecoveryBuffer(0),
+	  mRecoveryBufferStart(-1),
+	  mLastBlockHasSize(LastBlockHasSize),
+	  mEOF(false)
+{
+	// Make sure size of the IOStream::pos_type matches the pos_type used
+#ifdef PLATFORM_LINUX
+	ASSERT(sizeof(pos_type) >= sizeof(off_t));
+#else
+	ASSERT(sizeof(pos_type) == sizeof(off_t));
+#endif
+	
+	// Sanity check handles
+	if(mStripe1Handle != -1 && mStripe2Handle != -1)
+	{
+		// Everything is lovely, got two perfect files
+	}
+	else
+	{
+		// Check we have at least one stripe and a parity file
+		if((mStripe1Handle == -1 && mStripe2Handle == -1) || mParityHandle == -1)
+		{
+			// Should never have got this far
+			THROW_EXCEPTION(RaidFileException, Internal)
+		}
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::~RaidFileRead_Raid()
+//		Purpose: Destructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead_Raid::~RaidFileRead_Raid()
+{
+	Close();
+	if(mRecoveryBuffer != 0)
+	{
+		::free(mRecoveryBuffer);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::Read(const void *, int)
+//		Purpose: Reads bytes from the file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+int RaidFileRead_Raid::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// How many more bytes could we read?
+	unsigned int maxRead = mFileSize - mCurrentPosition;
+	if((unsigned int)NBytes > maxRead)
+	{
+		NBytes = maxRead;
+	}
+	
+	// Return immediately if there's nothing to read, and set EOF
+	if(NBytes == 0)
+	{
+		mEOF = true;
+		return 0;
+	}
+	
+	// Can we use the normal file reading routine?
+	if(mStripe1Handle == -1 || mStripe2Handle == -1)
+	{
+		// File is damaged, try a the recovery read function
+		return ReadRecovered(pBuffer, NBytes);
+	}
+	
+	// Vectors for reading stuff from the files
+	struct iovec stripe1Reads[READV_MAX_BLOCKS];
+	struct iovec stripe2Reads[READV_MAX_BLOCKS];
+	struct iovec *stripeReads[2] = {stripe1Reads, stripe2Reads};
+	unsigned int stripeReadsDataSize[2] = {0, 0};
+	unsigned int stripeReadsSize[2] = {0, 0};
+	int stripeHandles[2] = {mStripe1Handle, mStripe2Handle};
+	
+	// Which block are we doing?
+	unsigned int currentBlock = mCurrentPosition / mBlockSize;
+	unsigned int bytesLeftInCurrentBlock = mBlockSize - (mCurrentPosition % mBlockSize);
+	ASSERT(bytesLeftInCurrentBlock > 0)
+	unsigned int leftToRead = NBytes;
+	char *bufferPtr = (char*)pBuffer;
+	
+	// Now... add some whole block entries in...
+	try
+	{
+		while(leftToRead > 0)
+		{
+			int whichStripe = (currentBlock & 1);
+			size_t rlen = mBlockSize;
+			// Adjust if it's the first block
+			if(bytesLeftInCurrentBlock != 0)
+			{
+				rlen = bytesLeftInCurrentBlock;
+				bytesLeftInCurrentBlock = 0;
+			}
+			// Adjust if we're out of bytes
+			if(rlen > leftToRead)
+			{
+				rlen = leftToRead;
+			}
+			stripeReads[whichStripe][stripeReadsSize[whichStripe]].iov_base = bufferPtr;
+			stripeReads[whichStripe][stripeReadsSize[whichStripe]].iov_len = rlen;
+			stripeReadsSize[whichStripe]++;
+			stripeReadsDataSize[whichStripe] += rlen;
+			leftToRead -= rlen;
+			bufferPtr += rlen;
+			currentBlock++;
+
+			// Read data?
+			for(int s = 0; s < 2; ++s)
+			{
+				if((leftToRead == 0 || stripeReadsSize[s] >= READV_MAX_BLOCKS) && stripeReadsSize[s] > 0)
+				{
+					int r = ::readv(stripeHandles[s], stripeReads[s], stripeReadsSize[s]);
+					if(r == -1)
+					{
+						// Bad news... IO error?
+						if(errno == EIO)
+						{
+							// Attempt to recover from this failure
+							AttemptToRecoverFromIOError((s == 0) /* is stripe 1 */);
+							// Retry
+							return Read(pBuffer, NBytes, Timeout);
+						}
+						else
+						{
+							// Can't do anything, throw
+							THROW_EXCEPTION(RaidFileException, OSError)
+						}
+					}
+					else if(r != (int)stripeReadsDataSize[s])
+					{
+						// Got the file sizes wrong/logic error!
+						THROW_EXCEPTION(RaidFileException, Internal)
+					}
+					stripeReadsSize[s] = 0;
+					stripeReadsDataSize[s] = 0;
+				}
+			}
+		}
+	}
+	catch(...)
+	{
+		// Get file pointers to right place (to meet exception safe stuff)
+		SetPosition(mCurrentPosition);
+		
+		throw;
+	}
+	
+	// adjust current position
+	mCurrentPosition += NBytes;
+
+	return NBytes;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::MoveDamagedFileAlertDaemon(bool)
+//		Purpose: Moves a file into the damaged directory, and alerts the Daemon to recover it properly later.
+//		Created: 2003/07/22
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::MoveDamagedFileAlertDaemon(int SetNumber, const std::string &Filename, bool Stripe1)
+{
+	// Move the dodgy file away
+	// Get the controller and the disc set we're on
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+	if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size())
+	{
+		THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet)
+	}
+	// Start disc
+	int startDisc = rdiscSet.GetSetNumForWriteFiles(Filename);
+	int errOnDisc = (startDisc + (Stripe1?0:1)) % READ_NUMBER_DISCS_REQUIRED;
+	
+	// Make a munged filename for renaming
+	std::string mungeFn(Filename + RAIDFILE_EXTENSION);
+	std::string awayName;
+	for(std::string::const_iterator i = mungeFn.begin(); i != mungeFn.end(); ++i)
+	{
+		char c = (*i);
+		if(c == DIRECTORY_SEPARATOR_ASCHAR)
+		{
+			awayName += '_';
+		}
+		else if(c == '_')
+		{
+			awayName += "__";
+		}
+		else
+		{
+			awayName += c;
+		}
+	}
+	// Make sure the error files directory exists
+	std::string dirname(rdiscSet[errOnDisc] + DIRECTORY_SEPARATOR ".raidfile-unreadable");
+	int mdr = ::mkdir(dirname.c_str(), 0750);
+	if(mdr != 0 && errno != EEXIST)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	// Attempt to rename the file there -- ignore any return code here, as it's dubious anyway
+	std::string errorFile(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, errOnDisc));
+	::rename(errorFile.c_str(), (dirname + DIRECTORY_SEPARATOR_ASCHAR + awayName).c_str());
+
+	// TODO: Inform the recovery daemon
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::AttemptToRecoverFromIOError(bool)
+//		Purpose: Attempt to recover from an IO error, setting up to read from parity instead.
+//				 Will exception if this isn't possible.
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::AttemptToRecoverFromIOError(bool Stripe1)
+{
+	TRACE3("Attempting to recover from I/O error: %d %s, on stripe %d\n", mSetNumber, mFilename.c_str(), Stripe1?1:2);
+	::syslog(LOG_ERR | LOG_LOCAL5, "Attempting to recover from I/O error: %d %s, on stripe %d\n", mSetNumber, mFilename.c_str(), Stripe1?1:2);
+
+	// Close offending file
+	if(Stripe1)
+	{
+		if(mStripe1Handle != -1)
+		{
+			::close(mStripe1Handle);
+			mStripe1Handle = -1;
+		}
+	}
+	else
+	{
+		if(mStripe2Handle != -1)
+		{
+			::close(mStripe2Handle);
+			mStripe2Handle = -1;
+		}
+	}
+
+	// Check...
+	ASSERT((Stripe1?mStripe2Handle:mStripe1Handle) != -1);
+
+	// Get rid of the damaged file
+	MoveDamagedFileAlertDaemon(mSetNumber, mFilename, Stripe1);
+
+	// Get the controller and the disc set we're on
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size())
+	{
+		THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet)
+	}
+	// Start disc
+	int startDisc = rdiscSet.GetSetNumForWriteFiles(mFilename);
+
+	// Mark as nothing in recovery buffer
+	mRecoveryBufferStart = -1;
+	
+	// Seek to zero on the remaining file -- get to nice state
+	if(::lseek(Stripe1?mStripe2Handle:mStripe1Handle, 0, SEEK_SET) == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	// Open the parity file
+	std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (2 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+	mParityHandle = ::open(parityFilename.c_str(), O_RDONLY, 0555);
+	if(mParityHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	// Work out whether or not there's a size XORed into the last block
+	unsigned int bytesInLastTwoBlocks = mFileSize % (mBlockSize * 2);
+	if(bytesInLastTwoBlocks > mBlockSize && bytesInLastTwoBlocks < ((mBlockSize * 2) - sizeof(FileSizeType)))
+	{
+		// Yes, there's something to XOR in the last block
+		mLastBlockHasSize = true;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::ReadRecovered(const void *, int)
+//		Purpose: Reads data recreating from the parity stripe
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+int RaidFileRead_Raid::ReadRecovered(void *pBuffer, int NBytes)
+{
+	// Note: NBytes has been adjusted to definately be a range
+	// inside the given file length.
+
+	// Make sure a buffer is allocated
+	if(mRecoveryBuffer == 0)
+	{
+		mRecoveryBuffer = (char*)::malloc(mBlockSize * 2);
+		if(mRecoveryBuffer == 0)
+		{
+			throw std::bad_alloc();
+		}
+	}
+	
+	// Which stripe?
+	int stripe = (mStripe1Handle != -1)?mStripe1Handle:mStripe2Handle;
+	if(stripe == -1)
+	{
+		// Not enough file handles around
+		THROW_EXCEPTION(RaidFileException, FileIsDamagedNotRecoverable)
+	}
+	
+	char *outptr = (char*)pBuffer;
+	int bytesToGo = NBytes;
+	
+	pos_type preservedCurrentPosition = mCurrentPosition;
+	
+	try
+	{
+		// Start offset within buffer
+		int offset = (mCurrentPosition - mRecoveryBufferStart);
+		// Let's go!
+		while(bytesToGo > 0)
+		{
+			int bytesLeftInBuffer = 0;
+			if(mRecoveryBufferStart != -1)
+			{
+				bytesLeftInBuffer = (mRecoveryBufferStart + (mBlockSize*2)) - mCurrentPosition;
+				ASSERT(bytesLeftInBuffer >= 0);
+			}
+			
+			// How many bytes can be copied out?
+			int toCopy = bytesLeftInBuffer;
+			if(toCopy > bytesToGo) toCopy = bytesToGo;
+			//printf("offset = %d, tocopy = %d, bytestogo = %d, leftinbuffer = %d\n", (int)offset, toCopy, bytesToGo, bytesLeftInBuffer);
+			if(toCopy > 0)
+			{
+				for(int l = 0; l < toCopy; ++l)
+				{
+					*(outptr++) = mRecoveryBuffer[offset++];
+				}
+				bytesToGo -= toCopy;
+				mCurrentPosition += toCopy;
+			}
+			
+			// Load in the next buffer?
+			if(bytesToGo > 0)
+			{
+				// Calculate the blocks within the file that are needed to be loaded.
+				pos_type fileBlock = mCurrentPosition / (mBlockSize * 2);
+				// Is this the last block
+				bool isLastBlock = (fileBlock == (mFileSize / (mBlockSize * 2)));
+				
+				// Need to reposition file pointers?
+				if(mRecoveryBufferStart == -1)
+				{
+					// Yes!
+					// And the offset from which to read it
+					pos_type filePos = fileBlock * mBlockSize;
+					// Then seek
+					if(::lseek(stripe, filePos, SEEK_SET) == -1
+						|| ::lseek(mParityHandle, filePos, SEEK_SET) == -1)
+					{
+						THROW_EXCEPTION(RaidFileException, OSError)
+					}
+				}
+				
+				// Load a block from each file, getting the ordering the right way round
+				int r1 = ::read((mStripe1Handle != -1)?stripe:mParityHandle, mRecoveryBuffer, mBlockSize);
+				int r2 = ::read((mStripe1Handle != -1)?mParityHandle:stripe, mRecoveryBuffer + mBlockSize, mBlockSize);				
+				if(r1 == -1 || r2 == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+
+				// error checking and manipulation
+				if(isLastBlock)
+				{
+					// Allow not full reads, and append zeros if necessary to fill the space.
+					int r1zeros = mBlockSize - r1;
+					if(r1zeros > 0)
+					{
+						::memset(mRecoveryBuffer + r1, 0, r1zeros);
+					}
+					int r2zeros = mBlockSize - r2;
+					if(r2zeros > 0)
+					{
+						::memset(mRecoveryBuffer + mBlockSize + r2, 0, r2zeros);
+					}
+					
+					// if it's got the file size in it, XOR it off
+					if(mLastBlockHasSize)
+					{
+						int sizeXorOffset = (mBlockSize - sizeof(FileSizeType)) + ((mStripe1Handle != -1)?mBlockSize:0);
+						*((FileSizeType*)(mRecoveryBuffer + sizeXorOffset)) ^= ntoh64(mFileSize);
+					}
+				}
+				else
+				{
+					// Must have got a full block, otherwise things are a bit bad here.
+					if(r1 != (int)mBlockSize || r2 != (int)mBlockSize)
+					{
+						THROW_EXCEPTION(RaidFileException, InvalidRaidFile)
+					}
+				}
+				
+				// Go XORing!
+				unsigned int *b1 = (unsigned int*)mRecoveryBuffer;
+				unsigned int *b2 = (unsigned int *)(mRecoveryBuffer + mBlockSize);
+				if((mStripe1Handle == -1))
+				{
+					b1 = b2;
+					b2 = (unsigned int*)mRecoveryBuffer;
+				}
+				for(int x = ((mBlockSize/sizeof(unsigned int)) - 1); x >= 0; --x)
+				{
+					*b2 = (*b1) ^ (*b2);
+					++b1;
+					++b2;
+				}
+				
+				// New block location
+				mRecoveryBufferStart = fileBlock * (mBlockSize * 2);
+				
+				// New offset withing block
+				offset = (mCurrentPosition - mRecoveryBufferStart);
+				ASSERT(offset >= 0);
+			}
+		}
+	}
+	catch(...)
+	{
+		// Change variables so 1) buffer is invalidated and 2) the file will be seeked properly the next time round
+		mRecoveryBufferStart = -1;
+		mCurrentPosition = preservedCurrentPosition;
+		throw;
+	}
+	
+	return NBytes;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::GetPosition()
+//		Purpose: Returns current position
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileRead_Raid::GetPosition() const
+{
+	return mCurrentPosition;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::Seek(RaidFileRead::pos_type, bool)
+//		Purpose: Seek within the file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::Seek(IOStream::pos_type Offset, int SeekType)
+{
+	pos_type newpos = mCurrentPosition;
+	switch(SeekType)
+	{
+	case IOStream::SeekType_Absolute:
+		newpos = Offset;
+		break;
+		
+	case IOStream::SeekType_Relative:
+		newpos += Offset;
+		break;
+		
+	case IOStream::SeekType_End:
+		newpos = mFileSize + Offset;
+		break;
+		
+	default:
+		THROW_EXCEPTION(CommonException, IOStreamBadSeekType)
+	}
+	
+	if(newpos != mCurrentPosition)
+	{
+		SetPosition(newpos);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::SetPosition(pos_type)
+//		Purpose: Move the file pointers
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::SetPosition(pos_type FilePosition)
+{
+	if(FilePosition > mFileSize)
+	{
+		FilePosition = mFileSize;
+	}
+
+	if(mStripe1Handle != -1 && mStripe2Handle != -1)
+	{
+		// right then... which block is it in?
+		pos_type block = FilePosition / mBlockSize;
+		pos_type offset = FilePosition % mBlockSize;
+		
+		// Calculate offsets for each file
+		pos_type basepos = (block / 2) * mBlockSize;
+		pos_type s1p, s2p;
+		if((block & 1) == 0)
+		{
+			s1p = basepos + offset;
+			s2p = basepos;
+		}
+		else
+		{
+			s1p = basepos + mBlockSize;
+			s2p = basepos + offset;
+		}
+		// Note: lseek isn't in the man pages to return EIO, but assuming that it can return this,
+		// as it calls various OS bits and returns their error codes, and those fns look like they might.
+		if(::lseek(mStripe1Handle, s1p, SEEK_SET) == -1)
+		{
+			if(errno == EIO)
+			{
+				TRACE3("I/O error when seeking in %d %s (to %d), stripe 1\n", mSetNumber, mFilename.c_str(), (int)FilePosition);
+				::syslog(LOG_ERR | LOG_LOCAL5, "I/O error when seeking in %d %s (to %d), stripe 1\n", mSetNumber, mFilename.c_str(), (int)FilePosition);
+				// Attempt to recover
+				AttemptToRecoverFromIOError(true /* is stripe 1 */);
+				ASSERT(mStripe1Handle == -1);
+				// Retry
+				SetPosition(FilePosition);
+				return;
+			}
+			else
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+		if(::lseek(mStripe2Handle, s2p, SEEK_SET) == -1)
+		{
+			if(errno == EIO)
+			{
+				TRACE3("I/O error when seeking in %d %s (to %d), stripe 2\n", mSetNumber, mFilename.c_str(), (int)FilePosition);
+				::syslog(LOG_ERR | LOG_LOCAL5, "I/O error when seeking in %d %s (to %d), stripe 2\n", mSetNumber, mFilename.c_str(), (int)FilePosition);
+				// Attempt to recover
+				AttemptToRecoverFromIOError(false /* is stripe 2 */);
+				ASSERT(mStripe2Handle == -1);
+				// Retry
+				SetPosition(FilePosition);
+				return;
+			}
+			else
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+		
+		// Store position
+		mCurrentPosition = FilePosition;
+	}
+	else
+	{
+		// Simply store, and mark the recovery buffer invalid
+		mCurrentPosition = FilePosition;
+		mRecoveryBufferStart = -1;
+	}
+
+	// not EOF any more
+	mEOF = false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::Close()
+//		Purpose: Close the file (automatically done by destructor)
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::Close()
+{
+	if(mStripe1Handle != -1)
+	{
+		::close(mStripe1Handle);
+		mStripe1Handle = -1;
+	}
+	if(mStripe2Handle != -1)
+	{
+		::close(mStripe2Handle);
+		mStripe2Handle = -1;
+	}
+	if(mParityHandle != -1)
+	{
+		::close(mParityHandle);
+		mParityHandle = -1;
+	}
+	
+	mEOF = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::StreamDataLeft()
+//		Purpose: Any data left?
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead_Raid::StreamDataLeft()
+{
+	return !mEOF;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::GetFileSize()
+//		Purpose: Returns file size.
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+RaidFileRead::pos_type RaidFileRead_Raid::GetFileSize() const
+{
+	return mFileSize;
+}
+
+
+// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::RaidFileRead(int, const std::string &)
+//		Purpose: Constructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead::RaidFileRead(int SetNumber, const std::string &Filename)
+	: mSetNumber(SetNumber),
+	  mFilename(Filename)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::~RaidFileRead()
+//		Purpose: Destructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead::~RaidFileRead()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::Open(int, const std::string &, int)
+//		Purpose: Opens a RaidFile for reading.
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string &Filename, int64_t *pRevisionID, int BufferSizeHint)
+{
+	// See what's available...
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+	if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size() && 1 != rdiscSet.size()) // allow non-RAID configurations
+	{
+		THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet)
+	}
+
+	// See if the file exists
+	int startDisc = 0, existingFiles = 0;
+	RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, Filename, &startDisc, &existingFiles, pRevisionID);
+	if(existance == RaidFileUtil::NoFile)
+	{
+		THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist)
+	}
+	else if(existance == RaidFileUtil::NonRaid)
+	{
+		// Simple non-RAID file so far...
+	
+		// Get the filename for the write file
+		std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, Filename));
+
+		// Attempt to open
+		int osFileHandle = ::open(writeFilename.c_str(), O_RDONLY, 0);
+		if(osFileHandle == -1)
+		{
+			THROW_EXCEPTION(RaidFileException, ErrorOpeningFileForRead)
+		}
+		
+		// Return a read object for this file
+		try
+		{
+			return std::auto_ptr<RaidFileRead>(new RaidFileRead_NonRaid(SetNumber, Filename, osFileHandle));
+		}
+		catch(...)
+		{
+			::close(osFileHandle);
+			throw;
+		}
+	}
+	else if(existance == RaidFileUtil::AsRaid
+		|| ((existingFiles & RaidFileUtil::Stripe1Exists) && (existingFiles & RaidFileUtil::Stripe2Exists)))
+	{
+		if(existance != RaidFileUtil::AsRaid)
+		{
+			TRACE2("Opening %d %s in normal mode, but parity file doesn't exist\n", SetNumber, Filename.c_str());
+			::syslog(LOG_ERR | LOG_LOCAL5, "Opening %d %s in normal mode, but parity file doesn't exist\n", SetNumber, Filename.c_str());
+			// TODO: Alert recovery daemon
+		}
+	
+		// Open the two stripe files
+		std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (0 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+		std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (1 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+		int stripe1 = -1;
+		int stripe1errno = 0;
+		int stripe2 = -1;
+		int stripe2errno = 0;
+		
+		try
+		{
+			// Open stripe1
+			stripe1 = ::open(stripe1Filename.c_str(), O_RDONLY, 0555);
+			if(stripe1 == -1)
+			{
+				stripe1errno = errno;
+			}
+			// Open stripe2
+			stripe2 = ::open(stripe2Filename.c_str(), O_RDONLY, 0555);
+			if(stripe2 == -1)
+			{
+				stripe2errno = errno;
+			}
+			if(stripe1errno != 0 || stripe2errno != 0)
+			{
+				THROW_EXCEPTION(RaidFileException, ErrorOpeningFileForRead)
+			}
+			
+			// stat stripe 1 to find ('half' of) length...
+			struct stat st;
+			if(::fstat(stripe1, &st) != 0)
+			{
+				stripe1errno = errno;
+			}
+			pos_type length = st.st_size;
+			
+			// stat stripe2 to find (other 'half' of) length...
+			if(::fstat(stripe2, &st) != 0)
+			{
+				stripe2errno = errno;
+			}
+			length += st.st_size;
+			
+			// Handle errors
+			if(stripe1errno != 0 || stripe2errno != 0)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			
+			// Make a nice object to represent this file
+			return std::auto_ptr<RaidFileRead>(new RaidFileRead_Raid(SetNumber, Filename, stripe1, stripe2, -1, length, rdiscSet.GetBlockSize(), false /* actually we don't know */));
+		}
+		catch(...)
+		{
+			// Close open files
+			if(stripe1 != -1)
+			{
+				::close(stripe1);
+				stripe1 = -1;
+			}
+			if(stripe2 != -1)
+			{
+				::close(stripe2);
+				stripe2 = -1;
+			}
+			
+			// Now... maybe we can try again with one less file?
+			bool oktotryagain = true;
+			if(stripe1errno == EIO)
+			{
+				TRACE2("I/O error on opening %d %s stripe 1, trying recovery mode\n", SetNumber, Filename.c_str());
+				::syslog(LOG_ERR | LOG_LOCAL5, "I/O error on opening %d %s stripe 1, trying recovery mode\n", SetNumber, Filename.c_str());
+				RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, true /* is stripe 1 */);
+
+				existingFiles = existingFiles & ~RaidFileUtil::Stripe1Exists;
+				existance = (existance == RaidFileUtil::AsRaidWithMissingReadable)
+					?RaidFileUtil::AsRaidWithMissingNotRecoverable
+					:RaidFileUtil::AsRaidWithMissingReadable;
+			}
+			else if(stripe1errno != 0)
+			{
+				oktotryagain = false;
+			}
+			
+			if(stripe2errno == EIO)
+			{
+				TRACE2("I/O error on opening %d %s stripe 2, trying recovery mode\n", SetNumber, Filename.c_str());
+				::syslog(LOG_ERR | LOG_LOCAL5, "I/O error on opening %d %s stripe 2, trying recovery mode\n", SetNumber, Filename.c_str());
+				RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, false /* is stripe 2 */);
+
+				existingFiles = existingFiles & ~RaidFileUtil::Stripe2Exists;
+				existance = (existance == RaidFileUtil::AsRaidWithMissingReadable)
+					?RaidFileUtil::AsRaidWithMissingNotRecoverable
+					:RaidFileUtil::AsRaidWithMissingReadable;
+			}
+			else if(stripe2errno != 0)
+			{
+				oktotryagain = false;
+			}
+			
+			if(!oktotryagain)
+			{
+				throw;
+			}
+		}
+	}
+
+	if(existance == RaidFileUtil::AsRaidWithMissingReadable)
+	{
+		TRACE3("Attempting to open RAID file %d %s in recovery mode (stripe %d present)\n", SetNumber, Filename.c_str(), (existingFiles & RaidFileUtil::Stripe1Exists)?1:2);
+		::syslog(LOG_ERR | LOG_LOCAL5, "Attempting to open RAID file %d %s in recovery mode (stripe %d present)\n", SetNumber, Filename.c_str(), (existingFiles & RaidFileUtil::Stripe1Exists)?1:2);
+	
+		// Generate the filenames of all the lovely files
+		std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (0 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+		std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (1 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+		std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (2 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+
+		int stripe1 = -1;
+		int stripe2 = -1;
+		int parity = -1;
+
+		try
+		{
+			// Open stripe1?
+			if(existingFiles & RaidFileUtil::Stripe1Exists)
+			{
+				stripe1 = ::open(stripe1Filename.c_str(), O_RDONLY, 0555);
+				if(stripe1 == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+			}
+			// Open stripe2?
+			if(existingFiles & RaidFileUtil::Stripe2Exists)
+			{
+				stripe2 = ::open(stripe2Filename.c_str(), O_RDONLY, 0555);
+				if(stripe2 == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+			}
+			// Open parity
+			parity = ::open(parityFilename.c_str(), O_RDONLY, 0555);
+			if(parity == -1)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			
+			// Find the length. This is slightly complex.
+			unsigned int blockSize = rdiscSet.GetBlockSize();
+			pos_type length = 0;
+			
+			// The easy one... if the parity file is of an integral block size + sizeof(FileSizeType)
+			// then it's stored at the end of the parity file
+			struct stat st;
+			if(::fstat(parity, &st) != 0)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			pos_type paritySize = st.st_size;
+			FileSizeType parityLastData = 0;
+			bool parityIntegralPlusOffT = ((paritySize % blockSize) == sizeof(FileSizeType));
+			if(paritySize >= static_cast<pos_type>(sizeof(parityLastData)) && (parityIntegralPlusOffT || stripe1 != -1))
+			{
+				// Seek to near the end
+				ASSERT(sizeof(FileSizeType) == 8); // compiler bug (I think) prevents from using 0 - sizeof(FileSizeType)...
+				if(::lseek(parity, -8 /*(0 - sizeof(FileSizeType))*/, SEEK_END) == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+				// Read it in
+				if(::read(parity, &parityLastData, sizeof(parityLastData)) != sizeof(parityLastData))
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+				// Set back to beginning of file
+				if(::lseek(parity, 0, SEEK_SET) == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+			}
+			
+			bool lastBlockHasSize = false;
+			if(parityIntegralPlusOffT)
+			{
+				// Wonderful! Have the value
+				length = ntoh64(parityLastData);
+			}
+			else
+			{
+				// Have to resort to more devious means.
+				if(existingFiles & RaidFileUtil::Stripe1Exists)
+				{
+					// Procedure for stripe 1 existence...
+					// 	Get size of stripe1.
+					//  If this is not an integral block size, then size can use this
+					//  to work out the size of the file.
+					//  Otherwise, read in the end of the last block, and use a bit of XORing
+					//  to get the size from the FileSizeType value at end of the file.
+					if(::fstat(stripe1, &st) != 0)
+					{
+						THROW_EXCEPTION(RaidFileException, OSError)
+					}
+					pos_type stripe1Size = st.st_size;
+					// Is size integral?
+					if((stripe1Size % ((pos_type)blockSize)) != 0)
+					{
+						// No, so know the size.
+						length = stripe1Size + ((stripe1Size / blockSize) * blockSize);
+					}
+					else
+					{
+						// Must read the last bit of data from the block and XOR.
+						FileSizeType stripe1LastData = 0;	// initialise to zero, as we may not read everything from it
+						
+						// Work out how many bytes to read
+						int btr = 0;			// bytes to read off end
+						unsigned int lbs = stripe1Size % blockSize;
+						if(lbs == 0 && stripe1Size > 0)
+						{
+							// integral size, need the entire bit
+							btr = sizeof(FileSizeType);
+						}
+						else if(lbs > (blockSize - sizeof(FileSizeType)))
+						{
+							btr = lbs - (blockSize - sizeof(FileSizeType));
+						}
+						
+						// Seek to near the end
+						if(btr > 0)
+						{
+							ASSERT(sizeof(FileSizeType) == 8); // compiler bug (I think) prevents from using 0 - sizeof(FileSizeType)...
+							ASSERT(btr <= (int)sizeof(FileSizeType));
+							if(::lseek(stripe1, 0 - btr, SEEK_END) == -1)
+							{
+								THROW_EXCEPTION(RaidFileException, OSError)
+							}
+							// Read it in
+							if(::read(stripe1, &stripe1LastData, btr) != btr)
+							{
+								THROW_EXCEPTION(RaidFileException, OSError)
+							}
+							// Set back to beginning of file
+							if(::lseek(stripe1, 0, SEEK_SET) == -1)
+							{
+								THROW_EXCEPTION(RaidFileException, OSError)
+							}
+						}
+						// Lovely!
+						length = stripe1LastData ^ parityLastData;
+						// Convert to host byte order
+						length = ntoh64(length);
+						ASSERT(length <= (paritySize + stripe1Size));
+						// Mark is as having this to aid code later
+						lastBlockHasSize = true;
+					}
+				}
+				else
+				{
+					ASSERT(existingFiles & RaidFileUtil::Stripe2Exists);
+				}
+
+				if(existingFiles & RaidFileUtil::Stripe2Exists)
+				{
+					// Get size of stripe2 file
+					if(::fstat(stripe2, &st) != 0)
+					{
+						THROW_EXCEPTION(RaidFileException, OSError)
+					}
+					pos_type stripe2Size = st.st_size;
+					
+					// Is it an integral size?
+					if(stripe2Size % blockSize != 0)
+					{
+						// No. Working out the size is easy.
+						length = stripe2Size + (((stripe2Size / blockSize)+1) * blockSize);
+						// Got last block size in there?
+						if((stripe2Size % blockSize) <= static_cast<pos_type>((blockSize - sizeof(pos_type))))
+						{
+							// Yes...
+							lastBlockHasSize = true;
+						}
+					}
+					else
+					{
+						// Yes. So we need to compare with the parity file to get a clue...
+						pos_type stripe2Blocks = stripe2Size / blockSize;
+						pos_type parityBlocks = paritySize / blockSize;
+						if(stripe2Blocks == parityBlocks)
+						{
+							// Same size, so stripe1 must be the same size
+							length = (stripe2Blocks * 2) * blockSize;
+						}
+						else
+						{
+							// Different size, so stripe1 must be one block bigger
+							ASSERT(stripe2Blocks < parityBlocks);
+							length = ((stripe2Blocks * 2)+1) * blockSize;
+						}
+						
+						// Then... add in the extra bit of the parity length
+						unsigned int lastBlockSize = paritySize % blockSize;
+						length += lastBlockSize;
+					}
+				}
+				else
+				{
+					ASSERT(existingFiles & RaidFileUtil::Stripe1Exists);
+				}
+			}
+
+			// Create a lovely object to return
+			return std::auto_ptr<RaidFileRead>(new RaidFileRead_Raid(SetNumber, Filename, stripe1, stripe2, parity, length, blockSize, lastBlockHasSize));
+		}
+		catch(...)
+		{
+			// Close open files
+			if(stripe1 != -1)
+			{
+				::close(stripe1);
+				stripe1 = -1;
+			}
+			if(stripe2 != -1)
+			{
+				::close(stripe2);
+				stripe2 = -1;
+			}
+			if(parity != -1)
+			{
+				::close(parity);
+				parity = -1;
+			}
+			throw;
+		}
+	}
+	
+	THROW_EXCEPTION(RaidFileException, FileIsDamagedNotRecoverable)
+	
+	// Avoid compiler warning -- it'll never get here...
+	return std::auto_ptr<RaidFileRead>();
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::DirectoryExists(int, const std::string &)
+//		Purpose: Returns true if the directory exists. Throws exception if it's partially in existence.
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::DirectoryExists(int SetNumber, const std::string &rDirName)
+{
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+
+	return DirectoryExists(rdiscSet, rDirName);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::DirectoryExists(const RaidFileDiscSet &, const std::string &)
+//		Purpose: Returns true if the directory exists. Throws exception if it's partially in existence.
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::string &rDirName)
+{
+	// For each directory, test to see if it exists
+	unsigned int nexist = 0;
+	for(unsigned int l = 0; l < rSet.size(); ++l)
+	{
+		// build name
+		std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName);
+		
+		// check for existence
+		struct stat st;
+		if(::stat(dn.c_str(), &st) == 0)
+		{
+			// Directory?
+			if(st.st_mode & S_IFDIR)
+			{
+				// yes
+				nexist++;
+			}
+			else
+			{
+				// No. It's a file. Bad!
+				THROW_EXCEPTION(RaidFileException, UnexpectedFileInDirPlace)
+			}
+		}
+		else
+		{
+			// Was it a non-exist error?
+			if(errno != ENOENT)
+			{
+				// No. Bad things.
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+	}
+	
+	// Were all of them found?
+	if(nexist == 0)
+	{
+		// None.
+		return false;
+	}
+	else if(nexist == rSet.size())
+	{
+		// All
+		return true;
+	}
+
+	// Some exist. We don't like this -- it shows something bad happened before
+	// TODO: notify recovery daemon
+	THROW_EXCEPTION(RaidFileException, DirectoryIncomplete)
+	return false;	// avoid compiler warning
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::FileExists(int, const std::string &, int64_t *)
+//		Purpose: Does a Raid file exist? Optionally return a revision number, which is
+//				 unique to this saving of the file. (revision number may change
+//				 after transformation to RAID -- so only use for cache control,
+//				 not detecting changes to content).
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::FileExists(int SetNumber, const std::string &rFilename, int64_t *pRevisionID)
+{
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+
+	return RaidFileUtil::RaidFileExists(rdiscSet, rFilename, 0, 0, pRevisionID) != RaidFileUtil::NoFile;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadDirectoryContents(int, const std::string &, int, std::vector<std::string> &)
+//		Purpose: Read directory contents, returning whether or not all entries are likely to be readable or not
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput)
+{
+	// Remove anything in the vector to begin with.
+	rOutput.clear();
+	
+	// Controller and set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+
+	// Collect the directory listings
+	std::map<std::string, unsigned int> counts;
+	
+	unsigned int numDiscs = rdiscSet.size();
+	
+	for(unsigned int l = 0; l < numDiscs; ++l)
+	{
+		// build name
+		std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName);
+		
+		// read the contents...
+		DIR *dirHandle = 0;
+		try
+		{
+			dirHandle = ::opendir(dn.c_str());
+			if(dirHandle == 0)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			
+			struct dirent *en = 0;
+			while((en = ::readdir(dirHandle)) != 0)
+			{
+#ifdef PLATFORM_LINUX
+			LinuxWorkaround_FinishDirentStruct(en, dn.c_str());
+#endif
+
+				if(en->d_name[0] == '.' && 
+					(en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0')))
+				{
+					// ignore, it's . or ..
+					continue;
+				}
+				
+				// stat the file to find out what type it is
+/*				struct stat st;
+				std::string fullName(dn + DIRECTORY_SEPARATOR + en->d_name);
+				if(::stat(fullName.c_str(), &st) != 0)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}*/
+				
+				// Entry...
+				std::string name;
+				unsigned int countToAdd = 1;
+				if(DirReadType == DirReadType_FilesOnly && en->d_type == DT_REG) // (st.st_mode & S_IFDIR) == 0)
+				{
+					// File. Complex, need to check the extension
+					int dot = -1;
+					int p = 0;
+					while(en->d_name[p] != '\0')
+					{
+						if(en->d_name[p] == '.')
+						{
+							// store location of dot
+							dot = p;
+						}
+						++p;
+					}
+					// p is length of string
+					if(dot != -1 && ((p - dot) == 3 || (p - dot) == 4)
+						&& en->d_name[dot+1] == 'r' && en->d_name[dot+2] == 'f'
+						&& (en->d_name[dot+3] == 'w' || en->d_name[dot+3] == '\0'))
+					{
+						// so has right extension
+						name.assign(en->d_name, dot);	/* get name up to last . */
+						// Was it a write file (which counts as everything)
+						if(en->d_name[dot+3] == 'w')
+						{
+							countToAdd = numDiscs;
+						}
+					}
+				}
+				if(DirReadType == DirReadType_DirsOnly && en->d_type == DT_DIR) // (st.st_mode & S_IFDIR))
+				{
+					// Directory, and we want directories
+					name = en->d_name;
+				}
+				// Eligable for entry?
+				if(!name.empty())
+				{
+					// add to map...
+					std::map<std::string, unsigned int>::iterator i = counts.find(name);
+					if(i != counts.end())
+					{
+						// add to count
+						i->second += countToAdd;
+					}
+					else
+					{
+						// insert into map
+						counts[name] = countToAdd;
+					}
+				}
+			}
+			
+			if(::closedir(dirHandle) != 0)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			dirHandle = 0;
+		}
+		catch(...)
+		{
+			if(dirHandle != 0)
+			{
+				::closedir(dirHandle);
+			}
+			throw;
+		}
+	}
+	
+	// Now go through the map, adding in entries
+	bool everythingReadable = true;
+	
+	for(std::map<std::string, unsigned int>::const_iterator i = counts.begin(); i != counts.end(); ++i)
+	{
+		if(i->second < (numDiscs - 1))
+		{
+			// Too few discs to be confident of reading everything
+			everythingReadable = false;
+		}
+		
+		// Add name to vector
+		rOutput.push_back(i->first);
+	}
+	
+	return everythingReadable;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::Write(const void *, int)
+//		Purpose: Not support, throws exception
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void RaidFileRead::Write(const void *pBuffer, int NBytes)
+{
+	THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::StreamClosed()
+//		Purpose: Never any data to write
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::StreamClosed()
+{
+	return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::BytesLeftToRead()
+//		Purpose: Can tell how many bytes there are to go
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileRead::BytesLeftToRead()
+{
+	return GetFileSize() - GetPosition();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::GetDiscUsageInBlocks()
+//		Purpose: Return how many blocks are used.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileRead::GetDiscUsageInBlocks()
+{
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	return RaidFileUtil::DiscUsageInBlocks(GetFileSize(), rdiscSet);
+}
+
+
+
+
diff --git a/lib/raidfile/RaidFileRead.h b/lib/raidfile/RaidFileRead.h
new file mode 100755
index 00000000..93bf7388
--- /dev/null
+++ b/lib/raidfile/RaidFileRead.h
@@ -0,0 +1,72 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileRead.h
+//		Purpose: Read Raid like Files
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+
+#ifndef RAIDFILEREAD__H
+#define RAIDFILEREAD__H
+
+#include <string>
+#include <memory>
+#include <vector>
+
+#include "IOStream.h"
+
+class RaidFileDiscSet;
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileRead
+//		Purpose: Read RAID like files
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+class RaidFileRead : public IOStream
+{
+protected:
+	RaidFileRead(int SetNumber, const std::string &Filename);
+public:
+	virtual ~RaidFileRead();
+private:
+	RaidFileRead(const RaidFileRead &rToCopy);
+	
+public:
+	// Open a raid file
+	static std::auto_ptr<RaidFileRead> Open(int SetNumber, const std::string &Filename, int64_t *pRevisionID = 0, int BufferSizeHint = 4096);
+
+	// Extra info
+	virtual pos_type GetFileSize() const = 0;
+
+	// Utility functions
+	static bool FileExists(int SetNumber, const std::string &rFilename, int64_t *pRevisionID = 0);
+	static bool DirectoryExists(const RaidFileDiscSet &rSet, const std::string &rDirName);
+	static bool DirectoryExists(int SetNumber, const std::string &rDirName);
+	enum
+	{
+		DirReadType_FilesOnly = 0,
+		DirReadType_DirsOnly = 1
+	};
+	static bool ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput);
+
+	// Common IOStream interface implementation
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual bool StreamClosed();
+	virtual pos_type BytesLeftToRead();
+
+	pos_type GetDiscUsageInBlocks();
+
+	typedef int64_t FileSizeType;
+
+protected:
+	int mSetNumber;
+	std::string mFilename;
+};
+
+#endif // RAIDFILEREAD__H
+
diff --git a/lib/raidfile/RaidFileUtil.cpp b/lib/raidfile/RaidFileUtil.cpp
new file mode 100755
index 00000000..9177c8ba
--- /dev/null
+++ b/lib/raidfile/RaidFileUtil.cpp
@@ -0,0 +1,188 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileUtil.cpp
+//		Purpose: Utilities for raid files
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "RaidFileUtil.h"
+#include "FileModificationTime.h"
+#include "RaidFileRead.h"	// for type definition
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileUtil::RaidFileExists(RaidFileDiscSet &, const std::string &)
+//		Purpose: Check to see the state of a RaidFile on disc (doesn't look at contents,
+//				 just at existense of files)
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pStartDisc, int *pExisitingFiles, int64_t *pRevisionID)
+{
+	if(pExisitingFiles)
+	{
+		*pExisitingFiles = 0;
+	}
+	
+	// For stat call, although the results are not examined
+	struct stat st;
+
+	// check various files
+	int startDisc = 0;
+	{
+		std::string writeFile(RaidFileUtil::MakeWriteFileName(rDiscSet, rFilename, &startDisc));
+		if(pStartDisc)
+		{
+			*pStartDisc = startDisc;
+		}
+		if(::stat(writeFile.c_str(), &st) == 0)
+		{
+			// write file exists, use that
+			
+			// Get unique ID
+			if(pRevisionID != 0)
+			{
+				(*pRevisionID) = FileModificationTime(st);
+#ifdef PLATFORM_LINUX
+				// On linux, the time resolution is very low for modification times.
+				// So add the size to it to give a bit more chance of it changing.
+				// TODO: Make this better.
+				(*pRevisionID) += st.st_size;
+#endif
+			}
+			
+			// return non-raid file
+			return NonRaid;
+		}
+	}
+	
+	// Now see how many of the raid components exist
+	int64_t revisionID = 0;
+	int setSize = rDiscSet.size();
+	int rfCount = 0;
+#ifdef PLATFORM_LINUX
+	// TODO: replace this with better linux revision ID detection
+	int64_t revisionIDplus = 0;
+#endif
+	for(int f = 0; f < setSize; ++f)
+	{
+		std::string componentFile(RaidFileUtil::MakeRaidComponentName(rDiscSet, rFilename, (f + startDisc) % setSize));
+		if(::stat(componentFile.c_str(), &st) == 0)
+		{
+			// Component file exists, add to count
+			rfCount++;
+			// Set flags for existance?
+			if(pExisitingFiles)
+			{
+				(*pExisitingFiles) |= (1 << f);
+			}
+			// Revision ID
+			if(pRevisionID != 0)
+			{
+				int64_t rid = FileModificationTime(st);
+				if(rid > revisionID) revisionID = rid;
+#ifdef PLATFORM_LINUX
+				revisionIDplus += st.st_size;
+#endif
+			}
+		}
+	}
+	if(pRevisionID != 0)
+	{
+		(*pRevisionID) = revisionID;
+#ifdef PLATFORM_LINUX
+		(*pRevisionID) += revisionIDplus;
+#endif
+	}
+	
+	// Return a status based on how many parts are available
+	if(rfCount == setSize)
+	{
+		return AsRaid;
+	}
+	else if((setSize > 1) && rfCount == (setSize - 1))
+	{
+		return AsRaidWithMissingReadable;
+	}
+	else if(rfCount > 0)
+	{
+		return AsRaidWithMissingNotRecoverable;
+	}
+	
+	return NoFile;	// Obviously doesn't exist
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileUtil::DiscUsageInBlocks(int64_t, const RaidFileDiscSet &)
+//		Purpose: Returns the size of the file in blocks, given the file size and disc set
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+int64_t RaidFileUtil::DiscUsageInBlocks(int64_t FileSize, const RaidFileDiscSet &rDiscSet)
+{
+	// Get block size
+	int blockSize = rDiscSet.GetBlockSize();
+
+	// OK... so as the size of the file is always sizes of stripe1 + stripe2, we can
+	// do a very simple calculation for the main data.
+	int64_t blocks = (FileSize + (((int64_t)blockSize) - 1)) / ((int64_t)blockSize);
+	
+	// It's just that simple calculation for non-RAID disc sets
+	if(rDiscSet.IsNonRaidSet())
+	{
+		return blocks;
+	}
+
+	// It's the parity which is mildly complex.
+	// First of all, add in size for all but the last two blocks.
+	int64_t parityblocks = (FileSize / ((int64_t)blockSize)) / 2;
+	blocks += parityblocks;
+	
+	// Work out how many bytes are left
+	int bytesOver = (int)(FileSize - (parityblocks * ((int64_t)(blockSize*2))));
+	
+	// Then... (let compiler optimise this out)
+	if(bytesOver == 0)
+	{
+		// Extra block for the size info
+		blocks++;
+	}
+	else if(bytesOver == sizeof(RaidFileRead::FileSizeType))
+	{
+		// For last block of parity, plus the size info
+		blocks += 2;
+	}
+	else if(bytesOver < blockSize)
+	{
+		// Just want the parity block
+		blocks += 1;
+	}
+	else if(bytesOver == blockSize || bytesOver >= ((blockSize*2)-((int)sizeof(RaidFileRead::FileSizeType))))
+	{
+		// Last block, plus size info
+		blocks += 2;
+	}
+	else
+	{
+		// Just want parity block
+		blocks += 1;
+	}
+	
+	return blocks;
+}
+
+
diff --git a/lib/raidfile/RaidFileUtil.h b/lib/raidfile/RaidFileUtil.h
new file mode 100755
index 00000000..16670bf1
--- /dev/null
+++ b/lib/raidfile/RaidFileUtil.h
@@ -0,0 +1,97 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileUtil.h
+//		Purpose: Utilities for the raid file classes
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+
+#ifndef RAIDFILEUTIL__H
+#define RAIDFILEUTIL__H
+
+#include "RaidFileController.h"
+#include "RaidFileException.h"
+
+// note: these are hardcoded into the directory searching code
+#define RAIDFILE_EXTENSION			".rf"
+#define RAIDFILE_WRITE_EXTENSION	".rfw"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileUtil
+//		Purpose: Utility functions for RaidFile classes
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+class RaidFileUtil
+{
+public:
+	typedef enum 
+	{
+		NoFile = 0,
+		NonRaid = 1,
+		AsRaid = 2,
+		AsRaidWithMissingReadable = 3,
+		AsRaidWithMissingNotRecoverable = 4
+	} ExistType;
+	
+	typedef enum
+	{
+		Stripe1Exists = 1,
+		Stripe2Exists = 2,
+		ParityExists = 4
+	};
+	
+	static ExistType RaidFileExists(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pStartDisc = 0, int *pExisitingFiles = 0, int64_t *pRevisionID = 0);
+	
+	static int64_t DiscUsageInBlocks(int64_t FileSize, const RaidFileDiscSet &rDiscSet);
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    std::string MakeRaidComponentName(RaidFileDiscSet &, const std::string &, int)
+	//		Purpose: Returns the OS filename for a file of part of a disc set
+	//		Created: 2003/07/11
+	//
+	// --------------------------------------------------------------------------	
+	static inline std::string MakeRaidComponentName(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int Disc)
+	{
+		if(Disc < 0 || Disc >= (int)rDiscSet.size())
+		{
+			THROW_EXCEPTION(RaidFileException, NoSuchDiscSet)
+		}
+		std::string r(rDiscSet[Disc]);
+		r += DIRECTORY_SEPARATOR_ASCHAR;
+		r += rFilename;
+		r += RAIDFILE_EXTENSION;
+		return r;
+	}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    std::string MakeWriteFileName(RaidFileDiscSet &, const std::string &)
+	//		Purpose: Returns the OS filename for the temporary write file
+	//		Created: 2003/07/11
+	//
+	// --------------------------------------------------------------------------	
+	static inline std::string MakeWriteFileName(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pOnDiscSet = 0)
+	{
+		int livesOnSet = rDiscSet.GetSetNumForWriteFiles(rFilename);
+		
+		// does the caller want to know which set it's on?
+		if(pOnDiscSet) *pOnDiscSet = livesOnSet;
+		
+		// Make the string
+		std::string r(rDiscSet[livesOnSet]);
+		r += DIRECTORY_SEPARATOR_ASCHAR;
+		r += rFilename;
+		r += RAIDFILE_WRITE_EXTENSION;
+		return r;
+	}
+};
+
+#endif // RAIDFILEUTIL__H
+
diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp
new file mode 100755
index 00000000..414f24e6
--- /dev/null
+++ b/lib/raidfile/RaidFileWrite.cpp
@@ -0,0 +1,817 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileWrite.cpp
+//		Purpose: Writing RAID like files
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/file.h>
+
+#include <stdio.h>
+#include <string.h>
+
+#include "Guards.h"
+#include "RaidFileWrite.h"
+#include "RaidFileController.h"
+#include "RaidFileException.h"
+#include "RaidFileUtil.h"
+#include "Utils.h"
+// For DirectoryExists fn
+#include "RaidFileRead.h"
+
+#include "MemLeakFindOn.h"
+
+// should be a multiple of 2
+#define TRANSFORM_BLOCKS_TO_LOAD		4
+// Must have this number of discs in the set
+#define TRANSFORM_NUMBER_DISCS_REQUIRED	3
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::RaidFileWrite(int, const std::string &)
+//		Purpose: Construtor, just stores requried details
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+RaidFileWrite::RaidFileWrite(int SetNumber, const std::string &Filename)
+	: mSetNumber(SetNumber),
+	  mFilename(Filename),
+	  mOSFileHandle(-1)		// not valid file handle
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::~RaidFileWrite()
+//		Purpose: Destructor (will discard written file if not commited)
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+RaidFileWrite::~RaidFileWrite()
+{
+	if(mOSFileHandle != -1)
+	{
+		Discard();
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Open()
+//		Purpose: Opens the file for writing
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Open(bool AllowOverwrite)
+{
+	if(mOSFileHandle != -1)
+	{
+		THROW_EXCEPTION(RaidFileException, AlreadyOpen)
+	}
+	
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+
+	// Check for overwriting? (step 1)
+	if(!AllowOverwrite)
+	{
+		// See if the file exists already -- can't overwrite existing files
+		RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename);
+		if(existance != RaidFileUtil::NoFile)
+		{
+			TRACE2("Trying to overwrite raidfile %d %s\n", mSetNumber, mFilename.c_str());
+			THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile)
+		}
+	}
+
+	// Get the filename for the write file
+	std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename));
+	// Add on a temporary extension
+	writeFilename += 'X';
+
+	// Attempt to open
+	mOSFileHandle = ::open(writeFilename.c_str(), O_WRONLY | O_CREAT,
+		S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFile)
+	}
+	
+	// Get a lock on the write file
+	if(::flock(mOSFileHandle, LOCK_EX | LOCK_NB) != 0)
+	{
+		// Lock was not obtained.
+		bool wasLocked = (errno == EWOULDBLOCK);
+		// Close the file
+		::close(mOSFileHandle);
+		mOSFileHandle = -1;
+		// Report an exception?
+		if(wasLocked)
+		{
+			THROW_EXCEPTION(RaidFileException, FileIsCurrentlyOpenForWriting)
+		}
+		else
+		{
+			// Random error occured
+			THROW_EXCEPTION(RaidFileException, OSError)
+		}
+	}
+	
+	// Truncate it to size zero
+	if(::ftruncate(mOSFileHandle, 0) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFileOnTruncate)
+	}
+	
+	// Done!
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Write(const void *, int)
+//		Purpose: Writes a block of data
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Write(const void *pBuffer, int Length)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Write data
+	int written = ::write(mOSFileHandle, pBuffer, Length);
+	if(written != Length)
+	{
+		TRACE3("RaidFileWrite::Write: Write failure, Length = %d, written = %d, errno = %d\n", Length, written, errno);
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::GetPosition()
+//		Purpose: Returns current position in file
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileWrite::GetPosition() const
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Use lseek to find the current file position
+	off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR);
+	if(p == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	return p;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Seek(RaidFileWrite::pos_type, bool)
+//		Purpose: Seeks in the file, relative to current position if Relative is true.
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Seek(IOStream::pos_type SeekTo, int SeekType)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Seek...
+	if(::lseek(mOSFileHandle, SeekTo, ConvertSeekTypeToOSWhence(SeekType)) == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Commit(bool)
+//		Purpose: Closes, and commits the written file
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Commit(bool ConvertToRaidNow)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Rename it into place -- BEFORE it's closed so lock remains
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	// Get the filename for the write file
+	std::string renameTo(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename));
+	// And the current name
+	std::string renameFrom(renameTo + 'X');
+	if(::rename(renameFrom.c_str(), renameTo.c_str()) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	// Close file...
+	if(::close(mOSFileHandle) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	mOSFileHandle = -1;
+	
+	// Raid it?
+	if(ConvertToRaidNow)
+	{
+		TransformToRaidStorage();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Discard()
+//		Purpose: Closes, discarding the data written.
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Discard()
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+
+	// Get the filename for the write file (temporary)
+	std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename));
+	writeFilename += 'X';
+	
+	// Unlink and close it
+	if((::unlink(writeFilename.c_str()) != 0)
+		|| (::close(mOSFileHandle) != 0))
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	// reset file handle
+	mOSFileHandle = -1;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::TransformToRaidStorage()
+//		Purpose: Turns the file into the RAID storage form
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::TransformToRaidStorage()
+{
+	// open?
+	if(mOSFileHandle != -1)
+	{
+		THROW_EXCEPTION(RaidFileException, WriteFileOpenOnTransform)
+	}
+
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	if(rdiscSet.IsNonRaidSet())
+	{
+		// Not in RAID mode -- do nothing
+		return;
+	}
+	// Otherwise check that it's the right sized set
+	if(TRANSFORM_NUMBER_DISCS_REQUIRED != rdiscSet.size())
+	{
+		THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet)
+	}
+	unsigned int blockSize = rdiscSet.GetBlockSize();
+
+	// Get the filename for the write file (and get the disc set name for the start disc)
+	int startDisc = 0;
+	std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename, &startDisc));
+	
+	// Open it
+	FileHandleGuard<> writeFile(writeFilename.c_str());
+
+	// Get file information for write file	
+	struct stat writeFileStat;
+	if(::fstat(writeFile, &writeFileStat) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+//	// DEBUG MODE -- check file system size block size is same as block size for files
+//	// doesn't really apply, as space benefits of using fragment size are worth efficiency,
+//	// and anyway, it'll be buffered eventually so it won't matter.
+//	#ifndef NDEBUG
+//	{
+//		if(writeFileStat.st_blksize != blockSize)
+//		{
+//			TRACE2("TransformToRaidStorage: optimal block size of file = %d, of set = %d, MISMATCH\n",
+//					writeFileStat.st_blksize, blockSize);
+//		}
+//	}
+//	#endif
+	
+	// How many blocks is the file? (rounding up)
+	int writeFileSizeInBlocks = (writeFileStat.st_size + (blockSize - 1)) / blockSize;
+	// And how big should the buffer be? (round up to multiple of 2, and no bigger than the preset limit)
+	int bufferSizeBlocks = (writeFileSizeInBlocks + 1) & ~1;
+	if(bufferSizeBlocks > TRANSFORM_BLOCKS_TO_LOAD) bufferSizeBlocks = TRANSFORM_BLOCKS_TO_LOAD;
+	// How big should the buffer be?
+	int bufferSize = (TRANSFORM_BLOCKS_TO_LOAD * blockSize);
+	
+	// Allocate buffer...
+	MemoryBlockGuard<char*> buffer(bufferSize);
+	
+	// Allocate buffer for parity file
+	MemoryBlockGuard<char*> parityBuffer(blockSize);
+	
+	// Get filenames of eventual files
+	std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 0) % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 1) % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 2) % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	// Make write equivalents
+	std::string stripe1FilenameW(stripe1Filename + 'P');
+	std::string stripe2FilenameW(stripe2Filename + 'P');
+	std::string parityFilenameW(parityFilename + 'P');
+	
+	
+	// Then open them all for writing (in strict order)
+	try
+	{
+#ifdef PLATFORM_LINUX
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> stripe1(stripe1FilenameW.c_str());
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> stripe2(stripe2FilenameW.c_str());
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> parity(parityFilenameW.c_str());
+#else
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> stripe1(stripe1FilenameW.c_str());
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> stripe2(stripe2FilenameW.c_str());
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> parity(parityFilenameW.c_str());
+#endif
+
+		// Then... read in data...
+		int bytesRead = -1;
+		bool sizeRecordRequired = false;
+		int blocksDone = 0;
+		while((bytesRead = ::read(writeFile, buffer, bufferSize)) > 0)
+		{
+			// Blocks to do...
+			int blocksToDo = (bytesRead + (blockSize - 1)) / blockSize;
+
+			// Need to add zeros to end?
+			int blocksRoundUp = (blocksToDo + 1) & ~1;
+			int zerosEnd = (blocksRoundUp * blockSize);
+			if(bytesRead != zerosEnd)
+			{
+				// Set the end of the blocks to zero
+				::memset(buffer + bytesRead, 0, zerosEnd - bytesRead);
+			}
+
+			// number of int's to XOR
+			unsigned int num = blockSize / sizeof(unsigned int);
+
+			// Then... calculate and write parity data
+			for(int b = 0; b < blocksToDo; b += 2)
+			{
+				// Calculate int pointers
+				unsigned int *pstripe1 = (unsigned int *)(buffer + (b * blockSize));
+				unsigned int *pstripe2 = (unsigned int *)(buffer + ((b+1) * blockSize));
+				unsigned int *pparity = (unsigned int *)((char*)parityBuffer);
+
+				// Do XOR
+				for(unsigned int n = 0; n < num; ++n)
+				{
+					pparity[n] = pstripe1[n] ^ pstripe2[n];
+				}
+				
+				// Size of parity to write...
+				int parityWriteSize = blockSize;
+				
+				// Adjust if it's the last block
+				if((blocksDone + (b + 2)) >= writeFileSizeInBlocks)
+				{
+					// Yes...
+					unsigned int bytesInLastTwoBlocks = bytesRead - (b * blockSize);
+					
+					// Some special cases...
+					// Zero will never happen... but in the (imaginary) case it does, the file size will be appended
+					// by the test at the end.
+					if(bytesInLastTwoBlocks == sizeof(RaidFileRead::FileSizeType)
+						|| bytesInLastTwoBlocks == blockSize)
+					{
+						// Write the entire block, and put the file size at end
+						sizeRecordRequired = true;
+					}
+					else if(bytesInLastTwoBlocks < blockSize)
+					{
+						// write only these bits
+						parityWriteSize = bytesInLastTwoBlocks;
+					}
+					else if(bytesInLastTwoBlocks < ((blockSize * 2) - sizeof(RaidFileRead::FileSizeType)))
+					{
+						// XOR in the size at the end of the parity block
+						ASSERT(sizeof(RaidFileRead::FileSizeType) == (2*sizeof(unsigned int)));
+#ifdef PLATFORM_LINUX
+						ASSERT(sizeof(RaidFileRead::FileSizeType) >= sizeof(off_t));
+#else
+						ASSERT(sizeof(RaidFileRead::FileSizeType) == sizeof(off_t));
+#endif
+						int sizePos = (blockSize/sizeof(unsigned int)) - 2;
+						RaidFileRead::FileSizeType sw = hton64(writeFileStat.st_size);
+						unsigned int *psize = (unsigned int *)(&sw);
+						pparity[sizePos+0] = pstripe1[sizePos+0] ^ psize[0];
+						pparity[sizePos+1] = pstripe1[sizePos+1] ^ psize[1];
+					}
+					else
+					{
+						// Write the entire block, and put the file size at end
+						sizeRecordRequired = true;
+					}
+				}
+
+				// Write block
+				if(::write(parity, parityBuffer, parityWriteSize) != parityWriteSize)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+			}
+
+			// Write stripes
+			char *writeFrom = buffer;
+			for(int l = 0; l < blocksToDo; ++l)
+			{
+				// Write the block
+				int toWrite = (l == (blocksToDo - 1))
+								?(bytesRead - ((blocksToDo-1)*blockSize))
+								:blockSize;
+				if(::write(((l&1)==0)?stripe1:stripe2, writeFrom, toWrite) != toWrite)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}			
+
+				// Next block
+				writeFrom += blockSize;
+			}
+			
+			// Count of blocks done
+			blocksDone += blocksToDo;
+		}
+		// Error on read?
+		if(bytesRead == -1)
+		{
+			THROW_EXCEPTION(RaidFileException, OSError)
+		}
+		
+		// Special case for zero length files
+		if(writeFileStat.st_size == 0)
+		{
+			sizeRecordRequired = true;
+		}
+
+		// Might need to write the file size to the end of the parity file
+		// if it can't be worked out some other means -- size is required to rebuild the file if one of the stripe files is missing
+		if(sizeRecordRequired)
+		{
+#ifdef PLATFORM_LINUX
+			ASSERT(sizeof(writeFileStat.st_size) <= sizeof(RaidFileRead::FileSizeType));
+#else
+			ASSERT(sizeof(writeFileStat.st_size) == sizeof(RaidFileRead::FileSizeType));
+#endif
+			RaidFileRead::FileSizeType sw = hton64(writeFileStat.st_size);
+			ASSERT((::lseek(parity, 0, SEEK_CUR) % blockSize) == 0);
+			if(::write(parity, &sw, sizeof(sw)) != sizeof(sw))
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+
+		// Then close the written files (note in reverse order of opening)
+		parity.Close();
+		stripe2.Close();
+		stripe1.Close();
+		
+		// Rename them into place
+		if(::rename(stripe1FilenameW.c_str(), stripe1Filename.c_str()) != 0
+			|| ::rename(stripe2FilenameW.c_str(), stripe2Filename.c_str()) != 0
+			|| ::rename(parityFilenameW.c_str(), parityFilename.c_str()) != 0)
+		{
+			THROW_EXCEPTION(RaidFileException, OSError)
+		}
+
+		// Close the write file
+		writeFile.Close();
+
+		// Finally delete the write file
+		if(::unlink(writeFilename.c_str()) != 0)
+		{
+			THROW_EXCEPTION(RaidFileException, OSError)
+		}
+	}
+	catch(...)
+	{
+		// Unlink all the dodgy files
+		::unlink(stripe1Filename.c_str());
+		::unlink(stripe2Filename.c_str());
+		::unlink(parityFilename.c_str());
+		::unlink(stripe1FilenameW.c_str());
+		::unlink(stripe2FilenameW.c_str());
+		::unlink(parityFilenameW.c_str());
+		
+		// and send the error on its way
+		throw;
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Delete()
+//		Purpose: Deletes a RAID file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Delete()
+{
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+
+	// See if the file exists already -- can't delete files which don't exist
+	RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename);
+	if(existance == RaidFileUtil::NoFile)
+	{
+		THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist)
+	}
+
+	// Get the filename for the write file
+	std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename));
+
+	// Attempt to delete it
+	bool deletedSomething = false;
+	if(::unlink(writeFilename.c_str()) == 0)
+	{
+		deletedSomething = true;
+	}
+	
+	// If we're not running in RAID mode, stop now
+	if(rdiscSet.size() == 1)
+	{
+		return;
+	}
+	
+	// Now the other files
+	std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 0 % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 1 % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 2 % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	if(::unlink(stripe1Filename.c_str()) == 0)
+	{
+		deletedSomething = true;
+	}
+	if(::unlink(stripe2Filename.c_str()) == 0)
+	{
+		deletedSomething = true;
+	}
+	if(::unlink(parityFilename.c_str()) == 0)
+	{
+		deletedSomething = true;
+	}
+	
+	// Check something happened
+	if(!deletedSomething)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::CreateDirectory(int, const std::string &, bool, int)
+//		Purpose: Creates a directory within the raid file directories with the given name.
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive, int mode)
+{
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+	// Pass on...
+	CreateDirectory(rdiscSet, rDirName, Recursive, mode);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::CreateDirectory(const RaidFileDiscSet &, const std::string &, bool, int)
+//		Purpose: Creates a directory within the raid file directories with the given name.
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive, int mode)
+{
+	if(Recursive)
+	{
+		// split up string
+		std::vector<std::string> elements;
+		SplitString(rDirName, DIRECTORY_SEPARATOR_ASCHAR, elements);
+		
+		// Do each element in turn
+		std::string pn;
+		for(unsigned int e = 0; e < elements.size(); ++e)
+		{
+			// Only do this if the element has some text in it
+			if(elements[e].size() > 0)
+			{
+				pn += elements[e];
+				if(!RaidFileRead::DirectoryExists(rSet, pn))
+				{
+					CreateDirectory(rSet, pn, false, mode);
+				}
+
+				// add separator
+				pn += DIRECTORY_SEPARATOR_ASCHAR;
+			}
+		}
+	
+		return;
+	}
+	
+	// Create a directory in every disc of the set
+	for(unsigned int l = 0; l < rSet.size(); ++l)
+	{
+		// build name
+		std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName);
+	
+		// attempt to create
+		if(::mkdir(dn.c_str(), mode) != 0)
+		{
+			if(errno == EEXIST)
+			{
+				// No. Bad things.
+				THROW_EXCEPTION(RaidFileException, FileExistsInDirectoryCreation)
+			}
+			else
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Read(void *, int, int)
+//		Purpose: Unsupported, will exception
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+int RaidFileWrite::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Close()
+//		Purpose: Close, discarding file.
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Close()
+{
+	TRACE0("Warning: RaidFileWrite::Close() called, discarding file\n");
+	if(mOSFileHandle != -1)
+	{
+		Discard();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::StreamDataLeft()
+//		Purpose: Never any data left to read!
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileWrite::StreamDataLeft()
+{
+	return false;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::StreamClosed()
+//		Purpose: Is stream closed for writing?
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileWrite::StreamClosed()
+{
+	return mOSFileHandle == -1;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::GetFileSize()
+//		Purpose: Returns the size of the file written.
+//				 Can only be used before the file is commited.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileWrite::GetFileSize()
+{
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, CanOnlyGetFileSizeBeforeCommit)
+	}
+	
+	// Stat to get size
+	struct stat st;
+	if(fstat(mOSFileHandle, &st) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	return st.st_size;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::GetDiscUsageInBlocks()
+//		Purpose: Returns the amount of disc space used, in blocks.
+//				 Can only be used before the file is commited.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileWrite::GetDiscUsageInBlocks()
+{
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, CanOnlyGetUsageBeforeCommit)
+	}
+	
+	// Stat to get size
+	struct stat st;
+	if(fstat(mOSFileHandle, &st) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	// Then return calculation
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	return RaidFileUtil::DiscUsageInBlocks(st.st_size, rdiscSet);
+}
+
+
diff --git a/lib/raidfile/RaidFileWrite.h b/lib/raidfile/RaidFileWrite.h
new file mode 100755
index 00000000..d7e51f21
--- /dev/null
+++ b/lib/raidfile/RaidFileWrite.h
@@ -0,0 +1,66 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileWrite.h
+//		Purpose: Writing RAID like files
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef RAIDFILEWRITE__H
+#define RAIDFILEWRITE__H
+
+#include <string>
+
+#include "IOStream.h"
+
+class RaidFileDiscSet;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileWrite
+//		Purpose: Writing RAID like files
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+class RaidFileWrite : public IOStream
+{
+public:
+	RaidFileWrite(int SetNumber, const std::string &Filename);
+	~RaidFileWrite();
+private:
+	RaidFileWrite(const RaidFileWrite &rToCopy);
+
+public:
+	// IOStream interface
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);	// will exception
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual pos_type GetPosition() const;
+	virtual void Seek(pos_type Offset, int SeekType);
+	virtual void Close();		// will discard the file! Use commit instead.
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+	// Extra bits
+	void Open(bool AllowOverwrite = false);
+	void Commit(bool ConvertToRaidNow = false);
+	void Discard();
+	void TransformToRaidStorage();
+	void Delete();
+	pos_type GetFileSize();
+	pos_type GetDiscUsageInBlocks();
+	
+	static void CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive = false, int mode = 0777);
+	static void CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive = false, int mode = 0777);
+	
+private:
+
+private:
+	int mSetNumber;
+	std::string mFilename;
+	int mOSFileHandle;
+};
+
+#endif // RAIDFILEWRITE__H
+
diff --git a/lib/raidfile/raidfile-config b/lib/raidfile/raidfile-config
new file mode 100755
index 00000000..b3f31077
--- /dev/null
+++ b/lib/raidfile/raidfile-config
@@ -0,0 +1,97 @@
+#!/usr/bin/perl
+use strict;
+
+# should be running as root
+if($> != 0)
+{
+	printf "\nWARNING: this should be run as root\n\n"
+}
+
+# check and get command line parameters
+if($#ARGV != 4 && $#ARGV != 2)
+{
+	print <<__E;
+
+Setup raidfile config utility.
+
+Bad command line parameters.
+Usage:
+	raidfile-config config-dir block-size dir0 [dir1 dir2]
+
+config-dir usually /etc/box
+block-size must be a power of two, and usually the block or fragment size of your filing system
+dir0, dir1, dir2 are the directories used as the root of the raid file system
+
+If only one directory is specified, then userland RAID is disabled. Specifying three directories
+enables it.
+
+__E
+	exit(1);
+}
+
+my ($config_dir,$block_size,@dirs) = @ARGV;
+
+my $conf = $config_dir . '/raidfile.conf';
+
+# check dirs are unique, and exist
+my %d;
+for(@dirs)
+{
+	die "$_ is used twice" if exists $d{$_};
+	die "$_ is not a directory" unless -d $_;
+	die "$_ should be an absolute path" unless m/\A\//;
+	$d{$_} = 1;
+}
+
+# check block size is OK
+$block_size = int($block_size);
+die "Bad block size" if $block_size <= 0;
+my $c = 1;
+while(1)
+{
+	last if $c == $block_size;
+	die "Block size $block_size is not a power of two" if $c > $block_size;
+	$c = $c * 2;
+}
+
+# check that it doesn't already exist
+if(-f $conf)
+{
+	die "$conf already exists. Delete and try again"
+}
+
+# create directory
+if(!-d $config_dir)
+{
+	print "Creating $config_dir...\n";
+	mkdir $config_dir,0755 or die "Can't create $config_dir";
+}
+
+# adjust if userland RAID is disabled
+if($#dirs == 0)
+{
+	$dirs[1] = $dirs[0];
+	$dirs[2] = $dirs[0];
+	print "WARNING: userland RAID is disabled.\n"
+}
+
+# write the file
+open CONFIG,">$conf" or die "Can't open $conf for writing";
+
+print CONFIG <<__E;
+
+disc0
+{
+	SetNumber = 0
+	BlockSize = $block_size
+	Dir0 = $dirs[0]
+	Dir1 = $dirs[1]
+	Dir2 = $dirs[2]
+}
+
+__E
+
+close CONFIG;
+
+print "Config file written.\n";
+
diff --git a/lib/server/ConnectionException.txt b/lib/server/ConnectionException.txt
new file mode 100644
index 00000000..5056754f
--- /dev/null
+++ b/lib/server/ConnectionException.txt
@@ -0,0 +1,27 @@
+EXCEPTION Connection 7
+
+# for historic reasons not all numbers are used
+
+SocketWriteError			6	Probably a network issue between client and server.
+SocketReadError				7	Probably a network issue between client and server.
+SocketNameLookupError		9	Check hostname specified.
+SocketShutdownError			12
+SocketConnectError			15	Probably a network issue between client and server, bad hostname, or server not running.
+TLSHandshakeFailed			30
+TLSShutdownFailed			32
+TLSWriteFailed				33	Probably a network issue between client and server.
+TLSReadFailed				34	Probably a network issue between client and server.
+TLSNoPeerCertificate		36
+TLSPeerCertificateInvalid	37	Check certification process
+TLSClosedWhenWriting		38
+TLSHandshakeTimedOut		39
+Protocol_Timeout			41	Probably a network issue between client and server.
+Protocol_ObjTooBig			42
+Protocol_BadCommandRecieved				44
+Protocol_UnknownCommandRecieved			45
+Protocol_TriedToExecuteReplyCommand		46
+Protocol_UnexpectedReply				47	Server probably reported an error.
+Protocol_HandshakeFailed				48
+Protocol_StreamWhenObjExpected			49
+Protocol_ObjWhenStreamExpected			50
+Protocol_TimeOutWhenSendingStream		52	Probably a network issue between client and server.
diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp
new file mode 100755
index 00000000..31997fb6
--- /dev/null
+++ b/lib/server/Daemon.cpp
@@ -0,0 +1,530 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Daemon.cpp
+//		Purpose: Basic daemon functionality
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <syslog.h>
+#include <signal.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include "Daemon.h"
+#include "Configuration.h"
+#include "ServerException.h"
+#include "Guards.h"
+#include "UnixUser.h"
+
+#include "MemLeakFindOn.h"
+
+Daemon *Daemon::spDaemon = 0;
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::Daemon()
+//		Purpose: Constructor
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+Daemon::Daemon()
+	: mpConfiguration(0),
+	  mReloadConfigWanted(false),
+	  mTerminateWanted(false)
+{
+	if(spDaemon != 0)
+	{
+		THROW_EXCEPTION(ServerException, AlreadyDaemonConstructed)
+	}
+	spDaemon = this;
+	
+	// And in debug builds, we'll switch on assert failure logging to syslog
+	ASSERT_FAILS_TO_SYSLOG_ON
+	// And trace goes to syslog too
+	TRACE_TO_SYSLOG(true)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::~Daemon()
+//		Purpose: Destructor
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+Daemon::~Daemon()
+{
+	if(mpConfiguration)
+	{
+		delete mpConfiguration;
+		mpConfiguration = 0;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::Main(const char *, int, const char *[])
+//		Purpose: Starts the daemon off -- equivalent of C main() function
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[])
+{
+	// Banner (optional)
+	{
+		const char *banner = DaemonBanner();
+		if(banner != 0)
+		{
+			printf("%s", banner);
+		}
+	}
+
+	std::string pidFileName;
+	const char *configfile = 0;
+
+	try
+	{
+		// Find filename of config file
+		configfile = DefaultConfigFile;
+		if(argc >= 2)
+		{
+			// First argument is config file, or it's -c and the next arg is the config file
+			if(::strcmp(argv[1], "-c") == 0 && argc >= 3)
+			{
+				configfile = argv[2];
+			}
+			else
+			{
+				configfile = argv[1];
+			}
+		}
+		
+		// Test mode with no daemonisation?
+		bool asDaemon = true;
+		if(argc >= 3)
+		{
+			if(::strcmp(argv[2], "SINGLEPROCESS") == 0)
+			{
+				asDaemon = false;
+			}
+		}
+
+		// Load the configuration file.
+		std::string errors;
+		std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(configfile, GetConfigVerify(), errors);
+
+		// Got errors?
+		if(pconfig.get() == 0 || !errors.empty())
+		{
+			// Tell user about errors
+			fprintf(stderr, "%s: Errors in config file %s:\n%s", DaemonName(), configfile, errors.c_str());
+			// And give up
+			return 1;
+		}
+		
+		// Store configuration
+		mpConfiguration = pconfig.release();
+		
+		// Server configuration
+		const Configuration &serverConfig(mpConfiguration->GetSubConfiguration("Server"));
+		
+		// Let the derived class have a go at setting up stuff in the initial process
+		SetupInInitialProcess();
+		
+		// Set signal handler
+		if(::signal(SIGHUP, SignalHandler) == SIG_ERR || ::signal(SIGTERM, SignalHandler) == SIG_ERR)
+		{
+			THROW_EXCEPTION(ServerException, DaemoniseFailed)
+		}
+		
+		// Open PID file for writing
+		pidFileName = serverConfig.GetKeyValue("PidFile");
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_TRUNC), (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)> pidFile(pidFileName.c_str());
+		
+		// Handle changing to a different user
+		if(serverConfig.KeyExists("User"))
+		{
+			// Config file specifies an user -- look up
+			UnixUser daemonUser(serverConfig.GetKeyValue("User").c_str());
+			
+			// Change the owner on the PID file, so it can be deleted properly on termination
+			if(::fchown(pidFile, daemonUser.GetUID(), daemonUser.GetGID()) != 0)
+			{
+				THROW_EXCEPTION(ServerException, CouldNotChangePIDFileOwner)
+			}
+			
+			// Change the process ID
+			daemonUser.ChangeProcessUser();
+		}
+	
+		if(asDaemon)
+		{
+			// Let's go... Daemonise...
+			switch(::fork())
+			{
+			case -1:
+				// error
+				THROW_EXCEPTION(ServerException, DaemoniseFailed)
+				break;
+
+			default:
+				// parent
+				_exit(0);
+				return 0;
+				break;
+
+			case 0:
+				// child
+				break;
+			}
+
+			// In child
+
+			// Set new session
+			if(::setsid() == -1)
+			{
+				::syslog(LOG_ERR, "can't setsid");
+				THROW_EXCEPTION(ServerException, DaemoniseFailed)
+			}
+
+			// Fork again...
+			switch(::fork())
+			{
+			case -1:
+				// error
+				THROW_EXCEPTION(ServerException, DaemoniseFailed)
+				break;
+
+			default:
+				// parent
+				_exit(0);
+				return 0;
+				break;
+
+			case 0:
+				// child
+				break;
+			}
+		}
+
+		// open the log
+		::openlog(DaemonName(), LOG_PID, LOG_LOCAL6);
+		// Log the start message
+		::syslog(LOG_INFO, "Starting daemon (config: %s) (version " BOX_VERSION ")", configfile);
+
+		// Write PID to file
+		char pid[32];
+		int pidsize = sprintf(pid, "%d", (int)getpid());
+		if(::write(pidFile, pid, pidsize) != pidsize)
+		{
+			::syslog(LOG_ERR, "can't write pid file");
+			THROW_EXCEPTION(ServerException, DaemoniseFailed)
+		}
+		
+		// Set up memory leak reporting
+		#ifdef BOX_MEMORY_LEAK_TESTING
+		{
+			char filename[256];
+			sprintf(filename, "%s.memleaks", DaemonName());
+			memleakfinder_setup_exit_report(filename, DaemonName());
+		}
+		#endif // BOX_MEMORY_LEAK_TESTING
+	
+		if(asDaemon)
+		{
+			// Close standard streams
+			::close(0);
+			::close(1);
+			::close(2);
+			
+			// Open and redirect them into /dev/null
+			int devnull = ::open(PLATFORM_DEV_NULL, O_RDWR, 0);
+			if(devnull == -1)
+			{
+				THROW_EXCEPTION(CommonException, OSFileError);
+			}
+			// Then duplicate them to all three handles
+			if(devnull != 0) dup2(devnull, 0);
+			if(devnull != 1) dup2(devnull, 1);
+			if(devnull != 2) dup2(devnull, 2);
+			// Close the original handle if it was opened above the std* range
+			if(devnull > 2)
+			{
+				::close(devnull);
+			}
+			
+			// And definately don't try and send anything to those file descriptors
+			// -- this has in the past sent text to something which isn't expecting it.
+			TRACE_TO_STDOUT(false);
+		}		
+	}
+	catch(BoxException &e)
+	{
+		fprintf(stderr, "%s: exception %s (%d/%d)\n", DaemonName(), e.what(), e.GetType(), e.GetSubType());
+		return 1;
+	}
+	catch(std::exception &e)
+	{
+		fprintf(stderr, "%s: exception %s\n", DaemonName(), e.what());
+		return 1;
+	}
+	catch(...)
+	{
+		fprintf(stderr, "%s: unknown exception\n", DaemonName());
+		return 1;
+	}
+	
+	// Main Daemon running
+	try
+	{
+		while(!mTerminateWanted)
+		{
+			Run();
+			
+			if(mReloadConfigWanted && !mTerminateWanted)
+			{
+				// Need to reload that config file...
+				::syslog(LOG_INFO, "Reloading configuration (config: %s)", configfile);
+				std::string errors;
+				std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(configfile, GetConfigVerify(), errors);
+
+				// Got errors?
+				if(pconfig.get() == 0 || !errors.empty())
+				{
+					// Tell user about errors
+					::syslog(LOG_ERR, "Errors in config file %s:\n%s", configfile, errors.c_str());
+					// And give up
+					return 1;
+				}
+				
+				// delete old configuration
+				delete mpConfiguration;
+				mpConfiguration = 0;
+
+				// Store configuration
+				mpConfiguration = pconfig.release();
+				
+				// Stop being marked for loading config again
+				mReloadConfigWanted = false;
+			}
+		}
+		
+		// Delete the PID file
+		::unlink(pidFileName.c_str());
+		
+		// Log
+		::syslog(LOG_INFO, "Terminating daemon");
+	}
+	catch(BoxException &e)
+	{
+		::syslog(LOG_ERR, "exception %s (%d/%d) -- terminating", e.what(), e.GetType(), e.GetSubType());
+		return 1;
+	}
+	catch(std::exception &e)
+	{
+		::syslog(LOG_ERR, "exception %s -- terminating", e.what());
+		return 1;
+	}
+	catch(...)
+	{
+		::syslog(LOG_ERR, "unknown exception -- terminating");
+		return 1;
+	}
+	
+	return 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::EnterChild()
+//		Purpose: Sets up for a child task of the main server. Call just after fork()
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void Daemon::EnterChild()
+{
+	// Unset signal handlers
+	::signal(SIGHUP, SIG_DFL);
+	::signal(SIGTERM, SIG_DFL);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::SignalHandler(int)
+//		Purpose: Signal handler
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+void Daemon::SignalHandler(int sigraised)
+{
+	if(spDaemon != 0)
+	{
+		switch(sigraised)
+		{
+		case SIGHUP:
+			spDaemon->mReloadConfigWanted = true;
+			break;
+			
+		case SIGTERM:
+			spDaemon->mTerminateWanted = true;
+			break;
+		
+		default:
+			break;
+		}
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::DaemonName() 
+//		Purpose: Returns name of the daemon
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+const char *Daemon::DaemonName() const
+{
+	return "generic-daemon";
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::DaemonBanner()
+//		Purpose: Returns the text banner for this daemon's startup
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+const char *Daemon::DaemonBanner() const
+{
+	return 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::Run()
+//		Purpose: Main run function after basic Daemon initialisation
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+void Daemon::Run()
+{
+	while(!StopRun())
+	{
+		::sleep(10);
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::GetConfigVerify()
+//		Purpose: Returns the configuration file verification structure for this daemon
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+const ConfigurationVerify *Daemon::GetConfigVerify() const
+{
+	static ConfigurationVerifyKey verifyserverkeys[] = 
+	{
+		DAEMON_VERIFY_SERVER_KEYS
+	};
+
+	static ConfigurationVerify verifyserver[] = 
+	{
+		{
+			"Server",
+			0,
+			verifyserverkeys,
+			ConfigTest_Exists | ConfigTest_LastEntry,
+			0
+		}
+	};
+
+	static ConfigurationVerify verify =
+	{
+		"root",
+		verifyserver,
+		0,
+		ConfigTest_Exists | ConfigTest_LastEntry,
+		0
+	};
+
+	return &verify;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::GetConfiguration()
+//		Purpose: Returns the daemon configuration object
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+const Configuration &Daemon::GetConfiguration() const
+{
+	if(mpConfiguration == 0)
+	{
+		// Shouldn't get anywhere near this if a configuration file can't be loaded
+		THROW_EXCEPTION(ServerException, Internal)
+	}
+	
+	return *mpConfiguration;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Daemon::SetupInInitialProcess()
+//		Purpose: A chance for the daemon to do something initial setting up in the process which
+//				 initiates everything, and after the configuration file has been read and verified.
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void Daemon::SetupInInitialProcess()
+{
+	// Base class doesn't do anything.
+}
+
+
+void Daemon::SetProcessTitle(const char *format, ...)
+{
+	// On OpenBSD, setproctitle() sets the process title to imagename: <text> (imagename)
+	// -- make sure other platforms include the image name somewhere so ps listings give
+	// useful information.
+
+#ifdef PLATFORM_HAVE_setproctitle
+	// optional arguments
+	va_list args;
+	va_start(args, format);
+
+	// Make the string
+	char title[256];
+	::vsnprintf(title, sizeof(title), format, args);
+	
+	// Set process title
+	::setproctitle("%s", title);
+	
+#endif // PLATFORM_HAVE_setproctitle
+}
+
+
diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h
new file mode 100755
index 00000000..a7b9488b
--- /dev/null
+++ b/lib/server/Daemon.h
@@ -0,0 +1,75 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Daemon.h
+//		Purpose: Basic daemon functionality
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+
+/*  NOTE: will log to local6: include a line like
+	local6.info                                             /var/log/box
+	in /etc/syslog.conf
+*/
+
+
+#ifndef DAEMON__H
+#define DAEMON__H
+
+class Configuration;
+class ConfigurationVerify;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    Daemon
+//		Purpose: Basic daemon functionality
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+class Daemon
+{
+public:
+	Daemon();
+	virtual ~Daemon();
+private:
+	Daemon(const Daemon &rToCopy);
+public:
+
+	int Main(const char *DefaultConfigFile, int argc, const char *argv[]);
+	
+	virtual void Run();
+	const Configuration &GetConfiguration() const;
+	
+	virtual const char *DaemonName() const;
+	virtual const char *DaemonBanner() const;
+	virtual const ConfigurationVerify *GetConfigVerify() const;
+	
+	bool StopRun() {return mReloadConfigWanted | mTerminateWanted;}
+	bool IsReloadConfigWanted() {return mReloadConfigWanted;}
+	bool IsTerminateWanted() {return mTerminateWanted;}
+
+	// To allow derived classes to get these signals in other ways
+	void SetReloadConfigWanted() {mReloadConfigWanted = true;}
+	void SetTerminateWanted() {mTerminateWanted = true;}
+	
+	virtual void SetupInInitialProcess();
+	virtual void EnterChild();
+	
+	static void SetProcessTitle(const char *format, ...);
+	
+private:
+	static void SignalHandler(int sigraised);
+	
+private:
+	Configuration *mpConfiguration;
+	bool mReloadConfigWanted;
+	bool mTerminateWanted;
+	static Daemon *spDaemon;
+};
+
+#define DAEMON_VERIFY_SERVER_KEYS 	{"PidFile", 0, ConfigTest_Exists, 0}, \
+									{"User", 0, ConfigTest_LastEntry, 0}
+
+#endif // DAEMON__H
+
diff --git a/lib/server/LocalProcessStream.cpp b/lib/server/LocalProcessStream.cpp
new file mode 100644
index 00000000..f2a97c56
--- /dev/null
+++ b/lib/server/LocalProcessStream.cpp
@@ -0,0 +1,101 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    LocalProcessStream.cpp
+//		Purpose: Opens a process, and presents stdin/stdout as a stream.
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include "LocalProcessStream.h"
+#include "SocketStream.h"
+#include "autogen_ServerException.h"
+#include "Utils.h"
+
+#include "MemLeakFindOn.h"
+
+#define MAX_ARGUMENTS	64
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    LocalProcessStream(const char *, pid_t &)
+//		Purpose: Run a new process, and return a stream giving access to it's
+//				 stdin and stdout. Returns the PID of the new process -- this
+//				 must be waited on at some point to avoid zombies.
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidOut)
+{
+	// Split up command
+	std::vector<std::string> command;
+	SplitString(std::string(CommandLine), ' ', command);
+	// Build arguments
+	char *args[MAX_ARGUMENTS + 4];
+	{
+		int a = 0;
+		std::vector<std::string>::const_iterator i(command.begin());
+		while(a < MAX_ARGUMENTS && i != command.end())
+		{
+			args[a++] = (char*)(*(i++)).c_str();
+		}
+		args[a] = NULL;
+	}
+
+	// Create a socket pair to communicate over.
+	int sv[2] = {-1,-1};
+	if(::socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, sv) != 0)
+	{
+		THROW_EXCEPTION(ServerException, SocketPairFailed)
+	}
+	
+	std::auto_ptr<IOStream> stream(new SocketStream(sv[0]));
+
+	// Fork
+	pid_t pid = 0;
+	switch(pid = vfork())
+	{
+	case -1:	// error
+		::close(sv[0]);
+		::close(sv[1]);
+		THROW_EXCEPTION(ServerException, ServerForkError)
+		break;
+		
+	case 0:		// child
+		// Close end of the socket not being used
+		::close(sv[0]);
+		// Duplicate the file handles to stdin and stdout
+		if(sv[1] != 0) ::dup2(sv[1], 0);
+		if(sv[1] != 1) ::dup2(sv[1], 1);
+		// Close the now redundant socket
+		if(sv[1] != 0 && sv[1] != 1)
+		{
+			::close(sv[1]);
+		}
+		// Execute command!
+		::execv(args[0], args);
+		::_exit(127);	// report error
+		break;
+	
+	default:
+		// just continue...
+		break;
+	}
+	
+	// Close the file descriptor not being used
+	::close(sv[1]);
+
+	// Return the stream object and PID
+	rPidOut = pid;
+	return stream;	
+}
+
+
+
+
diff --git a/lib/server/LocalProcessStream.h b/lib/server/LocalProcessStream.h
new file mode 100644
index 00000000..490c0f45
--- /dev/null
+++ b/lib/server/LocalProcessStream.h
@@ -0,0 +1,19 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    LocalProcessStream.h
+//		Purpose: Opens a process, and presents stdin/stdout as a stream.
+//		Created: 12/3/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef LOCALPROCESSSTREAM__H
+#define LOCALPROCESSSTREAM__H
+
+#include <memory>
+#include "IOStream.h"
+
+std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidOut);
+
+#endif // LOCALPROCESSSTREAM__H
+
diff --git a/lib/server/Makefile.extra b/lib/server/Makefile.extra
new file mode 100755
index 00000000..6cc0de2e
--- /dev/null
+++ b/lib/server/Makefile.extra
@@ -0,0 +1,11 @@
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_ServerException.h autogen_ServerException.cpp:	$(MAKEEXCEPTION) ServerException.txt
+	perl $(MAKEEXCEPTION) ServerException.txt
+
+# AUTOGEN SEEDING
+autogen_ConnectionException.h autogen_ConnectionException.cpp:	$(MAKEEXCEPTION) ConnectionException.txt
+	perl $(MAKEEXCEPTION) ConnectionException.txt
+
diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp
new file mode 100755
index 00000000..ca89bdf1
--- /dev/null
+++ b/lib/server/Protocol.cpp
@@ -0,0 +1,1120 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Protocol.cpp
+//		Purpose: Generic protocol support
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <new>
+
+#include "Protocol.h"
+#include "ProtocolWire.h"
+#include "IOStream.h"
+#include "ServerException.h"
+#include "PartialReadStream.h"
+#include "ProtocolUncertainStream.h"
+
+#include "MemLeakFindOn.h"
+
+#ifdef NDEBUG
+	#define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK	1024
+#else
+//	#define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK	1024
+	#define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK	4
+#endif
+
+#define UNCERTAIN_STREAM_SIZE_BLOCK	(64*1024)
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Protocol(IOStream &rStream)
+//		Purpose: Constructor
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+Protocol::Protocol(IOStream &rStream)
+	: mrStream(rStream),
+	  mHandshakeDone(false),
+	  mMaxObjectSize(PROTOCOL_DEFAULT_MAXOBJSIZE),
+	  mTimeout(PROTOCOL_DEFAULT_TIMEOUT),
+	  mpBuffer(0),
+	  mBufferSize(0),
+	  mReadOffset(-1),
+	  mWriteOffset(-1),
+	  mValidDataSize(-1),
+	  mLastErrorType(NoError),
+	  mLastErrorSubType(NoError)
+{
+	TRACE1("Send block allocation size is %d\n", PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::~Protocol()
+//		Purpose: Destructor
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+Protocol::~Protocol()
+{
+	// Free buffer?
+	if(mpBuffer != 0)
+	{
+		free(mpBuffer);
+		mpBuffer = 0;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::GetLastError(int &, int &)
+//		Purpose: Returns true if there was an error, and type and subtype if there was.
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+bool Protocol::GetLastError(int &rTypeOut, int &rSubTypeOut)
+{
+	if(mLastErrorType == NoError)
+	{
+		// no error.
+		return false;
+	}
+	
+	// Return type and subtype in args
+	rTypeOut = mLastErrorType;
+	rSubTypeOut = mLastErrorSubType;
+	
+	// and unset them
+	mLastErrorType = NoError;
+	mLastErrorSubType = NoError;
+	
+	return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Handshake()
+//		Purpose: Handshake with peer (exchange ident strings)
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void Protocol::Handshake()
+{
+	// Already done?
+	if(mHandshakeDone)
+	{
+		THROW_EXCEPTION(CommonException, Internal)
+	}
+
+	// Make handshake block
+	PW_Handshake hsSend;
+	::memset(&hsSend, 0, sizeof(hsSend));
+	// Copy in ident string
+	::strncpy(hsSend.mIdent, GetIdentString(), sizeof(hsSend.mIdent));
+	
+	// Send it
+	mrStream.Write(&hsSend, sizeof(hsSend));
+	mrStream.WriteAllBuffered();
+
+	// Receive a handshake from the peer
+	PW_Handshake hsReceive;
+	::memset(&hsReceive, 0, sizeof(hsReceive));
+	char *readInto = (char*)&hsReceive;
+	int bytesToRead = sizeof(hsReceive);
+	while(bytesToRead > 0)
+	{
+		// Get some data from the stream
+		int bytesRead = mrStream.Read(readInto, bytesToRead, mTimeout);
+		if(bytesRead == 0)
+		{
+			THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout)
+		}
+		readInto += bytesRead;
+		bytesToRead -= bytesRead;
+	}
+	ASSERT(bytesToRead == 0);
+	
+	// Are they the same?
+	if(::memcmp(&hsSend, &hsReceive, sizeof(hsSend)) != 0)
+	{
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_HandshakeFailed)
+	}
+
+	// Mark as done
+	mHandshakeDone = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::CheckAndReadHdr(void *)
+//		Purpose: Check read for recieve call and get object header from stream.
+//				 Don't use type here to avoid dependency in .h file.
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void Protocol::CheckAndReadHdr(void *hdr)
+{
+	// Check usage
+	if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1)
+	{
+		THROW_EXCEPTION(ServerException, Protocol_BadUsage)
+	}
+	
+	// Handshake done?
+	if(!mHandshakeDone)
+	{
+		Handshake();
+	}
+
+	// Get some data into this header
+	if(!mrStream.ReadFullBuffer(hdr, sizeof(PW_ObjectHeader), 0 /* not interested in bytes read if this fails */, mTimeout))
+	{
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Recieve()
+//		Purpose: Recieves an object from the stream, creating it from the factory object type
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<ProtocolObject> Protocol::Receive()
+{
+	// Get object header
+	PW_ObjectHeader objHeader;
+	CheckAndReadHdr(&objHeader);
+	
+	// Hope it's not a stream
+	if(ntohl(objHeader.mObjType) == SPECIAL_STREAM_OBJECT_TYPE)
+	{
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_StreamWhenObjExpected)
+	}
+	
+	// Check the object size
+	u_int32_t objSize = ntohl(objHeader.mObjSize);
+	if(objSize < sizeof(objHeader) || objSize > mMaxObjectSize)
+	{
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjTooBig)
+	}
+
+	// Create a blank object
+	std::auto_ptr<ProtocolObject> obj(MakeProtocolObject(ntohl(objHeader.mObjType)));
+
+	// Make sure memory is allocated to read it into
+	EnsureBufferAllocated(objSize);
+	
+	// Read data
+	if(!mrStream.ReadFullBuffer(mpBuffer, objSize - sizeof(objHeader), 0 /* not interested in bytes read if this fails */, mTimeout))
+	{
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout)
+	}
+
+	// Setup ready to read out data from the buffer
+	mValidDataSize = objSize - sizeof(objHeader);
+	mReadOffset = 0;
+
+	// Get the object to read its properties from the data recieved
+	try
+	{
+		obj->SetPropertiesFromStreamData(*this);
+	}
+	catch(...)
+	{
+		// Make sure state is reset!
+		mValidDataSize = -1;
+		mReadOffset = -1;
+		throw;
+	}
+	
+	// Any data left over?
+	bool dataLeftOver = (mValidDataSize != mReadOffset);
+	
+	// Unset read state, so future read calls don't fail
+	mValidDataSize = -1;
+	mReadOffset = -1;
+	
+	// Exception if not all the data was consumed
+	if(dataLeftOver)
+	{
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved)
+	}
+
+	return obj;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Send()
+//		Purpose: Send an object to the other side of the connection.
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Send(const ProtocolObject &rObject)
+{
+	// Check usage
+	if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1)
+	{
+		THROW_EXCEPTION(ServerException, Protocol_BadUsage)
+	}
+
+	// Handshake done?
+	if(!mHandshakeDone)
+	{
+		Handshake();
+	}
+
+	// Make sure there's a little bit of space allocated
+	EnsureBufferAllocated(((sizeof(PW_ObjectHeader) + PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK - 1) / PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK) * PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK);
+	ASSERT(mBufferSize >= (int)sizeof(PW_ObjectHeader));
+	
+	// Setup for write operation
+	mValidDataSize = 0;		// Not used, but must not be -1
+	mWriteOffset = sizeof(PW_ObjectHeader);
+	
+	try
+	{
+		rObject.WritePropertiesToStreamData(*this);
+	}
+	catch(...)
+	{
+		// Make sure state is reset!
+		mValidDataSize = -1;
+		mWriteOffset = -1;
+		throw;
+	}
+
+	// How big?
+	int writtenSize = mWriteOffset;
+
+	// Reset write state
+	mValidDataSize = -1;
+	mWriteOffset = -1;	
+	
+	// Make header in the existing block
+	PW_ObjectHeader *pobjHeader = (PW_ObjectHeader*)(mpBuffer);
+	pobjHeader->mObjSize = htonl(writtenSize);
+	pobjHeader->mObjType = htonl(rObject.GetType());
+
+	// Write data
+	mrStream.Write(mpBuffer, writtenSize);
+	mrStream.WriteAllBuffered();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::EnsureBufferAllocated(int)
+//		Purpose: Private. Ensures the buffer is at least the size requested.
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::EnsureBufferAllocated(int Size)
+{
+	if(mpBuffer != 0 && mBufferSize >= Size)
+	{
+		// Nothing to do!
+		return;
+	}
+	
+	// Need to allocate, or reallocate, the block
+	if(mpBuffer != 0)
+	{
+		// Reallocate
+		void *b = realloc(mpBuffer, Size);
+		if(b == 0)
+		{
+			throw std::bad_alloc();
+		}
+		mpBuffer = (char*)b;
+		mBufferSize = Size;
+	}
+	else
+	{
+		// Just allocate
+		mpBuffer = (char*)malloc(Size);
+		if(mpBuffer == 0)
+		{
+			throw std::bad_alloc();
+		}
+		mBufferSize = Size;
+	}
+}
+
+
+#define READ_START_CHECK														\
+	if(mValidDataSize == -1 || mWriteOffset != -1 || mReadOffset == -1)			\
+	{																			\
+		THROW_EXCEPTION(ServerException, Protocol_BadUsage)						\
+	}
+
+#define READ_CHECK_BYTES_AVAILABLE(bytesRequired)								\
+	if((mReadOffset + (int)(bytesRequired)) > mValidDataSize)					\
+	{																			\
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved)	\
+	}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Read(void *, int)
+//		Purpose: Read raw data from the stream (buffered)
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Read(void *Buffer, int Size)
+{
+	READ_START_CHECK
+	READ_CHECK_BYTES_AVAILABLE(Size)
+	
+	// Copy data out
+	::memmove(Buffer, mpBuffer + mReadOffset, Size);
+	mReadOffset += Size;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Read(std::string &, int)
+//		Purpose: Read raw data from the stream (buffered), into a std::string
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void Protocol::Read(std::string &rOut, int Size)
+{
+	READ_START_CHECK
+	READ_CHECK_BYTES_AVAILABLE(Size)
+	
+	rOut.assign(mpBuffer + mReadOffset, Size);
+	mReadOffset += Size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Read(int64_t &)
+//		Purpose: Read a value from the stream (buffered)
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Read(int64_t &rOut)
+{
+	READ_START_CHECK
+	READ_CHECK_BYTES_AVAILABLE(sizeof(int64_t))
+	
+	rOut = ntoh64(*((int64_t*)(mpBuffer + mReadOffset)));
+	mReadOffset += sizeof(int64_t);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Read(int32_t &)
+//		Purpose: Read a value from the stream (buffered)
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Read(int32_t &rOut)
+{
+	READ_START_CHECK
+	READ_CHECK_BYTES_AVAILABLE(sizeof(int32_t))
+	
+	rOut = ntohl(*((int32_t*)(mpBuffer + mReadOffset)));
+	mReadOffset += sizeof(int32_t);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Read(int16_t &)
+//		Purpose: Read a value from the stream (buffered)
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Read(int16_t &rOut)
+{
+	READ_START_CHECK
+	READ_CHECK_BYTES_AVAILABLE(sizeof(int16_t))
+
+	rOut = ntohs(*((int16_t*)(mpBuffer + mReadOffset)));
+	mReadOffset += sizeof(int16_t);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Read(int8_t &)
+//		Purpose: Read a value from the stream (buffered)
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Read(int8_t &rOut)
+{
+	READ_START_CHECK
+	READ_CHECK_BYTES_AVAILABLE(sizeof(int8_t))
+
+	rOut = *((int8_t*)(mpBuffer + mReadOffset));
+	mReadOffset += sizeof(int8_t);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Read(std::string &)
+//		Purpose: Read a value from the stream (buffered)
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Read(std::string &rOut)
+{
+	// READ_START_CHECK implied
+	int32_t size;
+	Read(size);
+	
+	READ_CHECK_BYTES_AVAILABLE(size)
+	
+	// initialise string
+	rOut.assign(mpBuffer + mReadOffset, size);
+	mReadOffset += size;
+}
+
+
+
+
+#define WRITE_START_CHECK														\
+	if(mValidDataSize == -1 || mWriteOffset == -1 || mReadOffset != -1)			\
+	{																			\
+		THROW_EXCEPTION(ServerException, Protocol_BadUsage)						\
+	}
+
+#define WRITE_ENSURE_BYTES_AVAILABLE(bytesToWrite)								\
+	if(mWriteOffset + (int)(bytesToWrite) > mBufferSize)						\
+	{																			\
+		EnsureBufferAllocated((((mWriteOffset + (int)(bytesToWrite)) + PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK - 1) / PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK) * PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); \
+		ASSERT(mWriteOffset + (int)(bytesToWrite) <= mBufferSize);			\
+	}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Write(const void *, int)
+//		Purpose: Writes the contents of a buffer to the stream
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Write(const void *Buffer, int Size)
+{
+	WRITE_START_CHECK
+	WRITE_ENSURE_BYTES_AVAILABLE(Size)
+	
+	::memmove(mpBuffer + mWriteOffset, Buffer, Size);
+	mWriteOffset += Size;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Write(int64_t)
+//		Purpose: Writes a value to the stream
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Write(int64_t Value)
+{
+	WRITE_START_CHECK
+	WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int64_t))
+
+	*((int64_t*)(mpBuffer + mWriteOffset)) = hton64(Value);
+	mWriteOffset += sizeof(int64_t);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Write(int32_t)
+//		Purpose: Writes a value to the stream
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Write(int32_t Value)
+{
+	WRITE_START_CHECK
+	WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int32_t))
+
+	*((int32_t*)(mpBuffer + mWriteOffset)) = htonl(Value);
+	mWriteOffset += sizeof(int32_t);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Write(int16_t)
+//		Purpose: Writes a value to the stream
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Write(int16_t Value)
+{
+	WRITE_START_CHECK
+	WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int16_t))
+
+	*((int16_t*)(mpBuffer + mWriteOffset)) = htons(Value);
+	mWriteOffset += sizeof(int16_t);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Write(int8_t)
+//		Purpose: Writes a value to the stream
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Write(int8_t Value)
+{
+	WRITE_START_CHECK
+	WRITE_ENSURE_BYTES_AVAILABLE(sizeof(int8_t))
+
+	*((int8_t*)(mpBuffer + mWriteOffset)) = Value;
+	mWriteOffset += sizeof(int8_t);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::Write(const std::string &)
+//		Purpose: Writes a value to the stream
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void Protocol::Write(const std::string &rValue)
+{
+	// WRITE_START_CHECK implied
+	Write((int32_t)(rValue.size()));
+	
+	WRITE_ENSURE_BYTES_AVAILABLE(rValue.size())
+	Write(rValue.c_str(), rValue.size());
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::ReceieveStream()
+//		Purpose: Receive a stream from the remote side
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<IOStream> Protocol::ReceiveStream()
+{
+	// Get object header
+	PW_ObjectHeader objHeader;
+	CheckAndReadHdr(&objHeader);
+	
+	// Hope it's not an object
+	if(ntohl(objHeader.mObjType) != SPECIAL_STREAM_OBJECT_TYPE)
+	{
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjWhenStreamExpected)
+	}
+	
+	// Get the stream size
+	u_int32_t streamSize = ntohl(objHeader.mObjSize);
+
+	// Inform sub class
+	InformStreamReceiving(streamSize);
+
+	// Return a stream object
+	return std::auto_ptr<IOStream>((streamSize == ProtocolStream_SizeUncertain)?
+		((IOStream*)(new ProtocolUncertainStream(mrStream)))
+		:((IOStream*)(new PartialReadStream(mrStream, streamSize))));
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::SendStream(IOStream &)
+//		Purpose: Send a stream to the remote side
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+void Protocol::SendStream(IOStream &rStream)
+{
+	// Check usage
+	if(mValidDataSize != -1 || mWriteOffset != -1 || mReadOffset != -1)
+	{
+		THROW_EXCEPTION(ServerException, Protocol_BadUsage)
+	}
+
+	// Handshake done?
+	if(!mHandshakeDone)
+	{
+		Handshake();
+	}
+
+	// How should this be streamed?
+	bool uncertainSize = false;
+	IOStream::pos_type streamSize = rStream.BytesLeftToRead();
+	if(streamSize == IOStream::SizeOfStreamUnknown
+		|| streamSize > 0x7fffffff)
+	{
+		// Can't send this using the fixed size header
+		uncertainSize = true;
+	}
+	
+	// Inform sub class
+	InformStreamSending(streamSize);
+	
+	// Make header
+	PW_ObjectHeader objHeader;
+	objHeader.mObjSize = htonl(uncertainSize?(ProtocolStream_SizeUncertain):streamSize);
+	objHeader.mObjType = htonl(SPECIAL_STREAM_OBJECT_TYPE);
+
+	// Write header
+	mrStream.Write(&objHeader, sizeof(objHeader));
+	// Could be sent in one of two ways
+	if(uncertainSize)
+	{
+		// Don't know how big this is going to be -- so send it in chunks
+		
+		// Allocate memory
+		uint8_t *blockA = (uint8_t *)malloc(UNCERTAIN_STREAM_SIZE_BLOCK + sizeof(int));
+		if(blockA == 0)
+		{
+			throw std::bad_alloc();
+		}
+		uint8_t *block = blockA + sizeof(int);	// so that everything is word aligned for reading, but can put the one byte header before it
+		
+		try
+		{
+			int bytesInBlock = 0;
+			while(rStream.StreamDataLeft())
+			{
+				// Read some of it
+				bytesInBlock += rStream.Read(block + bytesInBlock, UNCERTAIN_STREAM_SIZE_BLOCK - bytesInBlock);
+				
+				// Send as much as we can out
+				bytesInBlock -= SendStreamSendBlock(block, bytesInBlock);
+			}
+
+			// Everything recieved from stream, but need to send whatevers left in the block
+			while(bytesInBlock > 0)
+			{
+				bytesInBlock -= SendStreamSendBlock(block, bytesInBlock);
+			}
+			
+			// Send final byte to finish the stream
+			uint8_t endOfStream = ProtocolStreamHeader_EndOfStream;
+			mrStream.Write(&endOfStream, 1);
+		}
+		catch(...)
+		{
+			free(blockA);
+			throw;
+		}
+		
+		// Clean up
+		free(blockA);
+	}
+	else
+	{
+		// Fixed size stream, send it all in one go
+		if(!rStream.CopyStreamTo(mrStream, mTimeout, 4096 /* slightly larger buffer */))
+		{
+			THROW_EXCEPTION(ConnectionException, Conn_Protocol_TimeOutWhenSendingStream)
+		}
+	}
+	// Make sure everything is written
+	mrStream.WriteAllBuffered();
+	
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::SendStreamSendBlock(uint8_t *, int)
+//		Purpose: Sends as much of the block as can be sent, moves the remainer down to the beginning,
+//				 and returns the number of bytes sent. WARNING: Will write to Block[-1]
+//		Created: 5/12/03
+//
+// --------------------------------------------------------------------------
+int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock)
+{
+	// Quick sanity check
+	if(BytesInBlock == 0)
+	{
+		return 0;
+	}
+	
+	// Work out the header byte
+	uint8_t header = 0;
+	int writeSize = 0;
+	if(BytesInBlock >= (64*1024))
+	{
+		header = ProtocolStreamHeader_SizeIs64k;
+		writeSize = (64*1024);
+	}
+	else
+	{
+		// Scan the table to find the most that can be written
+		for(int s = ProtocolStreamHeader_MaxEncodedSizeValue; s > 0; --s)
+		{
+			if(sProtocolStreamHeaderLengths[s] <= BytesInBlock)
+			{
+				header = s;
+				writeSize = sProtocolStreamHeaderLengths[s];
+				break;
+			}
+		}
+	}
+	ASSERT(header > 0);
+	
+	// Store the header
+	Block[-1] = header;
+	
+	// Write everything out
+	mrStream.Write(Block - 1, writeSize + 1);
+	
+	// move the remainer to the beginning of the block for the next time round
+	if(writeSize != BytesInBlock)
+	{
+		::memmove(Block, Block + writeSize, BytesInBlock - writeSize);
+	}
+	
+	return writeSize;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::InformStreamReceiving(u_int32_t)
+//		Purpose: Informs sub classes about streams being received
+//		Created: 2003/10/27
+//
+// --------------------------------------------------------------------------
+void Protocol::InformStreamReceiving(u_int32_t Size)
+{
+	// Do nothing
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Protocol::InformStreamSending(u_int32_t)
+//		Purpose: Informs sub classes about streams being sent
+//		Created: 2003/10/27
+//
+// --------------------------------------------------------------------------
+void Protocol::InformStreamSending(u_int32_t Size)
+{
+	// Do nothing
+}
+
+
+/* 
+perl code to generate the table below
+
+#!/usr/bin/perl
+use strict;
+open OUT,">protolengths.txt";
+my $len = 0;
+for(0 .. 255)
+{
+	print OUT "\t$len,\t// $_\n";
+	my $inc = 1;
+	$inc = 8 if $_ >= 64;
+	$inc = 16 if $_ >= 96;
+	$inc = 32 if $_ >= 112;
+	$inc = 64 if $_ >= 128;
+	$inc = 128 if $_ >= 135;
+	$inc = 256 if $_ >= 147;
+	$inc = 512 if $_ >= 159;
+	$inc = 1024 if $_ >= 231;
+	$len += $inc;
+}
+close OUT;
+
+*/
+const uint16_t Protocol::sProtocolStreamHeaderLengths[256] =
+{
+	0,	// 0
+	1,	// 1
+	2,	// 2
+	3,	// 3
+	4,	// 4
+	5,	// 5
+	6,	// 6
+	7,	// 7
+	8,	// 8
+	9,	// 9
+	10,	// 10
+	11,	// 11
+	12,	// 12
+	13,	// 13
+	14,	// 14
+	15,	// 15
+	16,	// 16
+	17,	// 17
+	18,	// 18
+	19,	// 19
+	20,	// 20
+	21,	// 21
+	22,	// 22
+	23,	// 23
+	24,	// 24
+	25,	// 25
+	26,	// 26
+	27,	// 27
+	28,	// 28
+	29,	// 29
+	30,	// 30
+	31,	// 31
+	32,	// 32
+	33,	// 33
+	34,	// 34
+	35,	// 35
+	36,	// 36
+	37,	// 37
+	38,	// 38
+	39,	// 39
+	40,	// 40
+	41,	// 41
+	42,	// 42
+	43,	// 43
+	44,	// 44
+	45,	// 45
+	46,	// 46
+	47,	// 47
+	48,	// 48
+	49,	// 49
+	50,	// 50
+	51,	// 51
+	52,	// 52
+	53,	// 53
+	54,	// 54
+	55,	// 55
+	56,	// 56
+	57,	// 57
+	58,	// 58
+	59,	// 59
+	60,	// 60
+	61,	// 61
+	62,	// 62
+	63,	// 63
+	64,	// 64
+	72,	// 65
+	80,	// 66
+	88,	// 67
+	96,	// 68
+	104,	// 69
+	112,	// 70
+	120,	// 71
+	128,	// 72
+	136,	// 73
+	144,	// 74
+	152,	// 75
+	160,	// 76
+	168,	// 77
+	176,	// 78
+	184,	// 79
+	192,	// 80
+	200,	// 81
+	208,	// 82
+	216,	// 83
+	224,	// 84
+	232,	// 85
+	240,	// 86
+	248,	// 87
+	256,	// 88
+	264,	// 89
+	272,	// 90
+	280,	// 91
+	288,	// 92
+	296,	// 93
+	304,	// 94
+	312,	// 95
+	320,	// 96
+	336,	// 97
+	352,	// 98
+	368,	// 99
+	384,	// 100
+	400,	// 101
+	416,	// 102
+	432,	// 103
+	448,	// 104
+	464,	// 105
+	480,	// 106
+	496,	// 107
+	512,	// 108
+	528,	// 109
+	544,	// 110
+	560,	// 111
+	576,	// 112
+	608,	// 113
+	640,	// 114
+	672,	// 115
+	704,	// 116
+	736,	// 117
+	768,	// 118
+	800,	// 119
+	832,	// 120
+	864,	// 121
+	896,	// 122
+	928,	// 123
+	960,	// 124
+	992,	// 125
+	1024,	// 126
+	1056,	// 127
+	1088,	// 128
+	1152,	// 129
+	1216,	// 130
+	1280,	// 131
+	1344,	// 132
+	1408,	// 133
+	1472,	// 134
+	1536,	// 135
+	1664,	// 136
+	1792,	// 137
+	1920,	// 138
+	2048,	// 139
+	2176,	// 140
+	2304,	// 141
+	2432,	// 142
+	2560,	// 143
+	2688,	// 144
+	2816,	// 145
+	2944,	// 146
+	3072,	// 147
+	3328,	// 148
+	3584,	// 149
+	3840,	// 150
+	4096,	// 151
+	4352,	// 152
+	4608,	// 153
+	4864,	// 154
+	5120,	// 155
+	5376,	// 156
+	5632,	// 157
+	5888,	// 158
+	6144,	// 159
+	6656,	// 160
+	7168,	// 161
+	7680,	// 162
+	8192,	// 163
+	8704,	// 164
+	9216,	// 165
+	9728,	// 166
+	10240,	// 167
+	10752,	// 168
+	11264,	// 169
+	11776,	// 170
+	12288,	// 171
+	12800,	// 172
+	13312,	// 173
+	13824,	// 174
+	14336,	// 175
+	14848,	// 176
+	15360,	// 177
+	15872,	// 178
+	16384,	// 179
+	16896,	// 180
+	17408,	// 181
+	17920,	// 182
+	18432,	// 183
+	18944,	// 184
+	19456,	// 185
+	19968,	// 186
+	20480,	// 187
+	20992,	// 188
+	21504,	// 189
+	22016,	// 190
+	22528,	// 191
+	23040,	// 192
+	23552,	// 193
+	24064,	// 194
+	24576,	// 195
+	25088,	// 196
+	25600,	// 197
+	26112,	// 198
+	26624,	// 199
+	27136,	// 200
+	27648,	// 201
+	28160,	// 202
+	28672,	// 203
+	29184,	// 204
+	29696,	// 205
+	30208,	// 206
+	30720,	// 207
+	31232,	// 208
+	31744,	// 209
+	32256,	// 210
+	32768,	// 211
+	33280,	// 212
+	33792,	// 213
+	34304,	// 214
+	34816,	// 215
+	35328,	// 216
+	35840,	// 217
+	36352,	// 218
+	36864,	// 219
+	37376,	// 220
+	37888,	// 221
+	38400,	// 222
+	38912,	// 223
+	39424,	// 224
+	39936,	// 225
+	40448,	// 226
+	40960,	// 227
+	41472,	// 228
+	41984,	// 229
+	42496,	// 230
+	43008,	// 231
+	44032,	// 232
+	45056,	// 233
+	46080,	// 234
+	47104,	// 235
+	48128,	// 236
+	49152,	// 237
+	50176,	// 238
+	51200,	// 239
+	52224,	// 240
+	53248,	// 241
+	54272,	// 242
+	55296,	// 243
+	56320,	// 244
+	57344,	// 245
+	58368,	// 246
+	59392,	// 247
+	60416,	// 248
+	61440,	// 249
+	62464,	// 250
+	63488,	// 251
+	64512,	// 252
+	0,		// 253 = 65536 / 64k
+	0,		// 254 = special (reserved)
+	0		// 255 = special (reserved)
+};
+
+
+
+
diff --git a/lib/server/Protocol.h b/lib/server/Protocol.h
new file mode 100755
index 00000000..e037e33c
--- /dev/null
+++ b/lib/server/Protocol.h
@@ -0,0 +1,201 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Protocol.h
+//		Purpose: Generic protocol support
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+
+#ifndef PROTOCOL__H
+#define PROTOCOL__H
+
+#include <sys/types.h>
+
+class IOStream;
+#include "ProtocolObject.h"
+#include <memory>
+#include <vector>
+#include <string>
+
+// default timeout is 15 minutes
+#define PROTOCOL_DEFAULT_TIMEOUT	(15*60*1000)
+// 16 default maximum object size -- should be enough
+#define PROTOCOL_DEFAULT_MAXOBJSIZE	(16*1024)
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    Protocol
+//		Purpose: Generic command / response protocol support
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+class Protocol
+{
+public:
+	Protocol(IOStream &rStream);
+	virtual ~Protocol();
+	
+private:
+	Protocol(const Protocol &rToCopy);
+
+public:
+	void Handshake();
+	std::auto_ptr<ProtocolObject> Receive();
+	void Send(const ProtocolObject &rObject);
+	
+	std::auto_ptr<IOStream> ReceiveStream();
+	void SendStream(IOStream &rStream);
+	
+	enum
+	{
+		NoError = -1,
+		UnknownError = 0
+	};
+
+	bool GetLastError(int &rTypeOut, int &rSubTypeOut);
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Protocol::SetTimeout(int)
+	//		Purpose: Sets the timeout for sending and reciving
+	//		Created: 2003/08/19
+	//
+	// --------------------------------------------------------------------------	
+	void SetTimeout(int NewTimeout) {mTimeout = NewTimeout;}
+	
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Protocol::GetTimeout() 
+	//		Purpose: Get current timeout for sending and receiving
+	//		Created: 2003/09/06
+	//
+	// --------------------------------------------------------------------------
+	int GetTimeout() {return mTimeout;}
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Protocol::SetMaxObjectSize(int)
+	//		Purpose: Sets the maximum size of an object which will be accepted
+	//		Created: 2003/08/19
+	//
+	// --------------------------------------------------------------------------	
+	void SetMaxObjectSize(unsigned int NewMaxObjSize) {mMaxObjectSize = NewMaxObjSize;}
+
+	// For ProtocolObject derived classes
+	void Read(void *Buffer, int Size);
+	void Read(std::string &rOut, int Size);
+	void Read(int64_t &rOut);
+	void Read(int32_t &rOut);
+	void Read(int16_t &rOut);
+	void Read(int8_t &rOut);
+	void Read(bool &rOut) {int8_t read; Read(read); rOut = (read == true);}
+	void Read(std::string &rOut);
+	template<typename type>
+	void Read(type &rOut)
+	{
+		rOut.ReadFromProtocol(*this);
+	}
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Protocol::ReadVector(std::vector<> &)
+	//		Purpose: Reads a vector/list of items from the stream
+	//		Created: 2003/08/19
+	//
+	// --------------------------------------------------------------------------
+	template<typename type>
+	void ReadVector(std::vector<type> &rOut)
+	{
+		rOut.clear();
+		int16_t num = 0;
+		Read(num);
+		for(int16_t n = 0; n < num; ++n)
+		{
+			type v;
+			Read(v);
+			rOut.push_back(v);
+		}
+	}
+	
+	void Write(const void *Buffer, int Size);
+	void Write(int64_t Value);
+	void Write(int32_t Value);
+	void Write(int16_t Value);
+	void Write(int8_t Value);
+	void Write(bool Value) {int8_t write = Value; Write(write);}
+	void Write(const std::string &rValue);
+	template<typename type>
+	void Write(const type &rValue)
+	{
+		rValue.WriteToProtocol(*this);
+	}
+	template<typename type>
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    Protocol::WriteVector(const std::vector<> &)
+	//		Purpose: Writes a vector/list of items from the stream
+	//		Created: 2003/08/19
+	//
+	// --------------------------------------------------------------------------
+	void WriteVector(const std::vector<type> &rValue)
+	{
+		int16_t num = rValue.size();
+		Write(num);
+		for(int16_t n = 0; n < num; ++n)
+		{
+			Write(rValue[n]);
+		}
+	}
+
+public:
+	static const uint16_t sProtocolStreamHeaderLengths[256];
+	enum
+	{
+		ProtocolStreamHeader_EndOfStream = 0,
+		ProtocolStreamHeader_MaxEncodedSizeValue = 252,
+		ProtocolStreamHeader_SizeIs64k = 253,
+		ProtocolStreamHeader_Reserved1 = 254,
+		ProtocolStreamHeader_Reserved2 = 255
+	};
+	enum
+	{
+		ProtocolStream_SizeUncertain = 0xffffffff
+	};
+
+protected:	
+	virtual std::auto_ptr<ProtocolObject> MakeProtocolObject(int ObjType) = 0;
+	virtual const char *GetIdentString() = 0;
+	void SetError(int Type, int SubType) {mLastErrorType = Type; mLastErrorSubType = SubType;}
+	void CheckAndReadHdr(void *hdr);	// don't use type here to avoid dependency
+	
+	// Will be used for logging
+	virtual void InformStreamReceiving(u_int32_t Size);
+	virtual void InformStreamSending(u_int32_t Size);
+	
+private:
+	void EnsureBufferAllocated(int Size);
+	int SendStreamSendBlock(uint8_t *Block, int BytesInBlock);
+
+private:
+	IOStream &mrStream;
+	bool mHandshakeDone;
+	unsigned int mMaxObjectSize;
+	int mTimeout;
+	char *mpBuffer;
+	int mBufferSize;
+	int mReadOffset;
+	int mWriteOffset;
+	int mValidDataSize;
+	int mLastErrorType;
+	int mLastErrorSubType;
+};
+
+#endif // PROTOCOL__H
+
diff --git a/lib/server/ProtocolObject.cpp b/lib/server/ProtocolObject.cpp
new file mode 100755
index 00000000..fb09f820
--- /dev/null
+++ b/lib/server/ProtocolObject.cpp
@@ -0,0 +1,125 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ProtocolObject.h
+//		Purpose: Protocol object base class
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "ProtocolObject.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolObject::ProtocolObject()
+//		Purpose: Default constructor
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+ProtocolObject::ProtocolObject()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolObject::ProtocolObject()
+//		Purpose: Destructor
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+ProtocolObject::~ProtocolObject()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolObject::ProtocolObject()
+//		Purpose: Copy constructor
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+ProtocolObject::ProtocolObject(const ProtocolObject &rToCopy)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolObject::IsError(int &, int &)
+//		Purpose: Does this represent an error, and if so, what is the type and subtype?
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+bool ProtocolObject::IsError(int &rTypeOut, int &rSubTypeOut) const
+{
+	return false;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolObject::IsConversationEnd()
+//		Purpose: Does this command end the conversation?
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+bool ProtocolObject::IsConversationEnd() const
+{
+	return false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolObject::GetType()
+//		Purpose: Return type of the object
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+int ProtocolObject::GetType() const
+{
+	// This isn't implemented in the base class!
+	THROW_EXCEPTION(CommonException, Internal)
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolObject::SetPropertiesFromStreamData(Protocol &)
+//		Purpose: Set the properties of the object from the stream data ready in the Protocol object
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void ProtocolObject::SetPropertiesFromStreamData(Protocol &rProtocol)
+{
+	// This isn't implemented in the base class!
+	THROW_EXCEPTION(CommonException, Internal)
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolObject::WritePropertiesToStreamData(Protocol &)
+//		Purpose: Write the properties of the object into the stream data in the Protocol object
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+void ProtocolObject::WritePropertiesToStreamData(Protocol &rProtocol) const
+{
+	// This isn't implemented in the base class!
+	THROW_EXCEPTION(CommonException, Internal)
+}
+
+
+
diff --git a/lib/server/ProtocolObject.h b/lib/server/ProtocolObject.h
new file mode 100755
index 00000000..0a127ab5
--- /dev/null
+++ b/lib/server/ProtocolObject.h
@@ -0,0 +1,41 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ProtocolObject.h
+//		Purpose: Protocol object base class
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+
+#ifndef PROTOCOLOBJECT__H
+#define PROTOCOLOBJECT__H
+
+class Protocol;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    ProtocolObject
+//		Purpose: Basic object representation of objects to pass through a Protocol session
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+class ProtocolObject
+{
+public:
+	ProtocolObject();
+	virtual ~ProtocolObject();
+	ProtocolObject(const ProtocolObject &rToCopy);
+
+	// Info about this object
+	virtual int GetType() const;
+	virtual bool IsError(int &rTypeOut, int &rSubTypeOut) const;
+	virtual bool IsConversationEnd() const;
+
+	// reading and writing with Protocol objects
+	virtual void SetPropertiesFromStreamData(Protocol &rProtocol);
+	virtual void WritePropertiesToStreamData(Protocol &rProtocol) const;	
+};
+
+#endif // PROTOCOLOBJECT__H
+
diff --git a/lib/server/ProtocolUncertainStream.cpp b/lib/server/ProtocolUncertainStream.cpp
new file mode 100755
index 00000000..60c1fa1d
--- /dev/null
+++ b/lib/server/ProtocolUncertainStream.cpp
@@ -0,0 +1,189 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ProtocolUncertainStream.h
+//		Purpose: Read part of another stream
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "ProtocolUncertainStream.h"
+#include "ServerException.h"
+#include "Protocol.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolUncertainStream::ProtocolUncertainStream(IOStream &, int)
+//		Purpose: Constructor, taking another stream.
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+ProtocolUncertainStream::ProtocolUncertainStream(IOStream &rSource)
+	: mrSource(rSource),
+	  mBytesLeftInCurrentBlock(0),
+	  mFinished(false)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolUncertainStream::~ProtocolUncertainStream()
+//		Purpose: Destructor. Won't absorb any unread bytes.
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+ProtocolUncertainStream::~ProtocolUncertainStream()
+{
+	if(!mFinished)
+	{
+		TRACE0("ProtocolUncertainStream::~ProtocolUncertainStream() destroyed when stream not complete\n");
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolUncertainStream::Read(void *, int, int)
+//		Purpose: As interface.
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// Finished?
+	if(mFinished)
+	{
+		return 0;
+	}
+	
+	int read = 0;
+	while(read < NBytes)
+	{
+		// Anything we can get from the current block?
+		ASSERT(mBytesLeftInCurrentBlock >= 0);
+		if(mBytesLeftInCurrentBlock > 0)
+		{
+			// Yes, let's use some of these up
+			int toRead = (NBytes - read);
+			if(toRead > mBytesLeftInCurrentBlock)
+			{
+				// Adjust downwards to only read stuff out of the current block
+				toRead = mBytesLeftInCurrentBlock;
+			}
+			
+			// Read it
+			int r = mrSource.Read(((uint8_t*)pBuffer) + read, toRead, Timeout);
+			// Give up now if it didn't return anything
+			if(r == 0)
+			{
+				return read;
+			}
+			
+			// Adjust counts of bytes by the bytes recieved
+			read += r;
+			mBytesLeftInCurrentBlock -= r;
+			
+			// stop now if the stream returned less than we asked for -- avoid blocking
+			if(r != toRead)
+			{
+				return read;
+			}
+		}
+		else
+		{
+			// Read the header byte to find out how much there is in the next block
+			uint8_t header;
+			if(mrSource.Read(&header, 1, Timeout) == 0)
+			{
+				// Didn't get the byte, return now
+				return read;
+			}
+			
+			// Interpret the byte...
+			if(header == Protocol::ProtocolStreamHeader_EndOfStream)
+			{
+				// All done.
+				mFinished = true;
+				return read;
+			}
+			else if(header <= Protocol::ProtocolStreamHeader_MaxEncodedSizeValue)
+			{
+				// get size of the block from the Protocol's lovely list
+				mBytesLeftInCurrentBlock = Protocol::sProtocolStreamHeaderLengths[header];
+			}
+			else if(header == Protocol::ProtocolStreamHeader_SizeIs64k)
+			{
+				// 64k
+				mBytesLeftInCurrentBlock = (64*1024);
+			}
+			else
+			{
+				// Bad. It used the reserved values.
+				THROW_EXCEPTION(ServerException, ProtocolUncertainStreamBadBlockHeader)	
+			}
+		}
+	}
+
+	// Return the number read
+	return read;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolUncertainStream::BytesLeftToRead()
+//		Purpose: As interface.
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type ProtocolUncertainStream::BytesLeftToRead()
+{
+	// Only know how much is left if everything is finished
+	return mFinished?(0):(IOStream::SizeOfStreamUnknown);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolUncertainStream::Write(const void *, int)
+//		Purpose: As interface. But will exception.
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+void ProtocolUncertainStream::Write(const void *pBuffer, int NBytes)
+{
+	THROW_EXCEPTION(ServerException, CantWriteToProtocolUncertainStream)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolUncertainStream::StreamDataLeft()
+//		Purpose: As interface.
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+bool ProtocolUncertainStream::StreamDataLeft()
+{
+	return !mFinished;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ProtocolUncertainStream::StreamClosed()
+//		Purpose: As interface.
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+bool ProtocolUncertainStream::StreamClosed()
+{
+	// always closed
+	return true;
+}
+
diff --git a/lib/server/ProtocolUncertainStream.h b/lib/server/ProtocolUncertainStream.h
new file mode 100755
index 00000000..66397a39
--- /dev/null
+++ b/lib/server/ProtocolUncertainStream.h
@@ -0,0 +1,47 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    PartialReadStream.h
+//		Purpose: Read part of another stream
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+
+#ifndef PROTOCOLUNCERTAINSTREAM__H
+#define PROTOCOLUNCERTAINSTREAM__H
+
+#include "IOStream.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    PartialReadStream
+//		Purpose: Read part of another stream
+//		Created: 2003/12/05
+//
+// --------------------------------------------------------------------------
+class ProtocolUncertainStream : public IOStream
+{
+public:
+	ProtocolUncertainStream(IOStream &rSource);
+	~ProtocolUncertainStream();
+private:
+	// no copying allowed
+	ProtocolUncertainStream(const IOStream &);
+	ProtocolUncertainStream(const ProtocolUncertainStream &);
+	
+public:
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type BytesLeftToRead();
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+private:
+	IOStream &mrSource;
+	int mBytesLeftInCurrentBlock;
+	bool mFinished;
+};
+
+#endif // PROTOCOLUNCERTAINSTREAM__H
+
diff --git a/lib/server/ProtocolWire.h b/lib/server/ProtocolWire.h
new file mode 100755
index 00000000..b6d3bd37
--- /dev/null
+++ b/lib/server/ProtocolWire.h
@@ -0,0 +1,43 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ProtocolWire.h
+//		Purpose: On the wire structures for Protocol
+//		Created: 2003/08/19
+//
+// --------------------------------------------------------------------------
+
+#ifndef PROTOCOLWIRE__H
+#define PROTOCOLWIRE__H
+
+#include <sys/types.h>
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+typedef struct
+{
+	char mIdent[32];
+} PW_Handshake;
+
+typedef struct
+{
+	u_int32_t	mObjSize;
+	u_int32_t	mObjType;
+} PW_ObjectHeader;
+
+#define SPECIAL_STREAM_OBJECT_TYPE		0xffffffff
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#endif // PROTOCOLWIRE__H
+
diff --git a/lib/server/SSLLib.cpp b/lib/server/SSLLib.cpp
new file mode 100755
index 00000000..e9f3a59d
--- /dev/null
+++ b/lib/server/SSLLib.cpp
@@ -0,0 +1,83 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    SSLLib.cpp
+//		Purpose: Utility functions for dealing with the OpenSSL library
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#define TLS_CLASS_IMPLEMENTATION_CPP
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <openssl/rand.h>
+
+#include <syslog.h>
+
+#include "SSLLib.h"
+#include "ServerException.h"
+
+#include "MemLeakFindOn.h"
+
+#ifndef NDEBUG
+	bool SSLLib__TraceErrors = false;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SSLLib::Initialise()
+//		Purpose: Initialise SSL library
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+void SSLLib::Initialise()
+{
+	if(!::SSL_library_init())
+	{
+		LogError("Initialisation");
+		THROW_EXCEPTION(ServerException, SSLLibraryInitialisationError)
+	}
+	
+	// More helpful error messages
+	::SSL_load_error_strings();
+
+	// Extra seeding over and above what's already done by the library
+#ifndef PLATFORM_RANDOM_DEVICE_NONE
+	if(::RAND_load_file(PLATFORM_RANDOM_DEVICE, 1024) != 1024)
+	{
+		THROW_EXCEPTION(ServerException, SSLRandomInitFailed)
+	}
+#else
+	::fprintf(stderr, "No random device -- additional seeding of random number generator not performed.\n");
+#endif
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SSLLib::LogError(const char *)
+//		Purpose: Logs an error
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+void SSLLib::LogError(const char *ErrorDuringAction)
+{
+	unsigned long errcode;
+	char errname[256];		// SSL docs say at least 120 bytes
+	while((errcode = ERR_get_error()) != 0)
+	{
+		::ERR_error_string_n(errcode, errname, sizeof(errname));
+		#ifndef NDEBUG
+			if(SSLLib__TraceErrors)
+			{
+				TRACE2("SSL err during %s: %s\n", ErrorDuringAction, errname);
+			}
+		#endif
+		::syslog(LOG_ERR, "SSL err during %s: %s", ErrorDuringAction, errname);
+	}
+}
+
diff --git a/lib/server/SSLLib.h b/lib/server/SSLLib.h
new file mode 100755
index 00000000..cdff4f04
--- /dev/null
+++ b/lib/server/SSLLib.h
@@ -0,0 +1,36 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    SSLLib.h
+//		Purpose: Utility functions for dealing with the OpenSSL library
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+
+#ifndef SSLLIB__H
+#define SSLLIB__H
+
+#ifndef NDEBUG
+	extern bool SSLLib__TraceErrors;
+	#define SET_DEBUG_SSLLIB_TRACE_ERRORS {SSLLib__TraceErrors = true;}
+#else
+	#define SET_DEBUG_SSLLIB_TRACE_ERRORS
+#endif
+
+
+// --------------------------------------------------------------------------
+//
+// Namespace
+//		Name:    SSLLib
+//		Purpose: Utility functions for dealing with the OpenSSL library
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+namespace SSLLib
+{
+	void Initialise();
+	void LogError(const char *ErrorDuringAction);
+};
+
+#endif // SSLLIB__H
+
diff --git a/lib/server/ServerException.h b/lib/server/ServerException.h
new file mode 100755
index 00000000..8851b90a
--- /dev/null
+++ b/lib/server/ServerException.h
@@ -0,0 +1,46 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ServerException.h
+//		Purpose: Exception
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef SERVEREXCEPTION__H
+#define SERVEREXCEPTION__H
+
+// Compatibility header
+#include "autogen_ServerException.h"
+#include "autogen_ConnectionException.h"
+
+// Rename old connection exception names to new names without Conn_ prefix
+// This is all because ConnectionException used to be derived from ServerException
+// with some funky magic with subtypes. Perhaps a little unreliable, and the
+// usefulness of it never really was used.
+#define Conn_SocketWriteError			SocketWriteError	
+#define Conn_SocketReadError			SocketReadError	
+#define Conn_SocketNameLookupError		SocketNameLookupError
+#define Conn_SocketShutdownError		SocketShutdownError
+#define Conn_SocketConnectError			SocketConnectError	
+#define Conn_TLSHandshakeFailed			TLSHandshakeFailed	
+#define Conn_TLSShutdownFailed			TLSShutdownFailed	
+#define Conn_TLSWriteFailed				TLSWriteFailed		
+#define Conn_TLSReadFailed				TLSReadFailed		
+#define Conn_TLSNoPeerCertificate		TLSNoPeerCertificate
+#define Conn_TLSPeerCertificateInvalid	TLSPeerCertificateInvalid
+#define Conn_TLSClosedWhenWriting		TLSClosedWhenWriting
+#define Conn_TLSHandshakeTimedOut		TLSHandshakeTimedOut
+#define Conn_Protocol_Timeout			Protocol_Timeout	
+#define Conn_Protocol_ObjTooBig			Protocol_ObjTooBig	
+#define Conn_Protocol_BadCommandRecieved			Protocol_BadCommandRecieved	
+#define Conn_Protocol_UnknownCommandRecieved		Protocol_UnknownCommandRecieved
+#define Conn_Protocol_TriedToExecuteReplyCommand	Protocol_TriedToExecuteReplyCommand
+#define Conn_Protocol_UnexpectedReply				Protocol_UnexpectedReply		
+#define Conn_Protocol_HandshakeFailed				Protocol_HandshakeFailed		
+#define Conn_Protocol_StreamWhenObjExpected			Protocol_StreamWhenObjExpected	
+#define Conn_Protocol_ObjWhenStreamExpected			Protocol_ObjWhenStreamExpected	
+#define Conn_Protocol_TimeOutWhenSendingStream		Protocol_TimeOutWhenSendingStream
+
+#endif // SERVEREXCEPTION__H
+
diff --git a/lib/server/ServerException.txt b/lib/server/ServerException.txt
new file mode 100644
index 00000000..ed591b73
--- /dev/null
+++ b/lib/server/ServerException.txt
@@ -0,0 +1,39 @@
+EXCEPTION Server 3
+
+# for historic reasons, some codes are not used
+
+Internal						0
+FailedToLoadConfiguration		1
+DaemoniseFailed					2
+AlreadyDaemonConstructed		3
+BadSocketHandle					4
+DupError						5
+SocketAlreadyOpen				8
+SocketOpenError					10
+SocketPollError					11
+SocketCloseError				13
+SocketNameUNIXPathTooLong		14
+SocketBindError					16	Check the ListenAddresses directive in your config file -- must refer to local IP addresses only
+SocketAcceptError				17
+ServerStreamBadListenAddrs		18
+ServerForkError					19
+ServerWaitOnChildError			20
+TooManySocketsInMultiListen		21	There is a limit on how many addresses you can listen on simulatiously.
+ServerStreamTooManyListenAddresses	22
+TLSContextNotInitialised		23
+TLSAllocationFailed				24
+TLSLoadCertificatesFailed		25
+TLSLoadPrivateKeyFailed			26
+TLSLoadTrustedCAsFailed			27
+TLSSetCiphersFailed				28
+SSLLibraryInitialisationError	29
+TLSNoSSLObject					31
+TLSAlreadyHandshaked			35
+SocketSetNonBlockingFailed		40
+Protocol_BadUsage				43
+Protocol_UnsuitableStreamTypeForSending		51
+CantWriteToProtocolUncertainStream			53
+ProtocolUncertainStreamBadBlockHeader		54
+SocketPairFailed				55
+CouldNotChangePIDFileOwner		56
+SSLRandomInitFailed				57	Read from /dev/*random device failed
diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h
new file mode 100755
index 00000000..d087a321
--- /dev/null
+++ b/lib/server/ServerStream.h
@@ -0,0 +1,340 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ServerStream.h
+//		Purpose: Stream based server daemons
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef SERVERSTREAM__H
+#define SERVERSTREAM__H
+
+#include <syslog.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <sys/wait.h>
+
+#include "Daemon.h"
+#include "SocketListen.h"
+#include "Utils.h"
+#include "Configuration.h"
+#include "WaitForEvent.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    ServerStream
+//		Purpose: Stream based server daemon
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+template<typename StreamType, int Port, int ListenBacklog = 128, bool ForkToHandleRequests = true>
+class ServerStream : public Daemon
+{
+public:
+	ServerStream()
+	{
+	}
+	~ServerStream()
+	{
+		DeleteSockets();
+	}
+private:
+	ServerStream(const ServerStream &rToCopy)
+	{
+	}
+public:
+
+	virtual const char *DaemonName() const
+	{
+		return "generic-stream-server";
+	}
+
+	virtual void Run()
+	{
+		// Set process title as appropraite
+		SetProcessTitle(ForkToHandleRequests?"server":"idle");
+	
+		// Handle exceptions and child task quitting gracefully.
+		bool childExit = false;
+		try
+		{
+			Run2(childExit);
+		}
+		catch(BoxException &e)
+		{
+			if(childExit)
+			{
+				::syslog(LOG_ERR, "in server child, exception %s (%d/%d) -- terminating child", e.what(), e.GetType(), e.GetSubType());
+				_exit(1);
+			}
+			else throw;
+		}
+		catch(std::exception &e)
+		{
+			if(childExit)
+			{
+				::syslog(LOG_ERR, "in server child, exception %s -- terminating child", e.what());
+				_exit(1);
+			}
+			else throw;
+		}
+		catch(...)
+		{
+			if(childExit)
+			{
+				::syslog(LOG_ERR, "in server child, unknown exception -- terminating child");
+				_exit(1);
+			}
+			else throw;
+		}
+
+		// if it's a child fork, exit the process now
+		if(childExit)
+		{
+			// Child task, dump leaks to trace, which we make sure is on
+			#ifdef BOX_MEMORY_LEAK_TESTING
+				#ifndef NDEBUG
+					TRACE_TO_SYSLOG(true);
+					TRACE_TO_STDOUT(true);
+				#endif
+				memleakfinder_traceblocksinsection();
+			#endif
+
+			// If this is a child quitting, exit now to stop bad things happening
+			_exit(0);
+		}
+	}
+	
+	virtual void Run2(bool &rChildExit)
+	{
+		try
+		{
+			// Wait object with a timeout of 10 seconds, which is a reasonable time to wait before
+			// cleaning up finished child processes.
+			WaitForEvent connectionWait(10000);
+			
+			// BLOCK
+			{
+				// Get the address we need to bind to
+				// this-> in next line required to build under some gcc versions
+				const Configuration &config(this->GetConfiguration());
+				const Configuration &server(config.GetSubConfiguration("Server"));
+				std::string addrs = server.GetKeyValue("ListenAddresses");
+	
+				// split up the list of addresses
+				std::vector<std::string> addrlist;
+				SplitString(addrs, ',', addrlist);
+	
+				for(unsigned int a = 0; a < addrlist.size(); ++a)
+				{
+					// split the address up into components
+					std::vector<std::string> c;
+					SplitString(addrlist[a], ':', c);
+	
+					// listen!
+					SocketListen<StreamType, ListenBacklog> *psocket = new SocketListen<StreamType, ListenBacklog>;
+					try
+					{
+						if(c[0] == "inet")
+						{
+							// Check arguments
+							if(c.size() != 2 && c.size() != 3)
+							{
+								THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs)
+							}
+							
+							// Which port?
+							int port = Port;
+							
+							if(c.size() == 3)
+							{
+								// Convert to number
+								port = ::atol(c[2].c_str());
+								if(port <= 0 || port > ((64*1024)-1))
+								{
+									THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs)
+								}
+							}
+							
+							// Listen
+							psocket->Listen(Socket::TypeINET, c[1].c_str(), port);
+						}
+						else if(c[0] == "unix")
+						{
+							// Check arguments size
+							if(c.size() != 2)
+							{
+								THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs)
+							}
+
+							// unlink anything there
+							::unlink(c[1].c_str());
+							
+							psocket->Listen(Socket::TypeUNIX, c[1].c_str());
+						}
+						else
+						{
+							delete psocket;
+							THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs)
+						}
+						
+						// Add to list of sockets
+						mSockets.push_back(psocket);
+					}
+					catch(...)
+					{
+						delete psocket;
+						throw;
+					}
+
+					// Add to the list of things to wait on
+					connectionWait.Add(psocket);
+				}
+			}
+	
+			while(!StopRun())
+			{
+				// Wait for a connection, or timeout
+				SocketListen<StreamType, ListenBacklog> *psocket
+					= (SocketListen<StreamType, ListenBacklog> *)connectionWait.Wait();
+
+				if(psocket)
+				{
+					// Get the incomming connection (with zero wait time)
+					std::string logMessage;
+					std::auto_ptr<StreamType> connection(psocket->Accept(0, &logMessage));
+
+					// Was there one (there should be...)
+					if(connection.get())
+					{
+						// Since this is a template parameter, the if() will be optimised out by the compiler
+						if(ForkToHandleRequests)
+						{
+							pid_t pid = ::fork();
+							switch(pid)
+							{
+							case -1:
+								// Error!
+								THROW_EXCEPTION(ServerException, ServerForkError)
+								break;
+								
+							case 0:
+								// Child process
+								rChildExit = true;
+								// Close listening sockets
+								DeleteSockets();
+								
+								// Set up daemon
+								EnterChild();
+								SetProcessTitle("transaction");
+								
+								// Memory leak test the forked process
+								#ifdef BOX_MEMORY_LEAK_TESTING
+									memleakfinder_startsectionmonitor();
+								#endif
+								
+								// The derived class does some server magic with the connection
+								HandleConnection(*connection);
+								// Since rChildExit == true, the forked process will call _exit() on return from this fn
+								return;
+			
+							default:
+								// parent daemon process
+								break;
+							}
+							
+							// Log it
+							::syslog(LOG_INFO, "%s (handling in child %d)", logMessage.c_str(), pid);
+						}
+						else
+						{
+							// Just handle in this connection
+							SetProcessTitle("handling");
+							HandleConnection(*connection);
+							SetProcessTitle("idle");										
+						}
+					}
+				}
+				
+				// Clean up child processes (if forking daemon)
+				if(ForkToHandleRequests)
+				{
+					int status = 0;
+					int p = 0;
+					do
+					{
+						if((p = ::waitpid(0 /* any child in process group */, &status, WNOHANG)) == -1
+							&& errno != ECHILD && errno != EINTR)
+						{
+							THROW_EXCEPTION(ServerException, ServerWaitOnChildError)
+						}
+					} while(p > 0);
+				}
+			}
+		}
+		catch(...)
+		{
+			DeleteSockets();
+			throw;
+		}
+		
+		// Delete the sockets
+		DeleteSockets();
+	}
+
+	virtual void HandleConnection(StreamType &rStream)
+	{
+		Connection(rStream);
+	}
+
+	virtual void Connection(StreamType &rStream) = 0;
+	
+protected:
+	// For checking code in dervied classes -- use if you have an algorithm which
+	// depends on the forking model in case someone changes it later.
+	bool WillForkToHandleRequests()
+	{
+		return ForkToHandleRequests;
+	}
+
+private:
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    ServerStream::DeleteSockets()
+	//		Purpose: Delete sockets
+	//		Created: 9/3/04
+	//
+	// --------------------------------------------------------------------------
+	void DeleteSockets()
+	{
+		for(unsigned int l = 0; l < mSockets.size(); ++l)
+		{
+			if(mSockets[l])
+			{
+				mSockets[l]->Close();
+				delete mSockets[l];
+			}
+			mSockets[l] = 0;
+		}
+		mSockets.clear();
+	}
+
+private:
+	std::vector<SocketListen<StreamType, ListenBacklog> *> mSockets;
+};
+
+#define SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \
+											{"ListenAddresses", DEFAULT_ADDRESSES, 0, 0}, \
+											DAEMON_VERIFY_SERVER_KEYS 
+
+#include "MemLeakFindOff.h"
+
+#endif // SERVERSTREAM__H
+
+
+
diff --git a/lib/server/ServerTLS.h b/lib/server/ServerTLS.h
new file mode 100755
index 00000000..71d35380
--- /dev/null
+++ b/lib/server/ServerTLS.h
@@ -0,0 +1,80 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    ServerTLS.h
+//		Purpose: Implementation of a server using TLS streams
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+
+#ifndef SERVERTLS__H
+#define SERVERTLS__H
+
+#include "ServerStream.h"
+#include "SocketStreamTLS.h"
+#include "SSLLib.h"
+#include "TLSContext.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    ServerTLS
+//		Purpose: Implementation of a server using TLS streams
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+template<int Port, int ListenBacklog = 128, bool ForkToHandleRequests = true>
+class ServerTLS : public ServerStream<SocketStreamTLS, Port, ListenBacklog, ForkToHandleRequests>
+{
+public:
+	ServerTLS()
+	{
+		// Safe to call this here, as the Daemon class makes sure there is only one instance every of a Daemon.
+		SSLLib::Initialise();
+	}
+	
+	~ServerTLS()
+	{
+	}
+private:
+	ServerTLS(const ServerTLS &)
+	{
+	}
+public:
+
+	virtual void Run2(bool &rChildExit)
+	{
+		// First, set up the SSL context.
+		// Get parameters from the configuration
+		// this-> in next line required to build under some gcc versions
+		const Configuration &conf(this->GetConfiguration());
+		const Configuration &serverconf(conf.GetSubConfiguration("Server"));
+		std::string certFile(serverconf.GetKeyValue("CertificateFile"));
+		std::string keyFile(serverconf.GetKeyValue("PrivateKeyFile"));
+		std::string caFile(serverconf.GetKeyValue("TrustedCAsFile"));
+		mContext.Initialise(true /* as server */, certFile.c_str(), keyFile.c_str(), caFile.c_str());
+	
+		// Then do normal stream server stuff
+		ServerStream<SocketStreamTLS, Port, ListenBacklog>::Run2(rChildExit);
+	}
+	
+	virtual void HandleConnection(SocketStreamTLS &rStream)
+	{
+		rStream.Handshake(mContext, true /* is server */);
+		// this-> in next line required to build under some gcc versions
+		this->Connection(rStream);
+	}
+	
+private:
+	TLSContext mContext;
+};
+
+#define SERVERTLS_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \
+											{"CertificateFile", 0, ConfigTest_Exists, 0}, \
+											{"PrivateKeyFile", 0, ConfigTest_Exists, 0}, \
+											{"TrustedCAsFile", 0, ConfigTest_Exists, 0}, \
+											SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES)
+
+
+#endif // SERVERTLS__H
+
diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp
new file mode 100755
index 00000000..52eb79e3
--- /dev/null
+++ b/lib/server/Socket.cpp
@@ -0,0 +1,171 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Socket.cpp
+//		Purpose: Socket related stuff
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <syslog.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <string.h>
+#include <stdio.h>
+
+#include "Socket.h"
+#include "ServerException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Socket::NameLookupToSockAddr(SocketAllAddr &, int, char *, int)
+//		Purpose: Sets up a sockaddr structure given a name and type
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type, const char *Name, int Port, int &rSockAddrLenOut)
+{
+	int sockAddrLen = 0;
+
+	switch(Type)
+	{
+	case TypeINET:
+		sockDomain = AF_INET;
+		{
+			// Lookup hostname
+			struct hostent *phost = ::gethostbyname(Name);
+			if(phost != NULL)
+			{
+				if(phost->h_addr_list[0] != 0)
+				{
+					sockAddrLen = sizeof(addr.sa_inet);
+#ifndef PLATFORM_sockaddr_NO_len
+					addr.sa_inet.sin_len = sizeof(addr.sa_inet);
+#endif
+					addr.sa_inet.sin_family = PF_INET;
+					addr.sa_inet.sin_port = htons(Port);
+					addr.sa_inet.sin_addr = *((in_addr*)phost->h_addr_list[0]);
+					for(unsigned int l = 0; l < sizeof(addr.sa_inet.sin_zero); ++l)
+					{
+						addr.sa_inet.sin_zero[l] = 0;
+					}
+				}
+				else
+				{
+					THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError);
+				}
+			}
+			else
+			{
+				THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError);
+			}
+		}
+		break;
+	
+	case TypeUNIX:
+		sockDomain = AF_UNIX;
+		{
+			// Check length of name is OK
+			unsigned int nameLen = ::strlen(Name);
+			if(nameLen >= (sizeof(addr.sa_unix.sun_path) - 1))
+			{
+				THROW_EXCEPTION(ServerException, SocketNameUNIXPathTooLong);
+			}
+			sockAddrLen = nameLen + (((char*)(&(addr.sa_unix.sun_path[0]))) - ((char*)(&addr.sa_unix)));
+#ifndef PLATFORM_sockaddr_NO_len
+			addr.sa_unix.sun_len = sockAddrLen;
+#endif
+			addr.sa_unix.sun_family = PF_UNIX;
+			::strcpy(addr.sa_unix.sun_path, Name);
+		}
+		break;
+		
+	default:
+		THROW_EXCEPTION(CommonException, BadArguments)
+		break;
+	}
+	
+	// Return size of structure to caller
+	rSockAddrLenOut = sockAddrLen;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Socket::LogIncomingConnection(const struct sockaddr *, socklen_t)
+//		Purpose: Writes a message logging the connection to syslog
+//		Created: 2003/08/01
+//
+// --------------------------------------------------------------------------
+void Socket::LogIncomingConnection(const struct sockaddr *addr, socklen_t addrlen)
+{
+	if(addr == NULL) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+	switch(addr->sa_family)
+	{
+	case AF_UNIX:
+		::syslog(LOG_INFO, "Incoming connection from local (UNIX socket)");
+		break;		
+	
+	case AF_INET:
+		{
+			sockaddr_in *a = (sockaddr_in*)addr;
+			::syslog(LOG_INFO, "Incoming connection from %s port %d", inet_ntoa(a->sin_addr), ntohs(a->sin_port));
+		}
+		break;		
+	
+	default:
+		::syslog(LOG_INFO, "Incoming connection of unknown type");
+		break;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    Socket::IncomingConnectionLogMessage(const struct sockaddr *, socklen_t)
+//		Purpose: Returns a string for use in log messages
+//		Created: 2003/08/01
+//
+// --------------------------------------------------------------------------
+std::string Socket::IncomingConnectionLogMessage(const struct sockaddr *addr, socklen_t addrlen)
+{
+	if(addr == NULL) {THROW_EXCEPTION(CommonException, BadArguments)}
+
+	switch(addr->sa_family)
+	{
+	case AF_UNIX:
+		return std::string("Incoming connection from local (UNIX socket)");
+		break;		
+	
+	case AF_INET:
+		{
+			char msg[256];	// more than enough
+			sockaddr_in *a = (sockaddr_in*)addr;
+			sprintf(msg, "Incoming connection from %s port %d", inet_ntoa(a->sin_addr), ntohs(a->sin_port));
+			return std::string(msg);
+		}
+		break;		
+	
+	default:
+		return std::string("Incoming connection of unknown type");
+		break;
+	}
+	
+	// Dummy.
+	return std::string();
+}
+
diff --git a/lib/server/Socket.h b/lib/server/Socket.h
new file mode 100755
index 00000000..86a06097
--- /dev/null
+++ b/lib/server/Socket.h
@@ -0,0 +1,47 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    Socket.h
+//		Purpose: Socket related stuff
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef SOCKET__H
+#define SOCKET__H
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <sys/un.h>
+
+#include <string>
+
+typedef union {
+	struct sockaddr sa_generic;
+	struct sockaddr_in sa_inet;
+	struct sockaddr_un sa_unix;
+} SocketAllAddr;
+
+// --------------------------------------------------------------------------
+//
+// Namespace
+//		Name:    Socket
+//		Purpose: Socket utilities
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+namespace Socket
+{
+	enum
+	{
+		TypeINET = 1,
+		TypeUNIX = 2
+	};
+
+	void NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type, const char *Name, int Port, int &rSockAddrLenOut);
+	void LogIncomingConnection(const struct sockaddr *addr, socklen_t addrlen);
+	std::string IncomingConnectionLogMessage(const struct sockaddr *addr, socklen_t addrlen);
+};
+
+#endif // SOCKET__H
+
diff --git a/lib/server/SocketListen.h b/lib/server/SocketListen.h
new file mode 100755
index 00000000..f1f5e377
--- /dev/null
+++ b/lib/server/SocketListen.h
@@ -0,0 +1,265 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    SocketListen.h
+//		Purpose: Stream based sockets for servers
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef SOCKETLISTEN__H
+#define SOCKETLISTEN__H
+
+#include <errno.h>
+#include <unistd.h>
+#include <new>
+#include <poll.h>
+#include <memory>
+#include <string>
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	#include <sys/event.h>
+	#include <sys/time.h>
+#endif
+
+#include "Socket.h"
+#include "ServerException.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    _NoSocketLocking
+//		Purpose: Default locking class for SocketListen
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+class _NoSocketLocking
+{
+public:
+	_NoSocketLocking(int sock)
+	{
+	}
+	
+	~_NoSocketLocking()
+	{
+	}
+	
+	bool HaveLock()
+	{
+		return true;
+	}
+	
+private:
+	_NoSocketLocking(const _NoSocketLocking &rToCopy)
+	{
+	}
+};
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    SocketListen
+//		Purpose: 
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+template<typename SocketType, int ListenBacklog = 128, typename SocketLockingType = _NoSocketLocking, int MaxMultiListenSockets = 16>
+class SocketListen
+{
+public:
+	// Initialise
+	SocketListen()
+		: mSocketHandle(-1)
+	{
+	}
+	// Close socket nicely
+	~SocketListen()
+	{
+		Close();
+	}
+private:
+	SocketListen(const SocketListen &rToCopy)
+	{
+	}
+public:
+
+	enum
+	{
+		MaxMultipleListenSockets = MaxMultiListenSockets
+	};
+
+	void Close()
+	{
+		if(mSocketHandle != -1)
+		{
+			if(::close(mSocketHandle) == -1)
+			{
+				THROW_EXCEPTION(ServerException, SocketCloseError)
+			}
+		}
+		mSocketHandle = -1;
+	}
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    SocketListen::Listen(int, char*, int)
+	//		Purpose: Initialises, starts the socket listening.
+	//		Created: 2003/07/31
+	//
+	// --------------------------------------------------------------------------
+	void Listen(int Type, const char *Name, int Port = 0)
+	{
+		if(mSocketHandle != -1) {THROW_EXCEPTION(ServerException, SocketAlreadyOpen)}
+		
+		// Setup parameters based on type, looking up names if required
+		int sockDomain = 0;
+		SocketAllAddr addr;
+		int addrLen = 0;
+		Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, Port, addrLen);
+	
+		// Create the socket
+		mSocketHandle = ::socket(sockDomain, SOCK_STREAM, 0 /* let OS choose protocol */);
+		if(mSocketHandle == -1)
+		{
+			THROW_EXCEPTION(ServerException, SocketOpenError)
+		}
+		
+		// Set an option to allow reuse (useful for -HUP situations!)
+		int option = true;
+		if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, &option, sizeof(option)) == -1)
+		{
+			THROW_EXCEPTION(ServerException, SocketOpenError)
+		}
+
+		// Bind it to the right port, and start listening
+		if(::bind(mSocketHandle, &addr.sa_generic, addrLen) == -1
+			|| ::listen(mSocketHandle, ListenBacklog) == -1)
+		{
+			// Dispose of the socket
+			::close(mSocketHandle);
+			mSocketHandle = -1;
+			THROW_EXCEPTION(ServerException, SocketBindError)
+		}	
+	}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    SocketListen::Accept(int)
+	//		Purpose: Accepts a connection, returning a pointer to a class of
+	//				 the specified type. May return a null pointer if a signal happens,
+	//				 or there's a timeout. Timeout specified in milliseconds, defaults to infinite time.
+	//		Created: 2003/07/31
+	//
+	// --------------------------------------------------------------------------
+	std::auto_ptr<SocketType> Accept(int Timeout = INFTIM, std::string *pLogMsg = 0)
+	{
+		if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)}
+		
+		// Do the accept, using the supplied locking type
+		int sock;
+		struct sockaddr addr;
+		socklen_t addrlen = sizeof(addr);
+		// BLOCK
+		{
+			SocketLockingType socklock(mSocketHandle);
+			
+			if(!socklock.HaveLock())
+			{
+				// Didn't get the lock for some reason. Wait a while, then
+				// return nothing.
+				::sleep(1);
+				return std::auto_ptr<SocketType>();
+			}
+			
+			// poll this socket
+			struct pollfd p;
+			p.fd = mSocketHandle;
+			p.events = POLLIN;
+			p.revents = 0;
+			switch(::poll(&p, 1, Timeout))
+			{
+			case -1:
+				// signal?
+				if(errno == EINTR)
+				{
+					// return nothing
+					return std::auto_ptr<SocketType>();
+				}
+				else
+				{
+					THROW_EXCEPTION(ServerException, SocketPollError)
+				}
+				break;
+			case 0:	// timed out
+				return std::auto_ptr<SocketType>();
+				break;
+			default:	// got some thing...
+				// control flows on...
+				break;
+			}
+			
+			sock = ::accept(mSocketHandle, &addr, &addrlen);
+		}
+		// Got socket (or error), unlock (implcit in destruction)
+		if(sock == -1)
+		{
+			THROW_EXCEPTION(ServerException, SocketAcceptError)
+		}
+
+		// Log it
+		if(pLogMsg)
+		{
+			*pLogMsg = Socket::IncomingConnectionLogMessage(&addr, addrlen);
+		}
+		else
+		{
+			// Do logging ourselves
+			Socket::LogIncomingConnection(&addr, addrlen);
+		}
+
+		return std::auto_ptr<SocketType>(new SocketType(sock));
+	}
+	
+	// Functions to allow adding to WaitForEvent class, for efficient waiting
+	// on multiple sockets.
+#ifndef PLATFORM_KQUEUE_NOT_SUPPORTED
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    SocketListen::FillInKEevent
+	//		Purpose: Fills in a kevent structure for this socket
+	//		Created: 9/3/04
+	//
+	// --------------------------------------------------------------------------
+	void FillInKEvent(struct kevent &rEvent, int Flags = 0) const
+	{
+		EV_SET(&rEvent, mSocketHandle, EVFILT_READ, 0, 0, 0, (void*)this);
+	}
+#else
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    SocketListen::FillInPoll
+	//		Purpose: Fills in the data necessary for a poll operation
+	//		Created: 9/3/04
+	//
+	// --------------------------------------------------------------------------
+	void FillInPoll(int &fd, short &events, int Flags = 0) const
+	{
+		fd = mSocketHandle;
+		events = POLLIN;
+	}
+#endif
+	
+private:
+	int mSocketHandle;
+};
+
+#include "MemLeakFindOff.h"
+
+#endif // SOCKETLISTEN__H
+
diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp
new file mode 100755
index 00000000..3c8bf453
--- /dev/null
+++ b/lib/server/SocketStream.cpp
@@ -0,0 +1,405 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    SocketStream.cpp
+//		Purpose: I/O stream interface for sockets
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <unistd.h>
+#include <sys/types.h>
+#include <poll.h>
+#include <errno.h>
+
+#include "SocketStream.h"
+#include "ServerException.h"
+#include "CommonException.h"
+#include "Socket.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::SocketStream()
+//		Purpose: Constructor (create stream ready for Open() call)
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+SocketStream::SocketStream()
+	: mSocketHandle(-1),
+	  mReadClosed(false),
+	  mWriteClosed(false)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::SocketStream(int)
+//		Purpose: Create stream from existing socket handle
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+SocketStream::SocketStream(int socket)
+	: mSocketHandle(socket),
+	  mReadClosed(false),
+	  mWriteClosed(false)
+{
+	if(socket < 0)
+	{
+		THROW_EXCEPTION(ServerException, BadSocketHandle);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::SocketStream(const SocketStream &)
+//		Purpose: Copy constructor (dup()s socket)
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+SocketStream::SocketStream(const SocketStream &rToCopy)
+	: mSocketHandle(::dup(rToCopy.mSocketHandle)),
+	  mReadClosed(rToCopy.mReadClosed),
+	  mWriteClosed(rToCopy.mWriteClosed)
+
+{
+	if(rToCopy.mSocketHandle < 0)
+	{
+		THROW_EXCEPTION(ServerException, BadSocketHandle);
+	}
+	if(mSocketHandle == -1)
+	{
+		THROW_EXCEPTION(ServerException, DupError);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::~SocketStream()
+//		Purpose: Destructor, closes stream if open
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+SocketStream::~SocketStream()
+{
+	if(mSocketHandle != -1)
+	{
+		Close();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::Attach(int)
+//		Purpose: Attach a socket handle to this stream
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void SocketStream::Attach(int socket)
+{
+	if(mSocketHandle != -1) {THROW_EXCEPTION(ServerException, SocketAlreadyOpen)}
+
+	mSocketHandle = socket;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::Open(int, char *, int)
+//		Purpose: Opens a connection to a listening socket (INET or UNIX)
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void SocketStream::Open(int Type, const char *Name, int Port)
+{
+	if(mSocketHandle != -1) {THROW_EXCEPTION(ServerException, SocketAlreadyOpen)}
+	
+	// Setup parameters based on type, looking up names if required
+	int sockDomain = 0;
+	SocketAllAddr addr;
+	int addrLen = 0;
+	Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, Port, addrLen);
+
+	// Create the socket
+	mSocketHandle = ::socket(sockDomain, SOCK_STREAM, 0 /* let OS choose protocol */);
+	if(mSocketHandle == -1)
+	{
+		THROW_EXCEPTION(ServerException, SocketOpenError)
+	}
+	
+	// Connect it
+	if(::connect(mSocketHandle, &addr.sa_generic, addrLen) == -1)
+	{
+		// Dispose of the socket
+		::close(mSocketHandle);
+		mSocketHandle = -1;
+		THROW_EXCEPTION(ConnectionException, Conn_SocketConnectError)
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::Read(void *pBuffer, int NBytes)
+//		Purpose: Reads data from stream. Maybe returns less than asked for.
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+int SocketStream::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)}
+
+	if(Timeout != IOStream::TimeOutInfinite)
+	{
+		struct pollfd p;
+		p.fd = mSocketHandle;
+		p.events = POLLIN;
+		p.revents = 0;
+		switch(::poll(&p, 1, (Timeout == IOStream::TimeOutInfinite)?INFTIM:Timeout))
+		{
+		case -1:
+			// error
+			if(errno == EINTR)
+			{
+				// Signal. Just return 0 bytes
+				return 0;
+			}
+			else
+			{
+				// Bad!
+				THROW_EXCEPTION(ServerException, SocketPollError)
+			}
+			break;
+			
+		case 0:
+			// no data
+			return 0;
+			break;
+			
+		default:
+			// good to go!
+			break;
+		}
+	}
+
+	int r = ::read(mSocketHandle, pBuffer, NBytes);
+	if(r == -1)
+	{
+		if(errno == EINTR)
+		{
+			// Nothing could be read
+			return 0;
+		}
+		else
+		{
+			// Other error
+			THROW_EXCEPTION(ConnectionException, Conn_SocketReadError)
+		}
+	}
+	// Closed for reading?
+	if(r == 0)
+	{
+		mReadClosed = true;
+	}
+	
+	return r;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::Write(void *pBuffer, int NBytes)
+//		Purpose: Writes data, blocking until it's all done.
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void SocketStream::Write(const void *pBuffer, int NBytes)
+{
+	if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)}
+	
+	// Buffer in byte sized type.
+	ASSERT(sizeof(char) == 1);
+	const char *buffer = (char *)pBuffer;
+	
+	// Bytes left to send
+	int bytesLeft = NBytes;
+	
+	while(bytesLeft > 0)
+	{
+		// Try to send.
+		int sent = ::write(mSocketHandle, buffer, bytesLeft);
+		if(sent == -1)
+		{
+			// Error.
+			mWriteClosed = true;	// assume can't write again
+			THROW_EXCEPTION(ConnectionException, Conn_SocketWriteError)
+		}
+		
+		// Knock off bytes sent
+		bytesLeft -= sent;
+		// Move buffer pointer
+		buffer += sent;
+		
+		// Need to wait until it can send again?
+		if(bytesLeft > 0)
+		{
+			TRACE3("Waiting to send data on socket %d, (%d to send of %d)\n", mSocketHandle, bytesLeft, NBytes);
+			
+			// Wait for data to send.
+			struct pollfd p;
+			p.fd = mSocketHandle;
+			p.events = POLLOUT;
+			p.revents = 0;
+			
+			if(::poll(&p, 1, 16000 /* 16 seconds */) == -1)
+			{
+				// Don't exception if it's just a signal
+				if(errno != EINTR)
+				{
+					THROW_EXCEPTION(ServerException, SocketPollError)
+				}
+			}
+		}
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::Close()
+//		Purpose: Closes connection to remote socket
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void SocketStream::Close()
+{
+	if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)}
+	
+	if(::close(mSocketHandle) == -1)
+	{
+		THROW_EXCEPTION(ServerException, SocketCloseError)
+	}
+	mSocketHandle = -1;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::Shutdown(bool, bool)
+//		Purpose: Shuts down a socket for further reading and/or writing
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+void SocketStream::Shutdown(bool Read, bool Write)
+{
+	if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)}
+	
+	// Do anything?
+	if(!Read && !Write) return;
+	
+	int how = SHUT_RDWR;
+	if(Read && !Write) how = SHUT_RD;
+	if(!Read && Write) how = SHUT_WR;
+	
+	// Shut it down!
+	if(::shutdown(mSocketHandle, how) == -1)
+	{
+		THROW_EXCEPTION(ConnectionException, Conn_SocketShutdownError)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::StreamDataLeft()
+//		Purpose: Still capable of reading data?
+//		Created: 2003/08/02
+//
+// --------------------------------------------------------------------------
+bool SocketStream::StreamDataLeft()
+{
+	return !mReadClosed;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::StreamClosed()
+//		Purpose: Connection been closed?
+//		Created: 2003/08/02
+//
+// --------------------------------------------------------------------------
+bool SocketStream::StreamClosed()
+{
+	return mWriteClosed;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::GetSocketHandle()
+//		Purpose: Returns socket handle for this stream (derived classes only).
+//				 Will exception if there's no valid socket.
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+int SocketStream::GetSocketHandle()
+{
+	if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)}
+	return mSocketHandle;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStream::GetPeerCredentials(uid_t &, gid_t &)
+//		Purpose: Returns true if the peer credientials are available.
+//				 (will work on UNIX domain sockets only)
+//		Created: 19/2/04
+//
+// --------------------------------------------------------------------------
+bool SocketStream::GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut)
+{
+#ifdef PLATFORM_HAVE_getpeereid
+	uid_t remoteEUID = 0xffff;
+	gid_t remoteEGID = 0xffff;
+
+	if(::getpeereid(mSocketHandle, &remoteEUID, &remoteEGID) == 0)
+	{
+		rUidOut = remoteEUID;
+		rGidOut = remoteEGID;
+		return true;
+	}
+#endif // PLATFORM_HAVE_getpeereid
+
+#ifdef PLATFORM_HAVE_getsockopt_SO_PEERCRED
+	struct ucred cred;
+	socklen_t credLen = sizeof(cred);
+
+	if(::getsockopt(mSocketHandle, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0)
+	{
+		rUidOut = cred.uid;
+		rGidOut = cred.gid;
+		return true;
+	}
+#endif // PLATFORM_HAVE_getsockopt_SO_PEERCRED
+
+	// Not available
+	return false;
+}
+
+
+
+
diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h
new file mode 100755
index 00000000..9b16dbfc
--- /dev/null
+++ b/lib/server/SocketStream.h
@@ -0,0 +1,56 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    SocketStream.h
+//		Purpose: I/O stream interface for sockets
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+
+#ifndef SOCKETSTREAM__H
+#define SOCKETSTREAM__H
+
+#include "IOStream.h"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    SocketStream
+//		Purpose: Stream interface for sockets
+//		Created: 2003/07/31
+//
+// --------------------------------------------------------------------------
+class SocketStream : public IOStream
+{
+public:
+	SocketStream();
+	SocketStream(int socket);
+	SocketStream(const SocketStream &rToCopy);
+	~SocketStream();
+	
+	void Open(int Type, const char *Name, int Port = 0);
+	void Attach(int socket);
+
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual void Close();
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+	virtual void Shutdown(bool Read = true, bool Write = true);
+
+	virtual bool GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut);
+
+protected:
+	int GetSocketHandle();
+	void MarkAsReadClosed() {mReadClosed = true;}
+	void MarkAsWriteClosed() {mWriteClosed = true;}
+
+private:
+	int mSocketHandle;
+	bool mReadClosed;
+	bool mWriteClosed;
+};
+
+#endif // SOCKETSTREAM__H
+
diff --git a/lib/server/SocketStreamTLS.cpp b/lib/server/SocketStreamTLS.cpp
new file mode 100755
index 00000000..63ac7bb5
--- /dev/null
+++ b/lib/server/SocketStreamTLS.cpp
@@ -0,0 +1,457 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    SocketStreamTLS.cpp
+//		Purpose: Socket stream encrpyted and authenticated by TLS
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#define TLS_CLASS_IMPLEMENTATION_CPP
+#include <openssl/ssl.h>
+#include <openssl/bio.h>
+#include <poll.h>
+#include <errno.h>
+#include <sys/ioctl.h>
+
+#include "SocketStreamTLS.h"
+#include "SSLLib.h"
+#include "ServerException.h"
+#include "TLSContext.h"
+
+#include "MemLeakFindOn.h"
+
+// Allow 5 minutes to handshake (in milliseconds)
+#define TLS_HANDSHAKE_TIMEOUT (5*60*1000)
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::SocketStreamTLS()
+//		Purpose: Constructor
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+SocketStreamTLS::SocketStreamTLS()
+	: mpSSL(0), mpBIO(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::SocketStreamTLS(int)
+//		Purpose: Constructor, taking previously connected socket
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+SocketStreamTLS::SocketStreamTLS(int socket)
+	: SocketStream(socket),
+	  mpSSL(0), mpBIO(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::~SocketStreamTLS()
+//		Purpose: Destructor
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+SocketStreamTLS::~SocketStreamTLS()
+{
+	if(mpSSL)
+	{
+		// Attempt to close to avoid problems
+		Close();
+		
+		// And if that didn't work...
+		if(mpSSL)
+		{
+			::SSL_free(mpSSL);
+			mpSSL = 0;
+			mpBIO = 0;	// implicity freed by the SSL_free call
+		}
+	}
+	
+	// If we only got to creating that BIO.
+	if(mpBIO)
+	{
+		::BIO_free(mpBIO);
+		mpBIO = 0;
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::Open(const TLSContext &, int, const char *, int)
+//		Purpose: Open connection, and perform TLS handshake
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+void SocketStreamTLS::Open(const TLSContext &rContext, int Type, const char *Name, int Port)
+{
+	SocketStream::Open(Type, Name, Port);
+	Handshake(rContext);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::Handshake(const TLSContext &, bool)
+//		Purpose: Perform TLS handshake
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer)
+{
+	if(mpBIO || mpSSL) {THROW_EXCEPTION(ServerException, TLSAlreadyHandshaked)}
+
+	// Create a BIO for this socket
+	mpBIO = ::BIO_new(::BIO_s_socket());
+	if(mpBIO == 0)
+	{
+		SSLLib::LogError("Create socket bio");
+		THROW_EXCEPTION(ServerException, TLSAllocationFailed)
+	}
+	int socket = GetSocketHandle();
+	BIO_set_fd(mpBIO, socket, BIO_NOCLOSE);
+	
+	// Then the SSL object
+	mpSSL = ::SSL_new(rContext.GetRawContext());
+	if(mpSSL == 0)
+	{
+		SSLLib::LogError("Create ssl");
+		THROW_EXCEPTION(ServerException, TLSAllocationFailed)
+	}
+
+	// Make the socket non-blocking so timeouts on Read work
+	int nonblocking = true;
+	if(::ioctl(socket, FIONBIO, &nonblocking) == -1)
+	{
+		THROW_EXCEPTION(ServerException, SocketSetNonBlockingFailed)
+	}
+	
+	// Set the two to know about each other
+	::SSL_set_bio(mpSSL, mpBIO, mpBIO);
+
+	bool waitingForHandshake = true;
+	while(waitingForHandshake)
+	{
+		// Attempt to do the handshake
+		int r = 0;
+		if(IsServer)
+		{
+			r = ::SSL_accept(mpSSL);
+		}
+		else
+		{
+			r = ::SSL_connect(mpSSL);
+		}
+
+		// check return code
+		int se;
+		switch((se = ::SSL_get_error(mpSSL, r)))
+		{
+		case SSL_ERROR_NONE:
+			// No error, handshake succeeded
+			waitingForHandshake = false;
+			break;
+
+		case SSL_ERROR_WANT_READ:
+		case SSL_ERROR_WANT_WRITE:
+			// wait for the requried data
+			if(WaitWhenRetryRequired(se, TLS_HANDSHAKE_TIMEOUT) == false)
+			{
+				// timed out
+				THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeTimedOut)
+			}
+			break;
+			
+		default: // (and SSL_ERROR_ZERO_RETURN)
+			// Error occured
+			if(IsServer)
+			{
+				SSLLib::LogError("Accept");
+				THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed)
+			}
+			else
+			{
+				SSLLib::LogError("Connect");
+				THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed)
+			}
+		}
+	}
+	
+	// And that's it
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    WaitWhenRetryRequired(int, int)
+//		Purpose: Waits until the condition required by the TLS layer is met.
+//				 Returns true if the condition is met, false if timed out.
+//		Created: 2003/08/15
+//
+// --------------------------------------------------------------------------
+bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout)
+{
+	struct pollfd p;
+	p.fd = GetSocketHandle();
+	switch(SSLErrorCode)
+	{
+	case SSL_ERROR_WANT_READ:
+		p.events = POLLIN;
+		break;
+		
+	case SSL_ERROR_WANT_WRITE:
+		p.events = POLLOUT;
+		break;
+
+	default:
+		// Not good!
+		THROW_EXCEPTION(ServerException, Internal)
+		break;
+	}
+	p.revents = 0;
+	switch(::poll(&p, 1, (Timeout == IOStream::TimeOutInfinite)?INFTIM:Timeout))
+	{
+	case -1:
+		// error
+		if(errno == EINTR)
+		{
+			// Signal. Do "time out"
+			return false;
+		}
+		else
+		{
+			// Bad!
+			THROW_EXCEPTION(ServerException, SocketPollError)
+		}
+		break;
+
+	case 0:
+		// Condition not met, timed out
+		return false;
+		break;
+
+	default:
+		// good to go!
+		return true;
+		break;
+	}
+
+	return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::Read(void *, int, int Timeout)
+//		Purpose: See base class
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)}
+
+	// Make sure zero byte reads work as expected
+	if(NBytes == 0)
+	{
+		return 0;
+	}
+	
+	while(true)
+	{
+		int r = ::SSL_read(mpSSL, pBuffer, NBytes);
+
+		int se;
+		switch((se = ::SSL_get_error(mpSSL, r)))
+		{
+		case SSL_ERROR_NONE:
+			// No error, return number of bytes read
+			return r;
+			break;
+
+		case SSL_ERROR_ZERO_RETURN:
+			// Connection closed
+			MarkAsReadClosed();
+			return 0;
+			break;
+
+		case SSL_ERROR_WANT_READ:
+		case SSL_ERROR_WANT_WRITE:
+			// wait for the requried data
+			// Will only get once around this loop, so don't need to calculate timeout values
+			if(WaitWhenRetryRequired(se, Timeout) == false)
+			{
+				// timed out
+				return 0;
+			}
+			break;
+			
+		default:
+			SSLLib::LogError("Read");
+			THROW_EXCEPTION(ConnectionException, Conn_TLSReadFailed)
+			break;
+		}
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::Write(const void *, int)
+//		Purpose: See base class
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+void SocketStreamTLS::Write(const void *pBuffer, int NBytes)
+{
+	if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)}
+	
+	// Make sure zero byte writes work as expected
+	if(NBytes == 0)
+	{
+		return;
+	}
+	
+	// from man SSL_write
+	//
+	// SSL_write() will only return with success, when the
+	// complete contents of buf of length num has been written.
+	//
+	// So no worries about partial writes and moving the buffer around
+	
+	while(true)
+	{
+		// try the write
+		int r = ::SSL_write(mpSSL, pBuffer, NBytes);
+		
+		int se;
+		switch((se = ::SSL_get_error(mpSSL, r)))
+		{
+		case SSL_ERROR_NONE:
+			// No error, data sent, return success
+			return;
+			break;
+
+		case SSL_ERROR_ZERO_RETURN:
+			// Connection closed
+			MarkAsWriteClosed();
+			THROW_EXCEPTION(ConnectionException, Conn_TLSClosedWhenWriting)
+			break;
+
+		case SSL_ERROR_WANT_READ:
+		case SSL_ERROR_WANT_WRITE:
+			// wait for the requried data
+			{
+			#ifndef NDEBUG
+				bool conditionmet = 
+			#endif
+				WaitWhenRetryRequired(se, IOStream::TimeOutInfinite);
+				ASSERT(conditionmet);
+			}
+			break;
+		
+		default:
+			SSLLib::LogError("Write");
+			THROW_EXCEPTION(ConnectionException, Conn_TLSWriteFailed)
+			break;
+		}
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::Close()
+//		Purpose: See base class
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+void SocketStreamTLS::Close()
+{
+	if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)}
+
+	// Base class to close
+	SocketStream::Close();
+
+	// Free resources
+	::SSL_free(mpSSL);
+	mpSSL = 0;
+	mpBIO = 0;	// implicitly freed by SSL_free
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::Shutdown()
+//		Purpose: See base class
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+void SocketStreamTLS::Shutdown(bool Read, bool Write)
+{
+	if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)}
+
+	if(::SSL_shutdown(mpSSL) < 0)
+	{
+		SSLLib::LogError("Shutdown");
+		THROW_EXCEPTION(ConnectionException, Conn_TLSShutdownFailed)
+	}
+
+	// Don't ask the base class to shutdown -- BIO does this, apparently.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    SocketStreamTLS::GetPeerCommonName()
+//		Purpose: Returns the common name of the other end of the connection
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+std::string SocketStreamTLS::GetPeerCommonName()
+{
+	if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)}
+
+	// Get certificate
+	X509 *cert = ::SSL_get_peer_certificate(mpSSL);
+	if(cert == 0)
+	{
+		::X509_free(cert);
+		THROW_EXCEPTION(ConnectionException, Conn_TLSNoPeerCertificate)
+	}
+
+	// Subject details	
+	X509_NAME *subject = ::X509_get_subject_name(cert); 
+	if(subject == 0)
+	{
+		::X509_free(cert);
+		THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid)
+	}
+	
+	// Common name
+	char commonName[256];
+	if(::X509_NAME_get_text_by_NID(subject, NID_commonName, commonName, sizeof(commonName)) <= 0)
+	{
+		::X509_free(cert);
+		THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid)
+	}
+	// Terminate just in case
+	commonName[sizeof(commonName)-1] = '\0';
+	
+	// Done.
+	return std::string(commonName);
+}
+
+
diff --git a/lib/server/SocketStreamTLS.h b/lib/server/SocketStreamTLS.h
new file mode 100755
index 00000000..64e52833
--- /dev/null
+++ b/lib/server/SocketStreamTLS.h
@@ -0,0 +1,60 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    SocketStreamTLS.h
+//		Purpose: Socket stream encrpyted and authenticated by TLS
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+
+#ifndef SOCKETSTREAMTLS__H
+#define SOCKETSTREAMTLS__H
+
+#include <string>
+
+#include "SocketStream.h"
+
+class TLSContext;
+#ifndef TLS_CLASS_IMPLEMENTATION_CPP
+	class SSL;
+	class BIO;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    SocketStreamTLS
+//		Purpose: Socket stream encrpyted and authenticated by TLS
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+class SocketStreamTLS : public SocketStream
+{
+public:
+	SocketStreamTLS();
+	SocketStreamTLS(int socket);
+	~SocketStreamTLS();
+private:
+	SocketStreamTLS(const SocketStreamTLS &rToCopy);
+public:
+
+	void Open(const TLSContext &rContext, int Type, const char *Name, int Port = 0);
+	void Handshake(const TLSContext &rContext, bool IsServer = false);
+	
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual void Close();
+	virtual void Shutdown(bool Read = true, bool Write = true);
+
+	std::string GetPeerCommonName();
+
+private:
+	bool WaitWhenRetryRequired(int SSLErrorCode, int Timeout);
+
+private:
+	SSL *mpSSL;
+	BIO *mpBIO;
+};
+
+#endif // SOCKETSTREAMTLS__H
+
diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp
new file mode 100755
index 00000000..cc125d00
--- /dev/null
+++ b/lib/server/TLSContext.cpp
@@ -0,0 +1,120 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    TLSContext.h
+//		Purpose: TLS (SSL) context for connections
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#define TLS_CLASS_IMPLEMENTATION_CPP
+#include <openssl/ssl.h>
+
+#include "TLSContext.h"
+#include "ServerException.h"
+#include "SSLLib.h"
+#include "TLSContext.h"
+
+#include "MemLeakFindOn.h"
+
+#define MAX_VERIFICATION_DEPTH		2
+#define CIPHER_LIST					"ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    TLSContext::TLSContext()
+//		Purpose: Constructor
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+TLSContext::TLSContext()
+	: mpContext(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    TLSContext::~TLSContext()
+//		Purpose: Destructor
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+TLSContext::~TLSContext()
+{
+	if(mpContext != 0)
+	{
+		::SSL_CTX_free(mpContext);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    TLSContext::Initialise(bool, const char *, const char *, const char *)
+//		Purpose: Initialise the context, loading in the specified certificate and private key files
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const char *PrivateKeyFile, const char *TrustedCAsFile)
+{
+	mpContext = ::SSL_CTX_new(AsServer?TLSv1_server_method():TLSv1_client_method());
+	if(mpContext == NULL)
+	{
+		THROW_EXCEPTION(ServerException, TLSAllocationFailed)
+	}
+	
+	// Setup our identity
+	if(::SSL_CTX_use_certificate_chain_file(mpContext, CertificatesFile) != 1)
+	{
+		SSLLib::LogError("Load certificates");
+		THROW_EXCEPTION(ServerException, TLSLoadCertificatesFailed)
+	}
+	if(::SSL_CTX_use_PrivateKey_file(mpContext, PrivateKeyFile, SSL_FILETYPE_PEM) != 1)
+	{
+		SSLLib::LogError("Load private key");
+		THROW_EXCEPTION(ServerException, TLSLoadPrivateKeyFailed)
+	}
+	
+	// Setup the identify of CAs we trust
+	if(::SSL_CTX_load_verify_locations(mpContext, TrustedCAsFile, NULL) != 1)
+	{
+		SSLLib::LogError("Load CA cert");
+		THROW_EXCEPTION(ServerException, TLSLoadTrustedCAsFailed)
+	}
+	
+	// Setup options to require these certificates
+	::SSL_CTX_set_verify(mpContext, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
+	// and a sensible maximum depth
+	::SSL_CTX_set_verify_depth(mpContext, MAX_VERIFICATION_DEPTH);
+	
+	// Setup allowed ciphers
+	if(::SSL_CTX_set_cipher_list(mpContext, CIPHER_LIST) != 1)
+	{
+		SSLLib::LogError("Set cipher list");
+		THROW_EXCEPTION(ServerException, TLSSetCiphersFailed)
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    TLSContext::GetRawContext()
+//		Purpose: Get the raw context for OpenSSL API
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+SSL_CTX *TLSContext::GetRawContext() const
+{
+	if(mpContext == 0)
+	{
+		THROW_EXCEPTION(ServerException, TLSContextNotInitialised)
+	}
+	return mpContext;
+}
+
+
+
diff --git a/lib/server/TLSContext.h b/lib/server/TLSContext.h
new file mode 100755
index 00000000..f52f5457
--- /dev/null
+++ b/lib/server/TLSContext.h
@@ -0,0 +1,41 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    TLSContext.h
+//		Purpose: TLS (SSL) context for connections
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+
+#ifndef TLSCONTEXT__H
+#define TLSCONTEXT__H
+
+#ifndef TLS_CLASS_IMPLEMENTATION_CPP
+	class SSL_CTX;
+#endif
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    TLSContext
+//		Purpose: TLS (SSL) context for connections
+//		Created: 2003/08/06
+//
+// --------------------------------------------------------------------------
+class TLSContext
+{
+public:
+	TLSContext();
+	~TLSContext();
+private:
+	TLSContext(const TLSContext &);
+public:
+	void Initialise(bool AsServer, const char *CertificatesFile, const char *PrivateKeyFile, const char *TrustedCAsFile);
+	SSL_CTX *GetRawContext() const;
+
+private:
+	SSL_CTX *mpContext;
+};
+
+#endif // TLSCONTEXT__H
+
diff --git a/lib/server/makeprotocol.pl b/lib/server/makeprotocol.pl
new file mode 100755
index 00000000..2a69c59c
--- /dev/null
+++ b/lib/server/makeprotocol.pl
@@ -0,0 +1,993 @@
+#!/usr/bin/perl
+use strict;
+
+# Make protocol C++ classes from a protocol description file
+
+# built in type info (values are is basic type, C++ typename)
+# may get stuff added to it later if protocol uses extra types
+my %translate_type_info =
+(
+	'int64' => [1, 'int64_t'],
+	'int32' => [1, 'int32_t'],
+	'int16' => [1, 'int16_t'],
+	'int8' => [1, 'int8_t'],
+	'bool' => [1, 'bool'],
+	'string' => [0, 'std::string']
+);
+
+# built in instructions for logging various types
+# may be added to
+my %log_display_types = 
+(
+	'int64' => ['0x%llx', 'VAR'],
+	'int32' => ['0x%x', 'VAR'],
+	'int16' => ['0x%x', 'VAR'],
+	'int8' => ['0x%x', 'VAR'],
+	'bool' => ['%s', '((VAR)?"true":"false")'],
+	'string' => ['%s', 'VAR.c_str()']
+);
+
+
+
+my ($type, $file) = @ARGV;
+
+if($type ne 'Server' && $type ne 'Client')
+{
+	die "Neither Server or Client is specified on command line\n";
+}
+
+open IN, $file or die "Can't open input file $file\n";
+
+print "Making $type protocol classes from $file...\n";
+
+my @extra_header_files;
+
+my $implement_syslog = 0;
+my $implement_filelog = 0;
+
+# read attributes
+my %attr;
+while(<IN>)
+{
+	# get and clean line
+	my $l = $_; $l =~ s/#.*\Z//; $l =~ s/\A\s+//; $l =~ s/\s+\Z//; next unless $l =~ m/\S/;
+	
+	last if $l eq 'BEGIN_OBJECTS';
+	
+	my ($k,$v) = split /\s+/,$l,2;
+	
+	if($k eq 'ClientType')
+	{
+		add_type($v) if $type eq 'Client';
+	}
+	elsif($k eq 'ServerType')
+	{
+		add_type($v) if $type eq 'Server';
+	}
+	elsif($k eq 'ImplementLog')
+	{
+		my ($log_if_type,$log_type) = split /\s+/,$v;
+		if($type eq $log_if_type)
+		{
+			if($log_type eq 'syslog')
+			{
+				$implement_syslog = 1;
+			}
+			elsif($log_type eq 'file')
+			{
+				$implement_filelog = 1;
+			}
+			else
+			{
+				printf("ERROR: Unknown log type for implementation: $log_type\n");
+				exit(1);
+			}
+		}
+	}
+	elsif($k eq 'LogTypeToText')
+	{
+		my ($log_if_type,$type_name,$printf_format,$arg_template) = split /\s+/,$v;
+		if($type eq $log_if_type)
+		{
+			$log_display_types{$type_name} = [$printf_format,$arg_template]
+		}
+	}
+	else
+	{
+		$attr{$k} = $v;
+	}
+}
+
+sub add_type
+{
+	my ($protocol_name, $cpp_name, $header_file) = split /\s+/,$_[0];
+	
+	$translate_type_info{$protocol_name} = [0, $cpp_name];
+	push @extra_header_files, $header_file;
+}
+
+# check attributes
+for(qw/Name ServerContextClass IdentString/)
+{
+	if(!exists $attr{$_})
+	{
+		die "Attribute $_ is required, but not specified\n";
+	}
+}
+
+my $protocol_name = $attr{'Name'};
+my ($context_class, $context_class_inc) = split /\s+/,$attr{'ServerContextClass'};
+my $ident_string = $attr{'IdentString'};
+
+my $current_cmd = '';
+my %cmd_contents;
+my %cmd_attributes;
+my %cmd_constants;
+my %cmd_id;
+my @cmd_list;
+
+# read in the command definitions
+while(<IN>)
+{
+	# get and clean line
+	my $l = $_; $l =~ s/#.*\Z//; $l =~ s/\s+\Z//; next unless $l =~ m/\S/;
+	
+	# definitions or new command thing?
+	if($l =~ m/\A\s+/)
+	{
+		die "No command defined yet" if $current_cmd eq '';
+	
+		# definition of component
+		$l =~ s/\A\s+//;
+		
+		my ($type,$name,$value) = split /\s+/,$l;
+		if($type eq 'CONSTANT')
+		{
+			push @{$cmd_constants{$current_cmd}},"$name = $value"
+		}
+		else
+		{
+			push @{$cmd_contents{$current_cmd}},$type,$name;
+		}
+	}
+	else
+	{
+		# new command
+		my ($name,$id,@attributes) = split /\s+/,$l;
+		$cmd_attributes{$name} = [@attributes];
+		$cmd_id{$name} = int($id);
+		$current_cmd = $name;
+		push @cmd_list,$name;
+	}
+}
+
+close IN;
+
+
+
+# open files
+my $h_filename = 'autogen_'.$protocol_name.'Protocol'.$type.'.h';
+open CPP,'>autogen_'.$protocol_name.'Protocol'.$type.'.cpp';
+open H,">$h_filename";
+
+print CPP <<__E;
+
+// Auto-generated file -- do not edit
+
+#include "Box.h"
+#include "$h_filename"
+#include "IOStream.h"
+
+__E
+
+if($implement_syslog)
+{
+	print H qq~#include <syslog.h>\n~;
+}
+
+
+my $guardname = uc 'AUTOGEN_'.$protocol_name.'Protocol'.$type.'_H';
+print H <<__E;
+
+// Auto-generated file -- do not edit
+
+#ifndef $guardname
+#define $guardname
+
+#include "Protocol.h"
+#include "ProtocolObject.h"
+#include "ServerException.h"
+
+class IOStream;
+
+__E
+
+if($implement_filelog)
+{
+	print H qq~#include <stdio.h>\n~;
+}
+
+# extra headers
+for(@extra_header_files)
+{
+	print H qq~#include "$_"\n~
+}
+print H "\n";
+
+if($type eq 'Server')
+{
+	# need utils file for the server
+	print H '#include "Utils.h"',"\n\n"
+}
+
+
+my $derive_objects_from = 'ProtocolObject';
+my $objects_extra_h = '';
+my $objects_extra_cpp = '';
+if($type eq 'Server')
+{
+	# define the context
+	print H "class $context_class;\n\n";
+	print CPP "#include \"$context_class_inc\"\n\n";
+	
+	# change class we derive the objects from
+	$derive_objects_from = $protocol_name.'ProtocolObject';
+	
+	$objects_extra_h = <<__E;
+	virtual std::auto_ptr<ProtocolObject> DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext);
+__E
+	$objects_extra_cpp = <<__E;
+std::auto_ptr<ProtocolObject> ${derive_objects_from}::DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext)
+{
+	THROW_EXCEPTION(ConnectionException, Conn_Protocol_TriedToExecuteReplyCommand)
+}
+__E
+}
+
+print CPP qq~#include "MemLeakFindOn.h"\n~;
+
+if($type eq 'Client' && ($implement_syslog || $implement_filelog))
+{
+	# change class we derive the objects from
+	$derive_objects_from = $protocol_name.'ProtocolObjectCl';
+}
+if($implement_syslog)
+{
+	$objects_extra_h .= <<__E;
+	virtual void LogSysLog(const char *Action) const = 0;
+__E
+}
+if($implement_filelog)
+{
+	$objects_extra_h .= <<__E;
+	virtual void LogFile(const char *Action, FILE *file) const = 0;
+__E
+}
+
+if($derive_objects_from ne 'ProtocolObject')
+{	
+	# output a definition for the protocol object derviced class
+	print H <<__E;
+class ${protocol_name}ProtocolServer;
+	
+class $derive_objects_from : public ProtocolObject
+{
+public:
+	$derive_objects_from();
+	virtual ~$derive_objects_from();
+	$derive_objects_from(const $derive_objects_from &rToCopy);
+	
+$objects_extra_h
+};
+__E
+
+	# and some cpp definitions
+	print CPP <<__E;
+${derive_objects_from}::${derive_objects_from}()
+{
+}
+${derive_objects_from}::~${derive_objects_from}()
+{
+}
+${derive_objects_from}::${derive_objects_from}(const $derive_objects_from &rToCopy)
+{
+}
+$objects_extra_cpp
+__E
+}
+
+
+
+my $classname_base = $protocol_name.'Protocol'.$type;
+
+# output the classes
+for my $cmd (@cmd_list)
+{
+	print H <<__E;
+class $classname_base$cmd : public $derive_objects_from
+{
+public:
+	$classname_base$cmd();
+	$classname_base$cmd(const $classname_base$cmd &rToCopy);
+	~$classname_base$cmd();
+	int GetType() const;
+	enum
+	{
+		TypeID = $cmd_id{$cmd}
+	};
+__E
+	# constants
+	if(exists $cmd_constants{$cmd})
+	{
+		print H "\tenum\n\t{\n\t\t";
+		print H join(",\n\t\t",@{$cmd_constants{$cmd}});
+		print H "\n\t};\n";
+	}
+	# flags
+	if(obj_is_type($cmd,'EndsConversation'))
+	{
+		print H "\tbool IsConversationEnd() const;\n";
+	}
+	if(obj_is_type($cmd,'IsError'))
+	{
+		print H "\tbool IsError(int &rTypeOut, int &rSubTypeOut) const;\n";
+	}
+	if($type eq 'Server' && obj_is_type($cmd, 'Command'))
+	{
+		print H "\tstd::auto_ptr<ProtocolObject> DoCommand(${protocol_name}ProtocolServer &rProtocol, $context_class &rContext); // IMPLEMENT THIS\n"
+	}
+
+	# want to be able to read from streams?
+	my $read_from_streams = (obj_is_type($cmd,'Command') && $type eq 'Server') || (obj_is_type($cmd,'Reply') && $type eq 'Client');
+	my $write_to_streams = (obj_is_type($cmd,'Command') && $type eq 'Client') || (obj_is_type($cmd,'Reply') && $type eq 'Server');
+	
+	if($read_from_streams)
+	{
+		print H "\tvoid SetPropertiesFromStreamData(Protocol &rProtocol);\n";
+		
+		# write Get functions
+		for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2)
+		{
+			my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]);
+			
+			print H "\t".translate_type_to_arg_type($ty)." Get$nm() {return m$nm;}\n";
+		}
+	}
+	my $param_con_args = '';
+	if($write_to_streams)
+	{
+		# extra constructor?
+		if($#{$cmd_contents{$cmd}} >= 0)
+		{
+			my @a;
+			for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2)
+			{
+				my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]);
+
+				push @a,translate_type_to_arg_type($ty)." $nm";
+			}		
+			$param_con_args = join(', ',@a);
+			print H "\t$classname_base$cmd(".$param_con_args.");\n";
+		}
+		print H "\tvoid WritePropertiesToStreamData(Protocol &rProtocol) const;\n";
+		# set functions
+		for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2)
+		{
+			my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]);
+			
+			print H "\tvoid Set$nm(".translate_type_to_arg_type($ty)." $nm) {m$nm = $nm;}\n";
+		}
+	}
+	
+	if($implement_syslog)
+	{
+		print H "\tvirtual void LogSysLog(const char *Action) const;\n";
+	}
+	if($implement_filelog)
+	{
+		print H "\tvirtual void LogFile(const char *Action, FILE *file) const;\n";
+	}
+
+	
+	# write member variables and setup for cpp file
+	my @def_constructor_list;
+	my @copy_constructor_list;
+	my @param_constructor_list;
+	
+	print H "private:\n";
+	for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2)
+	{
+		my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]);
+
+		print H "\t".translate_type_to_member_type($ty)." m$nm;\n";
+		
+		my ($basic,$typename) = translate_type($ty);
+		if($basic)
+		{
+			push @def_constructor_list, "m$nm(0)";
+		}
+		push @copy_constructor_list, "m$nm(rToCopy.m$nm)";
+		push @param_constructor_list, "m$nm($nm)";
+	}
+	
+	# finish off
+	print H "};\n\n";
+	
+	# now the cpp file...
+	my $def_con_vars = join(",\n\t  ",@def_constructor_list);
+	$def_con_vars = "\n\t: ".$def_con_vars if $def_con_vars ne '';
+	my $copy_con_vars = join(",\n\t  ",@copy_constructor_list);
+	$copy_con_vars = "\n\t: ".$copy_con_vars if $copy_con_vars ne '';
+	my $param_con_vars = join(",\n\t  ",@param_constructor_list);
+	$param_con_vars = "\n\t: ".$param_con_vars if $param_con_vars ne '';
+
+	my $class = "$classname_base$cmd".'::';
+	print CPP <<__E;
+$class$classname_base$cmd()$def_con_vars
+{
+}
+$class$classname_base$cmd(const $classname_base$cmd &rToCopy)$copy_con_vars
+{
+}
+$class~$classname_base$cmd()
+{
+}
+int ${class}GetType() const
+{
+	return $cmd_id{$cmd};
+}
+__E
+	if($read_from_streams)
+	{
+		print CPP "void ${class}SetPropertiesFromStreamData(Protocol &rProtocol)\n{\n";
+		for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2)
+		{
+			my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]);
+			if($ty =~ m/\Avector/)
+			{
+				print CPP "\trProtocol.ReadVector(m$nm);\n";
+			}
+			else
+			{
+				print CPP "\trProtocol.Read(m$nm);\n";
+			}
+		}
+		print CPP "}\n";
+	}
+	if($write_to_streams)
+	{
+		# implement extra constructor?
+		if($param_con_vars ne '')
+		{
+			print CPP "$class$classname_base$cmd($param_con_args)$param_con_vars\n{\n}\n";
+		}
+		print CPP "void ${class}WritePropertiesToStreamData(Protocol &rProtocol) const\n{\n";
+		for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2)
+		{
+			my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]);
+			if($ty =~ m/\Avector/)
+			{
+				print CPP "\trProtocol.WriteVector(m$nm);\n";
+			}
+			else
+			{
+				print CPP "\trProtocol.Write(m$nm);\n";
+			}
+		}
+		print CPP "}\n";
+	}
+	if(obj_is_type($cmd,'EndsConversation'))
+	{
+		print CPP "bool ${class}IsConversationEnd() const\n{\n\treturn true;\n}\n";
+	}
+	if(obj_is_type($cmd,'IsError'))
+	{
+		# get parameters
+		my ($mem_type,$mem_subtype) = split /,/,obj_get_type_params($cmd,'IsError');
+		print CPP <<__E;
+bool ${class}IsError(int &rTypeOut, int &rSubTypeOut) const
+{
+	rTypeOut = m$mem_type;
+	rSubTypeOut = m$mem_subtype;
+	return true;
+}
+__E
+	}
+
+	if($implement_syslog)
+	{
+		my ($format,$args) = make_log_strings($cmd);
+		print CPP <<__E;
+void ${class}LogSysLog(const char *Action) const
+{
+	::syslog(LOG_INFO,"%s $format",Action$args);
+}
+__E
+	}
+	if($implement_filelog)
+	{
+		my ($format,$args) = make_log_strings($cmd);
+		print CPP <<__E;
+void ${class}LogFile(const char *Action, FILE *File) const
+{
+	::fprintf(File,"%s $format\\n",Action$args);
+	::fflush(File);
+}
+__E
+	}
+}
+
+# finally, the protocol object itself
+print H <<__E;
+class $classname_base : public Protocol
+{
+public:
+	$classname_base(IOStream &rStream);
+	virtual ~$classname_base();
+
+	std::auto_ptr<$derive_objects_from> Receive();
+	void Send(const ${derive_objects_from} &rObject);
+__E
+if($implement_syslog)
+{
+	print H "\tvoid SetLogToSysLog(bool Log = false) {mLogToSysLog = Log;}\n";
+}
+if($implement_filelog)
+{
+	print H "\tvoid SetLogToFile(FILE *File = 0) {mLogToFile = File;}\n";
+}
+if($type eq 'Server')
+{
+	# need to put in the conversation function
+	print H "\tvoid DoServer($context_class &rContext);\n\n";
+	# and the send vector thing
+	print H "\tvoid SendStreamAfterCommand(IOStream *pStream);\n\n";
+}
+if($type eq 'Client')
+{
+	# add plain object taking query functions
+	my $with_params;
+	for my $cmd (@cmd_list)
+	{
+		if(obj_is_type($cmd,'Command'))
+		{
+			my $has_stream = obj_is_type($cmd,'StreamWithCommand');
+			my $argextra = $has_stream?', IOStream &rStream':'';
+			my $queryextra = $has_stream?', rStream':'';
+			my $reply = obj_get_type_params($cmd,'Command');
+			print H "\tstd::auto_ptr<$classname_base$reply> Query(const $classname_base$cmd &rQuery$argextra);\n";
+			my @a;
+			my @na;
+			for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2)
+			{
+				my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]);
+				push @a,translate_type_to_arg_type($ty)." $nm";
+				push @na,"$nm";
+			}
+			my $ar = join(', ',@a);
+			my $nar = join(', ',@na);
+			$nar = "($nar)" if $nar ne '';
+
+			$with_params .= "\tinline std::auto_ptr<$classname_base$reply> Query$cmd($ar$argextra)\n\t{\n";
+			$with_params .= "\t\t$classname_base$cmd send$nar;\n";
+			$with_params .= "\t\treturn Query(send$queryextra);\n";
+			$with_params .= "\t}\n";
+		}
+	}
+	# quick hack to correct bad argument lists for commands with zero paramters but with streams
+	$with_params =~ s/\(, /(/g;
+	print H "\n",$with_params,"\n";
+}
+print H <<__E;
+private:
+	$classname_base(const $classname_base &rToCopy);
+__E
+if($type eq 'Server')
+{
+	# need to put the streams to send vector
+	print H "\tstd::vector<IOStream*> mStreamsToSend;\n\tvoid DeleteStreamsToSend();\n";
+}
+
+if($implement_filelog || $implement_syslog)
+{
+	print H <<__E;
+	virtual void InformStreamReceiving(u_int32_t Size);
+	virtual void InformStreamSending(u_int32_t Size);
+__E
+}
+
+if($implement_syslog)
+{
+	print H "private:\n\tbool mLogToSysLog;\n";
+}
+if($implement_filelog)
+{
+	print H "private:\n\tFILE *mLogToFile;\n";
+}
+print H <<__E;
+
+protected:	
+	virtual std::auto_ptr<ProtocolObject> MakeProtocolObject(int ObjType);
+	virtual const char *GetIdentString();
+};
+
+__E
+
+my $construtor_extra = '';
+$construtor_extra .= ', mLogToSysLog(false)' if $implement_syslog;
+$construtor_extra .= ', mLogToFile(0)' if $implement_filelog;
+
+my $destructor_extra = ($type eq 'Server')?"\n\tDeleteStreamsToSend();":'';
+
+my $prefix = $classname_base.'::';
+print CPP <<__E;
+$prefix$classname_base(IOStream &rStream)
+	: Protocol(rStream)$construtor_extra
+{
+}
+$prefix~$classname_base()
+{$destructor_extra
+}
+const char *${prefix}GetIdentString()
+{
+	return "$ident_string";
+}
+std::auto_ptr<ProtocolObject> ${prefix}MakeProtocolObject(int ObjType)
+{
+	switch(ObjType)
+	{
+__E
+
+# do objects within this
+for my $cmd (@cmd_list)
+{
+	print CPP <<__E;
+	case $cmd_id{$cmd}:
+		return std::auto_ptr<ProtocolObject>(new $classname_base$cmd);
+		break;
+__E
+}
+
+print CPP <<__E;
+	default:
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnknownCommandRecieved)
+	}
+}
+__E
+# write receieve and send functions
+print CPP <<__E;
+std::auto_ptr<$derive_objects_from> ${prefix}Receive()
+{
+	std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(Protocol::Receive().release()));
+
+__E
+	if($implement_syslog)
+	{
+		print CPP <<__E;
+	if(mLogToSysLog)
+	{
+		preply->LogSysLog("Receive");
+	}
+__E
+	}
+	if($implement_filelog)
+	{
+		print CPP <<__E;
+	if(mLogToFile != 0)
+	{
+		preply->LogFile("Receive", mLogToFile);
+	}
+__E
+	}
+print CPP <<__E;
+
+	return preply;
+}
+
+void ${prefix}Send(const ${derive_objects_from} &rObject)
+{
+__E
+	if($implement_syslog)
+	{
+		print CPP <<__E;
+	if(mLogToSysLog)
+	{
+		rObject.LogSysLog("Send");
+	}
+__E
+	}
+	if($implement_filelog)
+	{
+		print CPP <<__E;
+	if(mLogToFile != 0)
+	{
+		rObject.LogFile("Send", mLogToFile);
+	}
+__E
+	}
+
+print CPP <<__E;
+	Protocol::Send(rObject);
+}
+
+__E
+# write server function?
+if($type eq 'Server')
+{
+	print CPP <<__E;
+void ${prefix}DoServer($context_class &rContext)
+{
+	// Handshake with client
+	Handshake();
+
+	// Command processing loop
+	bool inProgress = true;
+	while(inProgress)
+	{
+		// Get an object from the conversation
+		std::auto_ptr<${derive_objects_from}> pobj(Receive());
+
+__E
+	if($implement_syslog)
+	{
+		print CPP <<__E;
+		if(mLogToSysLog)
+		{
+			pobj->LogSysLog("Receive");
+		}
+__E
+	}
+	if($implement_filelog)
+	{
+		print CPP <<__E;
+		if(mLogToFile != 0)
+		{
+			pobj->LogFile("Receive", mLogToFile);
+		}
+__E
+	}
+	print CPP <<__E;
+
+		// Run the command
+		std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(pobj->DoCommand(*this, rContext).release()));
+		
+__E
+	if($implement_syslog)
+	{
+		print CPP <<__E;
+		if(mLogToSysLog)
+		{
+			preply->LogSysLog("Send");
+		}
+__E
+	}
+	if($implement_filelog)
+	{
+		print CPP <<__E;
+		if(mLogToFile != 0)
+		{
+			preply->LogFile("Send", mLogToFile);
+		}
+__E
+	}
+	print CPP <<__E;
+
+		// Send the reply
+		Send(*(preply.get()));
+
+		// Send any streams
+		for(unsigned int s = 0; s < mStreamsToSend.size(); s++)
+		{
+			// Send the streams
+			SendStream(*mStreamsToSend[s]);
+		}
+		// Delete these streams
+		DeleteStreamsToSend();
+
+		// Does this end the conversation?
+		if(pobj->IsConversationEnd())
+		{
+			inProgress = false;
+		}
+	}	
+}
+
+void ${prefix}SendStreamAfterCommand(IOStream *pStream)
+{
+	ASSERT(pStream != NULL);
+	mStreamsToSend.push_back(pStream);
+}
+
+void ${prefix}DeleteStreamsToSend()
+{
+	for(std::vector<IOStream*>::iterator i(mStreamsToSend.begin()); i != mStreamsToSend.end(); ++i)
+	{
+		delete (*i);
+	}
+	mStreamsToSend.clear();
+}
+
+__E
+}
+
+# write logging functions?
+if($implement_filelog || $implement_syslog)
+{
+	my ($fR,$fS);
+
+	if($implement_syslog)
+	{
+		$fR .= qq~\tif(mLogToSysLog) { ::syslog(LOG_INFO, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain":"Receiving stream, size %d", Size); }\n~;
+		$fS .= qq~\tif(mLogToSysLog) { ::syslog(LOG_INFO, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain":"Sending stream, size %d", Size); }\n~;
+	}
+	if($implement_filelog)
+	{
+		$fR .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain":"Receiving stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~;
+		$fS .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain":"Sending stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~;
+	}
+
+	print CPP <<__E;
+
+void ${prefix}InformStreamReceiving(u_int32_t Size)
+{
+$fR}
+
+void ${prefix}InformStreamSending(u_int32_t Size)
+{
+$fS}
+
+__E
+}
+
+
+# write client Query functions?
+if($type eq 'Client')
+{
+	for my $cmd (@cmd_list)
+	{
+		if(obj_is_type($cmd,'Command'))
+		{
+			my $reply = obj_get_type_params($cmd,'Command');
+			my $reply_id = $cmd_id{$reply};
+			my $has_stream = obj_is_type($cmd,'StreamWithCommand');
+			my $argextra = $has_stream?', IOStream &rStream':'';
+			my $send_stream_extra = '';
+			if($has_stream)
+			{
+				$send_stream_extra = <<__E;
+	
+	// Send stream after the command
+	SendStream(rStream);
+__E
+			}
+			print CPP <<__E;
+std::auto_ptr<$classname_base$reply> ${classname_base}::Query(const $classname_base$cmd &rQuery$argextra)
+{
+	// Send query
+	Send(rQuery);
+	$send_stream_extra
+	// Wait for the reply
+	std::auto_ptr<${derive_objects_from}> preply(Receive().release());
+
+	if(preply->GetType() == $reply_id)
+	{
+		// Correct response
+		return std::auto_ptr<$classname_base$reply>(($classname_base$reply*)preply.release());
+	}
+	else
+	{
+		// Set protocol error
+		int type, subType;
+		if(preply->IsError(type, subType))
+		{
+			SetError(type, subType);
+			TRACE2("Protocol: Error received %d/%d\\n", type, subType);
+		}
+		else
+		{
+			SetError(Protocol::UnknownError, Protocol::UnknownError);
+		}
+		
+		// Throw an exception
+		THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnexpectedReply)
+	}
+}
+__E
+		}
+	}
+}
+
+
+
+print H <<__E;
+#endif // $guardname
+
+__E
+
+# close files
+close H;
+close CPP;
+
+
+sub obj_is_type
+{
+	my ($c,$ty) = @_;
+	for(@{$cmd_attributes{$c}})
+	{
+		return 1 if $_ =~ m/\A$ty/;
+	}
+	
+	return 0;
+}
+
+sub obj_get_type_params
+{
+	my ($c,$ty) = @_;
+	for(@{$cmd_attributes{$c}})
+	{
+		return $1 if $_ =~ m/\A$ty\((.+?)\)\Z/;
+	}
+	die "Can't find attribute $ty\n"
+}
+
+# returns (is basic type, typename)
+sub translate_type
+{
+	my $ty = $_[0];
+	
+	if($ty =~ m/\Avector\<(.+?)\>\Z/)
+	{
+		my $v_type = $1;
+		my (undef,$v_ty) = translate_type($v_type);
+		return (0, 'std::vector<'.$v_ty.'>')
+	}
+	else
+	{
+		if(!exists $translate_type_info{$ty})
+		{
+			die "Don't know about type name $ty\n";
+		}
+		return @{$translate_type_info{$ty}}
+	}
+}
+
+sub translate_type_to_arg_type
+{
+	my ($basic,$typename) = translate_type(@_);
+	return $basic?$typename:'const '.$typename.'&'
+}
+
+sub translate_type_to_member_type
+{
+	my ($basic,$typename) = translate_type(@_);
+	return $typename
+}
+
+sub make_log_strings
+{
+	my ($cmd) = @_;
+
+	my @str;
+	my @arg;
+	for(my $x = 0; $x < $#{$cmd_contents{$cmd}}; $x+=2)
+	{
+		my ($ty,$nm) = (${$cmd_contents{$cmd}}[$x], ${$cmd_contents{$cmd}}[$x+1]);
+		
+		if(exists $log_display_types{$ty})
+		{
+			# need to translate it
+			my ($format,$arg) = @{$log_display_types{$ty}};
+			push @str,$format;
+			$arg =~ s/VAR/m$nm/g;
+			push @arg,$arg;
+		}
+		else
+		{
+			# is opaque
+			push @str,'OPAQUE';
+		}
+	}
+	return ($cmd.'('.join(',',@str).')', join(',','',@arg));
+}
+
+
diff --git a/modules.txt b/modules.txt
new file mode 100755
index 00000000..ab39b428
--- /dev/null
+++ b/modules.txt
@@ -0,0 +1,49 @@
+
+# first entry is module name, next entries are dependencies or -l<libname> library includes
+
+# put !<platform-name>,<platform-name> after a module / library to exclude it from a particular platform
+# put !+<platform-name>,... to include it only on those platforms
+
+# -l libaries must be in the order they should appear on the command line.
+# Note that order is important on platforms which do not have shared libraries.
+
+# Generic support code and modules
+
+OMIT:CYGWIN
+lib/raidfile
+END-OMIT
+lib/crypto			-lcrypto
+lib/server			-lssl -lcrypto
+lib/compress		-lz
+test/common
+test/crypto			lib/crypto
+test/compress		lib/compress
+test/basicserver	lib/server
+OMIT:CYGWIN
+test/raidfile		lib/raidfile
+END-OMIT
+
+# IF_DISTRIBUTION(boxbackup)
+
+# Backup system
+
+lib/backupclient	lib/server	lib/crypto	lib/compress
+OMIT:CYGWIN
+lib/backupstore		lib/server	lib/raidfile	lib/backupclient
+bin/bbstored		lib/raidfile	lib/server	lib/backupstore	lib/backupclient
+bin/bbstoreaccounts	lib/raidfile	lib/backupstore
+bin/bbackupobjdump	lib/backupclient lib/backupstore
+END-OMIT
+bin/bbackupd		lib/server	lib/backupclient	-lLINUX_DB!+Linux
+bin/bbackupquery	-lreadline!Darwin,NetBSD -ledit!+NetBSD	-lcurses	lib/server	lib/backupclient
+bin/bbackupctl		lib/server	lib/backupclient
+OMIT:CYGWIN
+test/backupstore	bin/bbstored	bin/bbstoreaccounts lib/server lib/backupstore lib/backupclient lib/raidfile
+test/backupstorefix	bin/bbstored	bin/bbstoreaccounts	lib/backupstore	lib/raidfile	bin/bbackupquery	bin/bbackupd
+test/backupstorepatch	bin/bbstored	bin/bbstoreaccounts	lib/backupstore	lib/raidfile
+test/backupdiff		lib/backupclient
+test/bbackupd		bin/bbackupd	bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupctl lib/server lib/backupstore lib/backupclient
+END-OMIT
+
+# END_IF_DISTRIBUTION
+
diff --git a/parcels.txt b/parcels.txt
new file mode 100644
index 00000000..4ae309ec
--- /dev/null
+++ b/parcels.txt
@@ -0,0 +1,21 @@
+
+# this file describes which binaries and scripts go to make
+# up a parcel -- a group of files and scripts needed to perform
+# a particular task
+
+backup-client
+	bin bbackupd
+	bin bbackupquery
+	bin bbackupctl
+	script bin/bbackupd/bbackupd-config
+
+OMIT:CYGWIN
+
+backup-server
+	bin bbstored
+	bin bbstoreaccounts
+	script bin/bbstored/bbstored-certs
+	script bin/bbstored/bbstored-config
+	script lib/raidfile/raidfile-config
+
+END-OMIT
diff --git a/runtest.pl b/runtest.pl
new file mode 100755
index 00000000..4cda8554
--- /dev/null
+++ b/runtest.pl
@@ -0,0 +1,96 @@
+#!/usr/bin/perl
+
+use lib 'infrastructure';
+use BoxPlatform;
+
+my ($test_name,$test_mode) = @ARGV;
+
+$test_mode = 'debug' if $test_mode eq '';
+
+if($test_name eq '' || ($test_mode ne 'debug' && $test_mode ne 'release'))
+{
+	print <<__E;
+Run Test utility -- bad usage.
+
+runtest.pl (test|ALL) [release|debug]
+
+Mode defaults to debug.
+
+__E
+	exit(0);
+}
+
+my @results;
+
+if($test_name ne 'ALL')
+{
+	# run one test
+	runtest($test_name);
+}
+else
+{
+	# run all tests
+	my @tests;
+	open MODULES,'modules.txt' or die "Can't open modules file";
+	while(<MODULES>)
+	{
+		# omit bits on some platforms?
+		next if m/\AEND-OMIT/;
+		if(m/\AOMIT:(.+)/)
+		{
+			if($1 eq $build_os)
+			{
+				while(<MODULES>)
+				{
+					last if m/\AEND-OMIT/;	
+				}
+			}
+			next;
+		}
+		push @tests,$1 if m~\Atest/(\w+)\s~;
+	}
+	close MODULES;
+	
+	runtest($_) for(@tests)
+}
+
+# report results
+print "--------\n",join("\n",@results),"\n";
+
+sub runtest
+{
+	my ($t) = @_;
+
+	# attempt to make this test
+	my $flag = ($test_mode eq 'release')?(BoxPlatform::make_flag('RELEASE')):'';
+	my $make_res = system("cd test/$t ; $make_command $flag");
+	if($make_res != 0)
+	{
+		push @results,"$t: make failed";
+		return;
+	}
+	
+	# run it
+	my $test_res = system("cd $test_mode/test/$t ; ./t | tee ../../../temp.runtest");
+
+	# open test results
+	if(open RESULTS,'temp.runtest')
+	{
+		my $last;
+		while(<RESULTS>)
+		{
+			$last = $_ if m/\w/;
+		}
+		close RESULTS;
+		chomp $last;
+		push @results,"$t: $last";
+	}
+	else
+	{
+		push @results,"$t: output not found";
+	}
+	
+	# delete test results
+	unlink 'temp.runtest';
+}
+
diff --git a/test/backupdiff/difftestfiles.cpp b/test/backupdiff/difftestfiles.cpp
new file mode 100755
index 00000000..881876ca
--- /dev/null
+++ b/test/backupdiff/difftestfiles.cpp
@@ -0,0 +1,294 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    createtestfiles.cpp
+//		Purpose: Create the test files for the backupdiff test
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+
+#include "FileStream.h"
+#include "PartialReadStream.h"
+#include "Test.h"
+#include "RollingChecksum.h"
+
+#include "MemLeakFindOn.h"
+
+#define ACT_END		0
+#define ACT_COPY	1
+#define	ACT_NEW		2
+#define	ACT_SKIP	3
+#define ACT_COPYEND	4
+
+typedef struct
+{
+	int action, length, seed;
+} gen_action;
+
+#define INITIAL_FILE_LENGTH (128*1024 + 342)
+
+
+gen_action file1actions[] = {
+	{ACT_COPYEND, 0, 0},
+	{ACT_END, 0, 0} };
+
+gen_action file2actions[] = {
+	{ACT_COPY, 16*1024, 0},
+	// Do blocks on block boundaries, but swapped around a little
+	{ACT_SKIP, 4*1024, 0},
+	{ACT_COPY, 8*1024, 0},
+	{ACT_SKIP, -12*1024, 0},
+	{ACT_COPY, 4*1024, 0},
+	{ACT_SKIP, 8*1024, 0},
+	// Get rest of file with some new data inserted
+	{ACT_COPY, 37*1024 + 12, 0},
+	{ACT_NEW, 23*1024 + 129, 23990},
+	{ACT_COPYEND, 0, 0},
+	{ACT_END, 0, 0} };
+
+gen_action file3actions[] = {
+	{ACT_COPY, 12*1024 + 983, 0},
+	{ACT_SKIP, 37*1024 + 12, 0},
+	{ACT_COPYEND, 0, 0},
+	{ACT_END, 0, 0} };
+
+gen_action file4actions[] = {
+	{ACT_COPY, 20*1024 + 2385, 0},
+	{ACT_NEW, 12, 2334},
+	{ACT_COPY, 16*1024 + 385, 0},
+	{ACT_SKIP, 9*1024 + 42, 0},
+	{ACT_COPYEND, 0, 0},
+	{ACT_END, 0, 0} };
+
+// insert 1 byte a block into the file, between two other blocks
+gen_action file5actions[] = {
+	{ACT_COPY, 4*1024, 0},
+	{ACT_NEW, 1, 2334},
+	{ACT_COPYEND, 0, 0},
+	{ACT_END, 0, 0} };
+
+gen_action file6actions[] = {
+	{ACT_NEW, 6*1024, 12353452},
+	{ACT_COPYEND, 0, 0},
+	{ACT_END, 0, 0} };
+
+// but delete that one byte block, it's annoying
+gen_action file7actions[] = {
+	{ACT_COPY, 10*1024, 0},
+	{ACT_SKIP, 1, 0},
+	{ACT_COPYEND, 0, 0},
+	{ACT_NEW, 7*1024, 1235352},
+	{ACT_END, 0, 0} };
+
+gen_action file8actions[] = {
+	{ACT_NEW, 54*1024 + 9, 125352},
+	{ACT_END, 0, 0} };
+
+gen_action file9actions[] = {
+	{ACT_END, 0, 0} };
+
+gen_action *testfiles[] = {file1actions, file2actions, file3actions, file4actions,
+	file5actions, file6actions, file7actions, file8actions, file9actions, 0};
+
+
+// Nice random data for testing written files
+class R250 {
+public:
+	// Set up internal state table with 32-bit random numbers.  
+	// The bizarre bit-twiddling is because rand() returns 16 bits of which
+	// the bottom bit is always zero!  Hence, I use only some of the bits.
+	// You might want to do something better than this....
+
+	R250(int seed) : posn1(0), posn2(103)
+	{
+		// populate the state and incr tables
+		srand(seed);
+
+		for (int i = 0; i != stateLen; ++i)	{
+			state[i] = ((rand() >> 2) << 19) ^ ((rand() >> 2) << 11) ^ (rand() >> 2);
+			incrTable[i] = i == stateLen - 1 ? 0 : i + 1;
+		}
+
+		// stir up the numbers to ensure they're random
+
+		for (int j = 0; j != stateLen * 4; ++j)			
+			(void) next();
+	}
+
+	// Returns the next random number.  Xor together two elements separated
+	// by 103 mod 250, replacing the first element with the result.  Then
+	// increment the two indices mod 250.
+	inline int next()
+	{
+		int ret = (state[posn1] ^= state[posn2]);	// xor and replace element
+
+		posn1 = incrTable[posn1];		// increment indices using lookup table
+		posn2 = incrTable[posn2];
+
+		return ret;
+	}
+private:
+	enum { stateLen = 250 };	// length of the state table
+	int state[stateLen];		// holds the random number state
+	int incrTable[stateLen];	// lookup table: maps i to (i+1) % stateLen
+	int posn1, posn2;			// indices into the state table
+};
+
+void make_random_data(void *buffer, int size, int seed)
+{
+	R250 rand(seed);
+
+	int n = size / sizeof(int);
+	int *b = (int*)buffer;
+	for(int l = 0; l < n; ++l)
+	{
+		b[l] = rand.next();
+	}
+}
+
+void write_test_data(IOStream &rstream, int size, int seed)
+{
+	R250 rand(seed);
+	
+	while(size > 0)
+	{
+		// make a nice buffer of data
+		int buffer[2048/sizeof(int)];
+		for(unsigned int l = 0; l < (sizeof(buffer) / sizeof(int)); ++l)
+		{
+			buffer[l] = rand.next();
+		}
+		
+		// Write out...
+		unsigned int w = size;
+		if(w > sizeof(buffer)) w = sizeof(buffer);
+		rstream.Write(buffer, w);
+		
+		size -= w;
+	}	
+}
+
+void gen_varient(IOStream &out, char *sourcename, gen_action *pact)
+{
+	// Open source
+	FileStream source(sourcename);
+	
+	while(true)
+	{
+		switch(pact->action)
+		{
+		case ACT_END:
+			{
+				// all done
+				return;
+			}
+		case ACT_COPY:
+			{
+				PartialReadStream copy(source, pact->length);
+				copy.CopyStreamTo(out);
+				break;
+			}
+		case ACT_NEW:
+			{
+				write_test_data(out, pact->length, pact->seed);
+				break;
+			}
+		case ACT_SKIP:
+			{
+				source.Seek(pact->length, IOStream::SeekType_Relative);
+				break;
+			}
+		case ACT_COPYEND:
+			{
+				source.CopyStreamTo(out);
+				break;
+			}
+		}
+	
+		++pact;
+	}
+}
+
+void create_test_files()
+{
+	// First, the keys for the crypto
+	{
+		FileStream keys("testfiles/backup.keys", O_WRONLY | O_CREAT);
+		write_test_data(keys, 1024, 237);
+	}
+	
+	// Create the initial file -- needs various special properties...
+	// 1) Two blocks much be the different, but have the same weak checksum
+	// 2) A block must exist twice, but at an offset which isn't a multiple of the block size.
+	{
+		FileStream f0("testfiles/f0", O_WRONLY | O_CREAT);
+		// Write first bit.
+		write_test_data(f0, (16*1024), 20012);
+		// Now repeated checksum blocks
+		uint8_t blk[4096];
+		make_random_data(blk, sizeof(blk), 12201);
+		// Three magic numbers which make the checksum work: Use this perl to find them:
+		/*
+			for($z = 1; $z < 4096; $z++)
+			{
+				for($n = 0; $n <= 255; $n++)
+				{
+					for($m = 0; $m <= 255; $m++)
+					{
+						if($n != $m && (($n*4096 + $m*(4096-$z)) % (64*1024) == ($n*(4096-$z) + $m*4096) % (64*1024)))
+						{
+							print "$z: $n $m\n";
+						}
+					}
+				}
+			}
+		*/
+		blk[0] = 255;
+		blk[1024] = 191;
+		// Checksum to check
+		RollingChecksum c1(blk, sizeof(blk));
+		// Write
+		f0.Write(blk, sizeof(blk));
+		// Adjust block and write again
+		uint8_t blk2[4096];
+		memcpy(blk2, blk, sizeof(blk2));
+		blk2[1024] = 255;
+		blk2[0] = 191;
+		TEST_THAT(::memcmp(blk2, blk, sizeof(blk)) != 0);
+		RollingChecksum c2(blk2, sizeof(blk2));
+		f0.Write(blk2, sizeof(blk2));
+		// Check checksums
+		TEST_THAT(c1.GetChecksum() == c2.GetChecksum());
+		
+		// Another 4k block
+		write_test_data(f0, (4*1024), 99209);
+		// Offset block
+		make_random_data(blk, 2048, 1234199);
+		f0.Write(blk, 2048);
+		f0.Write(blk, 2048);
+		f0.Write(blk, 2048);
+		make_random_data(blk, 2048, 1343278);
+		f0.Write(blk, 2048);
+	
+		write_test_data(f0, INITIAL_FILE_LENGTH - (16*1024) - ((4*1024)*2) - (4*1024) - (2048*4), 202);
+	
+	}
+	
+	// Then... create the varients
+	for(int l = 0; testfiles[l] != 0; ++l)
+	{
+		char n1[256];
+		char n2[256];
+		sprintf(n1, "testfiles/f%d", l + 1);
+		sprintf(n2, "testfiles/f%d", l);
+
+		FileStream f1(n1, O_WRONLY | O_CREAT);
+		gen_varient(f1, n2, testfiles[l]);
+	}
+}
+
+
diff --git a/test/backupdiff/testbackupdiff.cpp b/test/backupdiff/testbackupdiff.cpp
new file mode 100755
index 00000000..d086253e
--- /dev/null
+++ b/test/backupdiff/testbackupdiff.cpp
@@ -0,0 +1,511 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testbackupdiff.cpp
+//		Purpose: Test diffing routines for backup store files
+//		Created: 12/1/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "Test.h"
+#include "BackupClientCryptoKeys.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFilenameClear.h"
+#include "FileStream.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreFileCryptVar.h"
+#include "BackupStoreException.h"
+#include "CollectInBufferStream.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BackupStoreFileCryptVar;
+
+
+// from another file
+void create_test_files();
+
+bool files_identical(const char *file1, const char *file2)
+{
+	FileStream f1(file1);
+	FileStream f2(file2);
+	
+	if(f1.BytesLeftToRead() != f2.BytesLeftToRead())
+	{
+		return false;
+	}
+	
+	while(f1.StreamDataLeft())
+	{
+		char buffer1[2048];
+		char buffer2[2048];
+		int s = f1.Read(buffer1, sizeof(buffer1));
+		if(f2.Read(buffer2, s) != s)
+		{
+			return false;
+		}
+		if(::memcmp(buffer1, buffer2, s) != 0)
+		{
+			return false;
+		}
+	}
+	
+	if(f2.StreamDataLeft())
+	{
+		return false;
+	}
+	
+	return true;
+}
+
+void make_file_of_zeros(const char *filename, int size)
+{
+	void *b = malloc(size);
+	memset(b, 0, size);
+	FILE *f = fopen(filename, "wb");
+	fwrite(b, size, 1, f);
+	fclose(f);
+	free(b);
+	TEST_THAT(TestGetFileSize(filename) == size);
+}
+
+
+void check_encoded_file(const char *filename, int64_t OtherFileID, int new_blocks_expected, int old_blocks_expected)
+{
+	FileStream enc(filename);
+	
+	// Use the interface verify routine
+	int64_t otherIDFromFile = 0;
+	TEST_THAT(BackupStoreFile::VerifyEncodedFileFormat(enc, &otherIDFromFile));
+	TEST_THAT(otherIDFromFile == OtherFileID);
+	
+	// Now do our own reading
+	enc.Seek(0, IOStream::SeekType_Absolute);
+	BackupStoreFile::MoveStreamPositionToBlockIndex(enc);
+	// Read in header to check magic value is as expected
+	file_BlockIndexHeader hdr;
+	TEST_THAT(enc.ReadFullBuffer(&hdr, sizeof(hdr), 0));
+	TEST_THAT(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1));
+	TEST_THAT((uint64_t)ntoh64(hdr.mOtherFileID) == (uint64_t)OtherFileID);
+	// number of blocks
+	int64_t nblocks = ntoh64(hdr.mNumBlocks);
+	TRACE2("Reading index from '%s', has %lld blocks\n", filename, nblocks);
+	TRACE0("======== ===== ========== ======== ========\n   Index Where  EncSz/Idx     Size  WChcksm\n");
+	// Read them all in
+	int64_t nnew = 0, nold = 0;
+	for(int64_t b = 0; b < nblocks; ++b)
+	{
+		file_BlockIndexEntry en;
+		TEST_THAT(enc.ReadFullBuffer(&en, sizeof(en), 0));
+		int64_t s = ntoh64(en.mEncodedSize);
+		if(s > 0)
+		{
+			nnew++;
+			TRACE2("%8lld this  s=%8lld", b, s);
+		}
+		else
+		{
+			nold++;
+			TRACE2("%8lld other i=%8lld", b, 0 - s);		
+		}
+		// Decode the rest
+		uint64_t iv = ntoh64(hdr.mEntryIVBase);
+		iv += b;
+		sBlowfishDecryptBlockEntry.SetIV(&iv);			
+		file_BlockIndexEntryEnc entryEnc;
+		sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
+				en.mEnEnc, sizeof(en.mEnEnc));
+		TRACE2(" %8d %08x\n", ntohl(entryEnc.mSize), ntohl(entryEnc.mWeakChecksum));
+		
+	}
+	TRACE0("======== ===== ========== ======== ========\n");
+	TEST_THAT(new_blocks_expected == nnew);
+	TEST_THAT(old_blocks_expected == nold);
+}
+
+void test_diff(int from, int to, int new_blocks_expected, int old_blocks_expected, bool expect_completely_different = false)
+{
+	// First, get the block index of the thing it's comparing against
+	char from_encoded[256];
+	sprintf(from_encoded, "testfiles/f%d.encoded", from);
+	FileStream blockindex(from_encoded);
+	BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex);
+
+	// make filenames
+	char from_orig[256];
+	sprintf(from_orig, "testfiles/f%d", from);
+	char to_encoded[256];
+	sprintf(to_encoded, "testfiles/f%d.encoded", to);
+	char to_diff[256];
+	sprintf(to_diff, "testfiles/f%d.diff", to);
+	char to_orig[256];
+	sprintf(to_orig, "testfiles/f%d", to);
+	char rev_diff[256];
+	sprintf(rev_diff, "testfiles/f%d.revdiff", to);
+	char from_rebuild[256];
+	sprintf(from_rebuild, "testfiles/f%d.rebuilt", to);
+	char from_rebuild_dec[256];
+	sprintf(from_rebuild_dec, "testfiles/f%d.rebuilt_dec", to);
+	
+	// Then call the encode varient for diffing files
+	bool completelyDifferent = !expect_completely_different;	// oposite of what we want
+	{
+		BackupStoreFilenameClear f1name("filename");
+		FileStream out(to_diff, O_WRONLY | O_CREAT | O_EXCL);
+		std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff(to_orig, 1 /* dir ID */, f1name,
+			1000 + from /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite,
+			0, &completelyDifferent));
+		encoded->CopyStreamTo(out);
+	}
+	TEST_THAT(completelyDifferent == expect_completely_different);
+	
+	// Test that the number of blocks in the file match what's expected
+	check_encoded_file(to_diff, expect_completely_different?(0):(1000 + from), new_blocks_expected, old_blocks_expected);
+
+	// filename
+	char to_testdec[256];
+	sprintf(to_testdec, "testfiles/f%d.testdec", to);
+	
+	if(!completelyDifferent)
+	{
+		// Then produce a combined file
+		{
+			FileStream diff(to_diff);
+			FileStream diff2(to_diff);
+			FileStream from(from_encoded);
+			FileStream out(to_encoded, O_WRONLY | O_CREAT | O_EXCL);
+			BackupStoreFile::CombineFile(diff, diff2, from, out);
+		}
+		
+		// And check it
+		check_encoded_file(to_encoded, 0, new_blocks_expected + old_blocks_expected, 0);
+	}
+	else
+	{
+		// Emulate the above stage!
+		char cmd[256];
+		sprintf(cmd, "cp testfiles/f%d.diff testfiles/f%d.encoded", to, to);
+		::system(cmd);
+	}
+
+	// Decode it
+	{
+		FileStream enc(to_encoded);
+		BackupStoreFile::DecodeFile(enc, to_testdec, IOStream::TimeOutInfinite);
+		TEST_THAT(files_identical(to_orig, to_testdec));
+	}
+	
+	// Then do some comparisons against the block index
+	{
+		FileStream index(to_encoded);
+		BackupStoreFile::MoveStreamPositionToBlockIndex(index);
+		TEST_THAT(BackupStoreFile::CompareFileContentsAgainstBlockIndex(to_orig, index, IOStream::TimeOutInfinite) == true);
+	}
+	{
+		char from_orig[256];
+		sprintf(from_orig, "testfiles/f%d", from);
+		FileStream index(to_encoded);
+		BackupStoreFile::MoveStreamPositionToBlockIndex(index);
+		TEST_THAT(BackupStoreFile::CompareFileContentsAgainstBlockIndex(from_orig, index, IOStream::TimeOutInfinite) == files_identical(from_orig, to_orig));
+	}
+	
+	// Check that combined index creation works as expected
+	{
+		// Load a combined index into memory
+		FileStream diff(to_diff);
+		FileStream from(from_encoded);
+		std::auto_ptr<IOStream> indexCmbStr(BackupStoreFile::CombineFileIndices(diff, from));
+		CollectInBufferStream indexCmb;
+		indexCmbStr->CopyStreamTo(indexCmb);
+		// Then check that it's as expected!
+		FileStream result(to_encoded);
+		BackupStoreFile::MoveStreamPositionToBlockIndex(result);
+		CollectInBufferStream index;
+		result.CopyStreamTo(index);
+		TEST_THAT(indexCmb.GetSize() == index.GetSize());
+		TEST_THAT(::memcmp(indexCmb.GetBuffer(), index.GetBuffer(), index.GetSize()) == 0);
+	}
+	
+	// Check that reverse delta can be made, and that it decodes OK
+	{
+		// Create reverse delta
+		{
+			bool reversedCompletelyDifferent = !completelyDifferent;
+			FileStream diff(to_diff);
+			FileStream from(from_encoded);
+			FileStream from2(from_encoded);
+			FileStream reversed(rev_diff, O_WRONLY | O_CREAT);
+			BackupStoreFile::ReverseDiffFile(diff, from, from2, reversed, to, &reversedCompletelyDifferent);
+			TEST_THAT(reversedCompletelyDifferent == completelyDifferent);
+		}
+		// Use it to combine a file
+		{
+			FileStream diff(rev_diff);
+			FileStream diff2(rev_diff);
+			FileStream from(to_encoded);
+			FileStream out(from_rebuild, O_WRONLY | O_CREAT | O_EXCL);
+			BackupStoreFile::CombineFile(diff, diff2, from, out);
+		}
+		// And then confirm that this file is actually the one we want
+		{
+			FileStream enc(from_rebuild);
+			BackupStoreFile::DecodeFile(enc, from_rebuild_dec, IOStream::TimeOutInfinite);
+			TEST_THAT(files_identical(from_orig, from_rebuild_dec));
+		}
+		// Do some extra checking
+		{
+			TEST_THAT(files_identical(from_rebuild, from_encoded));
+		}
+	}
+}
+
+void test_combined_diff(int version1, int version2, int serial)
+{
+	char combined_file[256];
+	char last_diff[256];
+	sprintf(last_diff, "testfiles/f%d.diff", version1 + 1);	// ie from version1 to version1 + 1
+
+	for(int v = version1 + 2; v <= version2; ++v)
+	{
+		FileStream diff1(last_diff);
+		char next_diff[256];
+		sprintf(next_diff, "testfiles/f%d.diff", v);
+		FileStream diff2(next_diff);
+		FileStream diff2b(next_diff);
+		sprintf(combined_file, "testfiles/comb%d_%d.cmbdiff", version1, v);
+		FileStream out(combined_file, O_WRONLY | O_CREAT);
+		BackupStoreFile::CombineDiffs(diff1, diff2, diff2b, out);
+		strcpy(last_diff, combined_file);
+	}
+
+	// Then do a combine on it, and check that it decodes to the right thing
+	char orig_enc[256];
+	sprintf(orig_enc, "testfiles/f%d.encoded", version1);	
+	char combined_out[256];
+	sprintf(combined_out, "testfiles/comb%d_%d.out", version1, version2);
+
+	{
+		FileStream diff(combined_file);
+		FileStream diff2(combined_file);
+		FileStream from(orig_enc);
+		FileStream out(combined_out, O_WRONLY | O_CREAT);
+		BackupStoreFile::CombineFile(diff, diff2, from, out);
+	}
+
+	char combined_out_dec[256];
+	sprintf(combined_out_dec, "testfiles/comb%d_%d_s%d.dec", version1, version2, serial);
+	char to_orig[256];
+	sprintf(to_orig, "testfiles/f%d", version2);
+
+	{
+		FileStream enc(combined_out);
+		BackupStoreFile::DecodeFile(enc, combined_out_dec, IOStream::TimeOutInfinite);
+		TEST_THAT(files_identical(to_orig, combined_out_dec));
+	}
+
+}
+
+#define MAX_DIFF 9
+void test_combined_diffs()
+{
+	int serial = 0;
+
+	// Number of items to combine at once
+	for(int stages = 2; stages <= 4; ++stages)
+	{
+		// Offset to get complete coverage
+		for(int offset = 0; offset < stages; ++offset)
+		{
+			// And then actual end file number
+			for(int f = 0; f <= (MAX_DIFF - stages - offset); ++f)
+			{
+				// And finally, do something!
+				test_combined_diff(offset + f, offset + f + stages, ++serial);
+			}
+		}
+	}
+}
+
+int test(int argc, const char *argv[])
+{
+	// Want to trace out all the details
+	#ifndef NDEBUG
+	BackupStoreFile::TraceDetailsOfDiffProcess = true;
+	#endif
+
+	// Create all the test files
+	create_test_files();
+
+	// Setup the crypto
+	BackupClientCryptoKeys_Setup("testfiles/backup.keys");	
+
+	// Encode the first file
+	{
+		BackupStoreFilenameClear f0name("f0");
+		FileStream out("testfiles/f0.encoded", O_WRONLY | O_CREAT | O_EXCL);
+		std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/f0", 1 /* dir ID */, f0name));
+		encoded->CopyStreamTo(out);
+		check_encoded_file("testfiles/f0.encoded", 0, 33, 0);
+	}
+	
+	// Check the "seek to index" code
+	{
+		FileStream enc("testfiles/f0.encoded");
+		BackupStoreFile::MoveStreamPositionToBlockIndex(enc);
+		// Read in header to check magic value is as expected
+		file_BlockIndexHeader hdr;
+		TEST_THAT(enc.ReadFullBuffer(&hdr, sizeof(hdr), 0));
+		TEST_THAT(hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1));
+	}
+
+	// Diff some files -- parameters are from number, to number,
+	// then the number of new blocks expected, and the number of old blocks expected.
+	
+	// Diff the original file to a copy of itself, and check that there is no data in the file
+	// This checks that the hash table is constructed properly, because two of the blocks share
+	// the same weak checksum.
+	test_diff(0, 1, 0, 33);
+
+	// Insert some new data
+	// Blocks from old file moved whole, but put in different order
+	test_diff(1, 2, 7, 32);
+
+	// Delete some data, but not on block boundaries
+	test_diff(2, 3, 1, 29);
+
+	// Add a very small amount of data, not on block boundary
+	// delete a little data
+	test_diff(3, 4, 3, 25);
+
+	// 1 byte insertion between two blocks
+	test_diff(4, 5, 1, 28);
+
+	// a file with some new content at the very beginning
+	// NOTE: You might expect the last numbers to be 2, 29, but the small 1 byte block isn't searched for
+	test_diff(5, 6, 3, 28);
+	
+	// some new content at the very end
+	// NOTE: 1 byte block deleted, so number aren't what you'd initial expect.
+	test_diff(6, 7, 2, 30);
+	
+	// a completely different file, with no blocks matching.
+	test_diff(7, 8, 14, 0, true /* completely different expected */);
+	
+	// diff to zero sized file
+	test_diff(8, 9, 0, 0, true /* completely different expected */);
+	
+	// Test that combining diffs works
+	test_combined_diffs();
+	
+	// Check zero sized file works OK to encode on its own, using normal encoding
+	{
+		{
+			// Encode
+			BackupStoreFilenameClear fn("filename");
+			FileStream out("testfiles/f9.zerotest", O_WRONLY | O_CREAT | O_EXCL);
+			std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/f9", 1 /* dir ID */, fn));
+			encoded->CopyStreamTo(out);
+			check_encoded_file("testfiles/f9.zerotest", 0, 0, 0);		
+		}
+		{
+			// Decode
+			FileStream enc("testfiles/f9.zerotest");
+			BackupStoreFile::DecodeFile(enc, "testfiles/f9.testdec.zero", IOStream::TimeOutInfinite);
+			TEST_THAT(files_identical("testfiles/f9", "testfiles/f9.testdec.zero"));
+		}
+	}
+	
+	// Check that symlinks aren't diffed
+	TEST_THAT(::symlink("f2", "testfiles/f2.symlink") == 0)
+	// And go and diff it against the previous encoded file
+	{
+		bool completelyDifferent = false;
+		{
+			FileStream blockindex("testfiles/f1.encoded");
+			BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex);
+			
+			BackupStoreFilenameClear f1name("filename");
+			FileStream out("testfiles/f2.symlink.diff", O_WRONLY | O_CREAT | O_EXCL);
+			std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff("testfiles/f2.symlink", 1 /* dir ID */, f1name,
+				1001 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite,
+				0, &completelyDifferent));
+			encoded->CopyStreamTo(out);
+		}
+		TEST_THAT(completelyDifferent == true);
+		check_encoded_file("testfiles/f2.symlink.diff", 0, 0, 0);		
+	}
+
+	// Check that diffing against a file which isn't "complete" and referes another isn't allowed
+	{
+		FileStream blockindex("testfiles/f1.diff");
+		BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex);
+		
+		BackupStoreFilenameClear f1name("filename");
+		FileStream out("testfiles/f2.testincomplete", O_WRONLY | O_CREAT | O_EXCL);
+		TEST_CHECK_THROWS(BackupStoreFile::EncodeFileDiff("testfiles/f2", 1 /* dir ID */, f1name,
+			1001 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite,
+			0, 0), BackupStoreException, CannotDiffAnIncompleteStoreFile);
+	}
+
+	// Found a nasty case where files of lots of the same thing sock up lots of processor
+	// time -- because of lots of matches found. Check this out!
+	make_file_of_zeros("testfiles/zero.0", 20*1024);
+	make_file_of_zeros("testfiles/zero.1", 200*1024);
+	// Generate a first encoded file
+	{
+		BackupStoreFilenameClear f0name("zero.0");
+		FileStream out("testfiles/zero.0.enc", O_WRONLY | O_CREAT | O_EXCL);
+		std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/zero.0", 1 /* dir ID */, f0name));
+		encoded->CopyStreamTo(out);
+	}
+	// Then diff from it -- time how long it takes...
+	{
+		int beginTime = time(0);
+		FileStream blockindex("testfiles/zero.0.enc");
+		BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex);
+
+		BackupStoreFilenameClear f1name("zero.1");
+		FileStream out("testfiles/zero.1.enc", O_WRONLY | O_CREAT | O_EXCL);
+		std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff("testfiles/zero.1", 1 /* dir ID */, f1name,
+			2000 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite,
+			0, 0));
+		encoded->CopyStreamTo(out);
+		TEST_THAT(time(0) < (beginTime + 20));
+	}
+
+#if 0
+	// Code for a nasty real world example! (16Mb files, won't include them in the distribution
+	// for obvious reasons...)
+	// Generate a first encoded file
+	{
+		BackupStoreFilenameClear f0name("0000000000000000.old");
+		FileStream out("testfiles/0000000000000000.enc.0", O_WRONLY | O_CREAT | O_EXCL);
+		std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("/Users/ben/Desktop/0000000000000000.old", 1 /* dir ID */, f0name));
+		encoded->CopyStreamTo(out);
+	}
+	// Then diff from it -- time how long it takes...
+	{
+		int beginTime = time(0);
+		FileStream blockindex("testfiles/0000000000000000.enc.0");
+		BackupStoreFile::MoveStreamPositionToBlockIndex(blockindex);
+
+		BackupStoreFilenameClear f1name("0000000000000000.new");
+		FileStream out("testfiles/0000000000000000.enc.1", O_WRONLY | O_CREAT | O_EXCL);
+		std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff("/Users/ben/Desktop/0000000000000000.new", 1 /* dir ID */, f1name,
+			2000 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite,
+			0, 0));
+		encoded->CopyStreamTo(out);
+		TEST_THAT(time(0) < (beginTime + 20));
+	}
+#endif // 0
+
+	return 0;
+}
+
+
diff --git a/test/backupdiff/testextra b/test/backupdiff/testextra
new file mode 100644
index 00000000..165cacb9
--- /dev/null
+++ b/test/backupdiff/testextra
@@ -0,0 +1,2 @@
+rm -rf testfiles
+mkdir testfiles
diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp
new file mode 100755
index 00000000..3757bc60
--- /dev/null
+++ b/test/backupstore/testbackupstore.cpp
@@ -0,0 +1,1902 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testbackupstore.cpp
+//		Purpose: Test backup store server
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "Test.h"
+#include "autogen_BackupProtocolClient.h"
+#include "SSLLib.h"
+#include "TLSContext.h"
+#include "SocketStreamTLS.h"
+#include "BoxPortsAndFiles.h"
+#include "BackupStoreConstants.h"
+#include "Socket.h"
+#include "BackupStoreFilenameClear.h"
+#include "CollectInBufferStream.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "FileStream.h"
+#include "RaidFileController.h"
+#include "RaidFileWrite.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreException.h"
+#include "RaidFileException.h"
+#include "MemBlockStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupClientCryptoKeys.h"
+
+#include "MemLeakFindOn.h"
+
+
+#define ENCFILE_SIZE	2765
+
+typedef struct
+{
+	BackupStoreFilenameClear fn;
+	box_time_t mod;
+	int64_t id;
+	int64_t size;
+	int16_t flags;
+	box_time_t attrmod;
+} dirtest;
+
+static dirtest ens[] =
+{
+	{BackupStoreFilenameClear(), 324324, 3432, 324, BackupStoreDirectory::Entry::Flags_File, 458763243422LL},
+	{BackupStoreFilenameClear(), 3432, 32443245645LL, 78, BackupStoreDirectory::Entry::Flags_Dir, 3248972347LL},
+	{BackupStoreFilenameClear(), 544435, 234234, 23324, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_Deleted, 2348974782LL},
+	{BackupStoreFilenameClear(), 234, 235436, 6523, BackupStoreDirectory::Entry::Flags_File, 32458923175634LL},
+	{BackupStoreFilenameClear(), 0x3242343532144LL, 8978979789LL, 21345, BackupStoreDirectory::Entry::Flags_File, 329483243432LL},
+	{BackupStoreFilenameClear(), 324265765734LL, 12312312321LL, 324987324329874LL, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_Deleted, 32489747234LL},
+	{BackupStoreFilenameClear(), 3452134, 7868578768LL, 324243, BackupStoreDirectory::Entry::Flags_Dir, 34786457432LL},
+	{BackupStoreFilenameClear(), 43543543, 324234, 21432, BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, 3489723478327LL},
+	{BackupStoreFilenameClear(), 325654765874324LL, 4353543, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 32489734789237LL},
+	{BackupStoreFilenameClear(), 32144325, 436547657, 9, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 234897347234LL}
+};
+static const char *ens_filenames[] = {"obj1ertewt", "obj2", "obj3", "obj4dfedfg43", "obj5", "obj6dfgs", "obj7", "obj8xcvbcx", "obj9", "obj10fgjhfg"};
+#define DIR_NUM 10
+#define DIR_DIRS 3
+#define DIR_FILES 7
+#define DIR_OLD 2
+#define DIR_DELETED 3
+
+typedef struct
+{
+	char *fnextra;
+	BackupStoreFilenameClear name;
+	int seed;
+	int size;
+	box_time_t mod_time;
+	int64_t allocated_objid;
+	bool should_be_old_version;
+	bool delete_file;
+} uploadtest;
+
+#define TEST_FILE_FOR_PATCHING	"testfiles/test2"
+// a few bytes will be inserted at this point:
+#define TEST_FILE_FOR_PATCHING_PATCH_AT	((64*1024)-128)
+#define TEST_FILE_FOR_PATCHING_SIZE ((128*1024)+2564)
+#define UPLOAD_PATCH_EN	2
+
+uploadtest uploads[] = 
+{
+	{"0", BackupStoreFilenameClear(), 324, 455, 0, 0, false, false},
+	{"1", BackupStoreFilenameClear(), 3232432, 2674, 0, 0, true, false},			// old ver
+	{"2", BackupStoreFilenameClear(), 234, TEST_FILE_FOR_PATCHING_SIZE, 0, 0, false, false},
+	{"3", BackupStoreFilenameClear(), 324324, 6763, 0, 0, false, false},
+	{"4", BackupStoreFilenameClear(), 23456, 124, 0, 0, true, false},			// old ver
+	{"5", BackupStoreFilenameClear(), 675745, 1, 0, 0, false, false},			// will upload new attrs for this one!
+	{"6", BackupStoreFilenameClear(), 345213, 0, 0, 0, false, false},
+	{"7", BackupStoreFilenameClear(), 12313, 3246, 0, 0, true, true},		// old ver, will get deleted
+	{"8", BackupStoreFilenameClear(), 457, 3434, 0, 0, false, false},			// overwrites
+	{"9", BackupStoreFilenameClear(), 12315, 446, 0, 0, false, false},
+	{"a", BackupStoreFilenameClear(), 3476, 2466, 0, 0, false, false},
+	{"b", BackupStoreFilenameClear(), 124334, 4562, 0, 0, false, false},
+	{"c", BackupStoreFilenameClear(), 45778, 234, 0, 0, false, false},		// overwrites
+	{"d", BackupStoreFilenameClear(), 2423425, 435, 0, 0, false, true}		// overwrites, will be deleted
+};
+static const char *uploads_filenames[] = {"49587fds", "cvhjhj324", "sdfcscs324", "dsfdsvsdc3214", "XXsfdsdf2342", "dsfdsc232",
+	"sfdsdce2345", "YYstfbdtrdf76", "cvhjhj324", "fbfd098.ycy", "dfs98732hj", "svd987kjsad", "XXsfdsdf2342", "YYstfbdtrdf76"};
+#define UPLOAD_NUM	14
+#define UPLOAD_LATEST_FILES	12
+// file we'll upload some new attributes for
+#define UPLOAD_ATTRS_EN		5
+#define UPLOAD_DELETE_EN	13
+// file which will be moved (as well as it's old version)
+#define UPLOAD_FILE_TO_MOVE	8
+
+
+// Nice random data for testing written files
+class R250 {
+public:
+	// Set up internal state table with 32-bit random numbers.  
+	// The bizarre bit-twiddling is because rand() returns 16 bits of which
+	// the bottom bit is always zero!  Hence, I use only some of the bits.
+	// You might want to do something better than this....
+
+	R250(int seed) : posn1(0), posn2(103)
+	{
+		// populate the state and incr tables
+		srand(seed);
+
+		for (int i = 0; i != stateLen; ++i)	{
+			state[i] = ((rand() >> 2) << 19) ^ ((rand() >> 2) << 11) ^ (rand() >> 2);
+			incrTable[i] = i == stateLen - 1 ? 0 : i + 1;
+		}
+
+		// stir up the numbers to ensure they're random
+
+		for (int j = 0; j != stateLen * 4; ++j)			
+			(void) next();
+	}
+
+	// Returns the next random number.  Xor together two elements separated
+	// by 103 mod 250, replacing the first element with the result.  Then
+	// increment the two indices mod 250.
+	inline int next()
+	{
+		int ret = (state[posn1] ^= state[posn2]);	// xor and replace element
+
+		posn1 = incrTable[posn1];		// increment indices using lookup table
+		posn2 = incrTable[posn2];
+
+		return ret;
+	}
+private:
+	enum { stateLen = 250 };	// length of the state table
+	int state[stateLen];		// holds the random number state
+	int incrTable[stateLen];	// lookup table: maps i to (i+1) % stateLen
+	int posn1, posn2;			// indices into the state table
+};
+
+
+int SkipEntries(int e, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
+{
+	if(e >= DIR_NUM) return e;
+
+	bool skip = false;
+	do
+	{
+		skip = false;
+		
+		if(FlagsMustBeSet != BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING)
+		{
+			if((ens[e].flags & FlagsMustBeSet) != FlagsMustBeSet)
+			{
+				skip = true;
+			}
+		}
+		if((ens[e].flags & FlagsNotToBeSet) != 0)
+		{
+			skip = true;
+		}
+		
+		if(skip)
+		{
+			++e;
+		}
+	} while(skip && e < DIR_NUM);
+	
+	return e;
+}
+
+void CheckEntries(BackupStoreDirectory &rDir, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet)
+{
+	int e = 0;
+	
+	BackupStoreDirectory::Iterator i(rDir);
+	BackupStoreDirectory::Entry *en = 0;
+	while((en = i.Next()) != 0)
+	{
+		TEST_THAT(e < DIR_NUM);
+
+		// Skip to entry in the ens array which matches
+		e = SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet);
+
+		// Does it match?
+		TEST_THAT(en->GetName() == ens[e].fn && en->GetModificationTime() == ens[e].mod && en->GetObjectID() == ens[e].id && en->GetFlags() == ens[e].flags && en->GetSizeInBlocks() == ens[e].size);
+
+		// next
+		++e;		
+	}
+	
+	// Got them all?
+	TEST_THAT(en == 0);
+	TEST_THAT(DIR_NUM == SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet));
+}
+
+int test1(int argc, const char *argv[])
+{
+	// Initialise the raid file controller
+	RaidFileController &rcontroller = RaidFileController::GetController();
+	rcontroller.Initialise("testfiles/raidfile.conf");
+
+	// test some basics -- encoding and decoding filenames
+	{
+		// Make some filenames in various ways
+		BackupStoreFilenameClear fn1;
+		fn1.SetClearFilename(std::string("filenameXYZ"));
+		BackupStoreFilenameClear fn2(std::string("filenameXYZ"));
+		BackupStoreFilenameClear fn3(fn1);
+		TEST_THAT(fn1 == fn2);
+		TEST_THAT(fn1 == fn3);
+		
+		// Check that it's been encrypted
+		TEST_THAT(fn2.find("name") == fn2.npos);
+			
+		// Bung it in a stream, get it out in a Clear filename
+		{
+			CollectInBufferStream stream;
+			fn1.WriteToStream(stream);
+			stream.SetForReading();
+			BackupStoreFilenameClear fn4;
+			fn4.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(fn4.GetClearFilename() == "filenameXYZ");
+			TEST_THAT(fn4 == fn1);
+		}
+		// Bung it in a stream, get it out in a server non-Clear filename (two of them into the same var)
+		{
+			BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf ");
+			CollectInBufferStream stream;
+			fn1.WriteToStream(stream);
+			fno.WriteToStream(stream);
+			stream.SetForReading();
+			BackupStoreFilename fn5;
+			fn5.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(fn5 == fn1);
+			fn5.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(fn5 == fno);
+		}
+		// Same again with clear strings
+		{
+			BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf ");
+			CollectInBufferStream stream;
+			fn1.WriteToStream(stream);
+			fno.WriteToStream(stream);
+			stream.SetForReading();
+			BackupStoreFilenameClear fn5;
+			fn5.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(fn5.GetClearFilename() == "filenameXYZ");
+			fn5.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(fn5.GetClearFilename() == "pinglet dksfnsf jksjdf ");
+		}
+		// Test a very big filename
+		{
+			const char *fnr = "01234567890123456789012345678901234567890123456789"
+				"01234567890123456789012345678901234567890123456789"
+				"01234567890123456789012345678901234567890123456789"
+				"01234567890123456789012345678901234567890123456789"
+				"01234567890123456789012345678901234567890123456789"
+				"01234567890123456789012345678901234567890123456789"
+				"01234567890123456789012345678901234567890123456789"
+				"01234567890123456789012345678901234567890123456789";
+			BackupStoreFilenameClear fnLong(fnr);
+			CollectInBufferStream stream;
+			fnLong.WriteToStream(stream);
+			stream.SetForReading();
+			BackupStoreFilenameClear fn9;
+			fn9.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(fn9.GetClearFilename() == fnr);
+			TEST_THAT(fn9 == fnLong);
+		}
+		// Test a filename which went wrong once
+		{
+			BackupStoreFilenameClear dodgy("content-negotiation.html");
+		}
+	}
+	return 0;
+}
+
+int test2(int argc, const char *argv[])
+{
+	{
+		// Now play with directories
+		
+		// Fill in...
+		BackupStoreDirectory dir1(12, 98);
+		for(int e = 0; e < DIR_NUM; ++e)
+		{
+			dir1.AddEntry(ens[e].fn, ens[e].mod, ens[e].id, ens[e].size, ens[e].flags, ens[e].attrmod);
+		}
+		// Got the right number
+		TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM);
+		
+		// Stick it into a stream and get it out again
+		{
+			CollectInBufferStream stream;
+			dir1.WriteToStream(stream);
+			stream.SetForReading();
+			BackupStoreDirectory dir2;
+			dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(dir2.GetNumberOfEntries() == DIR_NUM);
+			TEST_THAT(dir2.GetObjectID() == 12);
+			TEST_THAT(dir2.GetContainerID() == 98);
+			CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
+		}
+		
+		// Then do selective writes and reads
+		{
+			CollectInBufferStream stream;
+			dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File);
+			stream.SetForReading();
+			BackupStoreDirectory dir2;
+			dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES);
+			CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
+		}
+		{
+			CollectInBufferStream stream;
+			dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_File);
+			stream.SetForReading();
+			BackupStoreDirectory dir2;
+			dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(dir2.GetNumberOfEntries() == DIR_DIRS);
+			CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_Dir, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING);
+		}
+		{
+			CollectInBufferStream stream;
+			dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion);
+			stream.SetForReading();
+			BackupStoreDirectory dir2;
+			dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - DIR_OLD);
+			CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion);
+		}
+		
+		// Finally test deleting items
+		{
+			dir1.DeleteEntry(12312312321LL);
+			// Verify
+			TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM - 1);
+			CollectInBufferStream stream;
+			dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File);
+			stream.SetForReading();
+			BackupStoreDirectory dir2;
+			dir2.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - 1);
+		}
+		
+		// Check attributes
+		{
+			int attrI[4] = {1, 2, 3, 4};
+			StreamableMemBlock attr(attrI, sizeof(attrI));
+			BackupStoreDirectory d1(16, 546);
+			d1.SetAttributes(attr, 56234987324232LL);
+			TEST_THAT(d1.GetAttributes() == attr);
+			TEST_THAT(d1.GetAttributesModTime() == 56234987324232LL);
+			CollectInBufferStream stream;
+			d1.WriteToStream(stream);
+			stream.SetForReading();
+			BackupStoreDirectory d2;
+			d2.ReadFromStream(stream, IOStream::TimeOutInfinite);
+			TEST_THAT(d2.GetAttributes() == attr);
+			TEST_THAT(d2.GetAttributesModTime() == 56234987324232LL);
+		}
+	}
+	return 0;
+}
+
+void write_test_file(int t)
+{
+	std::string filename("testfiles/test");
+	filename += uploads[t].fnextra;
+	printf("%s\n", filename.c_str());
+	
+	FileStream write(filename.c_str(), O_WRONLY | O_CREAT);
+	
+	R250 r(uploads[t].seed);
+	
+	unsigned char *data = (unsigned char*)malloc(uploads[t].size);
+	for(int l = 0; l < uploads[t].size; ++l)
+	{
+		data[l] = r.next() & 0xff;
+	}
+	write.Write(data, uploads[t].size);
+	
+	free(data);
+}
+
+void test_test_file(int t, IOStream &rStream)
+{
+	// Decode to a file
+	BackupStoreFile::DecodeFile(rStream, "testfiles/test_download", IOStream::TimeOutInfinite);
+	
+	// Compare...
+	FileStream in("testfiles/test_download");
+	TEST_THAT(in.BytesLeftToRead() == uploads[t].size);
+	
+	R250 r(uploads[t].seed);
+	
+	unsigned char *data = (unsigned char*)malloc(uploads[t].size);
+	TEST_THAT(in.ReadFullBuffer(data, uploads[t].size, 0 /* not interested in bytes read if this fails */));
+	
+	for(int l = 0; l < uploads[t].size; ++l)
+	{
+		TEST_THAT(data[l] == (r.next() & 0xff));
+	}
+	
+	free(data);
+	unlink("testfiles/test_download");
+}
+
+void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID)
+{
+	printf("Test for del: %llx\n", DirID);
+	
+	// Command
+	std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocol.QueryListDirectory(
+			DirID,
+			BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+			BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+	// Stream
+	BackupStoreDirectory dir;
+	std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
+	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+	
+	BackupStoreDirectory::Iterator i(dir);
+	BackupStoreDirectory::Entry *en = 0;
+	int files = 0;
+	int dirs = 0;
+	while((en = i.Next()) != 0)
+	{
+		if(en->GetFlags() & BackupProtocolClientListDirectory::Flags_Dir)
+		{
+			dirs++;
+			// Recurse
+			test_everything_deleted(protocol, en->GetObjectID());
+		}
+		else
+		{
+			files++;
+		}
+		// Check it's deleted
+		TEST_THAT(en->GetFlags() & BackupProtocolClientListDirectory::Flags_Deleted);
+	}
+	
+	// Check there were the right number of files and directories
+	TEST_THAT(files == 3);
+	TEST_THAT(dirs == 0 || dirs == 2);
+}
+
+int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, const char *name, int depth)
+{
+	// Create a directory
+	int64_t subdirid = 0;
+	BackupStoreFilenameClear dirname(name);
+	{
+		// Create with dummy attributes
+		int attrS = 0;
+		MemBlockStream attr(&attrS, sizeof(attrS));
+		std::auto_ptr<BackupProtocolClientSuccess> dirCreate(protocol.QueryCreateDirectory(
+			indir,
+			9837429842987984LL, dirname, attr));
+		subdirid = dirCreate->GetObjectID(); 
+	}
+	
+	printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid);
+	
+	// Put more directories in it, if we haven't gone down too far
+	if(depth > 0)
+	{
+		create_test_data_subdirs(protocol, subdirid, "dir_One", depth - 1);
+		create_test_data_subdirs(protocol, subdirid, "dir_Two", depth - 1);
+	}
+	
+	// Stick some files in it
+	{
+		BackupStoreFilenameClear name("file_One");
+		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
+		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+			subdirid,
+			0x123456789abcdefLL,		/* modification time */
+			0x7362383249872dfLL,		/* attr hash */
+			0,							/* diff from ID */
+			name,
+			*upload));
+	}
+	{
+		BackupStoreFilenameClear name("file_Two");
+		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
+		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+			subdirid,
+			0x123456789abcdefLL,		/* modification time */
+			0x7362383249872dfLL,		/* attr hash */
+			0,							/* diff from ID */
+			name,
+			*upload));
+	}
+	{
+		BackupStoreFilenameClear name("file_Three");
+		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
+		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+			subdirid,
+			0x123456789abcdefLL,		/* modification time */
+			0x7362383249872dfLL,		/* attr hash */
+			0,							/* diff from ID */
+			name,
+			*upload));
+	}
+
+	return subdirid;
+}
+
+
+void check_dir_after_uploads(BackupProtocolClient &protocol, const StreamableMemBlock &Attributes)
+{
+	// Command
+	std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocol.QueryListDirectory(
+			BackupProtocolClientListDirectory::RootDirectory,
+			BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+			BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+	TEST_THAT(dirreply->GetObjectID() == BackupProtocolClientListDirectory::RootDirectory);
+	// Stream
+	BackupStoreDirectory dir;
+	std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
+	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+	TEST_THAT(dir.GetNumberOfEntries() == UPLOAD_NUM + 1 /* for the first test file */);
+	TEST_THAT(!dir.HasAttributes());
+
+	// Check them!
+	BackupStoreDirectory::Iterator i(dir);
+	// Discard first
+	BackupStoreDirectory::Entry *en = i.Next();
+	TEST_THAT(en != 0);
+
+	for(int t = 0; t < UPLOAD_NUM; ++t)
+	{
+		en = i.Next();
+		TEST_THAT(en != 0);
+		TEST_THAT(en->GetName() == uploads[t].name);
+		TEST_THAT(en->GetObjectID() == uploads[t].allocated_objid);
+		TEST_THAT(en->GetModificationTime() == uploads[t].mod_time);
+		int correct_flags = BackupProtocolClientListDirectory::Flags_File;
+		if(uploads[t].should_be_old_version) correct_flags |= BackupProtocolClientListDirectory::Flags_OldVersion;
+		if(uploads[t].delete_file) correct_flags |= BackupProtocolClientListDirectory::Flags_Deleted;
+		TEST_THAT(en->GetFlags() == correct_flags);
+		if(t == UPLOAD_ATTRS_EN)
+		{
+			TEST_THAT(en->HasAttributes());
+			TEST_THAT(en->GetAttributesHash() == 32498749832475LL);
+			TEST_THAT(en->GetAttributes() == Attributes);
+		}
+		else
+		{
+			// No attributes on this one
+			TEST_THAT(!en->HasAttributes());
+		}
+	}				
+	en = i.Next();
+	TEST_THAT(en == 0);
+}
+
+
+typedef struct
+{
+	int objectsNotDel;
+	int deleted;
+	int old;
+} recusive_count_objects_results;
+
+void recusive_count_objects_r(BackupProtocolClient &protocol, int64_t id, recusive_count_objects_results &results)
+{
+	// Command
+	std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocol.QueryListDirectory(
+			id,
+			BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+			BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+	// Stream
+	BackupStoreDirectory dir;
+	std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
+	dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+
+	// Check them!
+	BackupStoreDirectory::Iterator i(dir);
+	// Discard first
+	BackupStoreDirectory::Entry *en = 0;
+	
+	while((en = i.Next()) != 0)
+	{
+		if((en->GetFlags() & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) == 0) results.objectsNotDel++;
+		if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) results.deleted++;
+		if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) results.old++;
+		
+		if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir)
+		{
+			recusive_count_objects_r(protocol, en->GetObjectID(), results);
+		}
+	}
+}
+
+void recusive_count_objects(const char *hostname, int64_t id, recusive_count_objects_results &results)
+{
+	// Context
+	TLSContext context;
+	context.Initialise(false /* client */,
+			"testfiles/clientCerts.pem",
+			"testfiles/clientPrivKey.pem",
+			"testfiles/clientTrustedCAs.pem");
+
+	// Get a connection
+	SocketStreamTLS connReadOnly;
+	connReadOnly.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+	BackupProtocolClient protocolReadOnly(connReadOnly);
+
+	{
+		std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION));
+		TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+		std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolClientLogin::Flags_ReadOnly));
+	}
+	
+	// Count objects
+	recusive_count_objects_r(protocolReadOnly, id, results);
+
+	// Close it
+	protocolReadOnly.QueryFinished();
+}
+
+bool check_block_index(const char *encoded_file, IOStream &rBlockIndex)
+{
+	// Open file, and move to the right position
+	FileStream enc(encoded_file);
+	BackupStoreFile::MoveStreamPositionToBlockIndex(enc);
+
+	bool same = true;
+
+	// Now compare the two...
+	while(enc.StreamDataLeft())
+	{
+		char buffer1[2048];
+		char buffer2[2048];
+		int s = enc.Read(buffer1, sizeof(buffer1));
+		if(rBlockIndex.Read(buffer2, s) != s)
+		{
+			same = false;
+			break;
+		}
+		if(::memcmp(buffer1, buffer2, s) != 0)
+		{
+			same = false;
+			break;
+		}
+	}
+	
+	if(rBlockIndex.StreamDataLeft())
+	{
+		same = false;
+		
+		// Absorb all this excess data so procotol is in the first state
+		char buffer[2048];
+		while(rBlockIndex.StreamDataLeft())
+		{
+			rBlockIndex.Read(buffer, sizeof(buffer));
+		}
+	}
+	
+	return same;
+}
+
+bool check_files_same(const char *f1, const char *f2)
+{
+	// Open file, and move to the right position
+	FileStream f1s(f1);
+	FileStream f2s(f2);
+
+	bool same = true;
+
+	// Now compare the two...
+	while(f1s.StreamDataLeft())
+	{
+		char buffer1[2048];
+		char buffer2[2048];
+		int s = f1s.Read(buffer1, sizeof(buffer1));
+		if(f2s.Read(buffer2, s) != s)
+		{
+			same = false;
+			break;
+		}
+		if(::memcmp(buffer1, buffer2, s) != 0)
+		{
+			same = false;
+			break;
+		}
+	}
+	
+	if(f2s.StreamDataLeft())
+	{
+		same = false;
+	}
+	
+	return same;
+}
+
+
+void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protocolReadOnly)
+{
+	int encfile[ENCFILE_SIZE];
+	{
+		for(int l = 0; l < ENCFILE_SIZE; ++l)
+		{
+			encfile[l] = l * 173;
+		}
+
+		// Write this to a file
+		{
+			FileStream f("testfiles/file1", O_WRONLY | O_CREAT | O_EXCL);
+			f.Write(encfile, sizeof(encfile));
+		}
+		
+	}
+
+	// Read the root directory a few times (as it's cached, so make sure it doesn't hurt anything)
+	for(int l = 0; l < 3; ++l)
+	{
+		// Command
+		std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocol.QueryListDirectory(
+				BackupProtocolClientListDirectory::RootDirectory,
+				BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+				BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+		// Stream
+		BackupStoreDirectory dir;
+		std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
+		dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+		TEST_THAT(dir.GetNumberOfEntries() == 0);
+	}
+
+	// Read the dir from the readonly connection (make sure it gets in the cache)
+	{
+		// Command
+		std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocolReadOnly.QueryListDirectory(
+				BackupProtocolClientListDirectory::RootDirectory,
+				BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+				BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+		// Stream
+		BackupStoreDirectory dir;
+		std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
+		dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+		TEST_THAT(dir.GetNumberOfEntries() == 0);			
+	}
+
+	// Store a file -- first make the encoded file
+	BackupStoreFilenameClear store1name("testfiles/file1");
+	{
+		FileStream out("testfiles/file1_upload1", O_WRONLY | O_CREAT | O_EXCL);
+		std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/file1", BackupProtocolClientListDirectory::RootDirectory, store1name));
+		encoded->CopyStreamTo(out);
+	}
+
+//	printf("SKIPPING\n");
+//	goto skip; {
+	// Then send it
+	int64_t store1objid = 0;
+	{
+		FileStream upload("testfiles/file1_upload1");
+		std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+			BackupProtocolClientListDirectory::RootDirectory,
+			0x123456789abcdefLL,		/* modification time */
+			0x7362383249872dfLL,		/* attr hash */
+			0,							/* diff from ID */
+			store1name,
+			upload));
+		store1objid = stored->GetObjectID();
+		TEST_THAT(store1objid == 2);
+	}
+	// And retrieve it
+	{
+		// Retrieve as object
+		std::auto_ptr<BackupProtocolClientSuccess> getfile(protocol.QueryGetObject(store1objid));
+		TEST_THAT(getfile->GetObjectID() == store1objid);
+		// BLOCK
+		{
+			// Get stream
+			std::auto_ptr<IOStream> filestream(protocol.ReceiveStream());
+			// Need to put it in another stream, because it's not in stream order
+			CollectInBufferStream f;
+			filestream->CopyStreamTo(f);
+			f.SetForReading();
+			// Get and decode
+			BackupStoreFile::DecodeFile(f, "testfiles/file1_upload_retrieved", IOStream::TimeOutInfinite);
+		}
+
+		// Retrieve as file
+		std::auto_ptr<BackupProtocolClientSuccess> getobj(protocol.QueryGetFile(BackupProtocolClientListDirectory::RootDirectory, store1objid));
+		TEST_THAT(getobj->GetObjectID() == store1objid);
+		// BLOCK
+		{
+			// Get stream
+			std::auto_ptr<IOStream> filestream(protocol.ReceiveStream());
+			// Get and decode
+			BackupStoreFile::DecodeFile(*filestream, "testfiles/file1_upload_retrieved_str", IOStream::TimeOutInfinite);
+		}
+
+		// Read in rebuilt original, and compare contents
+		{
+			FileStream in("testfiles/file1_upload_retrieved");
+			int encfile_i[ENCFILE_SIZE];
+			in.Read(encfile_i, sizeof(encfile_i));
+			TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0);
+		}
+		{
+			FileStream in("testfiles/file1_upload_retrieved_str");
+			int encfile_i[ENCFILE_SIZE];
+			in.Read(encfile_i, sizeof(encfile_i));
+			TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0);
+		}
+		
+		// Retrieve the block index, by ID
+		{
+			std::auto_ptr<BackupProtocolClientSuccess> getblockindex(protocol.QueryGetBlockIndexByID(store1objid));
+			TEST_THAT(getblockindex->GetObjectID() == store1objid);
+			std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream());
+			// Check against uploaded file
+			TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream));
+		}
+		// and again, by name
+		{
+			std::auto_ptr<BackupProtocolClientSuccess> getblockindex(protocol.QueryGetBlockIndexByName(BackupProtocolClientListDirectory::RootDirectory, store1name));
+			TEST_THAT(getblockindex->GetObjectID() == store1objid);
+			std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream());
+			// Check against uploaded file
+			TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream));
+		}
+	}
+	// Get the directory again, and see if the entry is in it
+	{
+		// Command
+		std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocol.QueryListDirectory(
+				BackupProtocolClientListDirectory::RootDirectory,
+				BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+				BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+		// Stream
+		BackupStoreDirectory dir;
+		std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
+		dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+		TEST_THAT(dir.GetNumberOfEntries() == 1);
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = i.Next();
+		TEST_THAT(en != 0);
+		TEST_THAT(i.Next() == 0);
+		if(en != 0)
+		{
+			TEST_THAT(en->GetName() == store1name);
+			TEST_THAT(en->GetModificationTime() == 0x123456789abcdefLL);
+			TEST_THAT(en->GetAttributesHash() == 0x7362383249872dfLL);
+			TEST_THAT(en->GetObjectID() == store1objid);
+			TEST_THAT(en->GetSizeInBlocks() < ((ENCFILE_SIZE * 4 * 3) / 2 / 2048)+2);
+			TEST_THAT(en->GetFlags() == BackupStoreDirectory::Entry::Flags_File);
+		}
+	}
+
+	// Try using GetFile on a directory
+	{
+		TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolClientSuccess> getFile(protocol.QueryGetFile(BackupProtocolClientListDirectory::RootDirectory, BackupProtocolClientListDirectory::RootDirectory)),
+			ConnectionException, Conn_Protocol_UnexpectedReply);
+	}
+}
+
+
+int test_server(const char *hostname)
+{
+	// Context
+	TLSContext context;
+	context.Initialise(false /* client */,
+			"testfiles/clientCerts.pem",
+			"testfiles/clientPrivKey.pem",
+			"testfiles/clientTrustedCAs.pem");
+
+	// Make some test attributes
+	#define ATTR1_SIZE 	245
+	#define ATTR2_SIZE 	23
+	#define ATTR3_SIZE 	122
+	int attr1[ATTR1_SIZE];
+	int attr2[ATTR2_SIZE];
+	int attr3[ATTR3_SIZE];
+	{
+		R250 r(3465657);
+		for(int l = 0; l < ATTR1_SIZE; ++l) {attr1[l] = r.next();}
+		for(int l = 0; l < ATTR2_SIZE; ++l) {attr2[l] = r.next();}
+		for(int l = 0; l < ATTR3_SIZE; ++l) {attr3[l] = r.next();}
+	}
+
+	// BLOCK
+	{
+		// Open a connection to the server
+		SocketStreamTLS conn;
+		conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+
+		// Make a protocol
+		BackupProtocolClient protocol(conn);
+		
+		// Get it logging
+		FILE *protocolLog = ::fopen("testfiles/protocol.log", "w");
+		TEST_THAT(protocolLog != 0);
+		protocol.SetLogToFile(protocolLog);
+
+		// Check the version
+		std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
+		TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+
+		// Login
+		std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));
+		
+		// Check marker is 0
+		TEST_THAT(loginConf->GetClientStoreMarker() == 0);
+
+		// Check that we can't open a new connection which requests write permissions
+		{
+			SocketStreamTLS conn;
+			conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+			BackupProtocolClient protocol(conn);
+			std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
+			TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+			TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)),
+				ConnectionException, Conn_Protocol_UnexpectedReply);
+			protocol.QueryFinished();
+		}
+		
+		// Set the client store marker
+		protocol.QuerySetClientStoreMarker(0x8732523ab23aLL);
+
+		// Open a new connection which is read only
+		SocketStreamTLS connReadOnly;
+		connReadOnly.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+		BackupProtocolClient protocolReadOnly(connReadOnly);
+
+		// Get it logging
+		FILE *protocolReadOnlyLog = ::fopen("testfiles/protocolReadOnly.log", "w");
+		TEST_THAT(protocolReadOnlyLog != 0);
+		protocolReadOnly.SetLogToFile(protocolReadOnlyLog);
+
+		{
+			std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION));
+			TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+			std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolClientLogin::Flags_ReadOnly));
+			
+			// Check client store marker
+			TEST_THAT(loginConf->GetClientStoreMarker() == 0x8732523ab23aLL);
+		}
+
+		test_server_1(protocol, protocolReadOnly);
+
+
+		// Create and upload some test files
+		int64_t maxID = 0;
+		for(int t = 0; t < UPLOAD_NUM; ++t)
+		{
+			write_test_file(t);
+
+			std::string filename("testfiles/test");
+			filename += uploads[t].fnextra;
+			int64_t modtime = 0;
+			
+			std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), BackupProtocolClientListDirectory::RootDirectory, uploads[t].name, &modtime));
+			TEST_THAT(modtime != 0);
+			
+			std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+				BackupProtocolClientListDirectory::RootDirectory,
+				modtime,
+				modtime, /* use it for attr hash too */
+				0,							/* diff from ID */
+				uploads[t].name,
+				*upload));
+			uploads[t].allocated_objid = stored->GetObjectID();
+			uploads[t].mod_time = modtime;
+			if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID();
+		}
+
+		// Add some attributes onto one of them
+		{
+			MemBlockStream attrnew(attr3, sizeof(attr3));
+			std::auto_ptr<BackupProtocolClientSuccess> set(protocol.QuerySetReplacementFileAttributes(
+				BackupProtocolClientListDirectory::RootDirectory,
+				32498749832475LL,
+				uploads[UPLOAD_ATTRS_EN].name,
+				attrnew));
+			TEST_THAT(set->GetObjectID() == uploads[UPLOAD_ATTRS_EN].allocated_objid);
+		}
+		
+		// Delete one of them (will implicitly delete an old version)
+		{
+			std::auto_ptr<BackupProtocolClientSuccess> del(protocol.QueryDeleteFile(
+				BackupProtocolClientListDirectory::RootDirectory,
+				uploads[UPLOAD_DELETE_EN].name));
+			TEST_THAT(del->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid);
+		}
+		// Check that the block index can be obtained by name even though it's been deleted
+		{
+			// Fetch the raw object
+			{
+				FileStream out("testfiles/downloaddelobj", O_WRONLY | O_CREAT);
+				std::auto_ptr<BackupProtocolClientSuccess> getobj(protocol.QueryGetObject(uploads[UPLOAD_DELETE_EN].allocated_objid));
+				std::auto_ptr<IOStream> objstream(protocol.ReceiveStream());
+				objstream->CopyStreamTo(out);
+			}
+			// query index and test
+			std::auto_ptr<BackupProtocolClientSuccess> getblockindex(protocol.QueryGetBlockIndexByName(
+				BackupProtocolClientListDirectory::RootDirectory, uploads[UPLOAD_DELETE_EN].name));
+			TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid);
+			std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream());
+			TEST_THAT(check_block_index("testfiles/downloaddelobj", *blockIndexStream));
+		}
+
+		// Download them all... (even deleted files)
+		for(int t = 0; t < UPLOAD_NUM; ++t)
+		{
+			printf("%d\n", t);
+			std::auto_ptr<BackupProtocolClientSuccess> getFile(protocol.QueryGetFile(BackupProtocolClientListDirectory::RootDirectory, uploads[t].allocated_objid));
+			TEST_THAT(getFile->GetObjectID() == uploads[t].allocated_objid);
+			std::auto_ptr<IOStream> filestream(protocol.ReceiveStream());
+			test_test_file(t, *filestream);
+		}
+
+		{
+			StreamableMemBlock attrtest(attr3, sizeof(attr3));
+
+			// Use the read only connection to verify that the directory is as we expect
+			check_dir_after_uploads(protocolReadOnly, attrtest);
+			// And on the read/write one
+			check_dir_after_uploads(protocol, attrtest);
+		}
+		
+		// Check diffing and rsync like stuff...
+		// Build a modified file
+		{
+			// Basically just insert a bit in the middle
+			TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING) == TEST_FILE_FOR_PATCHING_SIZE);
+			FileStream in(TEST_FILE_FOR_PATCHING);
+			void *buf = ::malloc(TEST_FILE_FOR_PATCHING_SIZE);
+			FileStream out(TEST_FILE_FOR_PATCHING ".mod", O_WRONLY | O_CREAT | O_EXCL);
+			TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_PATCH_AT);
+			out.Write(buf, TEST_FILE_FOR_PATCHING_PATCH_AT);
+			char insert[13] = "INSERTINSERT";
+			out.Write(insert, sizeof(insert));
+			TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT);
+			out.Write(buf, TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT);
+			::free(buf);
+		}
+		{
+			// Fetch the block index for this one
+			std::auto_ptr<BackupProtocolClientSuccess> getblockindex(protocol.QueryGetBlockIndexByName(
+				BackupProtocolClientListDirectory::RootDirectory, uploads[UPLOAD_PATCH_EN].name));
+			TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_PATCH_EN].allocated_objid);
+			std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream());
+			
+			// Do the patching
+			bool isCompletelyDifferent = false;
+			int64_t modtime;
+			std::auto_ptr<IOStream> patchstream(BackupStoreFile::EncodeFileDiff(TEST_FILE_FOR_PATCHING ".mod", BackupProtocolClientListDirectory::RootDirectory,
+					uploads[UPLOAD_PATCH_EN].name, uploads[UPLOAD_PATCH_EN].allocated_objid, *blockIndexStream,
+					IOStream::TimeOutInfinite, &modtime, &isCompletelyDifferent));
+			TEST_THAT(isCompletelyDifferent == false);
+			// Sent this to a file, so we can check the size, rather than uploading it directly
+			{
+				FileStream patch(TEST_FILE_FOR_PATCHING ".patch", O_WRONLY | O_CREAT | O_EXCL);
+				patchstream->CopyStreamTo(patch);
+			}
+			// Make sure the stream is a plausible size for a patch containing only one new block
+			TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING ".patch") < (8*1024));
+			// Upload it
+			int64_t patchedID = 0;
+			{
+				FileStream uploadpatch(TEST_FILE_FOR_PATCHING ".patch");
+				std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+					BackupProtocolClientListDirectory::RootDirectory,
+					modtime,
+					modtime, /* use it for attr hash too */
+					uploads[UPLOAD_PATCH_EN].allocated_objid,		/* diff from ID */
+					uploads[UPLOAD_PATCH_EN].name,
+					uploadpatch));
+				TEST_THAT(stored->GetObjectID() > 0);
+				if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID();
+				patchedID = stored->GetObjectID();
+			}
+			// Then download it to check it's OK
+			std::auto_ptr<BackupProtocolClientSuccess> getFile(protocol.QueryGetFile(BackupProtocolClientListDirectory::RootDirectory, patchedID));
+			TEST_THAT(getFile->GetObjectID() == patchedID);
+			std::auto_ptr<IOStream> filestream(protocol.ReceiveStream());
+			BackupStoreFile::DecodeFile(*filestream, TEST_FILE_FOR_PATCHING ".downloaded", IOStream::TimeOutInfinite);
+			// Check it's the same
+			TEST_THAT(check_files_same(TEST_FILE_FOR_PATCHING ".downloaded", TEST_FILE_FOR_PATCHING ".mod"));
+		}
+
+		// Create a directory
+		int64_t subdirid = 0;
+		BackupStoreFilenameClear dirname("lovely_directory");
+		{
+			// Attributes
+			MemBlockStream attr(attr1, sizeof(attr1));
+			std::auto_ptr<BackupProtocolClientSuccess> dirCreate(protocol.QueryCreateDirectory(
+				BackupProtocolClientListDirectory::RootDirectory,
+				9837429842987984LL, dirname, attr));
+			subdirid = dirCreate->GetObjectID(); 
+			TEST_THAT(subdirid == maxID + 1);
+		}
+		// Stick a file in it
+		int64_t subdirfileid = 0;
+		{
+			std::string filename("testfiles/test0");
+			int64_t modtime;
+			std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), subdirid, uploads[0].name, &modtime));
+
+			std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+				subdirid,
+				modtime,
+				modtime, /* use for attr hash too */
+				0,							/* diff from ID */
+				uploads[0].name,
+				*upload));
+			subdirfileid = stored->GetObjectID();
+		}
+		// Check the directories on the read only connection
+		{
+			// Command
+			std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocolReadOnly.QueryListDirectory(
+					BackupProtocolClientListDirectory::RootDirectory,
+					BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+					BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); // Stream
+			BackupStoreDirectory dir;
+			std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
+			dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+			TEST_THAT(dir.GetNumberOfEntries() == UPLOAD_NUM + 3 /* for the first test file, the patched upload, and this new dir */);
+
+			// Check the last one...
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+			BackupStoreDirectory::Entry *t = 0;
+			while((t = i.Next()) != 0)
+			{
+				if(en != 0)
+				{
+					// here for all but last object
+					TEST_THAT(en->GetObjectID() != subdirid);
+					TEST_THAT(en->GetName() != dirname);
+				}
+				en = t;
+			}
+			// Does it look right?
+			TEST_THAT(en->GetName() == dirname);
+			TEST_THAT(en->GetFlags() == BackupProtocolClientListDirectory::Flags_Dir);
+			TEST_THAT(en->GetObjectID() == subdirid);
+			TEST_THAT(en->GetModificationTime() == 0);	// dirs don't have modification times.
+		}
+		{
+			// Command
+			std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocolReadOnly.QueryListDirectory(
+					subdirid,
+					BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+					BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, true /* get attributes */));
+			TEST_THAT(dirreply->GetObjectID() == subdirid);
+			// Stream
+			BackupStoreDirectory dir;
+			std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
+			dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+			TEST_THAT(dir.GetNumberOfEntries() == 1);
+
+			// Check the last one...
+			BackupStoreDirectory::Iterator i(dir);
+			// Discard first
+			BackupStoreDirectory::Entry *en = i.Next();
+			TEST_THAT(en != 0);
+			// Does it look right?
+			TEST_THAT(en->GetName() == uploads[0].name);
+			TEST_THAT(en->GetFlags() == BackupProtocolClientListDirectory::Flags_File);
+			TEST_THAT(en->GetObjectID() == subdirfileid);
+			TEST_THAT(en->GetModificationTime() != 0);
+
+			// Attributes
+			TEST_THAT(dir.HasAttributes());
+			TEST_THAT(dir.GetAttributesModTime() == 9837429842987984LL);
+			StreamableMemBlock attr(attr1, sizeof(attr1));
+			TEST_THAT(dir.GetAttributes() == attr);
+		}
+		// Check that we don't get attributes if we don't ask for them
+		{
+			// Command
+			std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocolReadOnly.QueryListDirectory(
+					subdirid,
+					BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+					BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */));
+			// Stream
+			BackupStoreDirectory dir;
+			std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
+			dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+			TEST_THAT(!dir.HasAttributes());
+		}
+		// Change attributes on the directory
+		{
+			MemBlockStream attrnew(attr2, sizeof(attr2));
+			std::auto_ptr<BackupProtocolClientSuccess> changereply(protocol.QueryChangeDirAttributes(
+					subdirid,
+					329483209443598LL,
+					attrnew));
+			TEST_THAT(changereply->GetObjectID() == subdirid);
+		}
+		// Check the new attributes
+		{
+			// Command
+			std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocolReadOnly.QueryListDirectory(
+					subdirid,
+					0,	// no flags
+					BackupProtocolClientListDirectory::Flags_EXCLUDE_EVERYTHING, true /* get attributes */));
+			// Stream
+			BackupStoreDirectory dir;
+			std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
+			dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+			TEST_THAT(dir.GetNumberOfEntries() == 0);
+
+			// Attributes
+			TEST_THAT(dir.HasAttributes());
+			TEST_THAT(dir.GetAttributesModTime() == 329483209443598LL);
+			StreamableMemBlock attrtest(attr2, sizeof(attr2));
+			TEST_THAT(dir.GetAttributes() == attrtest);
+		}
+		
+		// Test moving a file
+		{
+			BackupStoreFilenameClear newName("moved-files");
+		
+			std::auto_ptr<BackupProtocolClientSuccess> rep(protocol.QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid,
+				BackupProtocolClientListDirectory::RootDirectory,
+				subdirid, BackupProtocolClientMoveObject::Flags_MoveAllWithSameName, newName));
+			TEST_THAT(rep->GetObjectID() == uploads[UPLOAD_FILE_TO_MOVE].allocated_objid);
+		}
+		// Try some dodgy renames
+		{
+			BackupStoreFilenameClear newName("moved-files");
+			TEST_CHECK_THROWS(protocol.QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid,
+					BackupProtocolClientListDirectory::RootDirectory,
+					subdirid, BackupProtocolClientMoveObject::Flags_MoveAllWithSameName, newName),
+				ConnectionException, Conn_Protocol_UnexpectedReply);
+			TEST_CHECK_THROWS(protocol.QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid,
+					subdirid,
+					subdirid, BackupProtocolClientMoveObject::Flags_MoveAllWithSameName, newName),
+				ConnectionException, Conn_Protocol_UnexpectedReply);
+		}
+		// Rename within a directory
+		{
+			BackupStoreFilenameClear newName("moved-files-x");
+			protocol.QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid,
+				subdirid,
+				subdirid, BackupProtocolClientMoveObject::Flags_MoveAllWithSameName, newName);
+		}
+		// Check it's all gone from the root directory...
+		{
+			// Command
+			std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocolReadOnly.QueryListDirectory(
+					BackupProtocolClientListDirectory::RootDirectory,
+					BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+					BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+			// Stream
+			BackupStoreDirectory dir;
+			std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
+			dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+			// Read all entries
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+			while((en = i.Next()) != 0)
+			{
+				TEST_THAT(en->GetName() != uploads[UPLOAD_FILE_TO_MOVE].name);
+			}
+		}
+		// Check the old and new versions are in the other directory
+		{
+			BackupStoreFilenameClear lookFor("moved-files-x");
+
+			// Command
+			std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocolReadOnly.QueryListDirectory(
+					subdirid,
+					BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+					BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+			// Stream
+			BackupStoreDirectory dir;
+			std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
+			dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+			// Check entries
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = 0;
+			bool foundCurrent = false;
+			bool foundOld = false;
+			while((en = i.Next()) != 0)
+			{
+				if(en->GetName() == lookFor)
+				{
+					if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File)) foundCurrent = true;
+					if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion)) foundOld = true;
+				}
+			}
+			TEST_THAT(foundCurrent);
+			TEST_THAT(foundOld);
+		}
+		// make a little bit more of a thing to look at
+		int64_t subsubdirid = 0;
+		int64_t subsubfileid = 0;
+		{
+			BackupStoreFilenameClear nd("sub2");
+			// Attributes
+			MemBlockStream attr(attr1, sizeof(attr1));
+			std::auto_ptr<BackupProtocolClientSuccess> dirCreate(protocol.QueryCreateDirectory(
+				subdirid,
+				9837429842987984LL, nd, attr));
+			subsubdirid = dirCreate->GetObjectID(); 
+
+			FileStream upload("testfiles/file1_upload1");
+			BackupStoreFilenameClear nf("file2");
+			std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+				subsubdirid,
+				0x123456789abcdefLL,		/* modification time */
+				0x7362383249872dfLL,		/* attr hash */
+				0,							/* diff from ID */
+				nf,
+				upload));
+			subsubfileid = stored->GetObjectID();
+		}
+		// Query names -- test that invalid stuff returns not found OK
+		{
+			std::auto_ptr<BackupProtocolClientObjectName> nameRep(protocol.QueryGetObjectName(3248972347823478927LL, subsubdirid));
+			TEST_THAT(nameRep->GetNumNameElements() == 0);		
+		}
+		{
+			std::auto_ptr<BackupProtocolClientObjectName> nameRep(protocol.QueryGetObjectName(subsubfileid, 2342378424LL));
+			TEST_THAT(nameRep->GetNumNameElements() == 0);		
+		}
+		{
+			std::auto_ptr<BackupProtocolClientObjectName> nameRep(protocol.QueryGetObjectName(38947234789LL, 2342378424LL));
+			TEST_THAT(nameRep->GetNumNameElements() == 0);		
+		}
+		{
+			std::auto_ptr<BackupProtocolClientObjectName> nameRep(protocol.QueryGetObjectName(BackupProtocolClientGetObjectName::ObjectID_DirectoryOnly, 2234342378424LL));
+			TEST_THAT(nameRep->GetNumNameElements() == 0);		
+		}
+		// Query names... first, get info for the file
+		{
+			std::auto_ptr<BackupProtocolClientObjectName> nameRep(protocol.QueryGetObjectName(subsubfileid, subsubdirid));
+			std::auto_ptr<IOStream> namestream(protocol.ReceiveStream());
+		
+			TEST_THAT(nameRep->GetNumNameElements() == 3);
+			TEST_THAT(nameRep->GetFlags() == BackupProtocolClientListDirectory::Flags_File);
+			TEST_THAT(nameRep->GetModificationTime() == 0x123456789abcdefLL);
+			TEST_THAT(nameRep->GetAttributesHash() == 0x7362383249872dfLL);
+			static const char *testnames[] = {"file2","sub2","lovely_directory"};
+			for(int l = 0; l < nameRep->GetNumNameElements(); ++l)
+			{
+				BackupStoreFilenameClear fn;
+				fn.ReadFromStream(*namestream, 10000);
+				TEST_THAT(fn.GetClearFilename() == testnames[l]);
+			}
+		}
+		// Query names... secondly, for the directory
+		{
+			std::auto_ptr<BackupProtocolClientObjectName> nameRep(protocol.QueryGetObjectName(BackupProtocolClientGetObjectName::ObjectID_DirectoryOnly, subsubdirid));
+			std::auto_ptr<IOStream> namestream(protocol.ReceiveStream());
+		
+			TEST_THAT(nameRep->GetNumNameElements() == 2);
+			TEST_THAT(nameRep->GetFlags() == BackupProtocolClientListDirectory::Flags_Dir);
+			static const char *testnames[] = {"sub2","lovely_directory"};
+			for(int l = 0; l < nameRep->GetNumNameElements(); ++l)
+			{
+				BackupStoreFilenameClear fn;
+				fn.ReadFromStream(*namestream, 10000);
+				TEST_THAT(fn.GetClearFilename() == testnames[l]);
+			}
+		}
+		
+//}	skip:
+	
+		// Create some nice recursive directories
+		int64_t dirtodelete = create_test_data_subdirs(protocol,
+			BackupProtocolClientListDirectory::RootDirectory, "test_delete", 6 /* depth */);
+		
+		// And delete them
+		{
+			std::auto_ptr<BackupProtocolClientSuccess> dirdel(protocol.QueryDeleteDirectory(
+					dirtodelete));
+			TEST_THAT(dirdel->GetObjectID() == dirtodelete);
+		}
+
+		// Get the root dir, checking for deleted items
+		{
+			// Command
+			std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocolReadOnly.QueryListDirectory(
+					BackupProtocolClientListDirectory::RootDirectory,
+					BackupProtocolClientListDirectory::Flags_Dir | BackupProtocolClientListDirectory::Flags_Deleted,
+					BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+			// Stream
+			BackupStoreDirectory dir;
+			std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream());
+			dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+			
+			// Check there's only that one entry
+			TEST_THAT(dir.GetNumberOfEntries() == 1);
+			
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = i.Next();
+			TEST_THAT(en != 0);
+			if(en)
+			{
+				TEST_THAT(en->GetObjectID() == dirtodelete);
+				BackupStoreFilenameClear n("test_delete");
+				TEST_THAT(en->GetName() == n);
+			}
+			
+			// Then... check everything's deleted
+			test_everything_deleted(protocolReadOnly, dirtodelete);
+		}
+			
+		// Finish the connections
+		protocolReadOnly.QueryFinished();
+		protocol.QueryFinished();
+		
+		// Close logs
+		::fclose(protocolReadOnlyLog);
+		::fclose(protocolLog);
+	}
+	
+	return 0;
+}
+
+int test3(int argc, const char *argv[])
+{
+	// Now test encoded files
+	// TODO: This test needs to check failure situations as well as everything working,
+	// but this will be saved for the full implementation.
+	int encfile[ENCFILE_SIZE];
+	{
+		for(int l = 0; l < ENCFILE_SIZE; ++l)
+		{
+			encfile[l] = l * 173;
+		}
+		
+		// Encode and decode a small block (shouldn't be compressed)
+		{
+			#define SMALL_BLOCK_SIZE	251
+			int encBlockSize = BackupStoreFile::MaxBlockSizeForChunkSize(SMALL_BLOCK_SIZE);
+			TEST_THAT(encBlockSize > SMALL_BLOCK_SIZE);
+			BackupStoreFile::EncodingBuffer encoded;
+			encoded.Allocate(encBlockSize / 8);		// make sure reallocation happens
+			
+			// Encode!
+			int encSize = BackupStoreFile::EncodeChunk(encfile, SMALL_BLOCK_SIZE, encoded);
+			// Check the header says it's not been compressed
+			TEST_THAT((encoded.mpBuffer[0] & 1) == 0);
+			// Check the output size has been inflated (no compression)
+			TEST_THAT(encSize > SMALL_BLOCK_SIZE);
+			
+			// Decode it
+			int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(SMALL_BLOCK_SIZE);
+			TEST_THAT(decBlockSize > SMALL_BLOCK_SIZE);
+			uint8_t *decoded = (uint8_t*)malloc(decBlockSize);
+			int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize);
+			TEST_THAT(decSize < decBlockSize);
+			TEST_THAT(decSize == SMALL_BLOCK_SIZE);
+			
+			// Check it came out of the wash the same
+			TEST_THAT(::memcmp(encfile, decoded, SMALL_BLOCK_SIZE) == 0);
+			
+			free(decoded);
+		}
+
+		// Encode and decode a big block (should be compressed)
+		{
+			int encBlockSize = BackupStoreFile::MaxBlockSizeForChunkSize(ENCFILE_SIZE);
+			TEST_THAT(encBlockSize > ENCFILE_SIZE);
+			BackupStoreFile::EncodingBuffer encoded;
+			encoded.Allocate(encBlockSize / 8);		// make sure reallocation happens
+			
+			// Encode!
+			int encSize = BackupStoreFile::EncodeChunk(encfile, ENCFILE_SIZE, encoded);
+			// Check the header says it's compressed
+			TEST_THAT((encoded.mpBuffer[0] & 1) == 1);
+			// Check the output size make it likely that it's compressed (is very compressible data)
+			TEST_THAT(encSize < ENCFILE_SIZE);
+			
+			// Decode it
+			int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(ENCFILE_SIZE);
+			TEST_THAT(decBlockSize > ENCFILE_SIZE);
+			uint8_t *decoded = (uint8_t*)malloc(decBlockSize);
+			int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize);
+			TEST_THAT(decSize < decBlockSize);
+			TEST_THAT(decSize == ENCFILE_SIZE);
+
+			// Check it came out of the wash the same
+			TEST_THAT(::memcmp(encfile, decoded, ENCFILE_SIZE) == 0);
+			
+			free(decoded);
+		}
+		
+		// The test block to a file
+		{
+			FileStream f("testfiles/testenc1", O_WRONLY | O_CREAT | O_EXCL);
+			f.Write(encfile, sizeof(encfile));
+		}
+		
+		// Encode it
+		{
+			FileStream out("testfiles/testenc1_enc", O_WRONLY | O_CREAT | O_EXCL);
+			BackupStoreFilenameClear name("testfiles/testenc1");
+
+			std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testenc1", 32, name));
+			encoded->CopyStreamTo(out);
+		}
+		
+		// Verify it
+		{
+			FileStream enc("testfiles/testenc1_enc");
+			TEST_THAT(BackupStoreFile::VerifyEncodedFileFormat(enc) == true);
+		}
+		
+		// Decode it
+		{
+			FileStream enc("testfiles/testenc1_enc");
+			BackupStoreFile::DecodeFile(enc, "testfiles/testenc1_orig", IOStream::TimeOutInfinite);
+		}
+		
+		// Read in rebuilt original, and compare contents
+		{
+			TEST_THAT(TestGetFileSize("testfiles/testenc1_orig") == sizeof(encfile));
+			FileStream in("testfiles/testenc1_orig");
+			int encfile_i[ENCFILE_SIZE];
+			in.Read(encfile_i, sizeof(encfile_i));
+			TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0);
+		}
+		
+		// Check how many blocks it had, and test the stream based interface
+		{
+			FileStream enc("testfiles/testenc1_enc");
+			std::auto_ptr<BackupStoreFile::DecodedStream> decoded(BackupStoreFile::DecodeFileStream(enc, IOStream::TimeOutInfinite));
+			CollectInBufferStream d;
+			decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 971 /* buffer block size */);
+			d.SetForReading();
+			TEST_THAT(d.GetSize() == sizeof(encfile));
+			TEST_THAT(memcmp(encfile, d.GetBuffer(), sizeof(encfile)) == 0);
+
+			TEST_THAT(decoded->GetNumBlocks() == 3);
+		}
+		
+		// Test that the last block in a file, if less than 256 bytes, gets put into the last block
+		{
+			#define FILE_SIZE_JUST_OVER	((4096*2)+58)
+			FileStream f("testfiles/testenc2", O_WRONLY | O_CREAT | O_EXCL);
+			f.Write(encfile + 2, FILE_SIZE_JUST_OVER);
+			BackupStoreFilenameClear name("testenc2");
+			std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testenc2", 32, name));
+			CollectInBufferStream e;
+			encoded->CopyStreamTo(e);
+			e.SetForReading();
+			std::auto_ptr<BackupStoreFile::DecodedStream> decoded(BackupStoreFile::DecodeFileStream(e, IOStream::TimeOutInfinite));
+			CollectInBufferStream d;
+			decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 879 /* buffer block size */);
+			d.SetForReading();
+			TEST_THAT(d.GetSize() == FILE_SIZE_JUST_OVER);
+			TEST_THAT(memcmp(encfile + 2, d.GetBuffer(), FILE_SIZE_JUST_OVER) == 0);			
+
+			TEST_THAT(decoded->GetNumBlocks() == 2);
+		}
+		
+		// Test that reordered streams work too
+		{
+			FileStream enc("testfiles/testenc1_enc");
+			std::auto_ptr<IOStream> reordered(BackupStoreFile::ReorderFileToStreamOrder(&enc, false));
+			std::auto_ptr<BackupStoreFile::DecodedStream> decoded(BackupStoreFile::DecodeFileStream(*reordered, IOStream::TimeOutInfinite));
+			CollectInBufferStream d;
+			decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 971 /* buffer block size */);
+			d.SetForReading();
+			TEST_THAT(d.GetSize() == sizeof(encfile));
+			TEST_THAT(memcmp(encfile, d.GetBuffer(), sizeof(encfile)) == 0);
+
+			TEST_THAT(decoded->GetNumBlocks() == 3);
+		}
+		
+		// Try out doing this on a symlink
+		{
+			TEST_THAT(::symlink("does/not/exist", "testfiles/testsymlink") == 0);
+			BackupStoreFilenameClear name("testsymlink");
+			std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testsymlink", 32, name));
+			// Can't decode it from the stream, because it's in file order, and doesn't have the 
+			// required properties to be able to reorder it. So buffer it...
+			CollectInBufferStream b;
+			encoded->CopyStreamTo(b);
+			b.SetForReading();
+			// Decode it
+			BackupStoreFile::DecodeFile(b, "testfiles/testsymlink_2", IOStream::TimeOutInfinite);
+		}
+	}
+
+	// Store info
+	{
+		RaidFileWrite::CreateDirectory(0, "test-info");
+		BackupStoreInfo::CreateNew(76, "test-info/", 0, 3461231233455433LL, 2934852487LL);
+		TEST_CHECK_THROWS(BackupStoreInfo::CreateNew(76, "test-info/", 0, 0, 0), RaidFileException, CannotOverwriteExistingFile);
+		std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(76, "test-info/", 0, true));
+		TEST_CHECK_THROWS(info->Save(), BackupStoreException, StoreInfoIsReadOnly);
+		TEST_CHECK_THROWS(info->ChangeBlocksUsed(1), BackupStoreException, StoreInfoIsReadOnly);
+		TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(1), BackupStoreException, StoreInfoIsReadOnly);
+		TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(1), BackupStoreException, StoreInfoIsReadOnly);
+		TEST_CHECK_THROWS(info->RemovedDeletedDirectory(2), BackupStoreException, StoreInfoIsReadOnly);
+		TEST_CHECK_THROWS(info->AddDeletedDirectory(2), BackupStoreException, StoreInfoIsReadOnly);
+	}
+	{
+		std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(76, "test-info/", 0, false));
+		info->ChangeBlocksUsed(8);
+		info->ChangeBlocksInOldFiles(9);
+		info->ChangeBlocksInDeletedFiles(10);
+		info->ChangeBlocksUsed(-1);
+		info->ChangeBlocksInOldFiles(-4);
+		info->ChangeBlocksInDeletedFiles(-9);
+		TEST_CHECK_THROWS(info->ChangeBlocksUsed(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative);
+		TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative);
+		TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative);
+		info->AddDeletedDirectory(2);
+		info->AddDeletedDirectory(3);
+		info->AddDeletedDirectory(4);
+		info->RemovedDeletedDirectory(3);
+		TEST_CHECK_THROWS(info->RemovedDeletedDirectory(9), BackupStoreException, StoreInfoDirNotInList);
+		info->Save();
+	}
+	{
+		std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(76, "test-info/", 0, true));
+		TEST_THAT(info->GetBlocksUsed() == 7);
+		TEST_THAT(info->GetBlocksInOldFiles() == 5);
+		TEST_THAT(info->GetBlocksInDeletedFiles() == 1);
+		TEST_THAT(info->GetBlocksSoftLimit() == 3461231233455433LL);
+		TEST_THAT(info->GetBlocksHardLimit() == 2934852487LL);
+		const std::vector<int64_t> &delfiles(info->GetDeletedDirectories());
+		TEST_THAT(delfiles.size() == 2);
+		TEST_THAT(delfiles[0] == 2);
+		TEST_THAT(delfiles[1] == 4);
+	}
+
+//printf("SKIPPINGTESTS---------\n");
+//return 0;
+
+	// Context
+	TLSContext context;
+	context.Initialise(false /* client */,
+			"testfiles/clientCerts.pem",
+			"testfiles/clientPrivKey.pem",
+			"testfiles/clientTrustedCAs.pem");
+
+	// First, try logging in without an account having been created... just make sure login fails.
+	int pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid");
+	TEST_THAT(pid != -1 && pid != 0);
+	if(pid > 0)
+	{
+		::sleep(1);
+		TEST_THAT(ServerIsAlive(pid));
+
+		// BLOCK
+		{
+			// Open a connection to the server
+			SocketStreamTLS conn;
+			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+
+			// Make a protocol
+			BackupProtocolClient protocol(conn);
+
+			// Check the version
+			std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
+			TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+
+			// Login
+			TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)),
+				ConnectionException, Conn_Protocol_UnexpectedReply);
+			
+			// Finish the connection
+			protocol.QueryFinished();
+		}
+
+		// Create an account for the test client
+		TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf create 01234567 0 10000B 20000B") == 0);
+		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+		TEST_THAT(TestDirExists("testfiles/0_0/backup/01234567"));
+		TEST_THAT(TestDirExists("testfiles/0_1/backup/01234567"));
+		TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567"));
+		TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8);	// make sure something is written to it
+
+		TEST_THAT(ServerIsAlive(pid));
+
+		TEST_THAT(test_server("localhost") == 0);
+		
+		// Test the deletion of objects by the housekeeping system
+		// First, things as they are now.
+		recusive_count_objects_results before = {0,0,0};
+
+		recusive_count_objects("localhost", BackupProtocolClientListDirectory::RootDirectory, before);
+		
+		TEST_THAT(before.objectsNotDel != 0);
+		TEST_THAT(before.deleted != 0);
+		TEST_THAT(before.old != 0);
+
+		// Kill it
+		TEST_THAT(KillServer(pid));
+		::sleep(1);
+		TEST_THAT(!ServerIsAlive(pid));
+		TestRemoteProcessMemLeaks("bbstored.memleaks");
+		
+		// Set a new limit on the account -- leave the hard limit high to make sure the target for
+		// freeing space is the soft limit.
+		TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf setlimit 01234567 10B 20000B") == 0);
+		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+
+		// Start things up
+		pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid");
+		::sleep(1);
+		TEST_THAT(ServerIsAlive(pid));
+		
+		// wait for housekeeping to happen
+		printf("waiting for housekeeping:\n");
+		for(int l = 0; l < 12; ++l)
+		{
+			::sleep(1);
+			printf(".");
+			fflush(stdout);
+		}
+		printf("\n");
+
+		// Count the objects again
+		recusive_count_objects_results after = {0,0,0};
+		recusive_count_objects("localhost", BackupProtocolClientListDirectory::RootDirectory, after);
+		
+		TEST_THAT(after.objectsNotDel == before.objectsNotDel);
+		TEST_THAT(after.deleted == 0);
+		TEST_THAT(after.old == 0);
+		
+		// Set a really small hard limit
+		TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf setlimit 01234567 10B 20B") == 0);
+		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+
+		// Try to upload a file and create a directory, and check an error is generated
+		{
+			// Open a connection to the server
+			SocketStreamTLS conn;
+			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+
+			// Make a protocol
+			BackupProtocolClient protocol(conn);
+
+			// Check the version
+			std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
+			TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+
+			// Login
+			std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));
+			
+			int64_t modtime = 0;
+			
+			BackupStoreFilenameClear fnx("exceed-limit");
+			std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/test3", BackupProtocolClientListDirectory::RootDirectory, fnx, &modtime));
+			TEST_THAT(modtime != 0);
+
+			TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+					BackupProtocolClientListDirectory::RootDirectory,
+					modtime,
+					modtime, /* use it for attr hash too */
+					0,							/* diff from ID */
+					fnx,
+					*upload)),
+				ConnectionException, Conn_Protocol_UnexpectedReply);
+
+			MemBlockStream attr(&modtime, sizeof(modtime));
+			BackupStoreFilenameClear fnxd("exceed-limit-dir");
+			TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolClientSuccess> dirCreate(protocol.QueryCreateDirectory(
+					BackupProtocolClientListDirectory::RootDirectory,
+					9837429842987984LL, fnxd, attr)),
+				ConnectionException, Conn_Protocol_UnexpectedReply);
+
+
+			// Finish the connection
+			protocol.QueryFinished();
+		}
+
+		// Kill it again
+		TEST_THAT(KillServer(pid));
+		::sleep(1);
+		TEST_THAT(!ServerIsAlive(pid));
+		TestRemoteProcessMemLeaks("bbstored.memleaks");
+	}
+
+	return 0;
+}
+
+int multi_server()
+{
+	printf("Starting server for connection from remote machines...\n");
+
+	// Create an account for the test client
+	TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf create 01234567 0 30000B 40000B") == 0);
+	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+
+	// First, try logging in without an account having been created... just make sure login fails.
+	int pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored_multi.conf", "testfiles/bbstored.pid");
+	TEST_THAT(pid != -1 && pid != 0);
+	if(pid > 0)
+	{
+		::sleep(1);
+		TEST_THAT(ServerIsAlive(pid));
+
+		// Wait for a keypress
+		printf("Press ENTER to terminate the server\n");
+		char line[512];
+		fgets(line, 512, stdin);
+		printf("Terminating server...\n");
+
+		// Kill it
+		TEST_THAT(KillServer(pid));
+		::sleep(1);
+		TEST_THAT(!ServerIsAlive(pid));
+		TestRemoteProcessMemLeaks("bbstored.memleaks");
+	}
+
+
+	return 0;
+}
+
+int test(int argc, const char *argv[])
+{
+	// SSL library
+	SSLLib::Initialise();
+	
+	// Give a test key for the filenames
+//	BackupStoreFilenameClear::SetBlowfishKey(FilenameEncodingKey, sizeof(FilenameEncodingKey));
+	// And set the encoding to blowfish
+//	BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish);
+	
+	// And for directory attributes -- need to set it, as used in file encoding
+//	BackupClientFileAttributes::SetBlowfishKey(AttributesEncodingKey, sizeof(AttributesEncodingKey));
+	
+	// And finally for file encoding
+//	BackupStoreFile::SetBlowfishKeys(FileEncodingKey, sizeof(FileEncodingKey), FileBlockEntryEncodingKey, sizeof(FileBlockEntryEncodingKey));
+
+	// Use the setup crypto command to set up all these keys, so that the bbackupquery command can be used
+	// for seeing what's going on.
+	BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys");	
+	
+	// encode in some filenames -- can't do static initialisation because the key won't be set up when these are initialised
+	for(unsigned int l = 0; l < sizeof(ens_filenames) / sizeof(ens_filenames[0]); ++l)
+	{
+		ens[l].fn = BackupStoreFilenameClear(ens_filenames[l]);
+	}
+	for(unsigned int l = 0; l < sizeof(uploads_filenames) / sizeof(uploads_filenames[0]); ++l)
+	{
+		uploads[l].name = BackupStoreFilenameClear(uploads_filenames[l]);
+	}
+	
+	// Trace errors out
+	SET_DEBUG_SSLLIB_TRACE_ERRORS
+
+	if(argc == 2 && strcmp(argv[1], "server") == 0)
+	{
+		return multi_server();
+	}
+	if(argc == 3 && strcmp(argv[1], "client") == 0)
+	{
+		return test_server(argv[2]);
+	}
+// large file test	
+/*	{
+		int64_t modtime = 0;
+		std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("/Users/ben/temp/large.tar",
+			BackupProtocolClientListDirectory::RootDirectory, uploads[0].name, &modtime));
+		TEST_THAT(modtime != 0);
+		FileStream write("testfiles/large.enc", O_WRONLY | O_CREAT);
+		upload->CopyStreamTo(write);
+	}	
+printf("SKIPPING TESTS ------------------------------------------------------\n");
+return 0;*/
+	int r = 0;
+	r = test1(argc, argv);
+	if(r != 0) return r;
+	r = test2(argc, argv);
+	if(r != 0) return r;
+	r = test3(argc, argv);
+	if(r != 0) return r;
+	return 0;
+}
+
diff --git a/test/backupstore/testextra b/test/backupstore/testextra
new file mode 100755
index 00000000..798c8c67
--- /dev/null
+++ b/test/backupstore/testextra
@@ -0,0 +1,4 @@
+mkdir testfiles/0_0
+mkdir testfiles/0_1
+mkdir testfiles/0_2
+mkdir testfiles/bbackupd-data
diff --git a/test/backupstore/testfiles/accounts.txt b/test/backupstore/testfiles/accounts.txt
new file mode 100755
index 00000000..e69de29b
--- /dev/null
+++ b/test/backupstore/testfiles/accounts.txt
diff --git a/test/backupstore/testfiles/bbackupd.keys b/test/backupstore/testfiles/bbackupd.keys
new file mode 100644
index 00000000..4c58fc22
--- /dev/null
+++ b/test/backupstore/testfiles/bbackupd.keys
diff --git a/test/backupstore/testfiles/bbstored.conf b/test/backupstore/testfiles/bbstored.conf
new file mode 100755
index 00000000..4862033b
--- /dev/null
+++ b/test/backupstore/testfiles/bbstored.conf
@@ -0,0 +1,17 @@
+
+RaidFileConf = testfiles/raidfile.conf
+AccountDatabase = testfiles/accounts.txt
+
+ExtendedLogging = yes
+
+TimeBetweenHousekeeping = 10
+
+Server
+{
+	PidFile = testfiles/bbstored.pid
+	ListenAddresses = inet:localhost
+	CertificateFile = testfiles/serverCerts.pem
+	PrivateKeyFile = testfiles/serverPrivKey.pem
+	TrustedCAsFile = testfiles/serverTrustedCAs.pem
+}
+
diff --git a/test/backupstore/testfiles/bbstored_multi.conf b/test/backupstore/testfiles/bbstored_multi.conf
new file mode 100755
index 00000000..73c70aa9
--- /dev/null
+++ b/test/backupstore/testfiles/bbstored_multi.conf
@@ -0,0 +1,16 @@
+
+RaidFileConf = testfiles/raidfile.conf
+AccountDatabase = testfiles/accounts.txt
+
+TimeBetweenHousekeeping = 5
+
+Server
+{
+	PidFile = testfiles/bbstored.pid
+	# 0.0.0.0 is the 'any' address, allowing connections from things other than localhost
+	ListenAddresses = inet:0.0.0.0
+	CertificateFile = testfiles/serverCerts.pem
+	PrivateKeyFile = testfiles/serverPrivKey.pem
+	TrustedCAsFile = testfiles/serverTrustedCAs.pem
+}
+
diff --git a/test/backupstore/testfiles/clientCerts.pem b/test/backupstore/testfiles/clientCerts.pem
new file mode 100644
index 00000000..c1f14fa7
--- /dev/null
+++ b/test/backupstore/testfiles/clientCerts.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBmDCCAQECAQMwDQYJKoZIhvcNAQEFBQAwDzENMAsGA1UEAxMEUk9PVDAeFw0w
+MzEwMDcwOTAwMDRaFw0zMTAyMjIwOTAwMDRaMBoxGDAWBgNVBAMTD0JBQ0tVUC0w
+MTIzNDU2NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvptM6A++ZdkxYN92
+OI6d0O32giRdybSdUVNJk09V1pdVJFhXr4owhtVv6d8yDnPaNOgS1LlxZ9CHcR5A
+LtFwI9wmGHBc5a2uFCZGORTaSggntCythvRV3DGm/fU7mRME7Le1/tWWxjycnk2k
+Rez6d7Ffj56SXDFoxY2dK8MwRasCAwEAATANBgkqhkiG9w0BAQUFAAOBgQB4D3LU
+knCM4UZHMJhlbGnvc+N4O5SGrNKrHs94juMF8dPXJNgboBflkYJKNx1qDf47C/Cx
+hxXjju2ucGHytNQ8kiWsz7vCzeS7Egkl0QhFcBcYVCeXNn7zc34aAUyVlLCuas2o
+EGpfF4se7D3abg7J/3ioW0hx8bSal7kROleKCQ==
+-----END CERTIFICATE-----
diff --git a/test/backupstore/testfiles/clientPrivKey.pem b/test/backupstore/testfiles/clientPrivKey.pem
new file mode 100644
index 00000000..34b1af2a
--- /dev/null
+++ b/test/backupstore/testfiles/clientPrivKey.pem
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXAIBAAKBgQC+m0zoD75l2TFg33Y4jp3Q7faCJF3JtJ1RU0mTT1XWl1UkWFev
+ijCG1W/p3zIOc9o06BLUuXFn0IdxHkAu0XAj3CYYcFzlra4UJkY5FNpKCCe0LK2G
+9FXcMab99TuZEwTst7X+1ZbGPJyeTaRF7Pp3sV+PnpJcMWjFjZ0rwzBFqwIDAQAB
+AoGAMW8Lqh/zLG0A/nPWMGLkkTw2M5iE7nw2VNI6AceQpqAHB+8VhsRbQ4z1gn1N
+eSwYyqHpyFv0Co2touvKj5nn8CJfMmm571cvdOlD/n/mQsW+xZqd9WmvSE8Jh4Qq
+iOQqwbwJlTYTV4BEo90qtfR+MDqffSCB8bHh4l3oO3fSp4kCQQDgbllQeq2kwlLp
+81oDfrk+J7vpiq9hZ/HxFY1fZAOa6iylazZz0JSzvNAtQNLI1LeKAzBc8FuPPSG9
+qSHAKoDHAkEA2Wrziib5OgY/G86yAWVn2hPM7Ky6wGtsJxYnObXUiTwVM7lM1nZU
+LpQaq//vzVDcWggqyEBTYkVcdEPYIJn3/QJBAL3e/bblowRx1p3Q4MV2L5gTG5pQ
+V2HsA7c3yZv7TEWCenUUSEQhIb0SL3kpj2qS9BhR7FekjYGYcXQ4o7IlAz8CQD1B
+BJxHnq/aUq1i7oO2Liwip/mGMJdFrJLWivaXY+nGI7MO4bcKX21ADMOot8cAoRQ8
+eNEyTkvBfurCsoF834ECQCPejz6x1bh/H7SeeANP17HKlwx1Lshw2JzxfF96MA26
+Eige4f0ttKHhMY/bnMcOzfPUSe/LvIN3AiMtphkl0pw=
+-----END RSA PRIVATE KEY-----
diff --git a/test/backupstore/testfiles/clientReq.pem b/test/backupstore/testfiles/clientReq.pem
new file mode 100644
index 00000000..8eee0b5f
--- /dev/null
+++ b/test/backupstore/testfiles/clientReq.pem
@@ -0,0 +1,10 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBWTCBwwIBADAaMRgwFgYDVQQDEw9CQUNLVVAtMDEyMzQ1NjcwgZ8wDQYJKoZI
+hvcNAQEBBQADgY0AMIGJAoGBAL6bTOgPvmXZMWDfdjiOndDt9oIkXcm0nVFTSZNP
+VdaXVSRYV6+KMIbVb+nfMg5z2jToEtS5cWfQh3EeQC7RcCPcJhhwXOWtrhQmRjkU
+2koIJ7QsrYb0Vdwxpv31O5kTBOy3tf7VlsY8nJ5NpEXs+nexX4+eklwxaMWNnSvD
+MEWrAgMBAAGgADANBgkqhkiG9w0BAQUFAAOBgQBtz10sGGYhbw9+7L8bOtOUV6j9
+46jnbHGXHmdBZsg8ZWgKBJQ61HwvKCNA+KAEeb9yMxWgpJRGqFk6yvPb62XXuRGl
+4RQN0/6rRc8GJh3Qi4oPV1GYnzyYg2+bjZAgeMoL6ro1YuH52CTHJpQ3Arg2Ortz
+xVxbWyMouzjc1g4gdw==
+-----END CERTIFICATE REQUEST-----
diff --git a/test/backupstore/testfiles/clientTrustedCAs.pem b/test/backupstore/testfiles/clientTrustedCAs.pem
new file mode 100644
index 00000000..2a065879
--- /dev/null
+++ b/test/backupstore/testfiles/clientTrustedCAs.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBjDCB9gIBADANBgkqhkiG9w0BAQUFADAPMQ0wCwYDVQQDEwRST09UMB4XDTAz
+MTAwNzA4NTkzMloXDTMxMDIyMjA4NTkzMlowDzENMAsGA1UEAxMEUk9PVDCBnzAN
+BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtZypR/5m5fuNMPNrSLdzwmXKqhdVZj/e
+cZHUZvVuXQZboosAznDrbh8HgpuTw5vaZDEz8VfPwgIaROZDT3ztFIedLapJ7Ot9
+I4JNqSv/y3V9MKb7trTSPVvyYLqk9isLmw8wmEidJiLbWbIc2cHFXDvWNqTr2jF6
+u4Q8DvdVfAECAwEAATANBgkqhkiG9w0BAQUFAAOBgQAL1lyJ/5y44yjk2BK+tnrZ
+hbK7Ghtqrq/uZ8RQq5sAme919TnPijh2tRBqSaUaD2K+Sgo3RNgUGbKhfHRU1pfM
+USllHskTKiJu74ix/T3UOnjpQ946OLSl5zNsOdOgbjBDnozfPSrKeEGN0huBbmmt
+SlL3iQzVXlF6NAhkzS54fQ==
+-----END CERTIFICATE-----
diff --git a/test/backupstore/testfiles/query.conf b/test/backupstore/testfiles/query.conf
new file mode 100755
index 00000000..984ace6c
--- /dev/null
+++ b/test/backupstore/testfiles/query.conf
@@ -0,0 +1,34 @@
+
+# this is a dummy config file so that bbackupquery can be used
+
+
+CertificateFile = testfiles/clientCerts.pem
+PrivateKeyFile = testfiles/clientPrivKey.pem
+TrustedCAsFile = testfiles/clientTrustedCAs.pem
+
+KeysFile = testfiles/bbackupd.keys
+
+DataDirectory = testfiles/bbackupd-data
+
+StoreHostname = localhost
+AccountNumber = 0x01234567
+UpdateStoreInterval = 3
+MinimumFileAge = 4
+MaxUploadWait = 24
+FileTrackingSizeThreshold = 1024
+DiffingUploadSizeThreshold = 1024
+
+Server
+{
+	PidFile = testfiles/bbackupd.pid
+}
+
+# this is just a dummy entry
+BackupLocations
+{
+	test_delete
+	{
+		Path = testfiles/test_delete
+	}
+}
+
diff --git a/test/backupstore/testfiles/raidfile.conf b/test/backupstore/testfiles/raidfile.conf
new file mode 100755
index 00000000..641872b0
--- /dev/null
+++ b/test/backupstore/testfiles/raidfile.conf
@@ -0,0 +1,10 @@
+
+disc0
+{
+	SetNumber = 0
+	BlockSize = 2048
+	Dir0 = testfiles/0_0
+	Dir1 = testfiles/0_1
+	Dir2 = testfiles/0_2
+}
+
diff --git a/test/backupstore/testfiles/root.pem b/test/backupstore/testfiles/root.pem
new file mode 100644
index 00000000..b7fa6a17
--- /dev/null
+++ b/test/backupstore/testfiles/root.pem
@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIBjDCB9gIBADANBgkqhkiG9w0BAQUFADAPMQ0wCwYDVQQDEwRST09UMB4XDTAz
+MDgyMDExNTEyN1oXDTAzMDkxOTExNTEyN1owDzENMAsGA1UEAxMEUk9PVDCBnzAN
+BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtZypR/5m5fuNMPNrSLdzwmXKqhdVZj/e
+cZHUZvVuXQZboosAznDrbh8HgpuTw5vaZDEz8VfPwgIaROZDT3ztFIedLapJ7Ot9
+I4JNqSv/y3V9MKb7trTSPVvyYLqk9isLmw8wmEidJiLbWbIc2cHFXDvWNqTr2jF6
+u4Q8DvdVfAECAwEAATANBgkqhkiG9w0BAQUFAAOBgQCPbEXLzpItnnh1kUPy0vui
+atzeQoTFzgEybKLqgM4irWUjUnVdcSFEJFgddABpMOlGymu/6NuqqVQR8OUUOUrk
+BUlucY1m3BuCJBsADKWXVBOky4aQ7oo7BZZUh7e9NeKHfu7u1+0kvIQlTc+1Xnub
+uAQzwDRZ5vAFMWzzvh5BtA==
+-----END CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
+MIICXgIBAAKBgQC1nKlH/mbl+40w82tIt3PCZcqqF1VmP95xkdRm9W5dBluiiwDO
+cOtuHweCm5PDm9pkMTPxV8/CAhpE5kNPfO0Uh50tqkns630jgk2pK//LdX0wpvu2
+tNI9W/JguqT2KwubDzCYSJ0mIttZshzZwcVcO9Y2pOvaMXq7hDwO91V8AQIDAQAB
+AoGBAIz2UB5lRBD2MxzvkzIZ0mvs/mUPP2Xh5RJZkndnwIXLzYxYQAP8eYA77WRe
+xU5qxhRGbH7DHasEXsdjwpML8CdT9aAMRHwcUt76F5ENMOq2Zc5cnmsQeDjSiZfi
+wxpixqxt3ookk4fw9LZgScJ7YQeYrHQfn4BddbV/brXMVF3BAkEA45FUmRqWMBK0
+5WIbkuZJERtOJEaYa1+9Uwqa87Vf4kTiskOGpA73h6y4Lrx97Opvfpq11aELWy01
+TcSZ0ru0zQJBAMxNdArmyVTGeO9h0wZB87sAXmG1qdZdViEXES8tSAcGS+B20nUe
+k2W2UGb4tnk5M4Jzdkf03uqk9NgslgA2xAUCQQCFqU20I36FO+eON0KU1Lej2ZLb
+Ea/imTgdN0Rt0mFACE/SfoDtiXDv+o2vvbyE0+mqxfn5QP7njbUaOVhUAzYdAkAO
+Fl0lD0rcrJ7UKtOpP8z1nQ3lAOjIHkF9IKEPtribu2RqAud6KfSR8+NRZl72tuoF
+Wb7TMWBZn6w+Z7ykISKdAkEAhoNryreYb+BAl51M/Xn60EyDBBTRgw2hyUi6xEHe
+3dPZnU8YjJNd/9sXPnn8bEqSWRaUyDGEf1BFfbuoYb1c/w==
+-----END RSA PRIVATE KEY-----
diff --git a/test/backupstore/testfiles/root.srl b/test/backupstore/testfiles/root.srl
new file mode 100644
index 00000000..eeee65ec
--- /dev/null
+++ b/test/backupstore/testfiles/root.srl
@@ -0,0 +1 @@
+05
diff --git a/test/backupstore/testfiles/rootcert.pem b/test/backupstore/testfiles/rootcert.pem
new file mode 100644
index 00000000..2a065879
--- /dev/null
+++ b/test/backupstore/testfiles/rootcert.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBjDCB9gIBADANBgkqhkiG9w0BAQUFADAPMQ0wCwYDVQQDEwRST09UMB4XDTAz
+MTAwNzA4NTkzMloXDTMxMDIyMjA4NTkzMlowDzENMAsGA1UEAxMEUk9PVDCBnzAN
+BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtZypR/5m5fuNMPNrSLdzwmXKqhdVZj/e
+cZHUZvVuXQZboosAznDrbh8HgpuTw5vaZDEz8VfPwgIaROZDT3ztFIedLapJ7Ot9
+I4JNqSv/y3V9MKb7trTSPVvyYLqk9isLmw8wmEidJiLbWbIc2cHFXDvWNqTr2jF6
+u4Q8DvdVfAECAwEAATANBgkqhkiG9w0BAQUFAAOBgQAL1lyJ/5y44yjk2BK+tnrZ
+hbK7Ghtqrq/uZ8RQq5sAme919TnPijh2tRBqSaUaD2K+Sgo3RNgUGbKhfHRU1pfM
+USllHskTKiJu74ix/T3UOnjpQ946OLSl5zNsOdOgbjBDnozfPSrKeEGN0huBbmmt
+SlL3iQzVXlF6NAhkzS54fQ==
+-----END CERTIFICATE-----
diff --git a/test/backupstore/testfiles/rootkey.pem b/test/backupstore/testfiles/rootkey.pem
new file mode 100644
index 00000000..7ce55861
--- /dev/null
+++ b/test/backupstore/testfiles/rootkey.pem
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXgIBAAKBgQC1nKlH/mbl+40w82tIt3PCZcqqF1VmP95xkdRm9W5dBluiiwDO
+cOtuHweCm5PDm9pkMTPxV8/CAhpE5kNPfO0Uh50tqkns630jgk2pK//LdX0wpvu2
+tNI9W/JguqT2KwubDzCYSJ0mIttZshzZwcVcO9Y2pOvaMXq7hDwO91V8AQIDAQAB
+AoGBAIz2UB5lRBD2MxzvkzIZ0mvs/mUPP2Xh5RJZkndnwIXLzYxYQAP8eYA77WRe
+xU5qxhRGbH7DHasEXsdjwpML8CdT9aAMRHwcUt76F5ENMOq2Zc5cnmsQeDjSiZfi
+wxpixqxt3ookk4fw9LZgScJ7YQeYrHQfn4BddbV/brXMVF3BAkEA45FUmRqWMBK0
+5WIbkuZJERtOJEaYa1+9Uwqa87Vf4kTiskOGpA73h6y4Lrx97Opvfpq11aELWy01
+TcSZ0ru0zQJBAMxNdArmyVTGeO9h0wZB87sAXmG1qdZdViEXES8tSAcGS+B20nUe
+k2W2UGb4tnk5M4Jzdkf03uqk9NgslgA2xAUCQQCFqU20I36FO+eON0KU1Lej2ZLb
+Ea/imTgdN0Rt0mFACE/SfoDtiXDv+o2vvbyE0+mqxfn5QP7njbUaOVhUAzYdAkAO
+Fl0lD0rcrJ7UKtOpP8z1nQ3lAOjIHkF9IKEPtribu2RqAud6KfSR8+NRZl72tuoF
+Wb7TMWBZn6w+Z7ykISKdAkEAhoNryreYb+BAl51M/Xn60EyDBBTRgw2hyUi6xEHe
+3dPZnU8YjJNd/9sXPnn8bEqSWRaUyDGEf1BFfbuoYb1c/w==
+-----END RSA PRIVATE KEY-----
diff --git a/test/backupstore/testfiles/rootreq.pem b/test/backupstore/testfiles/rootreq.pem
new file mode 100644
index 00000000..2ac6293c
--- /dev/null
+++ b/test/backupstore/testfiles/rootreq.pem
@@ -0,0 +1,10 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBTjCBuAIBADAPMQ0wCwYDVQQDEwRST09UMIGfMA0GCSqGSIb3DQEBAQUAA4GN
+ADCBiQKBgQC1nKlH/mbl+40w82tIt3PCZcqqF1VmP95xkdRm9W5dBluiiwDOcOtu
+HweCm5PDm9pkMTPxV8/CAhpE5kNPfO0Uh50tqkns630jgk2pK//LdX0wpvu2tNI9
+W/JguqT2KwubDzCYSJ0mIttZshzZwcVcO9Y2pOvaMXq7hDwO91V8AQIDAQABoAAw
+DQYJKoZIhvcNAQEFBQADgYEAarbwMXzojqzCzQLakpX8hMDiBnGb80M4au+r8MXI
+g492CbH+PgpSus4g58na+1S1xAV2a7kDN6udss+OjHvukePybWUkkR6DAfXVJuxO
+FrchOTv6Pwj1p4FZGzocnJ2sIp4fe+2p2ge2oAHw7EIX+1IhQUObGI/q7zEVDctK
+5Fg=
+-----END CERTIFICATE REQUEST-----
diff --git a/test/backupstore/testfiles/serverCerts.pem b/test/backupstore/testfiles/serverCerts.pem
new file mode 100644
index 00000000..92467618
--- /dev/null
+++ b/test/backupstore/testfiles/serverCerts.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBlzCCAQACAQQwDQYJKoZIhvcNAQEFBQAwDzENMAsGA1UEAxMEUk9PVDAeFw0w
+MzEwMDcwOTAwMTFaFw0zMTAyMjIwOTAwMTFaMBkxFzAVBgNVBAMTDlNUT1JFLTAw
+MDAwMDA4MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDNj1fGSCaSl/1w1lRV
+I8qE6BqjvT6R0XXGdIV+dk/mHmE3NOCPcBq/gxZOYevp+QnwMc+nUSS7Px/n+q92
+cl3a8ttInfZjLqg9o/wpd6dBfH4gLTG4bEujhMt1x4bEUJk/uWfnk5FhsJXDBrlH
+RJZNiS9Asme+5Zvjfz3Phy0YWwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBABhmdun/
+myn3l4SbH+PxSUaW/mSvBubFhbbl9wolwhzvGCrtY968jn464JUP1UwUnnvePUU2
+SSVPZOVCvobCfM6s20aOdlKvnn+7GZkjoFONuCw3O+1hIFTSyXFcJWBaYLuczVk1
+HfdIKKcVZ1CpAfnMhMxuu+nA7fjor4p1/K0t
+-----END CERTIFICATE-----
diff --git a/test/backupstore/testfiles/serverPrivKey.pem b/test/backupstore/testfiles/serverPrivKey.pem
new file mode 100644
index 00000000..fd87607d
--- /dev/null
+++ b/test/backupstore/testfiles/serverPrivKey.pem
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDNj1fGSCaSl/1w1lRVI8qE6BqjvT6R0XXGdIV+dk/mHmE3NOCP
+cBq/gxZOYevp+QnwMc+nUSS7Px/n+q92cl3a8ttInfZjLqg9o/wpd6dBfH4gLTG4
+bEujhMt1x4bEUJk/uWfnk5FhsJXDBrlHRJZNiS9Asme+5Zvjfz3Phy0YWwIDAQAB
+AoGBAI88mjo1noM528Wb4+nr5bvVDHMadJYhccMXAMqNYMGGW9GfS/dHc6wNiSaX
+P0+rVIyF+R+rAEBmDTKV0Vxk9xZQuAaDKjLluDkxSxSR869D2YOWYUfvjDo3OFlT
+LMZf0eE7u/3Pm0MtxPctXszqvNnmb+IvPXzttGRgUfU5G+tJAkEA+IphkGMI4A3l
+4KfxotZZU+HiJbRDFpm81RzCc2709KCMkXMEz/+xkvnqlo28jqOf7PRBeq/ecsZN
+8BGvtyoqVQJBANO6uj6sPI66GaRqxV83VyUUdMmL9uFOccIMqW5q0rx5UDi0mG7t
+Pjjz+ul1D247+dvVxnEBeW4C85TSNbbKR+8CQQChpV7PCZo8Hs3jz1bZEZAHfmIX
+I6Z+jH7EHHBbo06ty72g263FmgdkECcCxCxemQzqj/IGWVvUSiVmfhpKhqIBAkAl
+XbjswpzVW4aW+7jlevDIPHn379mcHan54x4rvHKAjLBZsZWNThVDG9vWQ7B7dd48
+q9efrfDuN1shko+kOMLFAkAGIc5w0bJNC4eu91Wr6AFgTm2DntyVQ9keVhYbrwrE
+xY37dgVhAWVeLDOk6eVOVSYqEI1okXPVqvfOIoRJUYkn
+-----END RSA PRIVATE KEY-----
diff --git a/test/backupstore/testfiles/serverReq.pem b/test/backupstore/testfiles/serverReq.pem
new file mode 100644
index 00000000..7475d406
--- /dev/null
+++ b/test/backupstore/testfiles/serverReq.pem
@@ -0,0 +1,10 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBWDCBwgIBADAZMRcwFQYDVQQDEw5TVE9SRS0wMDAwMDAwODCBnzANBgkqhkiG
+9w0BAQEFAAOBjQAwgYkCgYEAzY9Xxkgmkpf9cNZUVSPKhOgao70+kdF1xnSFfnZP
+5h5hNzTgj3Aav4MWTmHr6fkJ8DHPp1Ekuz8f5/qvdnJd2vLbSJ32Yy6oPaP8KXen
+QXx+IC0xuGxLo4TLdceGxFCZP7ln55ORYbCVwwa5R0SWTYkvQLJnvuWb4389z4ct
+GFsCAwEAAaAAMA0GCSqGSIb3DQEBBQUAA4GBAIdlFo8gbik1K/+4Ra87cQDZzn0L
+wE9bZrxRMPXqGjCQ8HBCfvQMFa1Oc6fEczCJ/nmmd76j0HIXW7uYOELIT8L/Zvf5
+jw/z9/OvEOQal7H2JN2d6W4ZmYpQko5+e/bJmlrOxyBpcXk34BvyQen9pTmI6J4E
+pkBN/5XUUvVJSM67
+-----END CERTIFICATE REQUEST-----
diff --git a/test/backupstore/testfiles/serverTrustedCAs.pem b/test/backupstore/testfiles/serverTrustedCAs.pem
new file mode 100644
index 00000000..2a065879
--- /dev/null
+++ b/test/backupstore/testfiles/serverTrustedCAs.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBjDCB9gIBADANBgkqhkiG9w0BAQUFADAPMQ0wCwYDVQQDEwRST09UMB4XDTAz
+MTAwNzA4NTkzMloXDTMxMDIyMjA4NTkzMlowDzENMAsGA1UEAxMEUk9PVDCBnzAN
+BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtZypR/5m5fuNMPNrSLdzwmXKqhdVZj/e
+cZHUZvVuXQZboosAznDrbh8HgpuTw5vaZDEz8VfPwgIaROZDT3ztFIedLapJ7Ot9
+I4JNqSv/y3V9MKb7trTSPVvyYLqk9isLmw8wmEidJiLbWbIc2cHFXDvWNqTr2jF6
+u4Q8DvdVfAECAwEAATANBgkqhkiG9w0BAQUFAAOBgQAL1lyJ/5y44yjk2BK+tnrZ
+hbK7Ghtqrq/uZ8RQq5sAme919TnPijh2tRBqSaUaD2K+Sgo3RNgUGbKhfHRU1pfM
+USllHskTKiJu74ix/T3UOnjpQ946OLSl5zNsOdOgbjBDnozfPSrKeEGN0huBbmmt
+SlL3iQzVXlF6NAhkzS54fQ==
+-----END CERTIFICATE-----
diff --git a/test/backupstorefix/testbackupstorefix.cpp b/test/backupstorefix/testbackupstorefix.cpp
new file mode 100644
index 00000000..a4edfd31
--- /dev/null
+++ b/test/backupstorefix/testbackupstorefix.cpp
@@ -0,0 +1,580 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testbackupstorefix.cpp
+//		Purpose: Test BackupStoreCheck functionality
+//		Created: 23/4/04
+//
+// --------------------------------------------------------------------------
+
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <string>
+#include <map>
+
+#include "Test.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "FileStream.h"
+#include "RaidFileController.h"
+#include "RaidFileWrite.h"
+#include "RaidFileRead.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreException.h"
+#include "RaidFileException.h"
+#include "StoreStructure.h"
+#include "BackupStoreFileWire.h"
+
+#include "MemLeakFindOn.h"
+
+/* 
+
+Errors checked:
+
+make some BackupDirectoryStore objects, CheckAndFix(), then verify
+	- multiple objects with same ID
+	- wrong order of old flags
+	- all old flags
+
+delete store info
+add suprious file
+delete directory (should appear again)
+change container ID of directory
+delete a file
+double reference to a file inside a single dir
+modify the object ID of a directory
+delete directory, which has no members (will be removed)
+extra reference to a file in another dir (higher ID to allow consistency -- use something in subti)
+delete dir + dir2 in dir/dir2/file where nothing in dir2 except file, file should end up in lost+found
+similarly with a dir, but that should get a dirxxx name
+corrupt dir
+corrupt file
+delete root, copy a file to it instead (equivalent to deleting it too)
+
+*/
+
+std::string storeRoot("backup/01234567/");
+int discSetNum = 0;
+
+std::map<std::string, int32_t> nameToID;
+std::map<int32_t, bool> objectIsDir;
+
+#define RUN_CHECK	\
+	::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf check 01234567"); \
+	::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf check 01234567 fix");
+
+// Wait a given number of seconds for something to complete
+void wait_for_operation(int seconds)
+{
+	printf("waiting: ");
+	fflush(stdout);
+	for(int l = 0; l < seconds; ++l)
+	{
+		sleep(1);
+		printf(".");
+		fflush(stdout);
+	}
+	printf("\n");
+}
+
+// Get ID of an object given a filename
+int32_t getID(const char *name)
+{
+	std::map<std::string, int32_t>::iterator i(nameToID.find(std::string(name)));
+	TEST_THAT(i != nameToID.end());
+	if(i == nameToID.end()) return -1;
+	
+	return i->second;
+}
+
+// Get the RAID filename of an object
+std::string getObjectName(int32_t id)
+{
+	std::string fn;
+	StoreStructure::MakeObjectFilename(id, storeRoot, discSetNum, fn, false);
+	return fn;
+}
+
+// Delete an object
+void DeleteObject(const char *name)
+{
+	RaidFileWrite del(discSetNum, getObjectName(getID(name)));
+	del.Delete();
+}
+
+// Load a directory
+void LoadDirectory(const char *name, BackupStoreDirectory &rDir)
+{
+	std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(discSetNum, getObjectName(getID(name))));
+	rDir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+}
+// Save a directory back again
+void SaveDirectory(const char *name, const BackupStoreDirectory &rDir)
+{
+	RaidFileWrite d(discSetNum, getObjectName(getID(name)));
+	d.Open(true /* allow overwrite */);
+	rDir.WriteToStream(d);
+	d.Commit(true /* write now! */);
+}
+
+void CorruptObject(const char *name, int start, const char *rubbish)
+{
+	int rubbish_len = ::strlen(rubbish);
+	std::string fn(getObjectName(getID(name)));
+	std::auto_ptr<RaidFileRead> r(RaidFileRead::Open(discSetNum, fn));
+	RaidFileWrite w(discSetNum, fn);
+	w.Open(true /* allow overwrite */);
+	// Copy beginning
+	char buf[2048];
+	r->Read(buf, start, IOStream::TimeOutInfinite);
+	w.Write(buf, start);
+	// Write rubbish
+	r->Seek(rubbish_len, IOStream::SeekType_Relative);
+	w.Write(rubbish, rubbish_len);
+	// Copy rest of file
+	r->CopyStreamTo(w);
+	// Commit
+	w.Commit(true /* convert now */);
+}
+
+BackupStoreFilename fnames[3];
+
+typedef struct
+{
+	int name;
+	int64_t id;
+	int flags;
+} dir_en_check;
+
+void check_dir(BackupStoreDirectory &dir, dir_en_check *ck)
+{
+	BackupStoreDirectory::Iterator i(dir);
+	BackupStoreDirectory::Entry *en;
+	
+	while((en = i.Next()) != 0)
+	{
+		TEST_THAT(ck->name != -1);
+		if(ck->name == -1)
+		{
+			break;
+		}
+		TEST_THAT(en->GetName() == fnames[ck->name]);
+		TEST_THAT(en->GetObjectID() == ck->id);
+		TEST_THAT(en->GetFlags() == ck->flags);
+		++ck;
+	}
+	
+	TEST_THAT(en == 0);
+	TEST_THAT(ck->name == -1);
+}
+
+typedef struct
+{
+	int64_t id, depNewer, depOlder;
+} checkdepinfoen;
+
+void check_dir_dep(BackupStoreDirectory &dir, checkdepinfoen *ck)
+{
+	BackupStoreDirectory::Iterator i(dir);
+	BackupStoreDirectory::Entry *en;
+	
+	while((en = i.Next()) != 0)
+	{
+		TEST_THAT(ck->id != -1);
+		if(ck->id == -1)
+		{
+			break;
+		}
+		TEST_THAT(en->GetObjectID() == ck->id);
+		TEST_THAT(en->GetDependsNewer() == ck->depNewer);
+		TEST_THAT(en->GetDependsOlder() == ck->depOlder);
+		++ck;
+	}
+	
+	TEST_THAT(en == 0);
+	TEST_THAT(ck->id == -1);
+}
+
+void test_dir_fixing()
+{
+	fnames[0].SetAsClearFilename("x1");
+	fnames[1].SetAsClearFilename("x2");
+	fnames[2].SetAsClearFilename("x3");
+
+	{
+		BackupStoreDirectory dir;
+		dir.AddEntry(fnames[0], 12, 2 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2);
+		dir.AddEntry(fnames[1], 12, 2 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2);
+		dir.AddEntry(fnames[0], 12, 3 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2);
+		dir.AddEntry(fnames[0], 12, 5 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2);
+		
+		dir_en_check ck[] = {
+			{1, 2, BackupStoreDirectory::Entry::Flags_File},
+			{0, 3, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion},
+			{0, 5, BackupStoreDirectory::Entry::Flags_File},
+			{-1, 0, 0}
+		};
+		
+		TEST_THAT(dir.CheckAndFix() == true);
+		TEST_THAT(dir.CheckAndFix() == false);
+		check_dir(dir, ck);
+	}
+	{
+		BackupStoreDirectory dir;
+		dir.AddEntry(fnames[0], 12, 2 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2);
+		dir.AddEntry(fnames[1], 12, 10 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_OldVersion, 2);
+		dir.AddEntry(fnames[0], 12, 3 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2);
+		dir.AddEntry(fnames[0], 12, 5 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2);
+		
+		dir_en_check ck[] = {
+			{0, 2, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion},
+			{1, 10, BackupStoreDirectory::Entry::Flags_Dir},
+			{0, 3, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion},
+			{0, 5, BackupStoreDirectory::Entry::Flags_File},
+			{-1, 0, 0}
+		};
+		
+		TEST_THAT(dir.CheckAndFix() == true);
+		TEST_THAT(dir.CheckAndFix() == false);
+		check_dir(dir, ck);
+	}
+
+	// Test dependency fixing
+	{
+		BackupStoreDirectory dir;
+		BackupStoreDirectory::Entry *e2
+		  = dir.AddEntry(fnames[0], 12, 2 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2);
+		TEST_THAT(e2 != 0);
+		e2->SetDependsNewer(3);
+		BackupStoreDirectory::Entry *e3
+		  = dir.AddEntry(fnames[0], 12, 3 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2);
+		TEST_THAT(e3 != 0);
+		e3->SetDependsNewer(4); e3->SetDependsOlder(2);
+		BackupStoreDirectory::Entry *e4
+		  = dir.AddEntry(fnames[0], 12, 4 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2);
+		TEST_THAT(e4 != 0);
+		e4->SetDependsNewer(5); e4->SetDependsOlder(3);
+		BackupStoreDirectory::Entry *e5
+		  = dir.AddEntry(fnames[0], 12, 5 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2);
+		TEST_THAT(e5 != 0);
+		e5->SetDependsOlder(4);
+		
+		// This should all be nice and valid
+		TEST_THAT(dir.CheckAndFix() == false);
+		static checkdepinfoen c1[] = {{2, 3, 0}, {3, 4, 2}, {4, 5, 3}, {5, 0, 4}, {-1, 0, 0}};
+		check_dir_dep(dir, c1);
+
+		// Check that dependency forwards are restored
+		e4->SetDependsOlder(34343);
+		TEST_THAT(dir.CheckAndFix() == true);
+		TEST_THAT(dir.CheckAndFix() == false);
+		check_dir_dep(dir, c1);
+
+		// Check that a suprious depends older ref is undone
+		e2->SetDependsOlder(1);
+		TEST_THAT(dir.CheckAndFix() == true);
+		TEST_THAT(dir.CheckAndFix() == false);
+		check_dir_dep(dir, c1);
+
+		// Now delete an entry, and check it's cleaned up nicely
+		dir.DeleteEntry(3);
+		TEST_THAT(dir.CheckAndFix() == true);
+		TEST_THAT(dir.CheckAndFix() == false);
+		static checkdepinfoen c2[] = {{4, 5, 0}, {5, 0, 4}, {-1, 0, 0}};
+		check_dir_dep(dir, c2);
+	}
+}
+
+int test(int argc, const char *argv[])
+{
+	// Test the backupstore directory fixing
+	test_dir_fixing();
+
+	// Initialise the raidfile controller
+	RaidFileController &rcontroller = RaidFileController::GetController();
+	rcontroller.Initialise("testfiles/raidfile.conf");
+
+	// Create an account
+	TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf create 01234567 0 10000B 20000B") == 0);
+	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+
+	// Start the bbstored server
+	int pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid");
+	TEST_THAT(pid != -1 && pid != 0);
+	if(pid > 0)
+	{
+		::sleep(1);
+		TEST_THAT(ServerIsAlive(pid));
+
+		// Run the perl script to create the initial directories
+		TEST_THAT_ABORTONFAIL(::system("perl testfiles/testbackupstorefix.pl init") == 0);
+
+		int bbackupd_pid = LaunchServer("../../bin/bbackupd/bbackupd testfiles/bbackupd.conf", "testfiles/bbackupd.pid");
+		TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0);
+		if(bbackupd_pid > 0)
+		{
+			::sleep(1);
+			TEST_THAT(ServerIsAlive(bbackupd_pid));
+	
+			// Create a nice store directory
+			wait_for_operation(30);
+
+			// That'll do nicely, stop the server	
+			TEST_THAT(KillServer(bbackupd_pid));
+			TestRemoteProcessMemLeaks("bbackupd.memleaks");
+		}
+		
+		// Generate a list of all the object IDs
+		TEST_THAT_ABORTONFAIL(::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf \"list -r\" quit > testfiles/initial-listing.txt") == 0);
+		// And load it in
+		{
+			FILE *f = ::fopen("testfiles/initial-listing.txt", "r");
+			TEST_THAT_ABORTONFAIL(f != 0);
+			char line[512];
+			int32_t id;
+			char flags[32];
+			char name[256];
+			while(::fgets(line, sizeof(line), f) != 0)
+			{
+				TEST_THAT(::sscanf(line, "%x %s %s", &id, flags, name) == 3);
+				bool isDir = (::strcmp(flags, "-d---") == 0);
+				//TRACE3("%x,%d,%s\n", id, isDir, name);
+				nameToID[std::string(name)] = id;
+				objectIsDir[id] = isDir;
+			}
+			::fclose(f);
+		}
+
+		// ------------------------------------------------------------------------------------------------		
+		::printf("  === Delete store info, add random file\n");
+		{
+			// Delete store info
+			RaidFileWrite del(discSetNum, storeRoot + "info");
+			del.Delete();
+		}
+		{
+			// Add a suprious file
+			RaidFileWrite random(discSetNum, storeRoot + "randomfile");
+			random.Open();
+			random.Write("test", 4);
+			random.Commit(true);
+		}
+		// Fix it
+		RUN_CHECK
+		// Check everything is as it was
+		TEST_THAT(::system("perl testfiles/testbackupstorefix.pl check 0") == 0);
+		// Check the random file doesn't exist
+		{
+			TEST_THAT(!RaidFileRead::FileExists(discSetNum, storeRoot + "01/randomfile"));
+		}
+
+		// ------------------------------------------------------------------------------------------------		
+		::printf("  === Delete an entry for an object from dir, change that object to be a patch, check it's deleted\n");
+		{
+			// Open dir and find entry
+			int64_t delID = getID("Test1/cannes/ict/metegoguered/oats");
+			{
+				BackupStoreDirectory dir;
+				LoadDirectory("Test1/cannes/ict/metegoguered", dir);
+				TEST_THAT(dir.FindEntryByID(delID) != 0);
+				dir.DeleteEntry(delID);
+				SaveDirectory("Test1/cannes/ict/metegoguered", dir);
+			}
+			
+			// Adjust that entry
+			//
+			// IMPORTANT NOTE: There's a special hack in testbackupstorefix.pl to make sure that
+			// the file we're modifiying has at least two blocks so we can modify it and produce a valid file
+			// which will pass the verify checks.
+			//
+			std::string fn(getObjectName(delID));
+			{
+				std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(discSetNum, fn));
+				RaidFileWrite f(discSetNum, fn);
+				f.Open(true /* allow overwrite */);
+				// Make a copy of the original
+				file->CopyStreamTo(f);
+				// Move to header in both
+				file->Seek(0, IOStream::SeekType_Absolute);
+				BackupStoreFile::MoveStreamPositionToBlockIndex(*file);
+				f.Seek(file->GetPosition(), IOStream::SeekType_Absolute);
+				// Read header
+				struct
+				{
+					file_BlockIndexHeader hdr;
+					file_BlockIndexEntry e[2];
+				} h;
+				TEST_THAT(file->Read(&h, sizeof(h)) == sizeof(h));
+				// Modify
+				TEST_THAT(ntoh64(h.hdr.mOtherFileID) == 0);
+				TEST_THAT(ntoh64(h.hdr.mNumBlocks) >= 2);
+				h.hdr.mOtherFileID = hton64(2345); // don't worry about endianness
+				h.e[0].mEncodedSize = hton64((ntoh64(h.e[0].mEncodedSize)) + (ntoh64(h.e[1].mEncodedSize)));
+				h.e[1].mOtherBlockIndex = hton64(-2);
+				// Write to modified file
+				f.Write(&h, sizeof(h));
+				// Commit new version
+				f.Commit(true /* write now! */);
+			}
+
+			// Fix it
+			RUN_CHECK
+			// Check
+			TEST_THAT(::system("perl testfiles/testbackupstorefix.pl check 1") == 0);
+
+			// Check the modified file doesn't exist
+			TEST_THAT(!RaidFileRead::FileExists(discSetNum, fn));
+		}
+		
+		// ------------------------------------------------------------------------------------------------		
+		::printf("  === Delete directory, change container ID of another, duplicate entry in dir, supurious file size, delete file\n");
+		{
+			BackupStoreDirectory dir;
+			LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir);
+			dir.SetContainerID(73773);
+			SaveDirectory("Test1/foreomizes/stemptinevidate/ict", dir);
+		}
+		int64_t duplicatedID = 0;
+		int64_t notSupriousFileSize = 0;
+		{
+			BackupStoreDirectory dir;
+			LoadDirectory("Test1/cannes/ict/peep", dir);
+			// Duplicate the second entry
+			{
+				BackupStoreDirectory::Iterator i(dir);
+				i.Next();
+				BackupStoreDirectory::Entry *en = i.Next();
+				TEST_THAT(en != 0);
+				duplicatedID = en->GetObjectID();
+				dir.AddEntry(*en);
+			}
+			// Adjust file size of first file
+			{
+				BackupStoreDirectory::Iterator i(dir);
+				BackupStoreDirectory::Entry *en = i.Next(BackupStoreDirectory::Entry::Flags_File);
+				TEST_THAT(en != 0);
+				notSupriousFileSize = en->GetSizeInBlocks();
+				en->SetSizeInBlocks(3473874);
+				TEST_THAT(en->GetSizeInBlocks() == 3473874);
+			}
+			SaveDirectory("Test1/cannes/ict/peep", dir);
+		}
+		// Delete a directory
+		DeleteObject("Test1/pass/cacted/ming");
+		// Delete a file
+		DeleteObject("Test1/cannes/ict/scely");
+		// Fix it
+		RUN_CHECK
+		// Check everything is as it should be
+		TEST_THAT(::system("perl testfiles/testbackupstorefix.pl check 2") == 0);
+		{
+			BackupStoreDirectory dir;
+			LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir);
+			TEST_THAT(dir.GetContainerID() == getID("Test1/foreomizes/stemptinevidate"));
+		}
+		{
+			BackupStoreDirectory dir;
+			LoadDirectory("Test1/cannes/ict/peep", dir);
+			BackupStoreDirectory::Iterator i(dir);
+			// Count the number of entries with the ID which was duplicated
+			int count = 0;
+			BackupStoreDirectory::Entry *en = 0;
+			while((en = i.Next()) != 0)
+			{
+				if(en->GetObjectID() == duplicatedID)
+				{
+					++count;
+				}
+			}
+			TEST_THAT(count == 1);
+			// Check file size has changed
+			{
+				BackupStoreDirectory::Iterator i(dir);
+				BackupStoreDirectory::Entry *en = i.Next(BackupStoreDirectory::Entry::Flags_File);
+				TEST_THAT(en != 0);
+				TEST_THAT(en->GetSizeInBlocks() == notSupriousFileSize);
+			}
+		}
+
+		// ------------------------------------------------------------------------------------------------		
+		::printf("  === Modify the obj ID of dir, delete dir with no members, add extra reference to a file\n");
+		// Set bad object ID
+		{
+			BackupStoreDirectory dir;
+			LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir);
+			dir.TESTONLY_SetObjectID(73773);
+			SaveDirectory("Test1/foreomizes/stemptinevidate/ict", dir);
+		}
+		// Delete dir with no members
+		DeleteObject("Test1/dir-no-members");
+		// Add extra reference
+		{
+			BackupStoreDirectory dir;
+			LoadDirectory("Test1/divel", dir);
+			BackupStoreDirectory::Iterator i(dir);
+			BackupStoreDirectory::Entry *en = i.Next(BackupStoreDirectory::Entry::Flags_File);
+			TEST_THAT(en != 0);
+			BackupStoreDirectory dir2;
+			LoadDirectory("Test1/divel/torsines/cruishery", dir2);
+			dir2.AddEntry(*en);
+			SaveDirectory("Test1/divel/torsines/cruishery", dir2);
+		}
+		// Fix it
+		RUN_CHECK
+		// Check everything is as it should be
+		TEST_THAT(::system("perl testfiles/testbackupstorefix.pl check 3") == 0);
+		{
+			BackupStoreDirectory dir;
+			LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir);
+			TEST_THAT(dir.GetObjectID() == getID("Test1/foreomizes/stemptinevidate/ict"));
+		}
+		
+		// ------------------------------------------------------------------------------------------------		
+		::printf("  === Orphan files and dirs without being recoverable\n");
+		DeleteObject("Test1/dir1");		
+		DeleteObject("Test1/dir1/dir2");		
+		// Fix it
+		RUN_CHECK
+		// Check everything is where it is predicted to be
+		TEST_THAT(::system("perl testfiles/testbackupstorefix.pl check 4") == 0);
+
+		// ------------------------------------------------------------------------------------------------		
+		::printf("  === Corrupt file and dir\n");
+		// File
+		CorruptObject("Test1/foreomizes/stemptinevidate/algoughtnerge", 33, "34i729834298349283479233472983sdfhasgs");
+		// Dir
+		CorruptObject("Test1/cannes/imulatrougge/foreomizes", 23, "dsf32489sdnadf897fd2hjkesdfmnbsdfcsfoisufio2iofe2hdfkjhsf");
+		// Fix it
+		RUN_CHECK
+		// Check everything is where it should be
+		TEST_THAT(::system("perl testfiles/testbackupstorefix.pl check 5") == 0);
+
+		// ------------------------------------------------------------------------------------------------		
+		::printf("  === Overwrite root with a file\n");
+		{
+			std::auto_ptr<RaidFileRead> r(RaidFileRead::Open(discSetNum, getObjectName(getID("Test1/pass/shuted/brightinats/milamptimaskates"))));
+			RaidFileWrite w(discSetNum, getObjectName(1 /* root */));
+			w.Open(true /* allow overwrite */);
+			r->CopyStreamTo(w);
+			w.Commit(true /* convert now */);
+		}
+		// Fix it
+		RUN_CHECK
+		// Check everything is where it should be
+		TEST_THAT(::system("perl testfiles/testbackupstorefix.pl reroot 6") == 0);
+
+
+		// ------------------------------------------------------------------------------------------------		
+		// Stop server
+		TEST_THAT(KillServer(pid));
+		TestRemoteProcessMemLeaks("bbstored.memleaks");
+	}
+
+	return 0;
+}
+
diff --git a/test/backupstorefix/testextra b/test/backupstorefix/testextra
new file mode 100644
index 00000000..bf5719f9
--- /dev/null
+++ b/test/backupstorefix/testextra
@@ -0,0 +1,5 @@
+mkdir testfiles/0_0
+mkdir testfiles/0_1
+mkdir testfiles/0_2
+mkdir testfiles/bbackupd-data
+cp ../../../test/bbackupd/testfiles/*.* testfiles/
diff --git a/test/backupstorefix/testfiles/testbackupstorefix.pl b/test/backupstorefix/testfiles/testbackupstorefix.pl
new file mode 100755
index 00000000..99a4c041
--- /dev/null
+++ b/test/backupstorefix/testfiles/testbackupstorefix.pl
@@ -0,0 +1,213 @@
+#!/usr/bin/perl
+use strict;
+
+my @words = split /\s+/,<<__E;
+nes ment foreomizes restout offety nount stemptinevidate ristraigation algoughtnerge nont ict aduals backyalivers scely peep hyphs olworks ning dro rogarcer poducts eatinizers bank magird backs bud metegoguered con mes prisionsidenning oats nost vulgarmiscar pass red rad cacted ded oud ming red emeated compt sly thetter shuted defeve plagger wished brightinats tillishellult arreenies honing ation recyclingentivell milamptimaskates debaffessly battenteriset
+bostopring prearnies mailatrisepatheryingic divel ing middle torsines quarcharattendlegipsied resteivate acingladdrairevents cruishery flowdemobiologgermanciolt ents subver honer paulounces relessition dunhoutpositivessiveng suers emancess
+cons cheating winneggs flow ditiespaynes constrannotalimentievolutal ing repowellike stucablest ablemates impsychocks sorts degruman lace scons cords unsertracturce tumottenting locapersethithend pushotty polly red rialgolfillopmeninflirer skied relocis hetterabbed undaunatermisuresocioll cont posippory fibruting cannes storm callushlike warnook imulatrougge dicreamentsvily spical fishericating roes carlylisticaller
+__E
+
+my @check_add = (
+	[],
+	[],
+	[],
+	[],
+	[['+1', '-d---- lost+found0']],
+	[]
+);
+my @check_remove = (
+	[],
+	['Test1/cannes/ict/metegoguered/oats'],
+	['Test1/cannes/ict/scely'],
+	['Test1/dir-no-members'],
+	[qw`Test1/dir1 Test1/dir1/dir2`],
+	['Test1/foreomizes/stemptinevidate/algoughtnerge']
+);
+my @check_move = (
+	[],
+	[],
+	[],
+	[],
+	[['Test1/dir1/dir2/file1'=>'lost+found0/file1'], ['Test1/dir1/dir2/dir3/file2'=>'lost+found0/dir00000000/file2'], ['Test1/dir1/dir2/dir3'=>'lost+found0/dir00000000']],
+	[]
+);
+
+if($ARGV[0] eq 'init')
+{
+	# create the initial tree of words
+	make_dir('testfiles/TestDir1', 0, 4, 0);
+	
+	# add some useful extra bits to it
+	mkdir('testfiles/TestDir1/dir-no-members', 0755);
+	mkdir('testfiles/TestDir1/dir1', 0755);
+	mkdir('testfiles/TestDir1/dir1/dir2', 0755);
+	mkdir('testfiles/TestDir1/dir1/dir2/dir3', 0755);
+	make_file('testfiles/TestDir1/dir1/dir2/file1');
+	make_file('testfiles/TestDir1/dir1/dir2/dir3/file2');
+}
+elsif($ARGV[0] eq 'check')
+{
+	# build set of expected lines
+	my %expected;
+	my %filenames;
+	my $max_id_seen = 0;
+	open INITIAL,'testfiles/initial-listing.txt' or die "Can't open original listing";
+	while(<INITIAL>)
+	{
+		chomp;
+		$expected{$_} = 1;
+		m/\A(.+?) .+? (.+)\Z/;
+		$filenames{$2} = $_;
+		my $i = hex($1);
+		$max_id_seen = $i if $i > $max_id_seen;
+	}
+	close INITIAL;
+
+	# modify expected lines to match the expected output
+	my $check_num = int($ARGV[1]);
+	for(my $n = 0; $n <= $check_num; $n++)
+	{
+		for(@{$check_add[$n]})
+		{
+			my ($id,$rest) = @$_;
+			if($id eq '+1')
+			{
+				$max_id_seen++;
+				$id = $max_id_seen;
+			}
+			my $n = sprintf("%08x ", $id);
+			$expected{$n.$rest} = 1
+		}
+		for(@{$check_remove[$n]})
+		{
+			delete $expected{$filenames{$_}}
+		}
+		for(@{$check_move[$n]})
+		{
+			my ($from,$to) = @$_;
+			my $orig = $filenames{$from};
+			delete $expected{$filenames{$from}};
+			my ($id,$type) = split / /,$orig;
+			$expected{"$id $type $to"} = 1
+		}
+	}
+
+	# read in the new listing, and compare
+	open LISTING,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf \"list -r\" quit |" or die "Can't open list utility";
+	open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' or die "can't open copy listing file";
+	my $err = 0;
+	while(<LISTING>)
+	{
+		print LISTING_COPY;
+		chomp;
+		s/\[FILENAME NOT ENCRYPTED\]//;
+		if(exists $expected{$_})
+		{
+			delete $expected{$_}
+		}
+		else
+		{
+			$err = 1;
+			print "Unexpected object $_ in new output\n"
+		}
+	}
+	close LISTING_COPY;
+	close LISTING;
+	
+	# check for anything which didn't appear but should have done
+	for(keys %expected)
+	{
+		$err = 1;
+		print "Expected object $_ not found in new output\n"
+	}
+	
+	exit $err;
+}
+elsif($ARGV[0] eq 'reroot')
+{
+	open LISTING,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf \"list -r\" quit |" or die "Can't open list utility";
+	open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' or die "can't open copy listing file";
+	my $err = 0;
+	my $count = 0;
+	while(<LISTING>)
+	{
+		print LISTING_COPY;
+		chomp;
+		s/\[FILENAME NOT ENCRYPTED\]//;
+		my ($id,$type,$name) = split / /;
+		$count++;
+		if($name !~ /\Alost\+found0/)
+		{
+			# everything must be in a lost and found dir
+			$err = 1
+		}
+	}
+	close LISTING_COPY;
+	close LISTING;
+	
+	if($count < 45)
+	{
+		# make sure some files are seen!
+		$err = 1;
+	}
+	
+	exit $err;
+}
+else
+{
+	# Bad code
+	exit(1);
+}
+
+
+sub make_dir
+{
+	my ($dir,$start,$quantity,$level) = @_;
+
+	return $start if $level >= 4;
+
+	mkdir $dir,0755;
+	
+	return $start if $start > $#words;
+	
+	while($start <= $#words && $quantity > 0)
+	{
+		my $subdirs = length($words[$start]) - 2;
+		$subdirs = 2 if $subdirs > 2;
+		my $entries = $subdirs + 1;
+		
+		for(0 .. ($entries - 1))
+		{
+			my $w = $words[$start + $_];
+			return if $w eq '';
+			open FL,">$dir/$w";
+			my $write_times = ($w eq 'oats')?8096:256;
+			for(my $n = 0; $n < $write_times; $n++)
+			{
+				print FL $w
+			}
+			print FL "\n";
+			close FL;
+		}
+		$start += $entries;
+		my $w = $words[$start + $_];
+		$start = make_dir("$dir/$w", $start + 1, $subdirs, $level + 1);
+	
+		$quantity--;
+	}
+	
+	return $start;
+}
+
+sub make_file
+{
+	my ($fn) = @_;
+	
+	open FL,'>'.$fn or die "can't open $fn for writing";
+	for(0 .. 255)
+	{
+		print FL $fn
+	}
+	close FL;
+}
+
diff --git a/test/backupstorepatch/testbackupstorepatch.cpp b/test/backupstorepatch/testbackupstorepatch.cpp
new file mode 100644
index 00000000..a4a1ece3
--- /dev/null
+++ b/test/backupstorepatch/testbackupstorepatch.cpp
@@ -0,0 +1,613 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testbackupstorepatch.cpp
+//		Purpose: Test storage of patches on the backup store server
+//		Created: 13/7/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+
+#include "Test.h"
+#include "autogen_BackupProtocolClient.h"
+#include "SSLLib.h"
+#include "TLSContext.h"
+#include "SocketStreamTLS.h"
+#include "BoxPortsAndFiles.h"
+#include "BackupStoreConstants.h"
+#include "Socket.h"
+#include "BackupStoreFilenameClear.h"
+#include "CollectInBufferStream.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "FileStream.h"
+#include "RaidFileController.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreException.h"
+#include "RaidFileException.h"
+#include "MemBlockStream.h"
+#include "BackupClientFileAttributes.h"
+#include "BackupClientCryptoKeys.h"
+
+#include "MemLeakFindOn.h"
+
+typedef struct
+{
+	int ChangePoint, InsertBytes, DeleteBytes;
+	int64_t IDOnServer;
+	bool IsCompletelyDifferent;
+	bool HasBeenDeleted;
+	int64_t DepNewer, DepOlder;
+} file_info;
+
+file_info test_files[] =
+{
+//   ChPnt,	Insert,	Delete, ID, IsCDf,	BeenDel
+	{0, 	0,		0,		0,	false,	false},	// 0 dummy first entry
+	{32000,	2087,	0,		0,	false,	false}, // 1
+	{1000,	1998,	2976,	0,	false,	false}, // 2
+	{27800,	0,		288,	0,	false,	false}, // 3
+	{3208,	1087,	98,		0,	false,	false}, // 4
+	{56000,	23087,	98,		0,	false,	false}, // 5
+	{0,		98765,	9999999,0,	false,	false},	// 6 completely different, make a break in the storage
+	{9899,	9887,	2,		0,	false,	false}, // 7
+	{12984,	12345,	1234,	0,	false,	false}, // 8
+	{1209,	29885,	3498,	0,	false,	false}  // 9
+};
+
+// Order in which the files will be removed from the server
+int test_file_remove_order[] = {0, 2, 3, 5, 8, 1, 4, -1};
+
+#define NUMBER_FILES	((sizeof(test_files) / sizeof(test_files[0])))
+#define FIRST_FILE_SIZE	(64*1024+3)
+#define BUFFER_SIZE		(256*1024)
+
+// Chunk of memory to use for copying files, etc
+static void *buffer = 0;
+
+int64_t ModificationTime = 7766333330000LL;
+#define MODIFICATION_TIME_INC	10000000;
+
+// Nice random data for testing written files
+class R250 {
+public:
+	// Set up internal state table with 32-bit random numbers.  
+	// The bizarre bit-twiddling is because rand() returns 16 bits of which
+	// the bottom bit is always zero!  Hence, I use only some of the bits.
+	// You might want to do something better than this....
+
+	R250(int seed) : posn1(0), posn2(103)
+	{
+		// populate the state and incr tables
+		srand(seed);
+
+		for (int i = 0; i != stateLen; ++i)	{
+			state[i] = ((rand() >> 2) << 19) ^ ((rand() >> 2) << 11) ^ (rand() >> 2);
+			incrTable[i] = i == stateLen - 1 ? 0 : i + 1;
+		}
+
+		// stir up the numbers to ensure they're random
+
+		for (int j = 0; j != stateLen * 4; ++j)			
+			(void) next();
+	}
+
+	// Returns the next random number.  Xor together two elements separated
+	// by 103 mod 250, replacing the first element with the result.  Then
+	// increment the two indices mod 250.
+	inline int next()
+	{
+		int ret = (state[posn1] ^= state[posn2]);	// xor and replace element
+
+		posn1 = incrTable[posn1];		// increment indices using lookup table
+		posn2 = incrTable[posn2];
+
+		return ret;
+	}
+private:
+	enum { stateLen = 250 };	// length of the state table
+	int state[stateLen];		// holds the random number state
+	int incrTable[stateLen];	// lookup table: maps i to (i+1) % stateLen
+	int posn1, posn2;			// indices into the state table
+};
+
+// will overrun the buffer!
+void make_random_data(void *buffer, int size, int seed)
+{
+	R250 rand(seed);
+
+	int n = (size / sizeof(int)) + 1;
+	int *b = (int*)buffer;
+	for(int l = 0; l < n; ++l)
+	{
+		b[l] = rand.next();
+	}
+}
+
+bool files_identical(const char *file1, const char *file2)
+{
+	FileStream f1(file1);
+	FileStream f2(file2);
+	
+	if(f1.BytesLeftToRead() != f2.BytesLeftToRead())
+	{
+		return false;
+	}
+	
+	while(f1.StreamDataLeft())
+	{
+		char buffer1[2048];
+		char buffer2[2048];
+		int s = f1.Read(buffer1, sizeof(buffer1));
+		if(f2.Read(buffer2, s) != s)
+		{
+			return false;
+		}
+		if(::memcmp(buffer1, buffer2, s) != 0)
+		{
+			return false;
+		}
+	}
+	
+	if(f2.StreamDataLeft())
+	{
+		return false;
+	}
+	
+	return true;
+}
+
+
+
+void create_test_files()
+{
+	// Create first file
+	{
+		make_random_data(buffer, FIRST_FILE_SIZE, 98);
+		FileStream out("testfiles/0.test", O_WRONLY | O_CREAT);
+		out.Write(buffer, FIRST_FILE_SIZE);
+	}
+	
+	// Create other files
+	int seed = 987;
+	for(unsigned int f = 1; f < NUMBER_FILES; ++f)
+	{
+		// Open files
+		char fnp[64];
+		sprintf(fnp, "testfiles/%d.test", f - 1);
+		FileStream previous(fnp);
+		char fnt[64];
+		sprintf(fnt, "testfiles/%d.test", f);
+		FileStream out(fnt, O_WRONLY | O_CREAT);
+		
+		// Copy up to the change point
+		int b = previous.Read(buffer, test_files[f].ChangePoint, IOStream::TimeOutInfinite);
+		out.Write(buffer, b);
+		
+		// Add new bytes?
+		if(test_files[f].InsertBytes > 0)
+		{
+			make_random_data(buffer, test_files[f].InsertBytes, ++seed);
+			out.Write(buffer, test_files[f].InsertBytes);
+		}
+		// Delete bytes?
+		if(test_files[f].DeleteBytes > 0)
+		{
+			previous.Seek(test_files[f].DeleteBytes, IOStream::SeekType_Relative);
+		}
+		// Copy rest of data
+		b = previous.Read(buffer, BUFFER_SIZE, IOStream::TimeOutInfinite);
+		out.Write(buffer, b);
+	}
+}
+
+void test_depends_in_dirs()
+{
+	BackupStoreFilenameClear storeFilename("test");
+
+	{
+		// Save directory with no dependency info
+		BackupStoreDirectory dir(1000, 1001); // some random ids
+		dir.AddEntry(storeFilename, 1, 2, 3, BackupStoreDirectory::Entry::Flags_File, 4);
+		dir.AddEntry(storeFilename, 1, 3, 3, BackupStoreDirectory::Entry::Flags_File, 4);
+		dir.AddEntry(storeFilename, 1, 4, 3, BackupStoreDirectory::Entry::Flags_File, 4);
+		dir.AddEntry(storeFilename, 1, 5, 3, BackupStoreDirectory::Entry::Flags_File, 4);
+		{
+			FileStream out("testfiles/dir.0", O_WRONLY | O_CREAT);
+			dir.WriteToStream(out);
+		}
+		// Add some dependency info to one of them
+		BackupStoreDirectory::Entry *en = dir.FindEntryByID(3);
+		TEST_THAT(en != 0);
+		en->SetDependsNewer(4);
+		// Save again
+		{
+			FileStream out("testfiles/dir.1", O_WRONLY | O_CREAT);
+			dir.WriteToStream(out);
+		}
+		// Check that the file size increases as expected.
+		TEST_THAT(TestGetFileSize("testfiles/dir.1") == (TestGetFileSize("testfiles/dir.0") + (4*16)));
+	}
+	{
+		// Load the directory back in
+		BackupStoreDirectory dir2;
+		FileStream in("testfiles/dir.1");
+		dir2.ReadFromStream(in, IOStream::TimeOutInfinite);
+		// Check entries
+		TEST_THAT(dir2.GetNumberOfEntries() == 4);
+		for(int i = 2; i <= 5; ++i)
+		{
+			BackupStoreDirectory::Entry *en = dir2.FindEntryByID(i);
+			TEST_THAT(en != 0);
+			TEST_THAT(en->GetDependsNewer() == ((i == 3)?4:0));
+			TEST_THAT(en->GetDependsOlder() == 0);
+		}
+		dir2.Dump(0, true);
+		// Test that numbers go in and out as required
+		for(int i = 2; i <= 5; ++i)
+		{
+			BackupStoreDirectory::Entry *en = dir2.FindEntryByID(i);
+			TEST_THAT(en != 0);
+			en->SetDependsNewer(i + 1);
+			en->SetDependsOlder(i - 1);
+		}
+		// Save
+		{
+			FileStream out("testfiles/dir.2", O_WRONLY | O_CREAT);
+			dir2.WriteToStream(out);
+		}
+		// Load and check
+		{
+			BackupStoreDirectory dir3;
+			FileStream in("testfiles/dir.2");
+			dir3.ReadFromStream(in, IOStream::TimeOutInfinite);
+			dir3.Dump(0, true);
+			for(int i = 2; i <= 5; ++i)
+			{
+				BackupStoreDirectory::Entry *en = dir2.FindEntryByID(i);
+				TEST_THAT(en != 0);
+				TEST_THAT(en->GetDependsNewer() == (i + 1));
+				TEST_THAT(en->GetDependsOlder() == (i - 1));
+			}
+		}
+	}
+}
+
+
+int test(int argc, const char *argv[])
+{
+	// Allocate a buffer
+	buffer = ::malloc(BUFFER_SIZE);
+	TEST_THAT(buffer != 0);
+
+	// SSL library
+	SSLLib::Initialise();
+	
+	// Use the setup crypto command to set up all these keys, so that the bbackupquery command can be used
+	// for seeing what's going on.
+	BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys");	
+
+	// Trace errors out
+	SET_DEBUG_SSLLIB_TRACE_ERRORS
+
+	// Initialise the raid file controller
+	RaidFileController &rcontroller = RaidFileController::GetController();
+	rcontroller.Initialise("testfiles/raidfile.conf");
+
+	// Context
+	TLSContext context;
+	context.Initialise(false /* client */,
+			"testfiles/clientCerts.pem",
+			"testfiles/clientPrivKey.pem",
+			"testfiles/clientTrustedCAs.pem");
+
+	// Create an account
+	TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf create 01234567 0 30000B 40000B") == 0);
+	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+
+	// Create test files
+	create_test_files();
+	
+	// Check the basic directory stuff works
+	test_depends_in_dirs();
+
+	// First, try logging in without an account having been created... just make sure login fails.
+	int pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid");
+	TEST_THAT(pid != -1 && pid != 0);
+	if(pid > 0)
+	{
+		TEST_THAT(ServerIsAlive(pid));
+
+		{
+			// Open a connection to the server
+			SocketStreamTLS conn;
+			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+	
+			// Make a protocol
+			BackupProtocolClient protocol(conn);
+	
+			// Login
+			{
+				// Check the version
+				std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
+				TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+	
+				// Login
+				protocol.QueryLogin(0x01234567, 0);
+			}
+	
+			// Filename for server
+			BackupStoreFilenameClear storeFilename("test");
+	
+			// Upload the first file
+			{
+				std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/0.test",
+						BackupProtocolClientListDirectory::RootDirectory, storeFilename));
+				std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+						BackupProtocolClientListDirectory::RootDirectory, ModificationTime,
+						ModificationTime, 0 /* no diff from file ID */, storeFilename, *upload));
+				test_files[0].IDOnServer = stored->GetObjectID();
+				test_files[0].IsCompletelyDifferent = true;
+				ModificationTime += MODIFICATION_TIME_INC;
+			}
+			
+			// Upload the other files, using the diffing process
+			for(unsigned int f = 1; f < NUMBER_FILES; ++f)
+			{
+				// Get an index for the previous version
+				std::auto_ptr<BackupProtocolClientSuccess> getBlockIndex(protocol.QueryGetBlockIndexByName(
+						BackupProtocolClientListDirectory::RootDirectory, storeFilename));
+				int64_t diffFromID = getBlockIndex->GetObjectID();
+				TEST_THAT(diffFromID != 0);
+				
+				if(diffFromID != 0)
+				{
+					// Found an old version -- get the index
+					std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream());
+				
+					// Diff the file
+					char filename[64];
+					::sprintf(filename, "testfiles/%d.test", f);
+					bool isCompletelyDifferent = false;
+					std::auto_ptr<IOStream> patchStream(BackupStoreFile::EncodeFileDiff(filename,
+							BackupProtocolClientListDirectory::RootDirectory,	/* containing directory */
+							storeFilename, diffFromID, *blockIndexStream,
+							protocol.GetTimeout(), 0 /* not interested in the modification time */, &isCompletelyDifferent));
+		
+					// Upload the patch to the store
+					std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
+							BackupProtocolClientListDirectory::RootDirectory, ModificationTime,
+							ModificationTime, isCompletelyDifferent?(0):(diffFromID), storeFilename, *patchStream));
+					ModificationTime += MODIFICATION_TIME_INC;
+					
+					// Store details
+					test_files[f].IDOnServer = stored->GetObjectID();
+					test_files[f].IsCompletelyDifferent = isCompletelyDifferent;
+					printf("ID %lld, completely different: %s\n", test_files[f].IDOnServer,
+						test_files[f].IsCompletelyDifferent?"yes":"no");			
+				}
+				else
+				{
+					::printf("WARNING: Block index not obtained when diffing file %d!\n", f);
+				}
+			}
+			
+			// List the directory from the server, and check that no dependency info is sent -- waste of bytes
+			{
+				std::auto_ptr<BackupProtocolClientSuccess> dirreply(protocol.QueryListDirectory(
+						BackupProtocolClientListDirectory::RootDirectory,
+						BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING,
+						BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */));
+				// Stream
+				BackupStoreDirectory dir;
+				std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
+				dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite);
+				
+				BackupStoreDirectory::Iterator i(dir);
+				BackupStoreDirectory::Entry *en = 0;
+				while((en = i.Next()) != 0)
+				{
+					TEST_THAT(en->GetDependsNewer() == 0);
+					TEST_THAT(en->GetDependsOlder() == 0);
+				}
+			}
+
+			// Finish the connection
+			protocol.QueryFinished();
+			conn.Close();
+		}
+
+		// Fill in initial dependency information
+		for(unsigned int f = 0; f < NUMBER_FILES; ++f)
+		{
+			int64_t newer = (f < (NUMBER_FILES - 1))?test_files[f + 1].IDOnServer:0;
+			int64_t older = (f > 0)?test_files[f - 1].IDOnServer:0;
+			if(test_files[f].IsCompletelyDifferent)
+			{
+				older = 0;
+			}
+			if(f < (NUMBER_FILES - 1) && test_files[f + 1].IsCompletelyDifferent)
+			{
+				newer = 0;
+			}
+			test_files[f].DepNewer = newer;
+			test_files[f].DepOlder = older;
+		}
+
+		// Check the stuff on the server
+		int deleteIndex = 0;
+		while(true)
+		{
+			// Load up the root directory
+			BackupStoreDirectory dir;
+			{
+				std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(0, "backup/01234567/o01"));
+				dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite);
+				dir.Dump(0, true);
+				
+				// Check that dependency info is correct
+				for(unsigned int f = 0; f < NUMBER_FILES; ++f)
+				{
+					//TRACE1("t f = %d\n", f);
+					BackupStoreDirectory::Entry *en = dir.FindEntryByID(test_files[f].IDOnServer);
+					if(en == 0)
+					{
+						TEST_THAT(test_files[f].HasBeenDeleted);
+					}
+					else
+					{
+						TEST_THAT(!test_files[f].HasBeenDeleted);
+						TEST_THAT(en->GetDependsNewer() == test_files[f].DepNewer);
+						TEST_THAT(en->GetDependsOlder() == test_files[f].DepOlder);
+						// Test that size is plausible
+						if(en->GetDependsNewer() == 0)
+						{
+							// Should be a full file
+							TEST_THAT(en->GetSizeInBlocks() > 40);
+						}
+						else
+						{
+							// Should be a patch
+							TEST_THAT(en->GetSizeInBlocks() < 40);
+						}
+					}
+				}
+			}
+			
+			// Open a connection to the server (need to do this each time, otherwise housekeeping won't delete files)
+			SocketStreamTLS conn;
+			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+			BackupProtocolClient protocol(conn);
+			{
+				std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
+				TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+				protocol.QueryLogin(0x01234567, 0);
+			}
+
+			// Pull all the files down, and check that they match the files on disc
+			for(unsigned int f = 0; f < NUMBER_FILES; ++f)
+			{
+				::printf("r=%d, f=%d\n", deleteIndex, f);
+				
+				// Might have been deleted
+				if(test_files[f].HasBeenDeleted)
+				{
+					continue;
+				}
+			
+				// Filenames
+				char filename[64], filename_fetched[64];
+				::sprintf(filename, "testfiles/%d.test", f);
+				::sprintf(filename_fetched, "testfiles/%d.test.fetched", f);
+				::unlink(filename_fetched);
+	
+				// Fetch the file
+				{
+					std::auto_ptr<BackupProtocolClientSuccess> getobj(protocol.QueryGetFile(
+						BackupProtocolClientListDirectory::RootDirectory,
+						test_files[f].IDOnServer));
+					TEST_THAT(getobj->GetObjectID() == test_files[f].IDOnServer);
+					// BLOCK
+					{
+						// Get stream
+						std::auto_ptr<IOStream> filestream(protocol.ReceiveStream());
+						// Get and decode
+						BackupStoreFile::DecodeFile(*filestream, filename_fetched, IOStream::TimeOutInfinite);
+					}
+				}
+				// Test for identicalness
+				TEST_THAT(files_identical(filename_fetched, filename));
+				
+				// Download the index, and check it looks OK
+				{
+					std::auto_ptr<BackupProtocolClientSuccess> getblockindex(protocol.QueryGetBlockIndexByID(test_files[f].IDOnServer));
+					TEST_THAT(getblockindex->GetObjectID() == test_files[f].IDOnServer);
+					std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream());
+					TEST_THAT(BackupStoreFile::CompareFileContentsAgainstBlockIndex(filename, *blockIndexStream, IOStream::TimeOutInfinite));
+				}
+			}
+
+			// Close the connection			
+			protocol.QueryFinished();
+			conn.Close();
+
+			// Mark one of the elements as deleted
+			if(test_file_remove_order[deleteIndex] == -1)
+			{
+				// Nothing left to do
+				break;
+			}
+			int todel = test_file_remove_order[deleteIndex++];
+			
+			// Modify the entry
+			BackupStoreDirectory::Entry *pentry = dir.FindEntryByID(test_files[todel].IDOnServer);
+			TEST_THAT(pentry != 0);
+			pentry->AddFlags(BackupStoreDirectory::Entry::Flags_RemoveASAP);
+			// Save it back
+			{
+				RaidFileWrite writedir(0, "backup/01234567/o01");
+				writedir.Open(true /* overwrite */);
+				dir.WriteToStream(writedir);
+				writedir.Commit(true);
+			}
+
+			// Send the server a restart signal, so it does housekeeping immedaitely, and wait for it to happen
+			::sleep(1);	// wait for old connections to terminate
+			::kill(pid, SIGHUP);
+			// Get the revision number of the info file
+			int64_t first_revision = 0;
+			RaidFileRead::FileExists(0, "backup/01234567/o01", &first_revision);
+			for(int l = 0; l < 32; ++l)
+			{
+				// Sleep a while, and print a dot
+				::sleep(1);
+				::printf(".");
+				::fflush(stdout);
+				
+				// Early end?
+				if(l > 2)
+				{
+					int64_t revid = 0;
+					RaidFileRead::FileExists(0, "backup/01234567/o01", &revid);
+					if(revid != first_revision)
+					{
+						break;
+					}
+				}
+			}
+			::printf("\n");
+			
+			// Flag for test
+			test_files[todel].HasBeenDeleted = true;
+			// Update dependency info
+			int z = todel;
+			while(z > 0 && test_files[z].HasBeenDeleted && test_files[z].DepOlder != 0)
+			{
+				--z;
+			}
+			if(z >= 0) test_files[z].DepNewer = test_files[todel].DepNewer;
+			z = todel;
+			while(z < (int)NUMBER_FILES && test_files[z].HasBeenDeleted && test_files[z].DepNewer != 0)
+			{
+				++z;
+			}
+			if(z < (int)NUMBER_FILES) test_files[z].DepOlder = test_files[todel].DepOlder;
+		}
+		
+		// Kill store server
+		TEST_THAT(KillServer(pid));
+		TEST_THAT(!ServerIsAlive(pid));
+		TestRemoteProcessMemLeaks("bbstored.memleaks");
+	}
+	
+	::free(buffer);
+	
+	return 0;
+}
diff --git a/test/backupstorepatch/testextra b/test/backupstorepatch/testextra
new file mode 100644
index 00000000..cacda911
--- /dev/null
+++ b/test/backupstorepatch/testextra
@@ -0,0 +1,6 @@
+rm -rf testfiles
+mkdir testfiles
+mkdir testfiles/0_0
+mkdir testfiles/0_1
+mkdir testfiles/0_2
+cp ../../../test/backupstore/testfiles/*.* testfiles/
diff --git a/test/basicserver/Makefile.extra b/test/basicserver/Makefile.extra
new file mode 100755
index 00000000..51280db9
--- /dev/null
+++ b/test/basicserver/Makefile.extra
@@ -0,0 +1,21 @@
+
+MAKEPROTOCOL = ../../lib/server/makeprotocol.pl
+
+GEN_CMD_SRV = $(MAKEPROTOCOL) Server testprotocol.txt
+GEN_CMD_CLI = $(MAKEPROTOCOL) Client testprotocol.txt
+
+# AUTOGEN SEEDING
+autogen_TestProtocolServer.cpp:	$(MAKEPROTOCOL) testprotocol.txt
+	perl $(GEN_CMD_SRV)
+
+autogen_TestProtocolServer.h:	$(MAKEPROTOCOL) testprotocol.txt
+	perl $(GEN_CMD_SRV)
+
+
+# AUTOGEN SEEDING
+autogen_TestProtocolClient.cpp:	$(MAKEPROTOCOL) testprotocol.txt
+	perl $(GEN_CMD_CLI)
+
+autogen_TestProtocolClient.h:	$(MAKEPROTOCOL) testprotocol.txt
+	perl $(GEN_CMD_CLI)
+
diff --git a/test/basicserver/TestCommands.cpp b/test/basicserver/TestCommands.cpp
new file mode 100755
index 00000000..b18a3326
--- /dev/null
+++ b/test/basicserver/TestCommands.cpp
@@ -0,0 +1,99 @@
+
+#include "Box.h"
+
+#include <syslog.h>
+
+#include "autogen_TestProtocolServer.h"
+#include "CollectInBufferStream.h"
+
+#include "MemLeakFindOn.h"
+
+
+std::auto_ptr<ProtocolObject> TestProtocolServerHello::DoCommand(TestProtocolServer &rProtocol, TestContext &rContext)
+{
+	if(mNumber32 != 41 || mNumber16 != 87 || mNumber8 != 11 || mText != "pingu")
+	{
+		return std::auto_ptr<ProtocolObject>(new TestProtocolServerError(0, 0));
+	}
+	return std::auto_ptr<ProtocolObject>(new TestProtocolServerHello(12,89,22,std::string("Hello world!")));
+}
+
+std::auto_ptr<ProtocolObject> TestProtocolServerLists::DoCommand(TestProtocolServer &rProtocol, TestContext &rContext)
+{
+	return std::auto_ptr<ProtocolObject>(new TestProtocolServerListsReply(mLotsOfText.size()));
+}
+
+std::auto_ptr<ProtocolObject> TestProtocolServerQuit::DoCommand(TestProtocolServer &rProtocol, TestContext &rContext)
+{
+	return std::auto_ptr<ProtocolObject>(new TestProtocolServerQuit);
+}
+
+std::auto_ptr<ProtocolObject> TestProtocolServerSimple::DoCommand(TestProtocolServer &rProtocol, TestContext &rContext)
+{
+	return std::auto_ptr<ProtocolObject>(new TestProtocolServerSimpleReply(mValue+1));
+}
+
+class UncertainBufferStream : public CollectInBufferStream
+{
+public:
+	// make the collect in buffer stream pretend not to know how many bytes are left
+	pos_type BytesLeftToRead()
+	{
+		return IOStream::SizeOfStreamUnknown;
+	}
+};
+
+std::auto_ptr<ProtocolObject> TestProtocolServerGetStream::DoCommand(TestProtocolServer &rProtocol, TestContext &rContext)
+{
+	// make a new stream object
+	CollectInBufferStream *pstream = mUncertainSize?(new UncertainBufferStream):(new CollectInBufferStream);
+	
+	// Data.
+	int values[24273];
+	int v = mStartingValue;
+	for(int l = 0; l < 3; ++l)
+	{
+		for(int x = 0; x < 24273; ++x)
+		{
+			values[x] = v++;
+		}
+		pstream->Write(values, sizeof(values));
+	}
+	
+	// Finished
+	pstream->SetForReading();
+	
+	// Get it to be sent
+	rProtocol.SendStreamAfterCommand(pstream);
+
+	return std::auto_ptr<ProtocolObject>(new TestProtocolServerGetStream(mStartingValue, mUncertainSize));
+}
+
+std::auto_ptr<ProtocolObject> TestProtocolServerSendStream::DoCommand(TestProtocolServer &rProtocol, TestContext &rContext)
+{
+	if(mValue != 0x73654353298ffLL)
+	{
+		return std::auto_ptr<ProtocolObject>(new TestProtocolServerError(0, 0));
+	}
+	
+	// Get a stream
+	std::auto_ptr<IOStream> stream(rProtocol.ReceiveStream());
+	bool uncertain = (stream->BytesLeftToRead() == IOStream::SizeOfStreamUnknown);
+	
+	// Count how many bytes in it
+	int bytes = 0;
+	char buffer[125];
+	while(stream->StreamDataLeft())
+	{
+		bytes += stream->Read(buffer, sizeof(buffer));
+	}
+
+	// tell the caller how many bytes there were
+	return std::auto_ptr<ProtocolObject>(new TestProtocolServerGetStream(bytes, uncertain));
+}
+
+std::auto_ptr<ProtocolObject> TestProtocolServerString::DoCommand(TestProtocolServer &rProtocol, TestContext &rContext)
+{
+	return std::auto_ptr<ProtocolObject>(new TestProtocolServerString(mTest));
+}
+
diff --git a/test/basicserver/TestContext.cpp b/test/basicserver/TestContext.cpp
new file mode 100755
index 00000000..4b92766e
--- /dev/null
+++ b/test/basicserver/TestContext.cpp
@@ -0,0 +1,16 @@
+
+#include "Box.h"
+#include "Test.h"
+#include "TestContext.h"
+
+#include "MemLeakFindOn.h"
+
+TestContext::TestContext()
+{
+}
+
+TestContext::~TestContext()
+{
+}
+
+
diff --git a/test/basicserver/TestContext.h b/test/basicserver/TestContext.h
new file mode 100755
index 00000000..d0c28349
--- /dev/null
+++ b/test/basicserver/TestContext.h
@@ -0,0 +1,7 @@
+
+class TestContext
+{
+public:
+	TestContext();
+	~TestContext();
+};
diff --git a/test/basicserver/testbasicserver.cpp b/test/basicserver/testbasicserver.cpp
new file mode 100755
index 00000000..4d124b1d
--- /dev/null
+++ b/test/basicserver/testbasicserver.cpp
@@ -0,0 +1,627 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testbasicserver.cpp
+//		Purpose: Test basic server classes
+//		Created: 2003/07/29
+//
+// --------------------------------------------------------------------------
+
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <time.h>
+
+#include <typeinfo>
+
+#include "Test.h"
+#include "Daemon.h"
+#include "Configuration.h"
+#include "ServerStream.h"
+#include "SocketStream.h"
+#include "IOStreamGetLine.h"
+#include "ServerTLS.h"
+#include "CollectInBufferStream.h"
+
+#include "TestContext.h"
+#include "autogen_TestProtocolClient.h"
+#include "autogen_TestProtocolServer.h"
+
+#include "MemLeakFindOn.h"
+
+
+#define SERVER_LISTEN_PORT	2003
+
+// in ms
+#define COMMS_READ_TIMEOUT					4
+#define COMMS_SERVER_WAIT_BEFORE_REPLYING	40
+
+class basicdaemon : public Daemon
+{
+public:
+basicdaemon() {};
+~basicdaemon() {}
+virtual void Run();
+};
+
+void basicdaemon::Run()
+{
+	// Write a file to check it's done...
+	const Configuration &c(GetConfiguration());
+	
+	FILE *f = fopen(c.GetKeyValue("TestFile").c_str(), "w");
+	fclose(f);
+
+	while(!StopRun())
+	{
+		::sleep(10);
+	}
+}
+
+void testservers_pause_before_reply()
+{
+	 struct timespec t;
+	 t.tv_sec = 0;
+	 t.tv_nsec = COMMS_SERVER_WAIT_BEFORE_REPLYING * 1000 * 1000;	// convert to ns
+	 ::nanosleep(&t, NULL);
+}
+
+#define LARGE_DATA_BLOCK_SIZE 19870
+#define LARGE_DATA_SIZE (LARGE_DATA_BLOCK_SIZE*1000)
+
+void testservers_connection(SocketStream &rStream)
+{
+	IOStreamGetLine getline(rStream);
+
+	if(typeid(rStream) == typeid(SocketStreamTLS))
+	{
+		// need to wait for some data before sending stuff, otherwise timeout test doesn't work
+		std::string line;
+		while(!getline.GetLine(line))
+			;
+		SocketStreamTLS &rtls = (SocketStreamTLS&)rStream;
+		std::string line1("CONNECTED:");
+		line1 += rtls.GetPeerCommonName();
+		line1 += '\n';
+		testservers_pause_before_reply();
+		rStream.Write(line1.c_str(), line1.size());
+	}
+
+	while(!getline.IsEOF())
+	{
+		std::string line;
+		while(!getline.GetLine(line))
+			;
+		if(line == "QUIT")
+		{
+			break;
+		}
+		if(line == "LARGEDATA")
+		{
+			{
+				// Send lots of data
+				char data[LARGE_DATA_BLOCK_SIZE];
+				for(unsigned int y = 0; y < sizeof(data); y++)
+				{
+					data[y] = y & 0xff;
+				}
+				for(int s = 0; s < (LARGE_DATA_SIZE / LARGE_DATA_BLOCK_SIZE); ++s)
+				{
+					rStream.Write(data, sizeof(data));
+				}
+			}
+			{
+				// Receive lots of data
+				char buf[1024];
+				int total = 0;
+				int r = 0;
+				while(total < LARGE_DATA_SIZE && (r = rStream.Read(buf, sizeof(buf))) != 0)
+				{
+					total += r;
+				}
+				TEST_THAT(total == LARGE_DATA_SIZE);
+			}
+			
+			// next!
+			continue;
+		}
+		std::string backwards;
+		for(std::string::const_reverse_iterator i(line.end()); i != std::string::const_reverse_iterator(line.begin()); ++i)
+		{
+			backwards += (*i);
+		}
+		backwards += '\n';
+		testservers_pause_before_reply();
+		rStream.Write(backwards.c_str(), backwards.size());
+	}
+	rStream.Shutdown();
+	rStream.Close();
+}
+
+
+
+class testserver : public ServerStream<SocketStream, SERVER_LISTEN_PORT>
+{
+public:
+	testserver() {}
+	~testserver() {}
+	
+	void Connection(SocketStream &rStream);
+	
+	virtual const char *DaemonName() const
+	{
+		return "test-srv2";
+	}
+	const ConfigurationVerify *GetConfigVerify() const;
+
+};
+
+const ConfigurationVerify *testserver::GetConfigVerify() const
+{
+	static ConfigurationVerifyKey verifyserverkeys[] = 
+	{
+		SERVERSTREAM_VERIFY_SERVER_KEYS(0)	// no default addresses
+	};
+
+	static ConfigurationVerify verifyserver[] = 
+	{
+		{
+			"Server",
+			0,
+			verifyserverkeys,
+			ConfigTest_Exists | ConfigTest_LastEntry,
+			0
+		}
+	};
+
+	static ConfigurationVerify verify =
+	{
+		"root",
+		verifyserver,
+		0,
+		ConfigTest_Exists | ConfigTest_LastEntry,
+		0
+	};
+
+	return &verify;
+}
+
+void testserver::Connection(SocketStream &rStream)
+{
+	testservers_connection(rStream);
+}
+
+class testProtocolServer : public testserver
+{
+public:
+	testProtocolServer() {}
+	~testProtocolServer() {}
+
+	void Connection(SocketStream &rStream);
+	
+	virtual const char *DaemonName() const
+	{
+		return "test-srv4";
+	}
+};
+
+void testProtocolServer::Connection(SocketStream &rStream)
+{
+	TestProtocolServer server(rStream);
+	TestContext context;
+	server.DoServer(context);
+}
+
+
+class testTLSserver : public ServerTLS<SERVER_LISTEN_PORT>
+{
+public:
+	testTLSserver() {}
+	~testTLSserver() {}
+	
+	void Connection(SocketStreamTLS &rStream);
+	
+	virtual const char *DaemonName() const
+	{
+		return "test-srv3";
+	}
+	const ConfigurationVerify *GetConfigVerify() const;
+
+};
+
+const ConfigurationVerify *testTLSserver::GetConfigVerify() const
+{
+	static ConfigurationVerifyKey verifyserverkeys[] = 
+	{
+		SERVERTLS_VERIFY_SERVER_KEYS(0)	// no default listen addresses
+	};
+
+	static ConfigurationVerify verifyserver[] = 
+	{
+		{
+			"Server",
+			0,
+			verifyserverkeys,
+			ConfigTest_Exists | ConfigTest_LastEntry,
+			0
+		}
+	};
+
+	static ConfigurationVerify verify =
+	{
+		"root",
+		verifyserver,
+		0,
+		ConfigTest_Exists | ConfigTest_LastEntry,
+		0
+	};
+
+	return &verify;
+}
+
+void testTLSserver::Connection(SocketStreamTLS &rStream)
+{
+	testservers_connection(rStream);
+}
+
+
+void Srv2TestConversations(const std::vector<IOStream *> &conns)
+{
+	const static char *tosend[] = {
+		"test 1\n", "carrots\n", "pineapples\n", "booo!\n", 0
+	};
+	const static char *recieve[] = {
+		"1 tset", "storrac", "selppaenip", "!ooob", 0
+	};
+	
+	IOStreamGetLine **getline = new IOStreamGetLine*[conns.size()];
+	for(unsigned int c = 0; c < conns.size(); ++c)
+	{
+		getline[c] = new IOStreamGetLine(*conns[c]);
+		
+		bool hadTimeout = false;
+		if(typeid(*conns[c]) == typeid(SocketStreamTLS))
+		{
+			SocketStreamTLS *ptls = (SocketStreamTLS *)conns[c];
+			printf("Connected to '%s'\n", ptls->GetPeerCommonName().c_str());
+		
+			// Send some data, any data, to get the first response.
+			conns[c]->Write("Hello\n", 6);
+		
+			std::string line1;
+			while(!getline[c]->GetLine(line1, false, COMMS_READ_TIMEOUT))
+				hadTimeout = true;
+			TEST_THAT(line1 == "CONNECTED:CLIENT");
+			TEST_THAT(hadTimeout)
+		}
+	}
+	
+	for(int q = 0; tosend[q] != 0; ++q)
+	{
+		for(unsigned int c = 0; c < conns.size(); ++c)
+		{
+			//printf("%d: %s", c, tosend[q]);
+			conns[c]->Write(tosend[q], strlen(tosend[q]));
+			std::string rep;
+			bool hadTimeout = false;
+			while(!getline[c]->GetLine(rep, false, COMMS_READ_TIMEOUT))
+				hadTimeout = true;
+			TEST_THAT(rep == recieve[q]);
+			TEST_THAT(hadTimeout)
+		}
+	}
+	for(unsigned int c = 0; c < conns.size(); ++c)
+	{
+		conns[c]->Write("LARGEDATA\n", 10);
+	}
+	for(unsigned int c = 0; c < conns.size(); ++c)
+	{
+		// Receive lots of data
+		char buf[1024];
+		int total = 0;
+		int r = 0;
+		while(total < LARGE_DATA_SIZE && (r = conns[c]->Read(buf, sizeof(buf))) != 0)
+		{
+			total += r;
+		}
+		TEST_THAT(total == LARGE_DATA_SIZE);
+	}
+	for(unsigned int c = 0; c < conns.size(); ++c)
+	{
+		// Send lots of data
+		char data[LARGE_DATA_BLOCK_SIZE];
+		for(unsigned int y = 0; y < sizeof(data); y++)
+		{
+			data[y] = y & 0xff;
+		}
+		for(int s = 0; s < (LARGE_DATA_SIZE / LARGE_DATA_BLOCK_SIZE); ++s)
+		{
+			conns[c]->Write(data, sizeof(data));
+		}
+	}
+
+	for(unsigned int c = 0; c < conns.size(); ++c)
+	{
+		conns[c]->Write("QUIT\n", 5);
+	}
+	
+	for(unsigned int c = 0; c < conns.size(); ++c)
+	{
+		delete getline[c];
+	}
+	delete [] getline;
+}
+
+void TestStreamReceive(TestProtocolClient &protocol, int value, bool uncertainstream)
+{
+	std::auto_ptr<TestProtocolClientGetStream> reply(protocol.QueryGetStream(value, uncertainstream));
+	TEST_THAT(reply->GetStartingValue() == value);
+	
+	// Get a stream
+	std::auto_ptr<IOStream> stream(protocol.ReceiveStream());
+	
+	// check uncertainty
+	TEST_THAT(uncertainstream == (stream->BytesLeftToRead() == IOStream::SizeOfStreamUnknown));
+	
+	printf("stream is %s\n", uncertainstream?"uncertain size":"fixed size");
+	
+	// Then check the contents
+	int values[998];
+	int v = value;
+	int count = 0;
+	int bytesleft = 0;
+	int bytessofar = 0;
+	while(stream->StreamDataLeft())
+	{
+		// Read some data
+		int bytes = stream->Read(((char*)values) + bytesleft, sizeof(values) - bytesleft);
+		bytessofar += bytes;
+		bytes += bytesleft;
+		int n = bytes / 4;
+		//printf("read %d, n = %d, so far = %d\n", bytes, n, bytessofar);
+		for(int t = 0; t < n; ++t)
+		{
+			if(values[t] != v) printf("%d, %d, %d\n", t, values[t], v);
+			TEST_THAT(values[t] == v++);
+		}
+		count += n;
+		bytesleft = bytes - (n*4);
+		if(bytesleft) ::memmove(values, ((char*)values) + bytes - bytesleft, bytesleft);
+	}
+	
+	TEST_THAT(bytesleft == 0);
+	TEST_THAT(count == (24273*3));	// over 64 k of data, definately
+}
+
+
+int test(int argc, const char *argv[])
+{
+	// Server launching stuff
+	if(argc >= 2)
+	{
+		if(strcmp(argv[1], "srv1") == 0)
+		{
+			// Run very basic daemon
+			basicdaemon daemon;
+			return daemon.Main("doesnotexist", argc - 1, argv + 1);
+		}
+		else if(strcmp(argv[1], "srv2") == 0)
+		{
+			// Run daemon which accepts connections
+			testserver daemon;
+			return daemon.Main("doesnotexist", argc - 1, argv + 1);
+		}		
+		else if(strcmp(argv[1], "srv3") == 0)
+		{
+			testTLSserver daemon;
+			return daemon.Main("doesnotexist", argc - 1, argv + 1);
+		}
+		else if(strcmp(argv[1], "srv4") == 0)
+		{
+			testProtocolServer daemon;
+			return daemon.Main("doesnotexist", argc - 1, argv + 1);
+		}
+	}
+
+//printf("SKIPPING TESTS------------------------\n");
+//goto protocolserver;
+
+	// Launch a basic server
+	{
+		int pid = LaunchServer("./test srv1 testfiles/srv1.conf", "testfiles/srv1.pid");
+		TEST_THAT(pid != -1 && pid != 0);
+		if(pid > 0)
+		{
+			// Check that it's written the expected file
+			TEST_THAT(TestFileExists("testfiles/srv1.test1"));
+			TEST_THAT(ServerIsAlive(pid));
+			// Move the config file over
+			TEST_THAT(::rename("testfiles/srv1b.conf", "testfiles/srv1.conf") != -1);
+			// Get it to reread the config file
+			TEST_THAT(HUPServer(pid));
+			::sleep(1);
+			TEST_THAT(ServerIsAlive(pid));
+			// Check that new file exists
+			TEST_THAT(TestFileExists("testfiles/srv1.test2"));
+			// Kill it off
+			TEST_THAT(KillServer(pid));
+			TestRemoteProcessMemLeaks("generic-daemon.memleaks");
+		}
+	}
+	
+	// Launch a test forking server
+	{
+		int pid = LaunchServer("./test srv2 testfiles/srv2.conf", "testfiles/srv2.pid");
+		TEST_THAT(pid != -1 && pid != 0);
+		if(pid > 0)
+		{
+			// Will it restart?
+			TEST_THAT(ServerIsAlive(pid));
+			TEST_THAT(HUPServer(pid));
+			::sleep(1);
+			TEST_THAT(ServerIsAlive(pid));
+			// Make some connections
+			{
+				SocketStream conn1;
+				conn1.Open(Socket::TypeINET, "localhost", 2003);
+				SocketStream conn2;
+				conn2.Open(Socket::TypeUNIX, "testfiles/srv2.sock");
+				SocketStream conn3;
+				conn3.Open(Socket::TypeINET, "localhost", 2003);
+				// Quick check that reconnections fail
+				TEST_CHECK_THROWS(conn1.Open(Socket::TypeUNIX, "testfiles/srv2.sock");, ServerException, SocketAlreadyOpen);
+				// Stuff some data around
+				std::vector<IOStream *> conns;
+				conns.push_back(&conn1);
+				conns.push_back(&conn2);
+				conns.push_back(&conn3);
+				Srv2TestConversations(conns);
+				// Implicit close
+			}
+			// HUP again
+			TEST_THAT(HUPServer(pid));
+			::sleep(1);
+			TEST_THAT(ServerIsAlive(pid));
+			// Kill it
+			TEST_THAT(KillServer(pid));
+			::sleep(1);
+			TEST_THAT(!ServerIsAlive(pid));
+			TestRemoteProcessMemLeaks("test-srv2.memleaks");
+		}
+	}
+
+	// Launch a test SSL server
+	{
+		int pid = LaunchServer("./test srv3 testfiles/srv3.conf", "testfiles/srv3.pid");
+		TEST_THAT(pid != -1 && pid != 0);
+		if(pid > 0)
+		{
+			// Will it restart?
+			TEST_THAT(ServerIsAlive(pid));
+			TEST_THAT(HUPServer(pid));
+			::sleep(1);
+			TEST_THAT(ServerIsAlive(pid));
+			// Make some connections
+			{
+				// SSL library
+				SSLLib::Initialise();
+			
+				// Context first
+				TLSContext context;
+				context.Initialise(false /* client */,
+						"testfiles/clientCerts.pem",
+						"testfiles/clientPrivKey.pem",
+						"testfiles/clientTrustedCAs.pem");
+
+				SocketStreamTLS conn1;
+				conn1.Open(context, Socket::TypeINET, "localhost", 2003);
+				SocketStreamTLS conn2;
+				conn2.Open(context, Socket::TypeUNIX, "testfiles/srv3.sock");
+				SocketStreamTLS conn3;
+				conn3.Open(context, Socket::TypeINET, "localhost", 2003);
+				// Quick check that reconnections fail
+				TEST_CHECK_THROWS(conn1.Open(context, Socket::TypeUNIX, "testfiles/srv3.sock");, ServerException, SocketAlreadyOpen);
+				// Stuff some data around
+				std::vector<IOStream *> conns;
+				conns.push_back(&conn1);
+				conns.push_back(&conn2);
+				conns.push_back(&conn3);
+				Srv2TestConversations(conns);
+				// Implicit close
+			}
+			// HUP again
+			TEST_THAT(HUPServer(pid));
+			::sleep(1);
+			TEST_THAT(ServerIsAlive(pid));
+			// Kill it
+			TEST_THAT(KillServer(pid));
+			::sleep(1);
+			TEST_THAT(!ServerIsAlive(pid));
+			TestRemoteProcessMemLeaks("test-srv3.memleaks");
+		}
+	}
+	
+//protocolserver:
+	// Launch a test protocol handling server
+	{
+		int pid = LaunchServer("./test srv4 testfiles/srv4.conf", "testfiles/srv4.pid");
+		TEST_THAT(pid != -1 && pid != 0);
+		if(pid > 0)
+		{
+			::sleep(1);
+			TEST_THAT(ServerIsAlive(pid));
+
+			// Open a connection to it		
+			SocketStream conn;
+			conn.Open(Socket::TypeUNIX, "testfiles/srv4.sock");
+			
+			// Create a protocol
+			TestProtocolClient protocol(conn);
+
+			// Simple query
+			{
+				std::auto_ptr<TestProtocolClientSimpleReply> reply(protocol.QuerySimple(41));
+				TEST_THAT(reply->GetValuePlusOne() == 42);
+			}
+			{
+				std::auto_ptr<TestProtocolClientSimpleReply> reply(protocol.QuerySimple(809));
+				TEST_THAT(reply->GetValuePlusOne() == 810);
+			}
+			
+			// Streams, twice, both uncertain and certain sizes
+			TestStreamReceive(protocol, 374, false);
+			TestStreamReceive(protocol, 23983, true);
+			TestStreamReceive(protocol, 12098, false);
+			TestStreamReceive(protocol, 4342, true);
+			
+			// Try to send a stream
+			{
+				CollectInBufferStream s;
+				char buf[1663];
+				s.Write(buf, sizeof(buf));
+				s.SetForReading();
+				std::auto_ptr<TestProtocolClientGetStream> reply(protocol.QuerySendStream(0x73654353298ffLL, s));
+				TEST_THAT(reply->GetStartingValue() == sizeof(buf));
+			}
+
+			// Lots of simple queries
+			for(int q = 0; q < 514; q++)
+			{
+				std::auto_ptr<TestProtocolClientSimpleReply> reply(protocol.QuerySimple(q));
+				TEST_THAT(reply->GetValuePlusOne() == (q+1));
+			}
+			// Send a list of strings to it
+			{
+				std::vector<std::string> strings;
+				strings.push_back(std::string("test1"));
+				strings.push_back(std::string("test2"));
+				strings.push_back(std::string("test3"));
+				std::auto_ptr<TestProtocolClientListsReply> reply(protocol.QueryLists(strings));
+				TEST_THAT(reply->GetNumberOfStrings() == 3);
+			}
+			
+			// And another
+			{
+				std::auto_ptr<TestProtocolClientHello> reply(protocol.QueryHello(41,87,11,std::string("pingu")));
+				TEST_THAT(reply->GetNumber32() == 12);
+				TEST_THAT(reply->GetNumber16() == 89);
+				TEST_THAT(reply->GetNumber8() == 22);
+				TEST_THAT(reply->GetText() == "Hello world!");
+			}
+		
+			// Quit query to finish
+			protocol.QueryQuit();
+		
+			// Kill it
+			TEST_THAT(KillServer(pid));
+			::sleep(1);
+			TEST_THAT(!ServerIsAlive(pid));
+			TestRemoteProcessMemLeaks("test-srv4.memleaks");
+		}
+	}
+
+	return 0;
+}
+
diff --git a/test/basicserver/testfiles/clientCerts.pem b/test/basicserver/testfiles/clientCerts.pem
new file mode 100644
index 00000000..81d4c5cc
--- /dev/null
+++ b/test/basicserver/testfiles/clientCerts.pem
@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICOTCCAaICAQUwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCR0IxDzANBgNV
+BAgTBkxvbmRvbjEPMA0GA1UEBxMGTG9uZG9uMQ0wCwYDVQQKEwRUZXN0MRUwEwYD
+VQQLEwxiYXNpYyBzZXJ2ZXIxDTALBgNVBAMTBFJPT1QwHhcNMDMwOTA2MTYyNDQz
+WhcNMzEwMTIyMTYyNDQzWjBmMQswCQYDVQQGEwJHQjEPMA0GA1UECBMGTG9uZG9u
+MQ8wDQYDVQQHEwZMb25kb24xDTALBgNVBAoTBFRlc3QxFTATBgNVBAsTDGJhc2lj
+IHNlcnZlcjEPMA0GA1UEAxMGQ0xJRU5UMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
+iQKBgQC7AJDQJdGHi4HO7VXZJdi/3C8rQx1uTxMO6QHBFep0wQZ6I37Zcr+TRrHk
+Q8CelymIBx2ZfQXMLKsoB8FScIp0zIT/drK0AghuWE5UPU6dntPlrA65y417qk5z
+NjiOy6coWl+7ktZ0ItCuy7VHWrTmHRbNZeXKub7fjuccDJdiywIDAQABMA0GCSqG
+SIb3DQEBBQUAA4GBACYkSYlrKNv1v6lrES4j68S8u8SNlnSM+Z4pTHF/7K7SQeIn
+SKVV8EI8CLR5jIsQRRHKB9rYgYS4kB8SFbPyrsH8VKngjIUcjmTKLq9zpAt2zDNo
+m+y5SMXsaJF6Xbtbz+MSxXZZ6YBBuseY+Wkpz4ZGSVlQrHxjsuYdBFHIguM3
+-----END CERTIFICATE-----
diff --git a/test/basicserver/testfiles/clientPrivKey.pem b/test/basicserver/testfiles/clientPrivKey.pem
new file mode 100644
index 00000000..a4797b66
--- /dev/null
+++ b/test/basicserver/testfiles/clientPrivKey.pem
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQC7AJDQJdGHi4HO7VXZJdi/3C8rQx1uTxMO6QHBFep0wQZ6I37Z
+cr+TRrHkQ8CelymIBx2ZfQXMLKsoB8FScIp0zIT/drK0AghuWE5UPU6dntPlrA65
+y417qk5zNjiOy6coWl+7ktZ0ItCuy7VHWrTmHRbNZeXKub7fjuccDJdiywIDAQAB
+AoGAF92enbH158KaMnp/tlLqMrI7It5R5z4YRJLgMnBFl9j6pqPZEI9ge79N/L/Y
+2WSZXE7sLCaUktYwkc9LkOXkBYQI7EIOonLdmSsNCMbSBVbeczdM77dBscuCTKva
+nvre/2+hlmuWBNINqXlprBkvd5YF4Q/yeXzoXPuMIQ0tROECQQDqifOZOfCle8uA
+CgdHT9pO638PwrrldMHmZSK3gUmHmFe7ziGpNGCfKZ+wkSIvDg9INQvEXvQfLZiV
+n4J78IOHAkEAzB0SoUU0cL+wK3OQTTOlx4cgxaxgtsuvccIhqTh4Jp1Aj9iMKiPW
+yXvbGhDBTZP2IL5HoqSLc3SxfXgvn6O/nQJBALgJMYWdalBf2GoK9HUnmpTsw1I5
+qe/c8z13RIubvnfQuZ8be1xLRjn+LlkdOSaVMLanMSmQnJxOafmWJYxdSMcCQFBc
+5ffe8n2tyyPgdSEgQ5YiatHJQ67U1Te50lz44b16TnAUN2NkBu3/OM2zaRgtOEu9
+/yBXHpyPhk47Iqz84LUCQQCIDIKluoughLVjJS2eD28UJHM9Z+OvmyIE0fF0Q0vi
+E+Rn/+iWCoEJYa7WP5AEo/aeVXiCeHONXGF1AI8a8gb5
+-----END RSA PRIVATE KEY-----
diff --git a/test/basicserver/testfiles/clientReq.pem b/test/basicserver/testfiles/clientReq.pem
new file mode 100644
index 00000000..14e2c6df
--- /dev/null
+++ b/test/basicserver/testfiles/clientReq.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBpjCCAQ8CAQAwZjELMAkGA1UEBhMCR0IxDzANBgNVBAgTBkxvbmRvbjEPMA0G
+A1UEBxMGTG9uZG9uMQ0wCwYDVQQKEwRUZXN0MRUwEwYDVQQLEwxiYXNpYyBzZXJ2
+ZXIxDzANBgNVBAMTBkNMSUVOVDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+uwCQ0CXRh4uBzu1V2SXYv9wvK0Mdbk8TDukBwRXqdMEGeiN+2XK/k0ax5EPAnpcp
+iAcdmX0FzCyrKAfBUnCKdMyE/3aytAIIblhOVD1OnZ7T5awOucuNe6pOczY4jsun
+KFpfu5LWdCLQrsu1R1q05h0WzWXlyrm+347nHAyXYssCAwEAAaAAMA0GCSqGSIb3
+DQEBBQUAA4GBAKV3H/yWrYep6yfEDQp61zn60tEnJOS5LVbuV7ivNjAue0/09wBT
+PGzTblwx116AT9GbTcbERK/ll549+tziTLT9NUT12ZcvaRezYP2PpaD8fiDKHs3D
+vSwpFoihLmUnDeMWE9Vbt+b0Fl/mdsH6sm3Mo0COG/DkolOVsydOj2Hp
+-----END CERTIFICATE REQUEST-----
diff --git a/test/basicserver/testfiles/clientTrustedCAs.pem b/test/basicserver/testfiles/clientTrustedCAs.pem
new file mode 100644
index 00000000..d72b70e5
--- /dev/null
+++ b/test/basicserver/testfiles/clientTrustedCAs.pem
@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICNzCCAaACAQAwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCR0IxDzANBgNV
+BAgTBkxvbmRvbjEPMA0GA1UEBxMGTG9uZG9uMQ0wCwYDVQQKEwRUZXN0MRUwEwYD
+VQQLEwxiYXNpYyBzZXJ2ZXIxDTALBgNVBAMTBFJPT1QwHhcNMDMwOTA2MTYyNDA4
+WhcNMzEwMTIyMTYyNDA4WjBkMQswCQYDVQQGEwJHQjEPMA0GA1UECBMGTG9uZG9u
+MQ8wDQYDVQQHEwZMb25kb24xDTALBgNVBAoTBFRlc3QxFTATBgNVBAsTDGJhc2lj
+IHNlcnZlcjENMAsGA1UEAxMEUk9PVDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
+gYEAzGFyfiCNApPYnK8A3hspnWdwIe0Tpgt9i6Ut7EFXIUHe+djuLYMk1D+neO6y
+3TNsbFY3UR3m/QA/g1a8wzUVq7T2MUDMoz4V8HkM/48MQMlUHcmBCFJHnGAL1g8K
+bfX+sxeSKXKurnZMbRNyRwp0d9RDltQnHLfqcoPCgYI95FMCAwEAATANBgkqhkiG
+9w0BAQUFAAOBgQDIAhGUvs47CQeKiF6GDxFfSseCk6UWB1lFe154ZpexgMTp8Dgu
+leJOvnZPmkywovIcxr2YZAM33e+3+rKDJEy9PJ9mGLsrZMHSi4v3U0e9bBDGCkKH
+1sSrbEGIc02HIo8m3PGUdrNJ8GNJdcYUghtoZbe01sIqVmWWLA8XXDQmOQ==
+-----END CERTIFICATE-----
diff --git a/test/basicserver/testfiles/key-creation.txt b/test/basicserver/testfiles/key-creation.txt
new file mode 100755
index 00000000..51f4eb77
--- /dev/null
+++ b/test/basicserver/testfiles/key-creation.txt
@@ -0,0 +1,83 @@
+$ openssl genrsa -out rootkey.pem 1024
+
+$ openssl req -new -key rootkey.pem -sha1 -out rootreq.pem
+You are about to be asked to enter information that will be incorporated
+into your certificate request.
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank
+For some fields there will be a default value,
+If you enter '.', the field will be left blank.
+-----
+Country Name (2 letter code) []:GB
+State or Province Name (full name) []:London
+Locality Name (eg, city) []:London
+Organization Name (eg, company) []:Test
+Organizational Unit Name (eg, section) []:basic server
+Common Name (eg, fully qualified host name) []:ROOT
+Email Address []:
+
+Please enter the following 'extra' attributes
+to be sent with your certificate request
+A challenge password []:
+An optional company name []:
+
+$ openssl x509 -req -in rootreq.pem -sha1 -extensions v3_ca -signkey rootkey.pem -out rootcert.pem -days 10000
+Signature ok
+subject=/C=GB/ST=London/L=London/O=Test/OU=basic server/CN=ROOT
+Getting Private key
+
+$ cp rootcert.pem serverTrustedCAs.pem
+$ cp rootcert.pem clientTrustedCAs.pem
+
+$ openssl genrsa -out clientPrivKey.pem 1024
+$ openssl req -new -key clientPrivKey.pem -sha1 -out clientReq.pem
+You are about to be asked to enter information that will be incorporated
+into your certificate request.
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank
+For some fields there will be a default value,
+If you enter '.', the field will be left blank.
+-----
+Country Name (2 letter code) []:GB
+State or Province Name (full name) []:London
+Locality Name (eg, city) []:London
+Organization Name (eg, company) []:Test
+Organizational Unit Name (eg, section) []:basic server
+Common Name (eg, fully qualified host name) []:CLIENT
+Email Address []:
+
+Please enter the following 'extra' attributes
+to be sent with your certificate request
+A challenge password []:
+An optional company name []:
+
+$ cat rootcert.pem rootkey.pem > root.pem
+
+$ echo 01 > root.srl
+
+$ openssl x509 -req -in clientReq.pem -sha1 -extensions usr_crt -CA root.pem -CAkey root.pem -out clientCerts.pem -days 10000
+
+$ openssl genrsa -out serverPrivKey.pem 1024
+$ openssl req -new -key serverPrivKey.pem -sha1 -out serverReq.pem
+You are about to be asked to enter information that will be incorporated
+into your certificate request.
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank
+For some fields there will be a default value,
+If you enter '.', the field will be left blank.
+-----
+Country Name (2 letter code) []:GB
+State or Province Name (full name) []:London
+Locality Name (eg, city) []:London
+Organization Name (eg, company) []:Test
+Organizational Unit Name (eg, section) []:basic server
+Common Name (eg, fully qualified host name) []:SERVER
+Email Address []:
+
+Please enter the following 'extra' attributes
+to be sent with your certificate request
+A challenge password []:
+An optional company name []:
+
+$ openssl x509 -req -in serverReq.pem -sha1 -extensions usr_crt -CA root.pem -CAkey root.pem -out serverCerts.pem -days 10000
+
diff --git a/test/basicserver/testfiles/root.pem b/test/basicserver/testfiles/root.pem
new file mode 100644
index 00000000..020c25c3
--- /dev/null
+++ b/test/basicserver/testfiles/root.pem
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIICNzCCAaACAQAwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCR0IxDzANBgNV
+BAgTBkxvbmRvbjEPMA0GA1UEBxMGTG9uZG9uMQ0wCwYDVQQKEwRUZXN0MRUwEwYD
+VQQLEwxiYXNpYyBzZXJ2ZXIxDTALBgNVBAMTBFJPT1QwHhcNMDMwOTA2MTYyNDA4
+WhcNMzEwMTIyMTYyNDA4WjBkMQswCQYDVQQGEwJHQjEPMA0GA1UECBMGTG9uZG9u
+MQ8wDQYDVQQHEwZMb25kb24xDTALBgNVBAoTBFRlc3QxFTATBgNVBAsTDGJhc2lj
+IHNlcnZlcjENMAsGA1UEAxMEUk9PVDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
+gYEAzGFyfiCNApPYnK8A3hspnWdwIe0Tpgt9i6Ut7EFXIUHe+djuLYMk1D+neO6y
+3TNsbFY3UR3m/QA/g1a8wzUVq7T2MUDMoz4V8HkM/48MQMlUHcmBCFJHnGAL1g8K
+bfX+sxeSKXKurnZMbRNyRwp0d9RDltQnHLfqcoPCgYI95FMCAwEAATANBgkqhkiG
+9w0BAQUFAAOBgQDIAhGUvs47CQeKiF6GDxFfSseCk6UWB1lFe154ZpexgMTp8Dgu
+leJOvnZPmkywovIcxr2YZAM33e+3+rKDJEy9PJ9mGLsrZMHSi4v3U0e9bBDGCkKH
+1sSrbEGIc02HIo8m3PGUdrNJ8GNJdcYUghtoZbe01sIqVmWWLA8XXDQmOQ==
+-----END CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDMYXJ+II0Ck9icrwDeGymdZ3Ah7ROmC32LpS3sQVchQd752O4t
+gyTUP6d47rLdM2xsVjdRHeb9AD+DVrzDNRWrtPYxQMyjPhXweQz/jwxAyVQdyYEI
+UkecYAvWDwpt9f6zF5Ipcq6udkxtE3JHCnR31EOW1Ccct+pyg8KBgj3kUwIDAQAB
+AoGAFsGO3u4+5ReTGbb+kLxTgNwghxZ/hpBm9SJ6H4ES83gDHKyDsHuWoS9JNVTW
+g3yTSOi8lgKPUoIxkC0bLVz+wYF0UWysOzhxbTqq43CdJM/HDuHbFGHs2MAKyvdm
+ai7ccJMISDATN6XT7BLRBE5AAVqDhNllvmr92niZS51yzJECQQD4LQWdK9IUjsja
+pYEeQKZENmC2pstAVYDyd3wuXaE8wiiTG86L/5zVRfEVpbD3rKPZVjcZKx+VZoIw
+iyW9WntbAkEA0tL2fSeBC1V9Jcj8TOuMmEaoPMclJLUBDLJPxFmHCguwvcH8cgTb
+Nr08FFqz62gZxudcrl5nISw3G0Rm3UGkaQJALRfhIUHJFjsre67+2wRcMaC/yfBc
+lf/zQhs70SDqHyQYQ0KWMRHs6UOgHpLQqPARhXgI4uXXA0pw9WkTHmjGaQJBAJ1x
+fTEkQmPjeS2xtnH/ayUBh3y0QJH0Nw9zTszVC1s+NcTQzSWdaNStZ+PPhRQlzzJS
+8E0sJRqJ+bF8WNGdxxkCQQCTpEUpqsVykhucZ3GsCTlI4o3HNmYFarKDDEHgppLS
+GKoUzTX2UMPJgeRITwacIh3lFhAily2PMFmlF+B7b5ep
+-----END RSA PRIVATE KEY-----
diff --git a/test/basicserver/testfiles/root.srl b/test/basicserver/testfiles/root.srl
new file mode 100644
index 00000000..2c7456e3
--- /dev/null
+++ b/test/basicserver/testfiles/root.srl
@@ -0,0 +1 @@
+07
diff --git a/test/basicserver/testfiles/rootcert.pem b/test/basicserver/testfiles/rootcert.pem
new file mode 100644
index 00000000..d72b70e5
--- /dev/null
+++ b/test/basicserver/testfiles/rootcert.pem
@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICNzCCAaACAQAwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCR0IxDzANBgNV
+BAgTBkxvbmRvbjEPMA0GA1UEBxMGTG9uZG9uMQ0wCwYDVQQKEwRUZXN0MRUwEwYD
+VQQLEwxiYXNpYyBzZXJ2ZXIxDTALBgNVBAMTBFJPT1QwHhcNMDMwOTA2MTYyNDA4
+WhcNMzEwMTIyMTYyNDA4WjBkMQswCQYDVQQGEwJHQjEPMA0GA1UECBMGTG9uZG9u
+MQ8wDQYDVQQHEwZMb25kb24xDTALBgNVBAoTBFRlc3QxFTATBgNVBAsTDGJhc2lj
+IHNlcnZlcjENMAsGA1UEAxMEUk9PVDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
+gYEAzGFyfiCNApPYnK8A3hspnWdwIe0Tpgt9i6Ut7EFXIUHe+djuLYMk1D+neO6y
+3TNsbFY3UR3m/QA/g1a8wzUVq7T2MUDMoz4V8HkM/48MQMlUHcmBCFJHnGAL1g8K
+bfX+sxeSKXKurnZMbRNyRwp0d9RDltQnHLfqcoPCgYI95FMCAwEAATANBgkqhkiG
+9w0BAQUFAAOBgQDIAhGUvs47CQeKiF6GDxFfSseCk6UWB1lFe154ZpexgMTp8Dgu
+leJOvnZPmkywovIcxr2YZAM33e+3+rKDJEy9PJ9mGLsrZMHSi4v3U0e9bBDGCkKH
+1sSrbEGIc02HIo8m3PGUdrNJ8GNJdcYUghtoZbe01sIqVmWWLA8XXDQmOQ==
+-----END CERTIFICATE-----
diff --git a/test/basicserver/testfiles/rootkey.pem b/test/basicserver/testfiles/rootkey.pem
new file mode 100644
index 00000000..4eb0f59d
--- /dev/null
+++ b/test/basicserver/testfiles/rootkey.pem
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDMYXJ+II0Ck9icrwDeGymdZ3Ah7ROmC32LpS3sQVchQd752O4t
+gyTUP6d47rLdM2xsVjdRHeb9AD+DVrzDNRWrtPYxQMyjPhXweQz/jwxAyVQdyYEI
+UkecYAvWDwpt9f6zF5Ipcq6udkxtE3JHCnR31EOW1Ccct+pyg8KBgj3kUwIDAQAB
+AoGAFsGO3u4+5ReTGbb+kLxTgNwghxZ/hpBm9SJ6H4ES83gDHKyDsHuWoS9JNVTW
+g3yTSOi8lgKPUoIxkC0bLVz+wYF0UWysOzhxbTqq43CdJM/HDuHbFGHs2MAKyvdm
+ai7ccJMISDATN6XT7BLRBE5AAVqDhNllvmr92niZS51yzJECQQD4LQWdK9IUjsja
+pYEeQKZENmC2pstAVYDyd3wuXaE8wiiTG86L/5zVRfEVpbD3rKPZVjcZKx+VZoIw
+iyW9WntbAkEA0tL2fSeBC1V9Jcj8TOuMmEaoPMclJLUBDLJPxFmHCguwvcH8cgTb
+Nr08FFqz62gZxudcrl5nISw3G0Rm3UGkaQJALRfhIUHJFjsre67+2wRcMaC/yfBc
+lf/zQhs70SDqHyQYQ0KWMRHs6UOgHpLQqPARhXgI4uXXA0pw9WkTHmjGaQJBAJ1x
+fTEkQmPjeS2xtnH/ayUBh3y0QJH0Nw9zTszVC1s+NcTQzSWdaNStZ+PPhRQlzzJS
+8E0sJRqJ+bF8WNGdxxkCQQCTpEUpqsVykhucZ3GsCTlI4o3HNmYFarKDDEHgppLS
+GKoUzTX2UMPJgeRITwacIh3lFhAily2PMFmlF+B7b5ep
+-----END RSA PRIVATE KEY-----
diff --git a/test/basicserver/testfiles/rootreq.pem b/test/basicserver/testfiles/rootreq.pem
new file mode 100644
index 00000000..6da1e428
--- /dev/null
+++ b/test/basicserver/testfiles/rootreq.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBpDCCAQ0CAQAwZDELMAkGA1UEBhMCR0IxDzANBgNVBAgTBkxvbmRvbjEPMA0G
+A1UEBxMGTG9uZG9uMQ0wCwYDVQQKEwRUZXN0MRUwEwYDVQQLEwxiYXNpYyBzZXJ2
+ZXIxDTALBgNVBAMTBFJPT1QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMxh
+cn4gjQKT2JyvAN4bKZ1ncCHtE6YLfYulLexBVyFB3vnY7i2DJNQ/p3just0zbGxW
+N1Ed5v0AP4NWvMM1Fau09jFAzKM+FfB5DP+PDEDJVB3JgQhSR5xgC9YPCm31/rMX
+kilyrq52TG0TckcKdHfUQ5bUJxy36nKDwoGCPeRTAgMBAAGgADANBgkqhkiG9w0B
+AQUFAAOBgQCmy4L/D/m1Q23y+WB1Ub2u1efl0sb7zMWNzHsD/IR1CXSvXmAfPpr2
+hpJQj118ccaTqkRhA8gwhktMTBuGH5KiOLHYXRlniKo3G0yr0+fHWnjclZ+m6Bg1
+9HjJZYqIWRMQ78+wTpLCxliX6yp0JxMdx/v6/7jx3BtXz8cyU8ANAw==
+-----END CERTIFICATE REQUEST-----
diff --git a/test/basicserver/testfiles/serverCerts.pem b/test/basicserver/testfiles/serverCerts.pem
new file mode 100644
index 00000000..f61c554e
--- /dev/null
+++ b/test/basicserver/testfiles/serverCerts.pem
@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICOTCCAaICAQYwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCR0IxDzANBgNV
+BAgTBkxvbmRvbjEPMA0GA1UEBxMGTG9uZG9uMQ0wCwYDVQQKEwRUZXN0MRUwEwYD
+VQQLEwxiYXNpYyBzZXJ2ZXIxDTALBgNVBAMTBFJPT1QwHhcNMDMwOTA2MTYyNTA0
+WhcNMzEwMTIyMTYyNTA0WjBmMQswCQYDVQQGEwJHQjEPMA0GA1UECBMGTG9uZG9u
+MQ8wDQYDVQQHEwZMb25kb24xDTALBgNVBAoTBFRlc3QxFTATBgNVBAsTDGJhc2lj
+IHNlcnZlcjEPMA0GA1UEAxMGU0VSVkVSMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
+iQKBgQDLR7tFaeNvCdvC5nQgfYggFHxZM5NcsxJSYcF27GhPylHE40XsmCEdHnDl
+AjWs48GrYN7tfTa7/JEFM9s7sgF9Oxj+tshMTNZvx25uih8gHFCg0RrYaQkgME2O
+mPuPtFcA/isTMCKO7D/aG2SapjY8/Xke0TseKO3jfP9LtxZz7QIDAQABMA0GCSqG
+SIb3DQEBBQUAA4GBALgh7u/7GZUMjzOPGuIenkdrsP0Gbst7wuXrLaMrAMlAaWMH
+E9AgU/6Q9+2yFxisgAzRmyKydNP4E4YomsE8rbx08vGw/6Rc7L19/UsFJxeNC5Ue
+6hziI9boB9LL5em4N8v+z4yhGvj2CrKzBxLNy8MYPi2S3KfQ69FdipvRQRp/
+-----END CERTIFICATE-----
diff --git a/test/basicserver/testfiles/serverPrivKey.pem b/test/basicserver/testfiles/serverPrivKey.pem
new file mode 100644
index 00000000..f2d73fd4
--- /dev/null
+++ b/test/basicserver/testfiles/serverPrivKey.pem
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDLR7tFaeNvCdvC5nQgfYggFHxZM5NcsxJSYcF27GhPylHE40Xs
+mCEdHnDlAjWs48GrYN7tfTa7/JEFM9s7sgF9Oxj+tshMTNZvx25uih8gHFCg0RrY
+aQkgME2OmPuPtFcA/isTMCKO7D/aG2SapjY8/Xke0TseKO3jfP9LtxZz7QIDAQAB
+AoGBAJSH7zAC9OmXXHoGhWeQEbzO+yT6aHxdY8/KGeBZUMasYB7qqZb8eYWbToYm
+nS2cpVAh0gHZcfrdyuDwSQpPQIIA8gAPFHqR8T8VGrpChxgetYzkoPDapmcqKU4H
+YobFVA1gypK1IM5z3Z5kargqGmmzRIxX8BwWr6FGmFPp2+NBAkEA7A17g4JewNtY
+vtpM0NhIyw+7HN3ljf+pAvHM2pMw1Wk8TrbPJNQ20ZWnhGMdIvP0m25zna6pShL6
+0laf5EUWFQJBANx1SJ+Xb3P9IyrIlyMhrsYvAveezh6wimjAFFNYWmGEZ6uuHM5P
+eBSc3P0x0LbFKlGQWomxMb3ULwpjEueX9HkCQDMf0GpxJ/h5CUV8njp1PX7NT2c3
+H+qbPo2mtQl564+tFSSvLzn4xE6sLPXdSYgycf3f9CZol721UqGPpV2ZIOkCQQCQ
+trxxZmrW7LgFAZ+UhCvCFGISQcB0DNcOY+fzve+2S7/xxl1KYIgmn8HAws6K62oY
+GHYWJKbOQVaPrvFd7TWhAkA8VQPjDSRkdg2fU5RDTRfOQBczgc8aHTiqAv/S2g47
+lpsw8CLitobBvi3e5XuBKNIbnjeoZMbHcBZ+RXAAZe/Q
+-----END RSA PRIVATE KEY-----
diff --git a/test/basicserver/testfiles/serverReq.pem b/test/basicserver/testfiles/serverReq.pem
new file mode 100644
index 00000000..ce510fae
--- /dev/null
+++ b/test/basicserver/testfiles/serverReq.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIBpjCCAQ8CAQAwZjELMAkGA1UEBhMCR0IxDzANBgNVBAgTBkxvbmRvbjEPMA0G
+A1UEBxMGTG9uZG9uMQ0wCwYDVQQKEwRUZXN0MRUwEwYDVQQLEwxiYXNpYyBzZXJ2
+ZXIxDzANBgNVBAMTBlNFUlZFUjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+y0e7RWnjbwnbwuZ0IH2IIBR8WTOTXLMSUmHBduxoT8pRxONF7JghHR5w5QI1rOPB
+q2De7X02u/yRBTPbO7IBfTsY/rbITEzWb8duboofIBxQoNEa2GkJIDBNjpj7j7RX
+AP4rEzAijuw/2htkmqY2PP15HtE7Hijt43z/S7cWc+0CAwEAAaAAMA0GCSqGSIb3
+DQEBBQUAA4GBAGdUCS76aBzPw4zcU999r6gE7/F8/bYlT/tr2SEyKzF+vC0widZN
+P3bg9IaNAWi84vw8WEB+j2wM3TPB5/kSKFpO2MxOHPERX+aOXh6JkN6a/ay5CDOT
+r/wCERRkqY2gphU5m3/S0Gd7wLbH/neBgNsHUzbNwwQ+uqkF2NRGg0V/
+-----END CERTIFICATE REQUEST-----
diff --git a/test/basicserver/testfiles/serverTrustedCAs.pem b/test/basicserver/testfiles/serverTrustedCAs.pem
new file mode 100644
index 00000000..d72b70e5
--- /dev/null
+++ b/test/basicserver/testfiles/serverTrustedCAs.pem
@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICNzCCAaACAQAwDQYJKoZIhvcNAQEFBQAwZDELMAkGA1UEBhMCR0IxDzANBgNV
+BAgTBkxvbmRvbjEPMA0GA1UEBxMGTG9uZG9uMQ0wCwYDVQQKEwRUZXN0MRUwEwYD
+VQQLEwxiYXNpYyBzZXJ2ZXIxDTALBgNVBAMTBFJPT1QwHhcNMDMwOTA2MTYyNDA4
+WhcNMzEwMTIyMTYyNDA4WjBkMQswCQYDVQQGEwJHQjEPMA0GA1UECBMGTG9uZG9u
+MQ8wDQYDVQQHEwZMb25kb24xDTALBgNVBAoTBFRlc3QxFTATBgNVBAsTDGJhc2lj
+IHNlcnZlcjENMAsGA1UEAxMEUk9PVDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
+gYEAzGFyfiCNApPYnK8A3hspnWdwIe0Tpgt9i6Ut7EFXIUHe+djuLYMk1D+neO6y
+3TNsbFY3UR3m/QA/g1a8wzUVq7T2MUDMoz4V8HkM/48MQMlUHcmBCFJHnGAL1g8K
+bfX+sxeSKXKurnZMbRNyRwp0d9RDltQnHLfqcoPCgYI95FMCAwEAATANBgkqhkiG
+9w0BAQUFAAOBgQDIAhGUvs47CQeKiF6GDxFfSseCk6UWB1lFe154ZpexgMTp8Dgu
+leJOvnZPmkywovIcxr2YZAM33e+3+rKDJEy9PJ9mGLsrZMHSi4v3U0e9bBDGCkKH
+1sSrbEGIc02HIo8m3PGUdrNJ8GNJdcYUghtoZbe01sIqVmWWLA8XXDQmOQ==
+-----END CERTIFICATE-----
diff --git a/test/basicserver/testfiles/srv1.conf b/test/basicserver/testfiles/srv1.conf
new file mode 100755
index 00000000..ee68704e
--- /dev/null
+++ b/test/basicserver/testfiles/srv1.conf
@@ -0,0 +1,6 @@
+Server
+{
+	PidFile = testfiles/srv1.pid
+}
+
+TestFile = testfiles/srv1.test1
diff --git a/test/basicserver/testfiles/srv1b.conf b/test/basicserver/testfiles/srv1b.conf
new file mode 100755
index 00000000..d6d6eebd
--- /dev/null
+++ b/test/basicserver/testfiles/srv1b.conf
@@ -0,0 +1,6 @@
+Server
+{
+	PidFile = testfiles/srv1.pid
+}
+
+TestFile = testfiles/srv1.test2
diff --git a/test/basicserver/testfiles/srv2.conf b/test/basicserver/testfiles/srv2.conf
new file mode 100755
index 00000000..ef1d7c49
--- /dev/null
+++ b/test/basicserver/testfiles/srv2.conf
@@ -0,0 +1,6 @@
+Server
+{
+	PidFile = testfiles/srv2.pid
+	ListenAddresses = inet:localhost,unix:testfiles/srv2.sock
+}
+
diff --git a/test/basicserver/testfiles/srv3.conf b/test/basicserver/testfiles/srv3.conf
new file mode 100755
index 00000000..e2211553
--- /dev/null
+++ b/test/basicserver/testfiles/srv3.conf
@@ -0,0 +1,9 @@
+Server
+{
+	PidFile = testfiles/srv3.pid
+	ListenAddresses = inet:localhost,unix:testfiles/srv3.sock
+	CertificateFile = testfiles/serverCerts.pem
+	PrivateKeyFile = testfiles/serverPrivKey.pem
+	TrustedCAsFile = testfiles/serverTrustedCAs.pem
+}
+
diff --git a/test/basicserver/testfiles/srv4.conf b/test/basicserver/testfiles/srv4.conf
new file mode 100755
index 00000000..b4c5627c
--- /dev/null
+++ b/test/basicserver/testfiles/srv4.conf
@@ -0,0 +1,6 @@
+Server
+{
+	PidFile = testfiles/srv4.pid
+	ListenAddresses = unix:testfiles/srv4.sock
+}
+
diff --git a/test/basicserver/testprotocol.txt b/test/basicserver/testprotocol.txt
new file mode 100755
index 00000000..5bca9f49
--- /dev/null
+++ b/test/basicserver/testprotocol.txt
@@ -0,0 +1,42 @@
+# test protocol file
+
+Name 			Test
+IdentString		Test-0.00
+ServerContextClass	TestContext	TestContext.h
+
+BEGIN_OBJECTS
+
+Error		0	IsError(Type,SubType)	Reply
+	int32	Type
+	int32	SubType
+
+Hello		1	Command(Hello)	Reply
+	int32	Number32
+	int16	Number16
+	int8	Number8
+	string	Text
+
+Lists		2	Command(ListsReply)
+	vector<string>	LotsOfText
+
+ListsReply	3	Reply
+	int32	NumberOfStrings
+
+Quit		4	Command(Quit)	Reply	EndsConversation
+
+Simple		5	Command(SimpleReply)
+	int32	Value
+
+SimpleReply	6	Reply
+	int32	ValuePlusOne
+
+GetStream	7	Command(GetStream)	Reply
+	int32	StartingValue
+	bool	UncertainSize
+
+SendStream	8	Command(GetStream)	StreamWithCommand
+	int64	Value
+
+String		9	Command(String)	Reply
+	string	Test
+
diff --git a/test/bbackupd/testbbackupd.cpp b/test/bbackupd/testbbackupd.cpp
new file mode 100755
index 00000000..acd13a10
--- /dev/null
+++ b/test/bbackupd/testbbackupd.cpp
@@ -0,0 +1,829 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testbbackupd.cpp
+//		Purpose: test backup daemon (and associated client bits)
+//		Created: 2003/10/07
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <limits.h>
+#include <string.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "Test.h"
+#include "BackupClientFileAttributes.h"
+#include "CommonException.h"
+#include "BackupStoreException.h"
+#include "FileModificationTime.h"
+#include "autogen_BackupProtocolClient.h"
+#include "SSLLib.h"
+#include "TLSContext.h"
+#include "SocketStreamTLS.h"
+#include "BoxPortsAndFiles.h"
+#include "BackupStoreConstants.h"
+#include "Socket.h"
+#include "BackupClientRestore.h"
+#include "BackupStoreDirectory.h"
+#include "BackupClientCryptoKeys.h"
+#include "CollectInBufferStream.h"
+#include "Utils.h"
+#include "BoxTime.h"
+#include "BoxTimeToUnix.h"
+
+#include "MemLeakFindOn.h"
+
+// two cycles and a bit
+#define TIME_TO_WAIT_FOR_BACKUP_OPERATION	12
+
+void wait_for_backup_operation(int seconds = TIME_TO_WAIT_FOR_BACKUP_OPERATION)
+{
+	printf("waiting: ");
+	fflush(stdout);
+	for(int l = 0; l < seconds; ++l)
+	{
+		sleep(1);
+		printf(".");
+		fflush(stdout);
+	}
+	printf("\n");
+}
+
+int bbstored_pid = 0;
+
+bool attrmatch(const char *f1, const char *f2)
+{
+	struct stat s1, s2;
+	TEST_THAT(::lstat(f1, &s1) == 0);
+	TEST_THAT(::lstat(f2, &s2) == 0);
+
+	// if link, just make sure other file is a link too, and that the link to names match
+	if((s1.st_mode & S_IFMT) == S_IFLNK)
+	{
+		if((s2.st_mode & S_IFMT) != S_IFLNK) return false;
+		
+		char p1[PATH_MAX], p2[PATH_MAX];
+		int p1l = ::readlink(f1, p1, PATH_MAX);
+		int p2l = ::readlink(f2, p2, PATH_MAX);
+		TEST_THAT(p1l != -1 && p2l != -1);
+		// terminate strings properly
+		p1[p1l] = '\0';
+		p2[p2l] = '\0';
+		return strcmp(p1, p2) == 0;
+	}
+
+	// modification times
+	if(FileModificationTime(s1) != FileModificationTime(s2))
+	{
+		return false;
+	}
+
+	// compare the rest
+	return (s1.st_mode == s2.st_mode && s1.st_uid == s2.st_uid && s1.st_gid == s2.st_gid);
+}
+
+int test_basics()
+{
+	// Read attributes from files
+	BackupClientFileAttributes t1;
+	t1.ReadAttributes("testfiles/test1");
+	TEST_THAT(!t1.IsSymLink());
+	BackupClientFileAttributes t2;
+	t2.ReadAttributes("testfiles/test2");
+	TEST_THAT(t2.IsSymLink());
+	// Check that it's actually been encrypted (search for symlink name encoded in it)
+	void *te = ::memchr(t2.GetBuffer(), 't', t2.GetSize() - 3);
+	TEST_THAT(te == 0 || ::memcmp(te, "test", 4) != 0);
+	
+	BackupClientFileAttributes t3;
+	TEST_CHECK_THROWS(t3.ReadAttributes("doesn't exist"), CommonException, OSFileError);
+
+	// Create some more files
+	FILE *f = fopen("testfiles/test1_n", "w");
+	fclose(f);
+	f = fopen("testfiles/test2_n", "w");
+	fclose(f);
+	
+	// Apply attributes to these new files
+	t1.WriteAttributes("testfiles/test1_n");
+	t2.WriteAttributes("testfiles/test2_n");
+	TEST_CHECK_THROWS(t1.WriteAttributes("testfiles/test1_nXX"), CommonException, OSFileError);
+	TEST_CHECK_THROWS(t3.WriteAttributes("doesn't exist"), BackupStoreException, AttributesNotLoaded);
+
+	// Test that atttributes are vaguely similar
+	TEST_THAT(attrmatch("testfiles/test1", "testfiles/test1_n"));
+	TEST_THAT(attrmatch("testfiles/test2", "testfiles/test2_n"));
+	
+	// Check encryption, and recovery from encryption
+	// First, check that two attributes taken from the same thing have different encrypted values (think IV)
+	BackupClientFileAttributes t1b;
+	t1b.ReadAttributes("testfiles/test1");
+	TEST_THAT(::memcmp(t1.GetBuffer(), t1b.GetBuffer(), t1.GetSize()) != 0);
+	// But that comparing them works OK.
+	TEST_THAT(t1 == t1b);
+	// Then store them both to a stream
+	CollectInBufferStream stream;
+	t1.WriteToStream(stream);
+	t1b.WriteToStream(stream);
+	// Read them back again
+	stream.SetForReading();
+	BackupClientFileAttributes t1_r, t1b_r;
+	t1_r.ReadFromStream(stream, 1000);
+	t1b_r.ReadFromStream(stream, 1000);
+	TEST_THAT(::memcmp(t1_r.GetBuffer(), t1b_r.GetBuffer(), t1_r.GetSize()) != 0);
+	TEST_THAT(t1_r == t1b_r);
+	TEST_THAT(t1 == t1_r);
+	TEST_THAT(t1b == t1b_r);
+	TEST_THAT(t1_r == t1b);
+	TEST_THAT(t1b_r == t1);
+
+	return 0;
+}
+
+int test_setupaccount()
+{
+	TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf create 01234567 0 1000B 2000B") == 0);
+	TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+	return 0;
+}
+
+int test_run_bbstored()
+{
+	bbstored_pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid");
+	TEST_THAT(bbstored_pid != -1 && bbstored_pid != 0);
+	if(bbstored_pid > 0)
+	{
+		::sleep(1);
+		TEST_THAT(ServerIsAlive(bbstored_pid));
+		return 0;	// success
+	}
+	
+	return 1;
+}
+
+int test_kill_bbstored()
+{
+	TEST_THAT(KillServer(bbstored_pid));
+	::sleep(1);
+	TEST_THAT(!ServerIsAlive(bbstored_pid));
+	TestRemoteProcessMemLeaks("bbstored.memleaks");
+	return 0;
+}
+
+int64_t GetDirID(BackupProtocolClient &protocol, const char *name, int64_t InDirectory)
+{
+	protocol.QueryListDirectory(
+			InDirectory,
+			BackupProtocolClientListDirectory::Flags_Dir,
+			BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING,
+			true /* want attributes */);
+	
+	// Retrieve the directory from the stream following
+	BackupStoreDirectory dir;
+	std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream());
+	dir.ReadFromStream(*dirstream, protocol.GetTimeout());
+	
+	BackupStoreDirectory::Iterator i(dir);
+	BackupStoreDirectory::Entry *en = 0;
+	int64_t dirid = 0;
+	BackupStoreFilenameClear dirname(name);
+	while((en = i.Next()) != 0)
+	{
+		if(en->GetName() == dirname)
+		{
+			dirid = en->GetObjectID();
+		}
+	}
+	return dirid;
+}
+
+void terminate_on_alarm(int sigraised)
+{
+	abort();
+}
+
+void do_interrupted_restore(const TLSContext &context, int64_t restoredirid)
+{
+	int pid = 0;
+	switch((pid = fork()))
+	{
+	case 0:
+		// child process
+		{
+			// connect and log in
+			SocketStreamTLS conn;
+			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+			BackupProtocolClient protocol(conn);
+			protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION);
+			std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, BackupProtocolClientLogin::Flags_ReadOnly));
+			
+			// Test the restoration
+			TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-interrupt", true /* print progress dots */) == Restore_Complete);
+
+			// Log out
+			protocol.QueryFinished();
+		}
+		exit(0);
+		break;
+	
+	case -1:
+		{
+			printf("Fork failed\n");
+			exit(1);
+		}
+	
+	default:
+		{
+			// Wait until a resume file is written, then terminate the child
+			while(true)
+			{
+				// Test for existence of the result file
+				int64_t resumesize = 0;
+				if(FileExists("testfiles/restore-interrupt.boxbackupresume", &resumesize) && resumesize > 16)
+				{
+					// It's done something. Terminate it.	
+					::kill(pid, SIGTERM);
+					break;
+				}
+
+				// Process finished?
+				int status = 0;
+				if(waitpid(pid, &status, WNOHANG) != 0)
+				{
+					// child has finished anyway.
+					return;
+				}
+				
+				// Give up timeslot so as not to hog the processor
+				::sleep(0);
+			}
+			
+			// Just wait until the child has completed
+			int status = 0;
+			waitpid(pid, &status, 0);
+		}
+	}
+}
+
+
+int test_bbackupd()
+{
+//	// First, wait for a normal period to make sure the last changes attributes are within a normal backup timeframe.
+//	wait_for_backup_operation();
+
+	// Connection gubbins
+	TLSContext context;
+	context.Initialise(false /* client */,
+			"testfiles/clientCerts.pem",
+			"testfiles/clientPrivKey.pem",
+			"testfiles/clientTrustedCAs.pem");
+
+	// unpack the files for the initial test
+	TEST_THAT(::system("rm -rf testfiles/TestDir1") == 0);
+	TEST_THAT(::system("mkdir testfiles/TestDir1") == 0);
+	TEST_THAT(::system("tar xzf testfiles/spacetest1.tgz -C testfiles/TestDir1/") == 0);
+
+	int pid = LaunchServer("../../bin/bbackupd/bbackupd testfiles/bbackupd.conf", "testfiles/bbackupd.pid");
+	TEST_THAT(pid != -1 && pid != 0);
+	if(pid > 0)
+	{
+		::sleep(1);
+		TEST_THAT(ServerIsAlive(pid));
+
+		// First, check storage space handling -- wait for file to be uploaded
+		wait_for_backup_operation();
+		//TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf info 01234567") == 0);
+		// Set limit to something very small
+		// About 28 blocks will be used at this point. bbackupd will only pause if the size used is
+		// greater than soft limit + 1/3 of (hard - soft). Set small values for limits accordingly.
+		TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf setlimit 01234567 10B 40B") == 0);
+		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+
+		// Unpack some more files
+		TEST_THAT(::system("tar xzf testfiles/spacetest2.tgz -C testfiles/TestDir1/") == 0);
+		// Delete a file and a directory
+		TEST_THAT(::unlink("testfiles/TestDir1/spacetest/d1/f3") == 0);
+		TEST_THAT(::system("rm -rf testfiles/TestDir1/spacetest/d3/d4") == 0);
+		wait_for_backup_operation();
+
+		// Make sure there are some differences
+		int compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query0a.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 2*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+		// Put the limit back
+		TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf setlimit 01234567 1000B 2000B") == 0);
+		TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks");
+		
+		// Check that the notify script was run
+		TEST_THAT(TestFileExists("testfiles/notifyran.store-full.1"));
+		// But only once!
+		TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2"));
+		
+		// unpack the initial files again
+		TEST_THAT(::system("tar xzf testfiles/test_base.tgz -C testfiles/") == 0);
+
+		// wait for it to do it's stuff
+		wait_for_backup_operation();
+		
+		// Check that the contents of the store are the same as the contents
+		// of the disc (-a = all, -c = give result in return code)
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query1.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		
+		printf("Delete file and update another, create symlink.\n");
+		
+		// Delete a file
+		TEST_THAT(::unlink("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0);
+		// New symlink
+		TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/symlink-to-dir") == 0);
+		
+		// Update a file (will be uploaded as a diff)
+		{
+			// Check that the file is over the diffing threshold in the bbstored.conf file
+			TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") > 1024);
+			
+			// Add a bit to the end
+			FILE *f = ::fopen("testfiles/TestDir1/f45.df", "a");
+			TEST_THAT(f != 0);
+			::fprintf(f, "EXTRA STUFF");
+			::fclose(f);
+			TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") > 1024);
+		}
+	
+		// wait for backup daemon to do it's stuff, and compare again
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query2.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		// Try a quick compare, just for fun
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query2q.log \"compare -acq\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		
+		// Bad case: delete a file/symlink, replace it with a directory
+		printf("Replace symlink with directory, add new directory\n");
+		TEST_THAT(::unlink("testfiles/TestDir1/symlink-to-dir") == 0);
+		TEST_THAT(::mkdir("testfiles/TestDir1/symlink-to-dir", 0755) == 0);
+		TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", 0755) == 0);
+		// NOTE: create a file within the directory to avoid deletion by the housekeeping process later
+		TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file/contents") == 0);
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");		
+
+		// And the inverse, replace a directory with a file/symlink
+		printf("Replace directory with symlink\n");
+		TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file/contents") == 0);
+		TEST_THAT(::rmdir("testfiles/TestDir1/x1/dir-to-file") == 0);
+		TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file") == 0);
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		
+		// And then, put it back to how it was before.
+		printf("Replace symlink with directory (which was a symlink)\n");
+		TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file") == 0);
+		TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", 0755) == 0);
+		TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file/contents2") == 0);
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		
+		// And finally, put it back to how it was before it was put back to how it was before
+		// This gets lots of nasty things in the store with directories over other old directories.
+		printf("Put it all back to how it was\n");
+		TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file/contents2") == 0);
+		TEST_THAT(::rmdir("testfiles/TestDir1/x1/dir-to-file") == 0);
+		TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file") == 0);
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+		// case which went wrong: rename a tracked file over a deleted file
+		printf("Rename an existing file over a deleted file\n");
+		TEST_THAT(::rename("testfiles/TestDir1/df9834.dsf", "testfiles/TestDir1/x1/dsfdsfs98.fd") == 0);
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		
+		printf("Add files with old times, update attributes of one to latest time\n");
+
+		// Move that file back
+		TEST_THAT(::rename("testfiles/TestDir1/x1/dsfdsfs98.fd", "testfiles/TestDir1/df9834.dsf") == 0);
+		
+		// Add some more files
+		// Because the 'm' option is not used, these files will look very old to the daemon.
+		// Lucky it'll upload them then!
+		TEST_THAT(::system("tar xzf testfiles/test2.tgz -C testfiles/") == 0);
+		::chmod("testfiles/TestDir1/sub23/dhsfdss/blf.h", 0415);
+		
+		// Wait and test
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		
+		// Check that modifying files with old timestamps still get added
+		printf("Modify existing file, but change timestamp to rather old\n");
+		// Time critical, so sync
+		TEST_THAT(::system("../../bin/bbackupctl/bbackupctl -q -c testfiles/bbackupd.conf wait-for-sync") == 0);
+		TestRemoteProcessMemLeaks("bbackupctl.memleaks");
+		// Then wait a second, to make sure the scan is complete
+		::sleep(1);
+		// Then modify an existing file
+		{
+			chmod("testfiles/TestDir1/sub23/rand.h", 0777);	// in the archive, it's read only
+			FILE *f = fopen("testfiles/TestDir1/sub23/rand.h", "w+");
+			TEST_THAT(f != 0);
+			fprintf(f, "MODIFIED!\n");
+			fclose(f);
+			// and then move the time backwards!
+			struct timeval times[2];
+			BoxTimeToTimeval(SecondsToBoxTime((uint32_t)(365*24*60*60)), times[1]);
+			times[0] = times[1];
+			TEST_THAT(::utimes("testfiles/TestDir1/sub23/rand.h", times) == 0);
+		}
+		// Wait and test
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3e.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+		// Add some files and directories which are marked as excluded
+		printf("Add files and dirs for exclusion test\n");
+		TEST_THAT(::system("tar xzf testfiles/testexclude.tgz -C testfiles/") == 0);
+		// Wait and test
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3c.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3d.log \"compare -acE\" quit");
+		TEST_THAT(compareReturnValue == 2*256);	// should find differences
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+		// These tests only work as non-root users.
+		if(::getuid() != 0)
+		{
+			// Check that read errors are reported neatly
+			printf("Add unreadable files\n");
+			{
+				// Dir and file which can't be read
+				TEST_THAT(::mkdir("testfiles/TestDir1/sub23/read-fail-test-dir", 0000) == 0);
+				int fd = ::open("testfiles/TestDir1/read-fail-test-file", O_CREAT | O_WRONLY, 0000);
+				TEST_THAT(fd != -1);
+				::close(fd);
+			}
+			// Wait and test...
+			wait_for_backup_operation();
+			compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3e.log \"compare -ac\" quit");
+			TEST_THAT(compareReturnValue == 2*256);	// should find differences
+			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+			// Check that it was reported correctly
+			TEST_THAT(TestFileExists("testfiles/notifyran.read-error.1"));
+			TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.2"));
+			// Set permissions on file and dir to stop errors in the future
+			::chmod("testfiles/TestDir1/sub23/read-fail-test-dir", 0770);
+			::chmod("testfiles/TestDir1/read-fail-test-file", 0770);
+		}
+
+		printf("Continuously update file, check isn't uploaded\n");
+		
+		// Make sure everything happens at the same point in the sync cycle: wait until exactly the start of a sync
+		TEST_THAT(::system("../../bin/bbackupctl/bbackupctl -c testfiles/bbackupd.conf wait-for-sync") == 0);
+		TestRemoteProcessMemLeaks("bbackupctl.memleaks");
+		// Then wait a second, to make sure the scan is complete
+		::sleep(1);
+
+		{
+			// Open a file, then save something to it every second
+			for(int l = 0; l < 12; ++l)
+			{
+				FILE *f = ::fopen("testfiles/TestDir1/continousupdate", "w+");
+				TEST_THAT(f != 0);
+				fprintf(f, "Loop iteration %d\n", l);
+				fflush(f);
+				sleep(1);
+				printf(".");
+				fflush(stdout);
+				::fclose(f);
+			}
+			printf("\n");
+			
+			// Check there's a difference
+			compareReturnValue = ::system("testfiles/extcheck1.pl");
+			TEST_THAT(compareReturnValue == 1*256);
+			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+			printf("Keep on continuously updating file, check it is uploaded eventually\n");
+
+			for(int l = 0; l < 18; ++l)
+			{
+				FILE *f = ::fopen("testfiles/TestDir1/continousupdate", "w+");
+				TEST_THAT(f != 0);
+				fprintf(f, "Loop 2 iteration %d\n", l);
+				fflush(f);
+				sleep(1);
+				printf(".");
+				fflush(stdout);
+				::fclose(f);
+			}
+			printf("\n");
+
+			compareReturnValue = ::system("testfiles/extcheck2.pl");
+			TEST_THAT(compareReturnValue == 1*256);
+			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		}
+		
+		printf("Delete directory, change attributes\n");
+	
+		// Delete a directory
+		TEST_THAT(::system("rm -rf testfiles/TestDir1/x1") == 0);
+		// Change attributes on an original file.
+		::chmod("testfiles/TestDir1/df9834.dsf", 0423);
+		
+		// Wait and test
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+	
+		printf("Restore files and directories\n");
+		int64_t deldirid = 0;
+		int64_t restoredirid = 0;
+		{
+			// connect and log in
+			SocketStreamTLS conn;
+			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+			BackupProtocolClient protocol(conn);
+			protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION);
+			std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, BackupProtocolClientLogin::Flags_ReadOnly));
+
+			// Find the ID of the Test1 directory
+			restoredirid = GetDirID(protocol, "Test1", BackupProtocolClientListDirectory::RootDirectory);
+			TEST_THAT(restoredirid != 0);
+
+			// Test the restoration
+			TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-Test1", true /* print progress dots */) == Restore_Complete);
+
+			// Compare it
+			compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query10.log \"compare -cE Test1 testfiles/restore-Test1\" quit");
+			TEST_THAT(compareReturnValue == 1*256);
+			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+			// Make sure you can't restore a restored directory
+			TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-Test1", true /* print progress dots */) == Restore_TargetExists);
+			
+			// Find ID of the deleted directory
+			deldirid = GetDirID(protocol, "x1", restoredirid);
+			TEST_THAT(deldirid != 0);
+
+			// Just check it doesn't bomb out -- will check this properly later (when bbackupd is stopped)
+			TEST_THAT(BackupClientRestore(protocol, deldirid, "testfiles/restore-Test1-x1", true /* print progress dots */, true /* deleted files */) == Restore_Complete);
+
+			// Log out
+			protocol.QueryFinished();
+		}
+
+		printf("Add files with current time\n");
+	
+		// Add some more files and modify others
+		// Use the m flag this time so they have a recent modification time
+		TEST_THAT(::system("tar xzmf testfiles/test3.tgz -C testfiles/") == 0);
+		
+		// Wait and test
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query5.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		
+		// Rename directory
+		printf("Rename directory\n");
+		TEST_THAT(rename("testfiles/TestDir1/sub23/dhsfdss", "testfiles/TestDir1/renamed-dir") == 0);
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query6.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		// and again, but with quick flag
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query6q.log \"compare -acq\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+		// Rename some files -- one under the threshold, others above
+		printf("Rename files\n");
+		TEST_THAT(rename("testfiles/TestDir1/continousupdate", "testfiles/TestDir1/continousupdate-ren") == 0);
+		TEST_THAT(rename("testfiles/TestDir1/df324", "testfiles/TestDir1/df324-ren") == 0);
+		TEST_THAT(rename("testfiles/TestDir1/sub23/find2perl", "testfiles/TestDir1/find2perl-ren") == 0);
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query6.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+		// Check that modifying files with madly in the future timestamps still get added
+		printf("Create a file with timestamp to way ahead in the future\n");
+		// Time critical, so sync
+		TEST_THAT(::system("../../bin/bbackupctl/bbackupctl -q -c testfiles/bbackupd.conf wait-for-sync") == 0);
+		TestRemoteProcessMemLeaks("bbackupctl.memleaks");
+		// Then wait a second, to make sure the scan is complete
+		::sleep(1);
+		// Then modify an existing file
+		{
+			FILE *f = fopen("testfiles/TestDir1/sub23/in-the-future", "w");
+			TEST_THAT(f != 0);
+			fprintf(f, "Back to the future!\n");
+			fclose(f);
+			// and then move the time forwards!
+			struct timeval times[2];
+			BoxTimeToTimeval(GetCurrentBoxTime() + SecondsToBoxTime((uint32_t)(365*24*60*60)), times[1]);
+			times[0] = times[1];
+			TEST_THAT(::utimes("testfiles/TestDir1/sub23/in-the-future", times) == 0);
+		}
+		// Wait and test
+		wait_for_backup_operation();
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3e.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 1*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+		printf("Change client store marker\n");
+
+		// Then... connect to the server, and change the client store marker. See what that does!
+		{
+			bool done = false;
+			int tries = 4;
+			while(!done && tries > 0)
+			{
+				try
+				{
+					SocketStreamTLS conn;
+					conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+					BackupProtocolClient protocol(conn);
+					protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION);
+					std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));	// read-write
+					// Make sure the marker isn't zero, because that's the default, and it should have changed
+					TEST_THAT(loginConf->GetClientStoreMarker() != 0);
+					
+					// Change it to something else
+					protocol.QuerySetClientStoreMarker(12);
+					
+					// Success!
+					done = true;
+					
+					// Log out
+					protocol.QueryFinished();
+				}
+				catch(...)
+				{
+					tries--;
+				}
+			}
+			TEST_THAT(done);
+		}
+		
+		printf("Check change of store marker pauses daemon\n");
+		
+		// Make a change to a file, to detect whether or not it's hanging around
+		// waiting to retry.
+		{
+			FILE *f = ::fopen("testfiles/TestDir1/fileaftermarker", "w");
+			TEST_THAT(f != 0);
+			::fprintf(f, "Lovely file you got there.");
+			::fclose(f);
+		}
+
+		// Wait and test that there *are* differences
+		wait_for_backup_operation((TIME_TO_WAIT_FOR_BACKUP_OPERATION*3) / 2); // little bit longer than usual
+		compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query6.log \"compare -ac\" quit");
+		TEST_THAT(compareReturnValue == 2*256);
+		TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		
+		printf("Interrupted restore\n");
+		{
+			do_interrupted_restore(context, restoredirid);
+			int64_t resumesize = 0;
+			TEST_THAT(FileExists("testfiles/restore-interrupt.boxbackupresume", &resumesize));
+			TEST_THAT(resumesize > 16);	// make sure it has recorded something to resume
+
+			printf("\nResume restore\n");
+
+			SocketStreamTLS conn;
+			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+			BackupProtocolClient protocol(conn);
+			protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION);
+			std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));	// read-write
+
+			// Check that the restore fn returns resume possible, rather than doing anything
+			TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-interrupt", true /* print progress dots */) == Restore_ResumePossible);
+
+			// Then resume it
+			TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-interrupt", true /* print progress dots */, false /* deleted files */, false /* undelete server */, true /* resume */) == Restore_Complete);
+
+			protocol.QueryFinished();
+
+			// Then check it has restored the correct stuff
+			compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query14.log \"compare -cE Test1 testfiles/restore-interrupt\" quit");
+			TEST_THAT(compareReturnValue == 1*256);
+			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		}
+
+		printf("Check restore deleted files\n");
+		{
+			SocketStreamTLS conn;
+			conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED);
+			BackupProtocolClient protocol(conn);
+			protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION);
+			std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));	// read-write
+
+			// Do restore and undelete
+			TEST_THAT(BackupClientRestore(protocol, deldirid, "testfiles/restore-Test1-x1-2", true /* print progress dots */, true /* deleted files */, true /* undelete on server */) == Restore_Complete);
+
+			protocol.QueryFinished();
+
+			// Do a compare with the now undeleted files
+			compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query11.log \"compare -cE Test1/x1 testfiles/restore-Test1-x1-2\" quit");
+			TEST_THAT(compareReturnValue == 1*256);
+			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+		}
+		
+		// Final check on notifications
+		TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2"));
+		TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.2"));
+
+		// Kill the daemon
+		TEST_THAT(KillServer(pid));
+		::sleep(1);
+		TEST_THAT(!ServerIsAlive(pid));
+		TestRemoteProcessMemLeaks("bbackupd.memleaks");
+		
+		// Start it again
+		pid = LaunchServer("../../bin/bbackupd/bbackupd testfiles/bbackupd.conf", "testfiles/bbackupd.pid");
+		TEST_THAT(pid != -1 && pid != 0);
+		if(pid != -1 && pid != 0)
+		{
+			// Wait and comapre
+			wait_for_backup_operation((TIME_TO_WAIT_FOR_BACKUP_OPERATION*3) / 2); // little bit longer than usual
+			compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac\" quit");
+			TEST_THAT(compareReturnValue == 1*256);
+			TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+
+			// Kill it again
+			TEST_THAT(KillServer(pid));
+			::sleep(1);
+			TEST_THAT(!ServerIsAlive(pid));
+			TestRemoteProcessMemLeaks("bbackupd.memleaks");
+		}
+	}
+
+	// List the files on the server
+	::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/queryLIST.log \"list -rotdh\" quit");
+	TestRemoteProcessMemLeaks("bbackupquery.memleaks");
+	
+	if(::getuid() == 0)
+	{
+		::printf("WARNING: This test was run as root. Some tests have been omitted.\n");
+	}
+	
+	return 0;
+}
+
+int test(int argc, const char *argv[])
+{
+	// SSL library
+	SSLLib::Initialise();
+
+	// Keys for subsystems
+	BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys");
+
+	// Initial files
+	TEST_THAT(::system("tar xzf testfiles/test_base.tgz -C testfiles/") == 0);
+
+	// Do the tests
+
+	int r = test_basics();
+	if(r != 0) return r;
+	
+	r = test_setupaccount();
+	if(r != 0) return r;
+
+	r = test_run_bbstored();
+	if(r != 0) return r;
+	
+	r = test_bbackupd();
+	if(r != 0) return r;
+	
+	test_kill_bbstored();
+
+	return 0;
+}
+
diff --git a/test/bbackupd/testextra b/test/bbackupd/testextra
new file mode 100755
index 00000000..798c8c67
--- /dev/null
+++ b/test/bbackupd/testextra
@@ -0,0 +1,4 @@
+mkdir testfiles/0_0
+mkdir testfiles/0_1
+mkdir testfiles/0_2
+mkdir testfiles/bbackupd-data
diff --git a/test/bbackupd/testfiles/accounts.txt b/test/bbackupd/testfiles/accounts.txt
new file mode 100755
index 00000000..e69de29b
--- /dev/null
+++ b/test/bbackupd/testfiles/accounts.txt
diff --git a/test/bbackupd/testfiles/bbackupd.conf b/test/bbackupd/testfiles/bbackupd.conf
new file mode 100755
index 00000000..c25b4eab
--- /dev/null
+++ b/test/bbackupd/testfiles/bbackupd.conf
@@ -0,0 +1,50 @@
+
+CertificateFile = testfiles/clientCerts.pem
+PrivateKeyFile = testfiles/clientPrivKey.pem
+TrustedCAsFile = testfiles/clientTrustedCAs.pem
+
+KeysFile = testfiles/bbackupd.keys
+
+DataDirectory = testfiles/bbackupd-data
+
+StoreHostname = localhost
+AccountNumber = 0x01234567
+
+UpdateStoreInterval = 3
+MinimumFileAge = 4
+MaxUploadWait = 24
+
+FileTrackingSizeThreshold = 1024
+DiffingUploadSizeThreshold = 1024
+
+MaximumDiffingTime = 8
+
+ExtendedLogging = yes
+
+CommandSocket = testfiles/bbackupd.sock
+
+NotifyScript = perl testfiles/notifyscript.pl
+
+Server
+{
+	PidFile = testfiles/bbackupd.pid
+}
+
+BackupLocations
+{
+	Test1
+	{
+		Path = testfiles/TestDir1
+
+		ExcludeFile = testfiles/TestDir1/excluded_1
+		ExcludeFile = testfiles/TestDir1/excluded_2
+		ExcludeFilesRegex = \.excludethis$
+		ExcludeFilesRegex = EXCLUDE
+		AlwaysIncludeFile = testfiles/TestDir1/dont.excludethis
+		ExcludeDir = testfiles/TestDir1/exclude_dir
+		ExcludeDir = testfiles/TestDir1/exclude_dir_2
+		ExcludeDirsRegex = not_this_dir
+		AlwaysIncludeDirsRegex = ALWAYSINCLUDE
+	}
+}
+
diff --git a/test/bbackupd/testfiles/bbackupd.keys b/test/bbackupd/testfiles/bbackupd.keys
new file mode 100644
index 00000000..d9135b97
--- /dev/null
+++ b/test/bbackupd/testfiles/bbackupd.keys
diff --git a/test/bbackupd/testfiles/bbstored.conf b/test/bbackupd/testfiles/bbstored.conf
new file mode 100755
index 00000000..18c73a40
--- /dev/null
+++ b/test/bbackupd/testfiles/bbstored.conf
@@ -0,0 +1,17 @@
+
+RaidFileConf = testfiles/raidfile.conf
+AccountDatabase = testfiles/accounts.txt
+
+ExtendedLogging = no
+
+TimeBetweenHousekeeping = 5
+
+Server
+{
+	PidFile = testfiles/bbstored.pid
+	ListenAddresses = inet:localhost
+	CertificateFile = testfiles/serverCerts.pem
+	PrivateKeyFile = testfiles/serverPrivKey.pem
+	TrustedCAsFile = testfiles/serverTrustedCAs.pem
+}
+
diff --git a/test/bbackupd/testfiles/clientCerts.pem b/test/bbackupd/testfiles/clientCerts.pem
new file mode 100755
index 00000000..c1f14fa7
--- /dev/null
+++ b/test/bbackupd/testfiles/clientCerts.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBmDCCAQECAQMwDQYJKoZIhvcNAQEFBQAwDzENMAsGA1UEAxMEUk9PVDAeFw0w
+MzEwMDcwOTAwMDRaFw0zMTAyMjIwOTAwMDRaMBoxGDAWBgNVBAMTD0JBQ0tVUC0w
+MTIzNDU2NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvptM6A++ZdkxYN92
+OI6d0O32giRdybSdUVNJk09V1pdVJFhXr4owhtVv6d8yDnPaNOgS1LlxZ9CHcR5A
+LtFwI9wmGHBc5a2uFCZGORTaSggntCythvRV3DGm/fU7mRME7Le1/tWWxjycnk2k
+Rez6d7Ffj56SXDFoxY2dK8MwRasCAwEAATANBgkqhkiG9w0BAQUFAAOBgQB4D3LU
+knCM4UZHMJhlbGnvc+N4O5SGrNKrHs94juMF8dPXJNgboBflkYJKNx1qDf47C/Cx
+hxXjju2ucGHytNQ8kiWsz7vCzeS7Egkl0QhFcBcYVCeXNn7zc34aAUyVlLCuas2o
+EGpfF4se7D3abg7J/3ioW0hx8bSal7kROleKCQ==
+-----END CERTIFICATE-----
diff --git a/test/bbackupd/testfiles/clientPrivKey.pem b/test/bbackupd/testfiles/clientPrivKey.pem
new file mode 100755
index 00000000..34b1af2a
--- /dev/null
+++ b/test/bbackupd/testfiles/clientPrivKey.pem
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXAIBAAKBgQC+m0zoD75l2TFg33Y4jp3Q7faCJF3JtJ1RU0mTT1XWl1UkWFev
+ijCG1W/p3zIOc9o06BLUuXFn0IdxHkAu0XAj3CYYcFzlra4UJkY5FNpKCCe0LK2G
+9FXcMab99TuZEwTst7X+1ZbGPJyeTaRF7Pp3sV+PnpJcMWjFjZ0rwzBFqwIDAQAB
+AoGAMW8Lqh/zLG0A/nPWMGLkkTw2M5iE7nw2VNI6AceQpqAHB+8VhsRbQ4z1gn1N
+eSwYyqHpyFv0Co2touvKj5nn8CJfMmm571cvdOlD/n/mQsW+xZqd9WmvSE8Jh4Qq
+iOQqwbwJlTYTV4BEo90qtfR+MDqffSCB8bHh4l3oO3fSp4kCQQDgbllQeq2kwlLp
+81oDfrk+J7vpiq9hZ/HxFY1fZAOa6iylazZz0JSzvNAtQNLI1LeKAzBc8FuPPSG9
+qSHAKoDHAkEA2Wrziib5OgY/G86yAWVn2hPM7Ky6wGtsJxYnObXUiTwVM7lM1nZU
+LpQaq//vzVDcWggqyEBTYkVcdEPYIJn3/QJBAL3e/bblowRx1p3Q4MV2L5gTG5pQ
+V2HsA7c3yZv7TEWCenUUSEQhIb0SL3kpj2qS9BhR7FekjYGYcXQ4o7IlAz8CQD1B
+BJxHnq/aUq1i7oO2Liwip/mGMJdFrJLWivaXY+nGI7MO4bcKX21ADMOot8cAoRQ8
+eNEyTkvBfurCsoF834ECQCPejz6x1bh/H7SeeANP17HKlwx1Lshw2JzxfF96MA26
+Eige4f0ttKHhMY/bnMcOzfPUSe/LvIN3AiMtphkl0pw=
+-----END RSA PRIVATE KEY-----
diff --git a/test/bbackupd/testfiles/clientTrustedCAs.pem b/test/bbackupd/testfiles/clientTrustedCAs.pem
new file mode 100755
index 00000000..2a065879
--- /dev/null
+++ b/test/bbackupd/testfiles/clientTrustedCAs.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBjDCB9gIBADANBgkqhkiG9w0BAQUFADAPMQ0wCwYDVQQDEwRST09UMB4XDTAz
+MTAwNzA4NTkzMloXDTMxMDIyMjA4NTkzMlowDzENMAsGA1UEAxMEUk9PVDCBnzAN
+BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtZypR/5m5fuNMPNrSLdzwmXKqhdVZj/e
+cZHUZvVuXQZboosAznDrbh8HgpuTw5vaZDEz8VfPwgIaROZDT3ztFIedLapJ7Ot9
+I4JNqSv/y3V9MKb7trTSPVvyYLqk9isLmw8wmEidJiLbWbIc2cHFXDvWNqTr2jF6
+u4Q8DvdVfAECAwEAATANBgkqhkiG9w0BAQUFAAOBgQAL1lyJ/5y44yjk2BK+tnrZ
+hbK7Ghtqrq/uZ8RQq5sAme919TnPijh2tRBqSaUaD2K+Sgo3RNgUGbKhfHRU1pfM
+USllHskTKiJu74ix/T3UOnjpQ946OLSl5zNsOdOgbjBDnozfPSrKeEGN0huBbmmt
+SlL3iQzVXlF6NAhkzS54fQ==
+-----END CERTIFICATE-----
diff --git a/test/bbackupd/testfiles/extcheck1.pl b/test/bbackupd/testfiles/extcheck1.pl
new file mode 100755
index 00000000..7481593d
--- /dev/null
+++ b/test/bbackupd/testfiles/extcheck1.pl
@@ -0,0 +1,33 @@
+#!/usr/bin/perl
+use strict;
+
+unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac\" quit|")
+{
+	print "Couldn't open compare utility\n";
+	exit 2;
+}
+
+my $ret = 1;
+my $seen = 0;
+
+while(<IN>)
+{
+	next unless m/\S/;
+	if(m/continousupdate/)
+	{
+		$ret = 2 unless m/exists/;
+		$seen = 1;
+	}
+	else
+	{
+		$ret = 2 unless m/\AWARNING/ || m/\ADifferences/ || /might be reason/ || /probably due to file mod/;
+	}
+	print;
+}
+
+close IN;
+
+$ret = 2 unless $seen;
+
+exit $ret;
+
diff --git a/test/bbackupd/testfiles/extcheck2.pl b/test/bbackupd/testfiles/extcheck2.pl
new file mode 100755
index 00000000..a0c7d3cd
--- /dev/null
+++ b/test/bbackupd/testfiles/extcheck2.pl
@@ -0,0 +1,29 @@
+#!/usr/bin/perl
+use strict;
+
+unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac\" quit|")
+{
+	print "Couldn't open compare utility\n";
+	exit 2;
+}
+
+my $ret = 1;
+
+while(<IN>)
+{
+	next unless m/\S/;
+	if(m/continousupdate/)
+	{
+		$ret = 2 unless m/contents/ || m/attributes/;
+	}
+	else
+	{
+		$ret = 2 unless m/\AWARNING/ || m/\ADifferences/ || /might be reason/ || /probably due to file mod/;
+	}
+	print;
+}
+
+close IN;
+
+exit $ret;
+
diff --git a/test/bbackupd/testfiles/notifyscript.pl b/test/bbackupd/testfiles/notifyscript.pl
new file mode 100755
index 00000000..a31d072e
--- /dev/null
+++ b/test/bbackupd/testfiles/notifyscript.pl
@@ -0,0 +1,15 @@
+#!/usr/bin/perl
+
+
+my $f = 'testfiles/notifyran.'.$ARGV[0].'.';
+my $n = 1;
+
+while(-e $f.$n)
+{
+	$n ++;
+}
+
+open FL,'>'.$f.$n;
+print FL localtime();
+close FL;
+
diff --git a/test/bbackupd/testfiles/raidfile.conf b/test/bbackupd/testfiles/raidfile.conf
new file mode 100755
index 00000000..641872b0
--- /dev/null
+++ b/test/bbackupd/testfiles/raidfile.conf
@@ -0,0 +1,10 @@
+
+disc0
+{
+	SetNumber = 0
+	BlockSize = 2048
+	Dir0 = testfiles/0_0
+	Dir1 = testfiles/0_1
+	Dir2 = testfiles/0_2
+}
+
diff --git a/test/bbackupd/testfiles/serverCerts.pem b/test/bbackupd/testfiles/serverCerts.pem
new file mode 100755
index 00000000..92467618
--- /dev/null
+++ b/test/bbackupd/testfiles/serverCerts.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBlzCCAQACAQQwDQYJKoZIhvcNAQEFBQAwDzENMAsGA1UEAxMEUk9PVDAeFw0w
+MzEwMDcwOTAwMTFaFw0zMTAyMjIwOTAwMTFaMBkxFzAVBgNVBAMTDlNUT1JFLTAw
+MDAwMDA4MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDNj1fGSCaSl/1w1lRV
+I8qE6BqjvT6R0XXGdIV+dk/mHmE3NOCPcBq/gxZOYevp+QnwMc+nUSS7Px/n+q92
+cl3a8ttInfZjLqg9o/wpd6dBfH4gLTG4bEujhMt1x4bEUJk/uWfnk5FhsJXDBrlH
+RJZNiS9Asme+5Zvjfz3Phy0YWwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBABhmdun/
+myn3l4SbH+PxSUaW/mSvBubFhbbl9wolwhzvGCrtY968jn464JUP1UwUnnvePUU2
+SSVPZOVCvobCfM6s20aOdlKvnn+7GZkjoFONuCw3O+1hIFTSyXFcJWBaYLuczVk1
+HfdIKKcVZ1CpAfnMhMxuu+nA7fjor4p1/K0t
+-----END CERTIFICATE-----
diff --git a/test/bbackupd/testfiles/serverPrivKey.pem b/test/bbackupd/testfiles/serverPrivKey.pem
new file mode 100755
index 00000000..fd87607d
--- /dev/null
+++ b/test/bbackupd/testfiles/serverPrivKey.pem
@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDNj1fGSCaSl/1w1lRVI8qE6BqjvT6R0XXGdIV+dk/mHmE3NOCP
+cBq/gxZOYevp+QnwMc+nUSS7Px/n+q92cl3a8ttInfZjLqg9o/wpd6dBfH4gLTG4
+bEujhMt1x4bEUJk/uWfnk5FhsJXDBrlHRJZNiS9Asme+5Zvjfz3Phy0YWwIDAQAB
+AoGBAI88mjo1noM528Wb4+nr5bvVDHMadJYhccMXAMqNYMGGW9GfS/dHc6wNiSaX
+P0+rVIyF+R+rAEBmDTKV0Vxk9xZQuAaDKjLluDkxSxSR869D2YOWYUfvjDo3OFlT
+LMZf0eE7u/3Pm0MtxPctXszqvNnmb+IvPXzttGRgUfU5G+tJAkEA+IphkGMI4A3l
+4KfxotZZU+HiJbRDFpm81RzCc2709KCMkXMEz/+xkvnqlo28jqOf7PRBeq/ecsZN
+8BGvtyoqVQJBANO6uj6sPI66GaRqxV83VyUUdMmL9uFOccIMqW5q0rx5UDi0mG7t
+Pjjz+ul1D247+dvVxnEBeW4C85TSNbbKR+8CQQChpV7PCZo8Hs3jz1bZEZAHfmIX
+I6Z+jH7EHHBbo06ty72g263FmgdkECcCxCxemQzqj/IGWVvUSiVmfhpKhqIBAkAl
+XbjswpzVW4aW+7jlevDIPHn379mcHan54x4rvHKAjLBZsZWNThVDG9vWQ7B7dd48
+q9efrfDuN1shko+kOMLFAkAGIc5w0bJNC4eu91Wr6AFgTm2DntyVQ9keVhYbrwrE
+xY37dgVhAWVeLDOk6eVOVSYqEI1okXPVqvfOIoRJUYkn
+-----END RSA PRIVATE KEY-----
diff --git a/test/bbackupd/testfiles/serverTrustedCAs.pem b/test/bbackupd/testfiles/serverTrustedCAs.pem
new file mode 100755
index 00000000..2a065879
--- /dev/null
+++ b/test/bbackupd/testfiles/serverTrustedCAs.pem
@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBjDCB9gIBADANBgkqhkiG9w0BAQUFADAPMQ0wCwYDVQQDEwRST09UMB4XDTAz
+MTAwNzA4NTkzMloXDTMxMDIyMjA4NTkzMlowDzENMAsGA1UEAxMEUk9PVDCBnzAN
+BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtZypR/5m5fuNMPNrSLdzwmXKqhdVZj/e
+cZHUZvVuXQZboosAznDrbh8HgpuTw5vaZDEz8VfPwgIaROZDT3ztFIedLapJ7Ot9
+I4JNqSv/y3V9MKb7trTSPVvyYLqk9isLmw8wmEidJiLbWbIc2cHFXDvWNqTr2jF6
+u4Q8DvdVfAECAwEAATANBgkqhkiG9w0BAQUFAAOBgQAL1lyJ/5y44yjk2BK+tnrZ
+hbK7Ghtqrq/uZ8RQq5sAme919TnPijh2tRBqSaUaD2K+Sgo3RNgUGbKhfHRU1pfM
+USllHskTKiJu74ix/T3UOnjpQ946OLSl5zNsOdOgbjBDnozfPSrKeEGN0huBbmmt
+SlL3iQzVXlF6NAhkzS54fQ==
+-----END CERTIFICATE-----
diff --git a/test/bbackupd/testfiles/spacetest1.tgz b/test/bbackupd/testfiles/spacetest1.tgz
new file mode 100644
index 00000000..c653c0ae
--- /dev/null
+++ b/test/bbackupd/testfiles/spacetest1.tgz
diff --git a/test/bbackupd/testfiles/spacetest2.tgz b/test/bbackupd/testfiles/spacetest2.tgz
new file mode 100644
index 00000000..aa47312d
--- /dev/null
+++ b/test/bbackupd/testfiles/spacetest2.tgz
diff --git a/test/bbackupd/testfiles/test2.tgz b/test/bbackupd/testfiles/test2.tgz
new file mode 100755
index 00000000..ac7f18af
--- /dev/null
+++ b/test/bbackupd/testfiles/test2.tgz
diff --git a/test/bbackupd/testfiles/test3.tgz b/test/bbackupd/testfiles/test3.tgz
new file mode 100755
index 00000000..c7d60cd7
--- /dev/null
+++ b/test/bbackupd/testfiles/test3.tgz
diff --git a/test/bbackupd/testfiles/test_base.tgz b/test/bbackupd/testfiles/test_base.tgz
new file mode 100755
index 00000000..9c8ddfc0
--- /dev/null
+++ b/test/bbackupd/testfiles/test_base.tgz
diff --git a/test/bbackupd/testfiles/testexclude.tgz b/test/bbackupd/testfiles/testexclude.tgz
new file mode 100644
index 00000000..ac7329d8
--- /dev/null
+++ b/test/bbackupd/testfiles/testexclude.tgz
diff --git a/test/common/testcommon.cpp b/test/common/testcommon.cpp
new file mode 100755
index 00000000..c6d0e52f
--- /dev/null
+++ b/test/common/testcommon.cpp
@@ -0,0 +1,515 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testcommon.cpp
+//		Purpose: Tests for the code in lib/common
+//		Created: 2003/07/23
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "Test.h"
+#include "Configuration.h"
+#include "FdGetLine.h"
+#include "Guards.h"
+#include "FileStream.h"
+#include "IOStreamGetLine.h"
+#include "NamedLock.h"
+#include "ReadGatherStream.h"
+#include "MemBlockStream.h"
+#include "ExcludeList.h"
+#include "CommonException.h"
+#include "Conversion.h"
+#include "autogen_ConversionException.h"
+
+#include "MemLeakFindOn.h"
+
+using namespace BoxConvert;
+
+void test_conversions()
+{
+	TEST_THAT((Convert<int32_t, const std::string &>(std::string("32"))) == 32);
+	TEST_THAT((Convert<int32_t, const char *>("42")) == 42);
+	TEST_THAT((Convert<int32_t, const char *>("-42")) == -42);
+	TEST_CHECK_THROWS((Convert<int8_t, const char *>("500")), ConversionException, IntOverflowInConvertFromString);
+	TEST_CHECK_THROWS((Convert<int8_t, const char *>("pants")), ConversionException, BadStringRepresentationOfInt);
+	TEST_CHECK_THROWS((Convert<int8_t, const char *>("")), ConversionException, CannotConvertEmptyStringToInt);
+	
+	std::string a(Convert<std::string, int32_t>(63));
+	TEST_THAT(a == "63");
+	std::string b(Convert<std::string, int32_t>(-3473463));
+	TEST_THAT(b == "-3473463");
+	std::string c(Convert<std::string, int16_t>(344));
+	TEST_THAT(c == "344");
+}
+
+ConfigurationVerifyKey verifykeys1_1_1[] =
+{
+	{"bing", 0, ConfigTest_Exists, 0},
+	{"carrots", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"terrible", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+ConfigurationVerifyKey verifykeys1_1_2[] =
+{
+	{"fish", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"string", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+
+ConfigurationVerify verifysub1_1[] = 
+{
+	{
+		"*",
+		0,
+		verifykeys1_1_1,
+		ConfigTest_Exists,
+		0
+	},
+	{
+		"otherthing",
+		0,
+		verifykeys1_1_2,
+		ConfigTest_Exists | ConfigTest_LastEntry,
+		0
+	}
+};
+
+ConfigurationVerifyKey verifykeys1_1[] =
+{
+	{"value", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"string1", 0, ConfigTest_Exists, 0},
+	{"string2", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+ConfigurationVerifyKey verifykeys1_2[] = 
+{
+	{"carrots", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+	{"string", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0}
+};
+
+ConfigurationVerify verifysub1[] = 
+{
+	{
+		"test1",
+		verifysub1_1,
+		verifykeys1_1,
+		ConfigTest_Exists,
+		0
+	},
+	{
+		"ping",
+		0,
+		verifykeys1_2,
+		ConfigTest_Exists | ConfigTest_LastEntry,
+		0
+	}
+};
+
+ConfigurationVerifyKey verifykeys1[] =
+{
+		{"notExpected", 0, 0, 0},
+		{"HasDefaultValue", "Lovely default value", 0, 0},
+		{"MultiValue", 0, ConfigTest_MultiValueAllowed, 0},
+		{"BoolTrue1", 0, ConfigTest_IsBool, 0},
+		{"BoolTrue2", 0, ConfigTest_IsBool, 0},
+		{"BoolFalse1", 0, ConfigTest_IsBool, 0},
+		{"BoolFalse2", 0, ConfigTest_IsBool, 0},
+		{"TOPlevel", 0, ConfigTest_LastEntry | ConfigTest_Exists, 0}
+};
+
+ConfigurationVerify verify =
+{
+	"root",
+	verifysub1,
+	verifykeys1,
+	ConfigTest_Exists | ConfigTest_LastEntry,
+	0
+};
+
+int test(int argc, const char *argv[])
+{
+	// Test memory leak detection
+#ifdef BOX_MEMORY_LEAK_TESTING
+	{
+		TEST_THAT(memleakfinder_numleaks() == 0);
+		void *block = ::malloc(12);
+		TEST_THAT(memleakfinder_numleaks() == 1);
+		void *b2 = ::realloc(block, 128*1024);
+		TEST_THAT(block != b2);
+		TEST_THAT(memleakfinder_numleaks() == 1);
+		::free(b2);
+		TEST_THAT(memleakfinder_numleaks() == 0);
+		char *test = new char[1024];
+		TEST_THAT(memleakfinder_numleaks() == 1);
+		MemBlockStream *s = new MemBlockStream(test,12);
+		TEST_THAT(memleakfinder_numleaks() == 2);
+		delete s;
+		TEST_THAT(memleakfinder_numleaks() == 1);
+		delete [] test;
+		TEST_THAT(memleakfinder_numleaks() == 0);
+	}
+#endif // BOX_MEMORY_LEAK_TESTING
+	
+
+	static char *testfilelines[] =
+	{
+		"First line",
+		"Second line",
+		"Third",
+		"",
+		"",
+		"",
+		"sdf hjjk",
+		"",
+		"test",
+		"test#not comment",
+		"test#not comment",
+		"",
+		"nice line",
+		"fish",
+		"",
+		"ping",
+		"",
+		"",
+		"Nothing",
+		"Nothing",
+		0
+	};
+
+	// First, test the FdGetLine class -- rather important this works!
+	{
+		FileHandleGuard<O_RDONLY> file("testfiles/fdgetlinetest.txt");
+		FdGetLine getline(file);
+		
+  		int l = 0;
+  		while(testfilelines[l] != 0)
+  		{
+	  		TEST_THAT(!getline.IsEOF());
+	  		std::string line = getline.GetLine(true);
+	  		//printf("expected |%s| got |%s|\n", lines[l], line.c_str());
+	  		TEST_THAT(strcmp(testfilelines[l], line.c_str()) == 0);
+	  		l++;
+  		}
+  		TEST_THAT(getline.IsEOF());
+  		TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF);
+	}
+	// and again without pre-processing
+	{
+		FileHandleGuard<O_RDONLY> file("testfiles/fdgetlinetest.txt");
+		FILE *file2 = fopen("testfiles/fdgetlinetest.txt", "r");
+		TEST_THAT_ABORTONFAIL(file2 != 0);
+		FdGetLine getline(file);
+		char ll[512];
+		
+		while(!feof(file2))
+		{
+			fgets(ll, sizeof(ll), file2);
+			int e = strlen(ll);
+			while(e > 0 && (ll[e-1] == '\n' || ll[e-1] == '\r'))
+			{
+				e--;
+			}
+			ll[e] = '\0';
+		
+	  		TEST_THAT(!getline.IsEOF());
+	  		std::string line = getline.GetLine(false);
+	  		//printf("expected |%s| got |%s|\n", ll, line.c_str());
+	  		TEST_THAT(strcmp(ll, line.c_str()) == 0);
+		}
+  		TEST_THAT(getline.IsEOF());
+  		TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF);
+		
+		fclose(file2);
+	}
+	
+	// Then the IOStream version of get line, seeing as we're here...
+	{
+		FileStream file("testfiles/fdgetlinetest.txt", O_RDONLY);
+		IOStreamGetLine getline(file);
+		
+  		int l = 0;
+  		while(testfilelines[l] != 0)
+  		{
+	  		TEST_THAT(!getline.IsEOF());
+	  		std::string line;
+	  		while(!getline.GetLine(line, true))
+	  			;
+	  		//printf("expected |%s| got |%s|\n", lines[l], line.c_str());
+	  		TEST_THAT(strcmp(testfilelines[l], line.c_str()) == 0);
+	  		l++;
+  		}
+  		TEST_THAT(getline.IsEOF());
+  		std::string dummy;
+  		TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF);
+	}
+	// and again without pre-processing
+	{
+		FileStream file("testfiles/fdgetlinetest.txt", O_RDONLY);
+		IOStreamGetLine getline(file);
+
+		FILE *file2 = fopen("testfiles/fdgetlinetest.txt", "r");
+		TEST_THAT_ABORTONFAIL(file2 != 0);
+		char ll[512];
+		
+		while(!feof(file2))
+		{
+			fgets(ll, sizeof(ll), file2);
+			int e = strlen(ll);
+			while(e > 0 && (ll[e-1] == '\n' || ll[e-1] == '\r'))
+			{
+				e--;
+			}
+			ll[e] = '\0';
+		
+	  		TEST_THAT(!getline.IsEOF());
+	  		std::string line;
+	  		while(!getline.GetLine(line, false))
+	  			;
+	  		//printf("expected |%s| got |%s|\n", ll, line.c_str());
+	  		TEST_THAT(strcmp(ll, line.c_str()) == 0);
+		}
+  		TEST_THAT(getline.IsEOF());
+  		std::string dummy;
+  		TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF);
+		
+		fclose(file2);
+	}
+	
+	// Doesn't exist
+	{
+		std::string errMsg;
+		TEST_CHECK_THROWS(std::auto_ptr<Configuration> pconfig(Configuration::LoadAndVerify("testfiles/DOESNTEXIST", &verify, errMsg)), CommonException, OSFileOpenError);
+	}
+
+	// Basic configuration test
+	{
+		std::string errMsg;
+		std::auto_ptr<Configuration> pconfig(Configuration::LoadAndVerify("testfiles/config1.txt", &verify, errMsg));
+		if(!errMsg.empty())
+		{
+			printf("UNEXPECTED error msg is:\n------\n%s------\n", errMsg.c_str());
+		}
+		TEST_THAT_ABORTONFAIL(pconfig.get() != 0);
+		TEST_THAT(errMsg.empty());
+		TEST_THAT(pconfig->KeyExists("TOPlevel"));
+		TEST_THAT(pconfig->GetKeyValue("TOPlevel") == "value");
+		TEST_THAT(pconfig->KeyExists("MultiValue"));
+		TEST_THAT(pconfig->GetKeyValue("MultiValue") == "single");
+		TEST_THAT(!pconfig->KeyExists("not exist"));
+		TEST_THAT(pconfig->KeyExists("HasDefaultValue"));
+		TEST_THAT(pconfig->GetKeyValue("HasDefaultValue") == "Lovely default value");
+		TEST_CHECK_THROWS(pconfig->GetKeyValue("not exist"), CommonException, ConfigNoKey);
+		// list of keys
+		std::vector<std::string> keylist(pconfig->GetKeyNames());
+		TEST_THAT(keylist.size() == 3);
+		// will be sorted alphanumerically
+		TEST_THAT(keylist[2] == "TOPlevel" && keylist[1] == "MultiValue" && keylist[0] == "HasDefaultValue");
+		// list of sub configurations
+		std::vector<std::string> sublist(pconfig->GetSubConfigurationNames());
+		TEST_THAT(sublist.size() == 2);
+		TEST_THAT(sublist[0] == "test1");
+		TEST_THAT(sublist[1] == "ping");
+		TEST_THAT(pconfig->SubConfigurationExists("test1"));
+		TEST_THAT(pconfig->SubConfigurationExists("ping"));
+		TEST_CHECK_THROWS(pconfig->GetSubConfiguration("nosubconfig"), CommonException, ConfigNoSubConfig);
+		// Get a sub configuration
+		const Configuration &sub1 = pconfig->GetSubConfiguration("test1");
+		TEST_THAT(sub1.GetKeyValueInt("value") == 12);
+		std::vector<std::string> sublist2(sub1.GetSubConfigurationNames());
+		TEST_THAT(sublist2.size() == 4);
+		// And the sub-sub configs
+		const Configuration &sub1_1 = sub1.GetSubConfiguration("subconfig");
+		TEST_THAT(sub1_1.GetKeyValueInt("carrots") == 0x2356);
+		const Configuration &sub1_2 = sub1.GetSubConfiguration("subconfig2");
+		TEST_THAT(sub1_2.GetKeyValueInt("carrots") == -243895);
+		const Configuration &sub1_3 = sub1.GetSubConfiguration("subconfig3");
+		TEST_THAT(sub1_3.GetKeyValueInt("carrots") == 050);
+		TEST_THAT(sub1_3.GetKeyValue("terrible") == "absolutely");
+	}	
+
+	static const char *file[] =
+	{
+		"testfiles/config2.txt", // Value missing from root
+		"testfiles/config3.txt", // Unexpected {
+		"testfiles/config4.txt", // Missing }
+		"testfiles/config5.txt", // { expected, but wasn't there
+		"testfiles/config6.txt", // Duplicate key
+		"testfiles/config7.txt", // Invalid key (no name)
+		"testfiles/config8.txt", // Not all sub blocks terminated
+		"testfiles/config9.txt", // Not valid integer
+		"testfiles/config9b.txt", // Not valid integer
+		"testfiles/config9c.txt", // Not valid integer
+		"testfiles/config9d.txt", // Not valid integer
+		"testfiles/config10.txt", // Missing key (in subblock)
+		"testfiles/config11.txt", // Unknown key
+		"testfiles/config12.txt", // Missing block
+		"testfiles/config13.txt", // Subconfig (wildcarded) should exist, but missing (ie nothing present)
+		"testfiles/config16.txt", // bad boolean value
+		0
+	};
+
+	for(int l = 0; file[l] != 0; ++l)	
+	{
+		std::string errMsg;
+		std::auto_ptr<Configuration> pconfig(Configuration::LoadAndVerify(file[l], &verify, errMsg));
+		TEST_THAT(pconfig.get() == 0);
+		TEST_THAT(!errMsg.empty());
+		printf("(%s) Error msg is:\n------\n%s------\n", file[l], errMsg.c_str());
+	}
+	
+	// Check that multivalues happen as expected
+	// (single value in a multivalue already checked)
+	{
+		std::string errMsg;
+		std::auto_ptr<Configuration> pconfig(Configuration::LoadAndVerify("testfiles/config14.txt", &verify, errMsg));
+		TEST_THAT(pconfig.get() != 0);
+		TEST_THAT(errMsg.empty());
+		TEST_THAT(pconfig->KeyExists("MultiValue"));
+		// values are separated by a specific character
+		std::string expectedvalue("value1");
+		expectedvalue += Configuration::MultiValueSeparator;
+		expectedvalue += "secondvalue";
+		TEST_THAT(pconfig->GetKeyValue("MultiValue") == expectedvalue);
+	}
+
+	// Check boolean values	
+	{
+		std::string errMsg;
+		std::auto_ptr<Configuration> pconfig(Configuration::LoadAndVerify("testfiles/config15.txt", &verify, errMsg));
+		TEST_THAT(pconfig.get() != 0);
+		TEST_THAT(errMsg.empty());
+		TEST_THAT(pconfig->GetKeyValueBool("BoolTrue1") == true);
+		TEST_THAT(pconfig->GetKeyValueBool("BoolTrue2") == true);
+		TEST_THAT(pconfig->GetKeyValueBool("BoolFalse1") == false);
+		TEST_THAT(pconfig->GetKeyValueBool("BoolFalse2") == false);
+	}
+	
+	// Test named locks
+	{
+		NamedLock lock1;
+		// Try and get a lock on a name in a directory which doesn't exist
+		TEST_CHECK_THROWS(lock1.TryAndGetLock("testfiles/non-exist/lock"), CommonException, OSFileError);
+		// And a more resonable request
+		TEST_THAT(lock1.TryAndGetLock("testfiles/lock1") == true);
+		// Try to lock something using the same lock
+		TEST_CHECK_THROWS(lock1.TryAndGetLock("testfiles/non-exist/lock2"), CommonException, NamedLockAlreadyLockingSomething);		
+		// And again on that name
+		NamedLock lock2;
+		TEST_THAT(lock2.TryAndGetLock("testfiles/lock1") == false);
+	}
+	{
+		// Check that it unlocked when it went out of scope
+		NamedLock lock3;
+		TEST_THAT(lock3.TryAndGetLock("testfiles/lock1") == true);
+	}
+	{
+		// And unlocking works
+		NamedLock lock4;
+		TEST_CHECK_THROWS(lock4.ReleaseLock(), CommonException, NamedLockNotHeld);		
+		TEST_THAT(lock4.TryAndGetLock("testfiles/lock4") == true);
+		lock4.ReleaseLock();
+		NamedLock lock5;
+		TEST_THAT(lock5.TryAndGetLock("testfiles/lock4") == true);
+		// And can reuse it
+		TEST_THAT(lock4.TryAndGetLock("testfiles/lock5") == true);
+	}
+
+	// Test the ReadGatherStream
+	{
+		#define GATHER_DATA1 "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+		#define GATHER_DATA2 "ZYZWVUTSRQPOMNOLKJIHGFEDCBA9876543210zyxwvutsrqpomno"
+
+		// Make two streams
+		MemBlockStream s1(GATHER_DATA1, sizeof(GATHER_DATA1));
+		MemBlockStream s2(GATHER_DATA2, sizeof(GATHER_DATA2));
+
+		// And a gather stream
+		ReadGatherStream gather(false /* no deletion */);
+		
+		// Add the streams
+		int s1_c = gather.AddComponent(&s1);
+		int s2_c = gather.AddComponent(&s2);
+		TEST_THAT(s1_c == 0);
+		TEST_THAT(s2_c == 1);
+		
+		// Set up some blocks
+		gather.AddBlock(s1_c, 11);
+		gather.AddBlock(s1_c, 2);
+		gather.AddBlock(s1_c, 8, true, 2);
+		gather.AddBlock(s2_c, 20);
+		gather.AddBlock(s1_c, 20);
+		gather.AddBlock(s2_c, 25);
+		gather.AddBlock(s1_c, 10, true, 0);
+		#define GATHER_RESULT "0123456789abc23456789ZYZWVUTSRQPOMNOLKJIHabcdefghijklmnopqrstGFEDCBA9876543210zyxwvuts0123456789"
+
+		// Read them in...
+		char buffer[1024];
+		unsigned int r = 0;
+		while(r < sizeof(GATHER_RESULT) - 1)
+		{
+			int s = gather.Read(buffer + r, 7);
+			r += s;
+			
+			TEST_THAT(gather.GetPosition() == r);
+			if(r < sizeof(GATHER_RESULT) - 1)
+			{
+				TEST_THAT(gather.StreamDataLeft());
+				TEST_THAT(static_cast<size_t>(gather.BytesLeftToRead()) == sizeof(GATHER_RESULT) - 1 - r);
+			}
+			else
+			{
+				TEST_THAT(!gather.StreamDataLeft());
+				TEST_THAT(gather.BytesLeftToRead() == 0);
+			}
+		}
+		TEST_THAT(r == sizeof(GATHER_RESULT) - 1);
+		TEST_THAT(::memcmp(buffer, GATHER_RESULT, sizeof(GATHER_RESULT) - 1) == 0);
+	}
+	
+	// Test ExcludeList
+	{
+		ExcludeList elist;
+		// Check assumption
+		TEST_THAT(Configuration::MultiValueSeparator == '\x01');
+		// Add definite entries
+		elist.AddDefiniteEntries(std::string("\x01"));
+		elist.AddDefiniteEntries(std::string(""));
+		elist.AddDefiniteEntries(std::string("Definite1\x01/dir/DefNumberTwo\x01\x01ThingDefThree"));
+		elist.AddDefiniteEntries(std::string("AnotherDef"));
+		TEST_THAT(elist.SizeOfDefiniteList() == 4);
+
+		// Add regex entries
+		#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+			elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$"));
+			elist.AddRegexEntries(std::string(""));
+			TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[:not_valid")), CommonException, BadRegularExpression);
+			TEST_THAT(elist.SizeOfRegexList() == 3);
+		#else
+			TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")), CommonException, RegexNotSupportedOnThisPlatform);
+			TEST_THAT(elist.SizeOfRegexList() == 0);
+		#endif
+		// Try some matches!
+		TEST_THAT(elist.IsExcluded(std::string("Definite1")) == true);
+		TEST_THAT(elist.IsExcluded(std::string("/dir/DefNumberTwo")) == true);
+		TEST_THAT(elist.IsExcluded(std::string("ThingDefThree")) == true);
+		TEST_THAT(elist.IsExcluded(std::string("AnotherDef")) == true);
+		TEST_THAT(elist.IsExcluded(std::string("dir/DefNumberTwo")) == false);
+		#ifndef PLATFORM_REGEX_NOT_SUPPORTED
+			TEST_THAT(elist.IsExcluded(std::string("b.reg")) == true);
+			TEST_THAT(elist.IsExcluded(std::string("e.reg")) == false);
+			TEST_THAT(elist.IsExcluded(std::string("b.Reg")) == false);
+			TEST_THAT(elist.IsExcluded(std::string("DEfinite1")) == false);
+			TEST_THAT(elist.IsExcluded(std::string("DEXCLUDEfinite1")) == true);
+			TEST_THAT(elist.IsExcluded(std::string("DEfinitexclude1")) == false);
+			TEST_THAT(elist.IsExcluded(std::string("exclude")) == true);
+		#endif
+	}
+
+	test_conversions();
+
+	return 0;
+}
diff --git a/test/common/testfiles/config1.txt b/test/common/testfiles/config1.txt
new file mode 100755
index 00000000..d000f759
--- /dev/null
+++ b/test/common/testfiles/config1.txt
@@ -0,0 +1,40 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+MultiValue = single
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config10.txt b/test/common/testfiles/config10.txt
new file mode 100755
index 00000000..02aeec74
--- /dev/null
+++ b/test/common/testfiles/config10.txt
@@ -0,0 +1,37 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config11.txt b/test/common/testfiles/config11.txt
new file mode 100755
index 00000000..cafabe74
--- /dev/null
+++ b/test/common/testfiles/config11.txt
@@ -0,0 +1,39 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		NOTEXPECTED= 34234
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config12.txt b/test/common/testfiles/config12.txt
new file mode 100755
index 00000000..17ed34f1
--- /dev/null
+++ b/test/common/testfiles/config12.txt
@@ -0,0 +1,33 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config13.txt b/test/common/testfiles/config13.txt
new file mode 100755
index 00000000..8de8ea5b
--- /dev/null
+++ b/test/common/testfiles/config13.txt
@@ -0,0 +1,15 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config14.txt b/test/common/testfiles/config14.txt
new file mode 100755
index 00000000..d409beaa
--- /dev/null
+++ b/test/common/testfiles/config14.txt
@@ -0,0 +1,41 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+MultiValue = value1
+MultiValue = secondvalue
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config15.txt b/test/common/testfiles/config15.txt
new file mode 100755
index 00000000..bfa7b022
--- /dev/null
+++ b/test/common/testfiles/config15.txt
@@ -0,0 +1,45 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+MultiValue = single
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
+BoolTrue1 = true
+BoolTrue2 = yes
+BoolFalse1 = fAlse
+BoolFalse2 = nO
+
diff --git a/test/common/testfiles/config16.txt b/test/common/testfiles/config16.txt
new file mode 100755
index 00000000..566070f2
--- /dev/null
+++ b/test/common/testfiles/config16.txt
@@ -0,0 +1,42 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+MultiValue = single
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
+BoolTrue1 = not a valid value
+
diff --git a/test/common/testfiles/config2.txt b/test/common/testfiles/config2.txt
new file mode 100755
index 00000000..724c911a
--- /dev/null
+++ b/test/common/testfiles/config2.txt
@@ -0,0 +1,39 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+# make this value missing
+# TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config3.txt b/test/common/testfiles/config3.txt
new file mode 100755
index 00000000..688675a4
--- /dev/null
+++ b/test/common/testfiles/config3.txt
@@ -0,0 +1,39 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	{
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config4.txt b/test/common/testfiles/config4.txt
new file mode 100755
index 00000000..1563d8aa
--- /dev/null
+++ b/test/common/testfiles/config4.txt
@@ -0,0 +1,40 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+}
+
+
diff --git a/test/common/testfiles/config5.txt b/test/common/testfiles/config5.txt
new file mode 100755
index 00000000..d1821ecc
--- /dev/null
+++ b/test/common/testfiles/config5.txt
@@ -0,0 +1,37 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config6.txt b/test/common/testfiles/config6.txt
new file mode 100755
index 00000000..d7381738
--- /dev/null
+++ b/test/common/testfiles/config6.txt
@@ -0,0 +1,39 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		bing= something else
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config7.txt b/test/common/testfiles/config7.txt
new file mode 100755
index 00000000..6a24d036
--- /dev/null
+++ b/test/common/testfiles/config7.txt
@@ -0,0 +1,39 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		= invalid thing here!
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config8.txt b/test/common/testfiles/config8.txt
new file mode 100755
index 00000000..3a66bbb3
--- /dev/null
+++ b/test/common/testfiles/config8.txt
@@ -0,0 +1,37 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+
diff --git a/test/common/testfiles/config9.txt b/test/common/testfiles/config9.txt
new file mode 100755
index 00000000..936ad6ce
--- /dev/null
+++ b/test/common/testfiles/config9.txt
@@ -0,0 +1,38 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050X
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config9b.txt b/test/common/testfiles/config9b.txt
new file mode 100755
index 00000000..65c44a19
--- /dev/null
+++ b/test/common/testfiles/config9b.txt
@@ -0,0 +1,38 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=C-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config9c.txt b/test/common/testfiles/config9c.txt
new file mode 100755
index 00000000..d9be55ad
--- /dev/null
+++ b/test/common/testfiles/config9c.txt
@@ -0,0 +1,38 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=2430-895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =050
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/config9d.txt b/test/common/testfiles/config9d.txt
new file mode 100755
index 00000000..28ea300e
--- /dev/null
+++ b/test/common/testfiles/config9d.txt
@@ -0,0 +1,38 @@
+test1
+{
+	value=12
+	string1 = carrots in may  
+	string2 =on the string
+	subconfig
+	{
+		bing= nothing really
+		carrots =0x2356
+		terrible=lovely
+	}
+	subconfig2
+	{
+		bing= something
+		carrots=-243895
+		terrible=pgin!
+	}
+	subconfig3
+	{
+		bing= 435
+		carrots =090
+		terrible=absolutely
+	}
+	otherthing
+	{
+		string= ping
+		fish =0
+	}
+}
+
+TOPlevel= value
+
+ping
+{
+	carrots=324
+	string = casrts
+}
+
diff --git a/test/common/testfiles/fdgetlinetest.txt b/test/common/testfiles/fdgetlinetest.txt
new file mode 100755
index 00000000..f7b2c829
--- /dev/null
+++ b/test/common/testfiles/fdgetlinetest.txt
@@ -0,0 +1,20 @@
+First line
+	Second line    
+   Third
+# comment
+	# comment 		
+	
+	sdf hjjk
+	    
+	     test #coment  	
+	     test#not comment
+	     test#not comment    #comment
+	    	
+nice line	
+fish  
+
+ping
+#comment
+
+	Nothing
+  Nothing
\ No newline at end of file
diff --git a/test/compress/testcompress.cpp b/test/compress/testcompress.cpp
new file mode 100755
index 00000000..592dd641
--- /dev/null
+++ b/test/compress/testcompress.cpp
@@ -0,0 +1,260 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testcompress.cpp
+//		Purpose: Test lib/compress
+//		Created: 5/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "Test.h"
+#include "Compress.h"
+#include "CompressStream.h"
+#include "CollectInBufferStream.h"
+
+#include "MemLeakFindOn.h"
+
+#define DATA_SIZE			(1024*128+103)
+#define CHUNK_SIZE			2561
+#define DECOMP_CHUNK_SIZE	3
+
+// Stream for testing
+class CopyInToOutStream : public IOStream
+{
+public:
+	CopyInToOutStream() : currentBuffer(0) {buffers[currentBuffer].SetForReading();}
+	~CopyInToOutStream() {}
+	int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite)
+	{
+		if(buffers[currentBuffer].StreamDataLeft())
+		{
+			return buffers[currentBuffer].Read(pBuffer, NBytes, Timeout);
+		}
+		
+		// Swap buffers?
+		if(buffers[(currentBuffer + 1) & 1].GetSize() > 0)
+		{
+			buffers[currentBuffer].Reset();
+			currentBuffer = (currentBuffer + 1) & 1;
+			buffers[currentBuffer].SetForReading();
+			return buffers[currentBuffer].Read(pBuffer, NBytes, Timeout);		
+		}
+
+		return 0;
+	}
+	void Write(const void *pBuffer, int NBytes)
+	{
+		buffers[(currentBuffer + 1) & 1].Write(pBuffer, NBytes);
+	}
+	bool StreamDataLeft()
+	{
+		return buffers[currentBuffer].StreamDataLeft() || buffers[(currentBuffer + 1) % 1].GetSize() > 0;
+	}
+	bool StreamClosed()
+	{
+		return false;
+	}
+	int currentBuffer;
+	CollectInBufferStream buffers[2];
+};
+
+// Test stream based interface
+int test_stream()
+{
+	// Make a load of compressible data to compress
+	CollectInBufferStream source;
+	uint16_t data[1024];
+	for(int x = 0; x < 1024; ++x)
+	{
+		data[x] = x;
+	}
+	for(int x = 0; x < (32*1024); ++x)
+	{
+		source.Write(data, (x % 1024) * 2);
+	}
+	source.SetForReading();
+
+	// Straight compress from one stream to another
+	{
+		CollectInBufferStream *poutput = new CollectInBufferStream;
+		CompressStream compress(poutput, true /* take ownership */, false /* read */, true /* write */);
+		
+		source.CopyStreamTo(compress);
+		compress.Close();
+		poutput->SetForReading();
+		
+		// Check sizes
+		TEST_THAT(poutput->GetSize() < source.GetSize());
+		TRACE2("compressed size = %d, source size = %d\n", poutput->GetSize(), source.GetSize());
+		
+		// Decompress the data
+		{
+			CollectInBufferStream decompressed;
+			CompressStream decompress(poutput, false /* don't take ownership */, true /* read */, false /* write */);
+			decompress.CopyStreamTo(decompressed);
+			decompress.Close();
+			
+			TEST_THAT(decompressed.GetSize() == source.GetSize());
+			TEST_THAT(::memcmp(decompressed.GetBuffer(), source.GetBuffer(), decompressed.GetSize()) == 0);
+		}
+		
+		// Don't delete poutput, let mem leak testing ensure it's deleted.
+	}
+
+	// Set source to the beginning
+	source.Seek(0, IOStream::SeekType_Absolute);
+
+	// Test where the same stream compresses and decompresses, should be fun!
+	{
+		CollectInBufferStream output;
+		CopyInToOutStream copyer;
+		CompressStream compress(&copyer, false /* no ownership */, true, true);
+
+		bool done = false;
+		int count = 0;
+		int written = 0;
+		while(!done)
+		{
+			++count;
+			bool do_sync = (count % 256) == 0;
+			uint8_t buffer[4096];
+			int r = source.Read(buffer, sizeof(buffer), IOStream::TimeOutInfinite);
+			if(r == 0)
+			{
+				done = true;
+				compress.Close();
+			}
+			else
+			{
+				compress.Write(buffer, r);
+				written += r;
+				if(do_sync)
+				{
+					compress.WriteAllBuffered();
+				}
+			}
+
+			int r2 = 0;
+			do
+			{
+				r2 = compress.Read(buffer, sizeof(buffer), IOStream::TimeOutInfinite);
+				if(r2 > 0)
+				{
+					output.Write(buffer, r2);
+				}
+			} while(r2 > 0);
+			if(do_sync && r != 0)
+			{
+				// Check that everything is synced
+				TEST_THAT(output.GetSize() == written);
+				TEST_THAT(::memcmp(output.GetBuffer(), source.GetBuffer(), output.GetSize()) == 0);
+			}
+		}
+		output.SetForReading();
+		
+		// Test that it's the same
+		TEST_THAT(output.GetSize() == source.GetSize());
+		TEST_THAT(::memcmp(output.GetBuffer(), source.GetBuffer(), output.GetSize()) == 0);
+	}
+
+	return 0;
+}
+
+// Test basic interface
+int test(int argc, const char *argv[])
+{
+	// Bad data to compress!
+	char *data = (char *)malloc(DATA_SIZE);
+	for(int l = 0; l < DATA_SIZE; ++l)
+	{
+		data[l] = l*23;
+	}
+	
+	// parameters about compression	
+	int maxOutput = Compress_MaxSizeForCompressedData(DATA_SIZE);
+	TEST_THAT(maxOutput >= DATA_SIZE);
+
+	char *compressed = (char *)malloc(maxOutput);
+	int compressedSize = 0;
+	
+	// Do compression, in small chunks
+	{
+		Compress<true> compress;
+		
+		int in_loc = 0;
+		while(!compress.OutputHasFinished())
+		{
+			int ins = DATA_SIZE - in_loc;
+			if(ins > CHUNK_SIZE) ins = CHUNK_SIZE;
+			
+			if(ins == 0)
+			{
+				compress.FinishInput();
+			}
+			else
+			{
+				compress.Input(data + in_loc, ins);
+			}
+			in_loc += ins;
+			
+			// Get output data
+			int s = 0;
+			do
+			{
+				TEST_THAT(compressedSize < maxOutput);
+				s = compress.Output(compressed + compressedSize, maxOutput - compressedSize);
+				compressedSize += s;
+			} while(s > 0);
+		}
+	}
+	
+	// a reasonable test, especially given the compressability of the input data.
+	TEST_THAT(compressedSize < DATA_SIZE);
+
+	// decompression
+	char *decompressed = (char*)malloc(DATA_SIZE * 2);
+	int decomp_size = 0;
+	{	
+		Compress<false> decompress;
+		
+		int in_loc = 0;
+		while(!decompress.OutputHasFinished())
+		{
+			int ins = compressedSize - in_loc;
+			if(ins > DECOMP_CHUNK_SIZE) ins = DECOMP_CHUNK_SIZE;
+			
+			if(ins == 0)
+			{
+				decompress.FinishInput();
+			}
+			else
+			{
+				decompress.Input(compressed + in_loc, ins);
+			}
+			in_loc += ins;
+			
+			// Get output data
+			int s = 0;
+			do
+			{
+				TEST_THAT(decomp_size <= DATA_SIZE);
+				s = decompress.Output(decompressed + decomp_size, (DATA_SIZE*2) - decomp_size);
+				decomp_size += s;
+			} while(s > 0);
+		}
+	}
+	
+	TEST_THAT(decomp_size == DATA_SIZE);
+	TEST_THAT(::memcmp(data, decompressed, DATA_SIZE) == 0);
+
+	::free(data);
+	::free(compressed);
+	::free(decompressed);
+	
+	return test_stream();
+}
diff --git a/test/crypto/testcrypto.cpp b/test/crypto/testcrypto.cpp
new file mode 100755
index 00000000..2b055f91
--- /dev/null
+++ b/test/crypto/testcrypto.cpp
@@ -0,0 +1,308 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    testcrypto.cpp
+//		Purpose: test lib/crypto
+//		Created: 1/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <string.h>
+#include <strings.h>
+#include <openssl/rand.h>
+
+#include "Test.h"
+#include "CipherContext.h"
+#include "CipherBlowfish.h"
+#include "CipherAES.h"
+#include "CipherException.h"
+#include "RollingChecksum.h"
+#include "Random.h"
+
+#include "MemLeakFindOn.h"
+
+#define STRING1	"Mary had a little lamb"
+#define STRING2 "Skjdf sdjf sjksd fjkhsdfjk hsdfuiohcverfg sdfnj sdfgkljh sdfjb jlhdfvghsdip vjsdfv bsdfhjvg yuiosdvgpvj kvbn m,sdvb sdfuiovg sdfuivhsdfjkv"
+
+#define KEY "0123456701234567012345670123456"
+#define KEY2 "1234567012345670123456A"
+
+#define CHECKSUM_DATA_SIZE			(128*1024)
+#define CHECKSUM_BLOCK_SIZE_BASE	(65*1024)
+#define CHECKSUM_BLOCK_SIZE_LAST	(CHECKSUM_BLOCK_SIZE_BASE + 64)
+#define CHECKSUM_ROLLS				16
+
+void check_random_int(uint32_t max)
+{
+	for(int c = 0; c < 1024; ++c)
+	{
+		uint32_t v = Random::RandomInt(max);
+		TEST_THAT(v >= 0 && v <= max);
+	}
+}
+
+#define ZERO_BUFFER(x) ::bzero(x, sizeof(x));
+
+template<typename CipherType, int BLOCKSIZE>
+void test_cipher()
+{
+	{
+		// Make a couple of cipher contexts
+		CipherContext encrypt1;
+		encrypt1.Reset();
+		encrypt1.Init(CipherContext::Encrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		TEST_CHECK_THROWS(encrypt1.Init(CipherContext::Encrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY))),
+			CipherException, AlreadyInitialised);
+		// Encrpt something
+		char buf1[256];
+		unsigned int buf1_used = encrypt1.TransformBlock(buf1, sizeof(buf1), STRING1, sizeof(STRING1));
+		TEST_THAT(buf1_used >= sizeof(STRING1));
+		// Decrypt it
+		CipherContext decrypt1;
+		decrypt1.Init(CipherContext::Decrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		char buf1_de[256];
+		unsigned int buf1_de_used = decrypt1.TransformBlock(buf1_de, sizeof(buf1_de), buf1, buf1_used);
+		TEST_THAT(buf1_de_used == sizeof(STRING1));
+		TEST_THAT(memcmp(STRING1, buf1_de, sizeof(STRING1)) == 0);
+		
+		// Use them again...
+		char buf1_de2[256];
+		unsigned int buf1_de2_used = decrypt1.TransformBlock(buf1_de2, sizeof(buf1_de2), buf1, buf1_used);
+		TEST_THAT(buf1_de2_used == sizeof(STRING1));
+		TEST_THAT(memcmp(STRING1, buf1_de2, sizeof(STRING1)) == 0);
+		
+		// Test the interface
+		char buf2[256];
+		TEST_CHECK_THROWS(encrypt1.Transform(buf2, sizeof(buf2), STRING1, sizeof(STRING1)),
+			CipherException, BeginNotCalled);
+		TEST_CHECK_THROWS(encrypt1.Final(buf2, sizeof(buf2)),
+			CipherException, BeginNotCalled);
+		encrypt1.Begin();
+		int e = 0;
+		e = encrypt1.Transform(buf2, sizeof(buf2), STRING2, sizeof(STRING2) - 16);
+		e += encrypt1.Transform(buf2 + e, sizeof(buf2) - e, STRING2 + sizeof(STRING2) - 16, 16);
+		e += encrypt1.Final(buf2 + e, sizeof(buf2) - e);
+		TEST_THAT(e >= (int)sizeof(STRING2));
+		
+		// Then decrypt
+		char buf2_de[256];
+		decrypt1.Begin();
+		TEST_CHECK_THROWS(decrypt1.Transform(buf2_de, 2, buf2, e), CipherException, OutputBufferTooSmall);
+		TEST_CHECK_THROWS(decrypt1.Final(buf2_de, 2), CipherException, OutputBufferTooSmall);
+		int d = decrypt1.Transform(buf2_de, sizeof(buf2_de), buf2, e - 48);
+		d += decrypt1.Transform(buf2_de + d, sizeof(buf2_de) - d, buf2 + e - 48, 48);
+		d += decrypt1.Final(buf2_de + d, sizeof(buf2_de) - d);
+		TEST_THAT(d == sizeof(STRING2));
+		TEST_THAT(memcmp(STRING2, buf2_de, sizeof(STRING2)) == 0);
+		
+		// Try a reset and rekey
+		encrypt1.Reset();
+		encrypt1.Init(CipherContext::Encrypt, CipherType(CipherDescription::Mode_CBC, KEY2, sizeof(KEY2)));
+		buf1_used = encrypt1.TransformBlock(buf1, sizeof(buf1), STRING1, sizeof(STRING1));
+	}
+	
+	// Test initialisation vectors
+	{
+		// Init with random IV
+		CipherContext encrypt2;
+		encrypt2.Init(CipherContext::Encrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		int ivLen;
+		char iv2[BLOCKSIZE];
+		const void *ivGen = encrypt2.SetRandomIV(ivLen);
+		TEST_THAT(ivLen == BLOCKSIZE);	// block size
+		TEST_THAT(ivGen != 0);
+		memcpy(iv2, ivGen, ivLen);
+		
+		char buf3[256];
+		unsigned int buf3_used = encrypt2.TransformBlock(buf3, sizeof(buf3), STRING2, sizeof(STRING2));
+		
+		// Encrypt again with different IV
+		char iv3[BLOCKSIZE];
+		int ivLen3;
+		const void *ivGen3 = encrypt2.SetRandomIV(ivLen3);
+		TEST_THAT(ivLen3 == BLOCKSIZE);	// block size
+		TEST_THAT(ivGen3 != 0);
+		memcpy(iv3, ivGen3, ivLen3);
+		// Check the two generated IVs are different
+		TEST_THAT(memcmp(iv2, iv3, BLOCKSIZE) != 0);
+
+		char buf4[256];
+		unsigned int buf4_used = encrypt2.TransformBlock(buf4, sizeof(buf4), STRING2, sizeof(STRING2));
+
+		// check encryptions are different
+		TEST_THAT(buf3_used == buf4_used);
+		TEST_THAT(memcmp(buf3, buf4, buf3_used) != 0);
+		
+		// Test that decryption with the right IV works
+		CipherContext decrypt2;
+		decrypt2.Init(CipherContext::Decrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY), iv2));
+		char buf3_de[256];
+		unsigned int buf3_de_used = decrypt2.TransformBlock(buf3_de, sizeof(buf3_de), buf3, buf3_used);
+		TEST_THAT(buf3_de_used == sizeof(STRING2));
+		TEST_THAT(memcmp(STRING2, buf3_de, sizeof(STRING2)) == 0);
+		
+		// And that using the wrong one doesn't
+		decrypt2.SetIV(iv3);
+		buf3_de_used = decrypt2.TransformBlock(buf3_de, sizeof(buf3_de), buf3, buf3_used);
+		TEST_THAT(buf3_de_used == sizeof(STRING2));
+		TEST_THAT(memcmp(STRING2, buf3_de, sizeof(STRING2)) != 0);		
+	}
+	
+	// Test with padding off.
+	{
+		CipherContext encrypt3;
+		encrypt3.Init(CipherContext::Encrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		encrypt3.UsePadding(false);
+		
+		// Should fail because the encrypted size is not a multiple of the block size
+		char buf4[256];
+		encrypt3.Begin();
+		ZERO_BUFFER(buf4);
+		int buf4_used = encrypt3.Transform(buf4, sizeof(buf4), STRING2, 6);
+		TEST_CHECK_THROWS(encrypt3.Final(buf4, sizeof(buf4)), CipherException, EVPFinalFailure);
+		
+		// Check a nice encryption with the correct block size
+		CipherContext encrypt4;
+		encrypt4.Init(CipherContext::Encrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		encrypt4.UsePadding(false);
+		encrypt4.Begin();
+		ZERO_BUFFER(buf4);
+		buf4_used = encrypt4.Transform(buf4, sizeof(buf4), STRING2, 16);
+		buf4_used += encrypt4.Final(buf4+buf4_used, sizeof(buf4));
+		TEST_THAT(buf4_used == 16);
+
+		// Check it's encrypted to the same thing as when there's padding on
+		CipherContext encrypt4b;
+		encrypt4b.Init(CipherContext::Encrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		encrypt4b.Begin();
+		char buf4b[256];
+		ZERO_BUFFER(buf4b);
+		int buf4b_used = encrypt4b.Transform(buf4b, sizeof(buf4b), STRING2, 16);
+		buf4b_used += encrypt4b.Final(buf4b + buf4b_used, sizeof(buf4b));
+		TEST_THAT(buf4b_used == 16+BLOCKSIZE);
+		TEST_THAT(::memcmp(buf4, buf4b, 16) == 0);
+
+		// Decrypt
+		char buf4_de[256];
+		CipherContext decrypt4;
+		decrypt4.Init(CipherContext::Decrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		decrypt4.UsePadding(false);
+		decrypt4.Begin();
+		ZERO_BUFFER(buf4_de);
+		int buf4_de_used = decrypt4.Transform(buf4_de, sizeof(buf4_de), buf4, 16);
+		buf4_de_used += decrypt4.Final(buf4_de+buf4_de_used, sizeof(buf4_de));
+		TEST_THAT(buf4_de_used == 16);
+		TEST_THAT(::memcmp(buf4_de, STRING2, 16) == 0);
+		
+		// Test that the TransformBlock thing works as expected too with blocks the same size as the input
+		TEST_THAT(encrypt4.TransformBlock(buf4, 16, STRING2, 16) == 16);
+		// But that it exceptions if we try the trick with padding on
+		encrypt4.UsePadding(true);
+		TEST_CHECK_THROWS(encrypt4.TransformBlock(buf4, 16, STRING2, 16), CipherException, OutputBufferTooSmall);
+	}
+
+	// And again, but with different string size
+	{
+		char buf4[256];
+		int buf4_used;
+
+		// Check a nice encryption with the correct block size
+		CipherContext encrypt4;
+		encrypt4.Init(CipherContext::Encrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		encrypt4.UsePadding(false);
+		encrypt4.Begin();
+		ZERO_BUFFER(buf4);
+		buf4_used = encrypt4.Transform(buf4, sizeof(buf4), STRING2, (BLOCKSIZE*3)); // do three blocks worth
+		buf4_used += encrypt4.Final(buf4+buf4_used, sizeof(buf4));
+		TEST_THAT(buf4_used == (BLOCKSIZE*3));
+
+		// Check it's encrypted to the same thing as when there's padding on
+		CipherContext encrypt4b;
+		encrypt4b.Init(CipherContext::Encrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		encrypt4b.Begin();
+		char buf4b[256];
+		ZERO_BUFFER(buf4b);
+		int buf4b_used = encrypt4b.Transform(buf4b, sizeof(buf4b), STRING2, (BLOCKSIZE*3));
+		buf4b_used += encrypt4b.Final(buf4b + buf4b_used, sizeof(buf4b));
+		TEST_THAT(buf4b_used == (BLOCKSIZE*4));
+		TEST_THAT(::memcmp(buf4, buf4b, (BLOCKSIZE*3)) == 0);
+
+		// Decrypt
+		char buf4_de[256];
+		CipherContext decrypt4;
+		decrypt4.Init(CipherContext::Decrypt, CipherType(CipherDescription::Mode_CBC, KEY, sizeof(KEY)));
+		decrypt4.UsePadding(false);
+		decrypt4.Begin();
+		ZERO_BUFFER(buf4_de);
+		int buf4_de_used = decrypt4.Transform(buf4_de, sizeof(buf4_de), buf4, (BLOCKSIZE*3));
+		buf4_de_used += decrypt4.Final(buf4_de+buf4_de_used, sizeof(buf4_de));
+		TEST_THAT(buf4_de_used == (BLOCKSIZE*3));
+		TEST_THAT(::memcmp(buf4_de, STRING2, (BLOCKSIZE*3)) == 0);
+		
+		// Test that the TransformBlock thing works as expected too with blocks the same size as the input
+		TEST_THAT(encrypt4.TransformBlock(buf4, (BLOCKSIZE*3), STRING2, (BLOCKSIZE*3)) == (BLOCKSIZE*3));
+		// But that it exceptions if we try the trick with padding on
+		encrypt4.UsePadding(true);
+		TEST_CHECK_THROWS(encrypt4.TransformBlock(buf4, (BLOCKSIZE*3), STRING2, (BLOCKSIZE*3)), CipherException, OutputBufferTooSmall);
+	}
+}
+
+int test(int argc, const char *argv[])
+{
+	Random::Initialise();
+
+	// Cipher type
+	::printf("Blowfish...\n");
+	test_cipher<CipherBlowfish, 8>();
+#ifndef PLATFORM_OLD_OPENSSL
+	::printf("AES...\n");
+	test_cipher<CipherAES, 16>();
+#else
+	::printf("Skipping AES -- not supported by version of OpenSSL in use.\n");
+#endif
+	
+	::printf("Misc...\n");
+	// Check rolling checksums
+	uint8_t *checkdata_blk = (uint8_t *)malloc(CHECKSUM_DATA_SIZE);
+	uint8_t *checkdata = checkdata_blk;
+	RAND_pseudo_bytes(checkdata, CHECKSUM_DATA_SIZE);
+	for(int size = CHECKSUM_BLOCK_SIZE_BASE; size <= CHECKSUM_BLOCK_SIZE_LAST; ++size)
+	{
+		//printf("size = %d\n", size);
+		// Checksum to roll
+		RollingChecksum roll(checkdata, size);
+		
+		// Roll forward
+		for(int l = 0; l < CHECKSUM_ROLLS; ++l)
+		{			
+			// Calculate new one
+			RollingChecksum calc(checkdata, size);
+
+			//printf("%08X %08X %d %d\n", roll.GetChecksum(), calc.GetChecksum(), checkdata[0], checkdata[size]);
+
+			// Compare them!
+			TEST_THAT(calc.GetChecksum() == roll.GetChecksum());
+			
+			// Roll it onwards
+			roll.RollForward(checkdata[0], checkdata[size], size);
+	
+			// increment
+			++checkdata;
+		}
+	}
+	::free(checkdata_blk);
+
+	// Random integers
+	check_random_int(0);
+	check_random_int(1);
+	check_random_int(5);
+	check_random_int(15);	// all 1's
+	check_random_int(1022);
+
+	return 0;
+}
+
+
+
diff --git a/test/raidfile/Darwin-SYS.h b/test/raidfile/Darwin-SYS.h
new file mode 100755
index 00000000..da95c347
--- /dev/null
+++ b/test/raidfile/Darwin-SYS.h
@@ -0,0 +1,167 @@
+

+/* Taken from the Darwin Libc source

+*/

+

+/*

+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.

+ *

+ * @APPLE_LICENSE_HEADER_START@

+ * 

+ * The contents of this file constitute Original Code as defined in and

+ * are subject to the Apple Public Source License Version 1.1 (the

+ * "License").  You may not use this file except in compliance with the

+ * License.  Please obtain a copy of the License at

+ * http://www.apple.com/publicsource and read it before using this file.

+ * 

+ * This Original Code and all software distributed under the License are

+ * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER

+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,

+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

+ * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the

+ * License for the specific language governing rights and limitations

+ * under the License.

+ * 

+ * @APPLE_LICENSE_HEADER_END@

+ */

+/* Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved.

+ *

+ *	File:	SYS.h

+ *

+ *	Definition of the user side of the UNIX system call interface

+ *	for M98K.

+ *

+ *	Errors are flagged by the location of the trap return (ie., which

+ *	instruction is executed upon rfi):

+ *

+ *		SC PC + 4:	Error (typically branch to cerror())

+ *		SC PC + 8:	Success

+ *

+ * HISTORY

+ * 18-Nov-92	Ben Fathi (benf@next.com)

+ *	Ported to m98k.

+ *

+ *  9-Jan-92	Peter King (king@next.com)

+ *	Created.

+ */

+

+#define KERNEL_PRIVATE	1

+/*

+ * Header files.

+ */

+#import	<architecture/ppc/asm_help.h>

+#import	<architecture/ppc/pseudo_inst.h>

+#import	<mach/ppc/exception.h>

+#import	<sys/syscall.h>

+

+/* From rhapsody kernel mach/ppc/syscall_sw.h */

+#define	kernel_trap_args_0

+#define	kernel_trap_args_1

+#define	kernel_trap_args_2

+#define	kernel_trap_args_3

+#define	kernel_trap_args_4

+#define	kernel_trap_args_5

+#define	kernel_trap_args_6

+#define	kernel_trap_args_7

+

+/*

+ * simple_kernel_trap -- Mach system calls with 8 or less args

+ * Args are passed in a0 - a7, system call number in r0.

+ * Do a "sc" instruction to enter kernel.

+ */	

+#define simple_kernel_trap(trap_name, trap_number)	\

+	.globl	_##trap_name				@\

+_##trap_name:						@\

+	li	r0,trap_number				 @\

+	sc						 @\

+	blr						 @\

+	END(trap_name)

+

+#define kernel_trap_0(trap_name,trap_number)		 \

+	simple_kernel_trap(trap_name,trap_number)

+

+#define kernel_trap_1(trap_name,trap_number)		 \

+	simple_kernel_trap(trap_name,trap_number)

+

+#define kernel_trap_2(trap_name,trap_number)		 \

+	simple_kernel_trap(trap_name,trap_number)

+

+#define kernel_trap_3(trap_name,trap_number)		 \

+	simple_kernel_trap(trap_name,trap_number)

+

+#define kernel_trap_4(trap_name,trap_number)		 \

+	simple_kernel_trap(trap_name,trap_number)

+

+#define kernel_trap_5(trap_name,trap_number)		 \

+	simple_kernel_trap(trap_name,trap_number)

+

+#define kernel_trap_6(trap_name,trap_number)		 \

+	simple_kernel_trap(trap_name,trap_number)

+

+#define kernel_trap_7(trap_name,trap_number)		 \

+	simple_kernel_trap(trap_name,trap_number)

+

+#define kernel_trap_8(trap_name,trap_number)		 \

+        simple_kernel_trap(trap_name,trap_number)

+

+#define kernel_trap_9(trap_name,trap_number)		 \

+        simple_kernel_trap(trap_name,trap_number)

+

+/* End of rhapsody kernel mach/ppc/syscall_sw.h */

+

+/*

+ * Macros.

+ */

+

+#define	SYSCALL(name, nargs)			\

+	.globl	cerror				@\

+LEAF(_##name)					@\

+	kernel_trap_args_##nargs		@\

+	li	r0,SYS_##name			@\

+	sc					@\

+	b	1f   				@\

+	b	2f				@\

+1:	BRANCH_EXTERN(cerror)			@\

+.text						\

+2:	nop

+

+#define	SYSCALL_NONAME(name, nargs)		\

+	.globl	cerror				@\

+	kernel_trap_args_##nargs		@\

+	li	r0,SYS_##name			@\

+	sc					@\

+	b	1f   				@\

+	b	2f				@\

+1:	BRANCH_EXTERN(cerror)			@\

+.text						\

+2:	nop

+

+#define	PSEUDO(pseudo, name, nargs)		\

+LEAF(_##pseudo)					@\

+	SYSCALL_NONAME(name, nargs)

+

+

+#undef END

+#import	<mach/ppc/syscall_sw.h>

+

+#if !defined(SYS_getdirentriesattr)

+#define SYS_getdirentriesattr 222

+#endif

+

+#if !defined(SYS_semsys)

+#define SYS_semsys	251

+#define SYS_msgsys	252

+#define SYS_shmsys	253

+#define SYS_semctl	254

+#define SYS_semget	255

+#define SYS_semop	256

+#define SYS_semconfig	257

+#define SYS_msgctl	258

+#define SYS_msgget	259

+#define SYS_msgsnd	260

+#define SYS_msgrcv	261

+#define SYS_shmat	262

+#define SYS_shmctl	263

+#define SYS_shmdt	264

+#define SYS_shmget	265

+#endif

+ 

diff --git a/test/raidfile/Makefile.extra.Darwin b/test/raidfile/Makefile.extra.Darwin
new file mode 100755
index 00000000..dc3bad81
--- /dev/null
+++ b/test/raidfile/Makefile.extra.Darwin
@@ -0,0 +1,6 @@
+
+# link-extra: intercept-lseek.o intercept-close.o intercept-open.o intercept-read.o intercept-readv.o intercept-write.o
+
+$(OUTDIR)/intercept-lseek.o:	make-darwin-intercepts.pl $(OUTDIR)
+	./make-darwin-intercepts.pl $(OUTDIR)
+
diff --git a/test/raidfile/intercept.cpp b/test/raidfile/intercept.cpp
new file mode 100755
index 00000000..4f20ba0a
--- /dev/null
+++ b/test/raidfile/intercept.cpp
@@ -0,0 +1,252 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    intercept.cpp
+//		Purpose: Syscall interception code for the raidfile test
+//		Created: 2003/07/22
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/uio.h>
+#include <errno.h>
+
+#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE
+
+#ifdef PLATFORM_DARWIN
+	// For some reason, __syscall just doesn't work on Darwin
+	// so instead, we build functions using assembler in a varient
+	// of the technique used in the Darwin Libc
+	extern "C" int
+	TEST_open(const char *path, int flags, mode_t mode);
+	extern "C" int
+	TEST_close(int d);
+	extern "C" ssize_t
+	TEST_write(int d, const void *buf, size_t nbytes);
+	extern "C" ssize_t
+	TEST_read(int d, void *buf, size_t nbytes);
+	extern "C" ssize_t
+	TEST_readv(int d, const struct iovec *iov, int iovcnt);
+	extern "C" off_t
+	TEST_lseek(int fildes, off_t offset, int whence);
+#else
+	#ifdef PLATFORM_LINUX
+		#undef __syscall
+		#define __syscall syscall
+	#else
+		// Need this, not declared in syscall.h
+		extern "C" off_t __syscall(quad_t number, ...);
+	#endif
+#endif
+
+#include <string.h>
+#include <stdio.h>
+
+#include "MemLeakFindOn.h"
+
+bool intercept_enabled = false;
+const char *intercept_filename = 0;
+int intercept_filedes = -1;
+unsigned int intercept_errorafter = 0;
+int intercept_errno = 0;
+int intercept_syscall = 0;
+off_t intercept_filepos = 0;
+
+#define SIZE_ALWAYS_ERROR	-773
+
+void intercept_clear_setup()
+{
+	intercept_enabled = false;
+	intercept_filename = 0;
+	intercept_filedes = -1;
+	intercept_errorafter = 0;
+	intercept_syscall = 0;
+	intercept_filepos = 0;
+}
+
+bool intercept_triggered()
+{
+	return !intercept_enabled;
+}
+
+void intercept_setup_error(const char *filename, unsigned int errorafter, int errortoreturn, int syscalltoerror)
+{
+	TRACE4("Setup for error: %s, after %d, err %d, syscall %d\n", filename, errorafter, errortoreturn, syscalltoerror);
+	intercept_enabled = true;
+	intercept_filename = filename;
+	intercept_filedes = -1;
+	intercept_errorafter = errorafter;
+	intercept_syscall = syscalltoerror;
+	intercept_errno = errortoreturn;
+	intercept_filepos = 0;
+}
+
+bool intercept_errornow(int d, int size, int syscallnum)
+{
+	if(intercept_filedes != -1 && d == intercept_filedes && syscallnum == intercept_syscall)
+	{
+		//printf("Checking for err, %d, %d, %d\n", d, size, syscallnum);
+		if(size == SIZE_ALWAYS_ERROR)
+		{
+			// Looks good for an error!
+			TRACE2("Returning error %d for syscall %d\n", intercept_errno, syscallnum);
+			return true;
+		}
+		// where are we in the file?
+		if(intercept_filepos >= intercept_errorafter || intercept_filepos >= ((int)intercept_errorafter - size))
+		{
+			TRACE3("Returning error %d for syscall %d, file pos %d\n", intercept_errno, syscallnum, (int)intercept_filepos);
+			return true;
+		}
+	}
+	return false;	// no error please!
+}
+
+int intercept_reterr()
+{
+	intercept_enabled = false;
+	intercept_filename = 0;
+	intercept_filedes = -1;
+	intercept_errorafter = 0;
+	intercept_syscall = 0;
+	return intercept_errno;
+}
+
+#define CHECK_FOR_FAKE_ERROR_COND(D, S, CALL, FAILRES)	\
+	if(intercept_enabled)					\
+	{										\
+		if(intercept_errornow(D, S, CALL))	\
+		{									\
+			errno = intercept_reterr();		\
+			return FAILRES;					\
+		}									\
+	}
+
+extern "C" int
+open(const char *path, int flags, mode_t mode)
+{
+	if(intercept_enabled)
+	{
+		if(intercept_syscall == SYS_open && strcmp(path, intercept_filename) == 0)
+		{
+			errno = intercept_reterr();
+			return -1;
+		}
+	}
+#ifdef PLATFORM_DARWIN
+	int r = TEST_open(path, flags, mode);
+#else
+	int r = __syscall(SYS_open, path, flags, mode);
+#endif
+	if(intercept_enabled && intercept_filedes == -1)
+	{
+		// Right file?
+		if(strcmp(intercept_filename, path) == 0)
+		{
+			intercept_filedes = r;
+			//printf("Found file to intercept, h = %d\n", r);
+		}
+	}
+	return r;
+}
+
+extern "C" int
+close(int d)
+{
+	CHECK_FOR_FAKE_ERROR_COND(d, SIZE_ALWAYS_ERROR, SYS_close, -1);
+#ifdef PLATFORM_DARWIN
+	int r = TEST_close(d);
+#else
+	int r = __syscall(SYS_close, d);
+#endif
+	if(r == 0)
+	{
+		if(d == intercept_filedes)
+		{
+			intercept_filedes = -1;
+		}
+	}
+	return r;
+}
+
+extern "C" ssize_t
+write(int d, const void *buf, size_t nbytes)
+{
+	CHECK_FOR_FAKE_ERROR_COND(d, nbytes, SYS_write, -1);
+#ifdef PLATFORM_DARWIN
+	int r = TEST_write(d, buf, nbytes);
+#else
+	int r = __syscall(SYS_write, d, buf, nbytes);
+#endif
+	if(r != -1)
+	{
+		intercept_filepos += r;
+	}
+	return r;
+}
+
+extern "C" ssize_t
+read(int d, void *buf, size_t nbytes)
+{
+	CHECK_FOR_FAKE_ERROR_COND(d, nbytes, SYS_read, -1);
+#ifdef PLATFORM_DARWIN
+	int r = TEST_read(d, buf, nbytes);
+#else
+	int r = __syscall(SYS_read, d, buf, nbytes);
+#endif
+	if(r != -1)
+	{
+		intercept_filepos += r;
+	}
+	return r;
+}
+
+extern "C" ssize_t
+readv(int d, const struct iovec *iov, int iovcnt)
+{
+	// how many bytes?
+	int nbytes = 0;
+	for(int b = 0; b < iovcnt; ++b)
+	{
+		nbytes += iov[b].iov_len;
+	}
+
+	CHECK_FOR_FAKE_ERROR_COND(d, nbytes, SYS_readv, -1);
+#ifdef PLATFORM_DARWIN
+	int r = TEST_readv(d, iov, iovcnt);
+#else
+	int r = __syscall(SYS_readv, d, iov, iovcnt);
+#endif
+	if(r != -1)
+	{
+		intercept_filepos += r;
+	}
+	return r;
+}
+
+extern "C" off_t
+lseek(int fildes, off_t offset, int whence)
+{
+	// random magic for lseek syscall, see /usr/src/lib/libc/sys/lseek.c
+	CHECK_FOR_FAKE_ERROR_COND(fildes, 0, SYS_lseek, -1);
+#ifdef PLATFORM_DARWIN
+	int r = TEST_lseek(fildes, offset, whence);
+#else
+	#ifdef PLATFORM_LINUX
+		off_t r = __syscall(SYS_lseek, fildes, offset, whence);
+	#else
+		off_t r = __syscall(SYS_lseek, fildes, 0 /* extra 0 required here! */, offset, whence);
+	#endif
+#endif
+	if(r != -1)
+	{
+		intercept_filepos = r;
+	}
+	return r;
+}
+
+#endif // n PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE
diff --git a/test/raidfile/make-darwin-intercepts.pl b/test/raidfile/make-darwin-intercepts.pl
new file mode 100755
index 00000000..2f19e090
--- /dev/null
+++ b/test/raidfile/make-darwin-intercepts.pl
@@ -0,0 +1,46 @@
+#!/usr/bin/perl
+use strict;
+
+my $out = $ARGV[0];
+die "No out directory specified" unless $out ne '';
+
+my @calls = split /[\r\n]+/,<<__E;
+lseek SYSCALL_TEST(lseek, 3)
+open SYSCALL_TEST(open, 3)
+close SYSCALL_TEST(close, 1)
+write SYSCALL_TEST(write, 3)
+read SYSCALL_TEST(read, 3)
+readv SYSCALL_TEST(readv, 3)
+__E
+
+for(@calls)
+{
+	my ($name,$line) = split / /,$_,2;
+	
+	open FL,">$out/intercept-$name.s" or die "Can't open out file";
+	print FL <<'__S';
+#include "../../../test/raidfile/Darwin-SYS.h"
+
+#define	SYSCALL_TEST(name, nargs)			\
+	.globl	cerror				@\
+LEAF(_TEST_##name)					@\
+	kernel_trap_args_##nargs		@\
+	li	r0,SYS_##name			@\
+	sc					@\
+	b	1f   				@\
+	b	2f				@\
+1:	BRANCH_EXTERN(cerror)			@\
+.text						\
+2:	nop
+
+__S
+	print FL $line,"\n\tblr\n\n";
+
+	close FL;
+	
+	if(system("gcc -c $out/intercept-$name.s -o $out/intercept-$name.o") != 0)
+	{
+		die "Assembly failed\n";
+	}
+}
+
diff --git a/test/raidfile/testextra b/test/raidfile/testextra
new file mode 100755
index 00000000..1f4fbcb2
--- /dev/null
+++ b/test/raidfile/testextra
@@ -0,0 +1,7 @@
+mkdir testfiles/0_0
+mkdir testfiles/0_1
+mkdir testfiles/0_2
+mkdir testfiles/1_0
+mkdir testfiles/1_1
+mkdir testfiles/1_2
+mkdir testfiles/2
diff --git a/test/raidfile/testfiles/raidfile.conf b/test/raidfile/testfiles/raidfile.conf
new file mode 100755
index 00000000..6c6d02f9
--- /dev/null
+++ b/test/raidfile/testfiles/raidfile.conf
@@ -0,0 +1,30 @@
+
+disc0
+{
+	SetNumber = 0
+	BlockSize = 2048
+	Dir0 = testfiles/0_0
+	Dir1 = testfiles/0_1
+	Dir2 = testfiles/0_2
+}
+
+disc1
+{
+	SetNumber = 1
+	BlockSize = 2048
+	Dir0 = testfiles/1_0
+	Dir1 = testfiles/1_1
+	Dir2 = testfiles/1_2
+}
+
+disc2
+{
+	SetNumber = 2
+	BlockSize = 2048
+	Dir0 = testfiles/2
+	Dir1 = testfiles/2
+	Dir2 = testfiles/2
+}
+
+
+
diff --git a/test/raidfile/testraidfile.cpp b/test/raidfile/testraidfile.cpp
new file mode 100755
index 00000000..13ae9083
--- /dev/null
+++ b/test/raidfile/testraidfile.cpp
@@ -0,0 +1,907 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    test/raidfile/test.cpp  
+//		Purpose: Test RaidFile system
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/syscall.h>
+
+#include <string.h>
+
+#include "Test.h"
+#include "RaidFileController.h"
+#include "RaidFileWrite.h"
+#include "RaidFileException.h"
+#include "RaidFileRead.h"
+#include "Guards.h"
+
+#include "MemLeakFindOn.h"
+
+#define RAID_BLOCK_SIZE	2048
+#define RAID_NUMBER_DISCS 3
+
+#define TEST_DATA_SIZE	(8*1024 + 173)
+
+#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE
+	#define	TRF_CAN_INTERCEPT
+#endif
+
+
+#ifdef TRF_CAN_INTERCEPT
+// function in intercept.cpp for setting up errors
+void intercept_setup_error(const char *filename, unsigned int errorafter, int errortoreturn, int syscalltoerror);
+bool intercept_triggered();
+void intercept_clear_setup();
+#endif
+
+// Nice random data for testing written files
+class R250 {
+public:
+	// Set up internal state table with 32-bit random numbers.  
+	// The bizarre bit-twiddling is because rand() returns 16 bits of which
+	// the bottom bit is always zero!  Hence, I use only some of the bits.
+	// You might want to do something better than this....
+
+	R250(int seed) : posn1(0), posn2(103)
+	{
+		// populate the state and incr tables
+		srand(seed);
+
+		for (int i = 0; i != stateLen; ++i)	{
+			state[i] = ((rand() >> 2) << 19) ^ ((rand() >> 2) << 11) ^ (rand() >> 2);
+			incrTable[i] = i == stateLen - 1 ? 0 : i + 1;
+		}
+
+		// stir up the numbers to ensure they're random
+
+		for (int j = 0; j != stateLen * 4; ++j)			
+			(void) next();
+	}
+
+	// Returns the next random number.  Xor together two elements separated
+	// by 103 mod 250, replacing the first element with the result.  Then
+	// increment the two indices mod 250.
+	inline int next()
+	{
+		int ret = (state[posn1] ^= state[posn2]);	// xor and replace element
+
+		posn1 = incrTable[posn1];		// increment indices using lookup table
+		posn2 = incrTable[posn2];
+
+		return ret;
+	}
+private:
+	enum { stateLen = 250 };	// length of the state table
+	int state[stateLen];		// holds the random number state
+	int incrTable[stateLen];	// lookup table: maps i to (i+1) % stateLen
+	int posn1, posn2;			// indices into the state table
+};
+
+void testReadingFileContents(int set, const char *filename, void *data, int datasize, bool TestRAIDProperties, int UsageInBlocks = -1)
+{
+	// Work out which disc is the "start" disc.
+	int h = 0;
+	int n = 0;
+	while(filename[n] != 0)
+	{
+		h += filename[n];
+		n++;
+	}
+	int startDisc = h % RAID_NUMBER_DISCS;
+
+//printf("UsageInBlocks = %d\n", UsageInBlocks);
+
+	// sizes of data to read
+	static int readsizes[] = {2047, 1, 1, 2047, 12, 1, 1, RAID_BLOCK_SIZE - (12+1+1), RAID_BLOCK_SIZE, RAID_BLOCK_SIZE + 246, (RAID_BLOCK_SIZE * 3) + 3, 243};
+	
+	// read the data in to test it
+	char testbuff[(RAID_BLOCK_SIZE * 3) + 128];	// bigger than the max request above!
+	std::auto_ptr<RaidFileRead> pread = RaidFileRead::Open(set, filename);
+	if(UsageInBlocks != -1)
+	{
+		TEST_THAT(UsageInBlocks == pread->GetDiscUsageInBlocks());
+	}
+	//printf("%d, %d\n", pread->GetFileSize(), datasize);
+	TEST_THAT(pread->GetFileSize() == datasize);
+	IOStream &readstream1 = *(pread.get());	
+	int dataread = 0;
+	int r;
+	int readsize = readsizes[0];
+	int bsc = 1;
+	while((r = readstream1.Read(testbuff, readsize)) > 0)
+	{
+		//printf("== read, asked: %d actual: %d\n", readsize, r);
+		TEST_THAT(((dataread+r) == datasize) || r == readsize);
+		TEST_THAT(r > 0);
+		TEST_THAT(readstream1.StreamDataLeft());		// check IOStream interface is correct
+		for(int z = 0; z < r; ++z)
+		{
+			TEST_THAT(((char*)data)[dataread+z] == testbuff[z]);
+			/*if(((char*)data)[dataread+z] != testbuff[z])
+			{
+				printf("z = %d\n", z);
+			}*/
+		}
+		// Next size...
+		if(bsc <= (int)((sizeof(readsizes) / sizeof(readsizes[0])) - 1))
+		{
+			readsize = readsizes[bsc++];
+		}
+		dataread += r;
+	}
+	TEST_THAT(dataread == datasize);
+	pread->Close();
+	
+	// open and close it...
+	pread.reset(RaidFileRead::Open(set, filename).release());
+	if(UsageInBlocks != -1)
+	{
+		TEST_THAT(UsageInBlocks == pread->GetDiscUsageInBlocks());
+	}
+	IOStream &readstream2 = *(pread.get());
+	
+	// positions to try seeking too..
+	static int seekpos[] = {0, 1, 2, 887, 887+256 /* no seek required */, RAID_BLOCK_SIZE, RAID_BLOCK_SIZE + 1, RAID_BLOCK_SIZE - 1, RAID_BLOCK_SIZE*3, RAID_BLOCK_SIZE + 23, RAID_BLOCK_SIZE * 4, RAID_BLOCK_SIZE * 4 + 1};
+	
+	for(unsigned int p = 0; p < (sizeof(seekpos) / sizeof(seekpos[0])); ++p)
+	{
+		//printf("== seekpos = %d\n", seekpos[p]);
+		// only try if test file size is big enough
+		if(seekpos[p]+256 > datasize) continue;
+		
+		readstream2.Seek(seekpos[p], IOStream::SeekType_Absolute);
+		TEST_THAT(readstream2.Read(testbuff, 256) == 256);
+		TEST_THAT(readstream2.GetPosition() == seekpos[p] + 256);
+		TEST_THAT(::memcmp(((char*)data) + seekpos[p], testbuff, 256) == 0);
+	}
+
+	// open and close it...
+	pread.reset(RaidFileRead::Open(set, filename).release());
+	if(UsageInBlocks != -1)
+	{
+		TEST_THAT(UsageInBlocks == pread->GetDiscUsageInBlocks());
+	}
+	IOStream &readstream3 = *(pread.get());	
+	
+	int pos = 0;
+	for(unsigned int p = 0; p < (sizeof(seekpos) / sizeof(seekpos[0])); ++p)
+	{
+		// only try if test file size is big enough
+		if(seekpos[p]+256 > datasize) continue;
+		
+		//printf("pos %d, seekpos %d, p %d\n", pos, seekpos[p], p);
+		
+		readstream3.Seek(seekpos[p] - pos, IOStream::SeekType_Relative);
+		TEST_THAT(readstream3.Read(testbuff, 256) == 256);
+		pos = seekpos[p] + 256;
+		TEST_THAT(readstream3.GetPosition() == pos);
+		TEST_THAT(::memcmp(((char*)data) + seekpos[p], testbuff, 256) == 0);
+	}
+	
+	// Straight read of file
+	pread.reset(RaidFileRead::Open(set, filename).release());
+	if(UsageInBlocks != -1)
+	{
+		TEST_THAT(UsageInBlocks == pread->GetDiscUsageInBlocks());
+	}
+	IOStream &readstream4 = *(pread.get());
+	pos = 0;
+	int bytesread = 0;
+	while((r = readstream4.Read(testbuff, 988)) != 0)
+	{
+		TEST_THAT(readstream4.StreamDataLeft());		// check IOStream interface is behaving as expected
+		
+		// check contents
+		TEST_THAT(::memcmp(((char*)data) + pos, testbuff, r) == 0);
+		
+		// move on
+		pos += r;
+		bytesread += r;
+	}
+	TEST_THAT(!readstream4.StreamDataLeft());		// check IOStream interface is correct
+
+	// Be nasty, and create some errors for the RAID stuff to recover from...
+	if(TestRAIDProperties)
+	{
+		char stripe1fn[256], stripe1fnRename[256];
+		sprintf(stripe1fn, "testfiles/%d_%d/%s.rf", set, startDisc, filename);
+		sprintf(stripe1fnRename, "testfiles/%d_%d/%s.rf-REMOVED", set, startDisc, filename);
+		char stripe2fn[256], stripe2fnRename[256];
+		sprintf(stripe2fn, "testfiles/%d_%d/%s.rf", set, (startDisc + 1) % RAID_NUMBER_DISCS, filename);
+		sprintf(stripe2fnRename, "testfiles/%d_%d/%s.rf-REMOVED", set, (startDisc + 1) % RAID_NUMBER_DISCS, filename);
+
+		// Read with stripe1 + parity		
+		TEST_THAT(::rename(stripe2fn, stripe2fnRename) == 0);
+		testReadingFileContents(set, filename, data, datasize, false /* avoid recursion! */, UsageInBlocks);
+		TEST_THAT(::rename(stripe2fnRename, stripe2fn) == 0);
+
+		// Read with stripe2 + parity
+		TEST_THAT(::rename(stripe1fn, stripe1fnRename) == 0);
+		testReadingFileContents(set, filename, data, datasize, false /* avoid recursion! */, UsageInBlocks);
+		TEST_THAT(::rename(stripe1fnRename, stripe1fn) == 0);
+
+		// Munged filename for avoidance
+		char mungefilename[256];
+		char filenamepart[256];
+		sprintf(filenamepart, "%s.rf", filename);
+		int m = 0, s = 0;
+		while(filenamepart[s] != '\0')
+		{
+			if(filenamepart[s] == '/')
+			{
+				mungefilename[m++] = '_';
+			}
+			else if(filenamepart[s] == '_')
+			{
+				mungefilename[m++] = '_';
+				mungefilename[m++] = '_';
+			}
+			else
+			{
+				mungefilename[m++] = filenamepart[s];
+			}
+			s++;
+		}
+		mungefilename[m++] = '\0';
+		char stripe1munge[256];
+		sprintf(stripe1munge, "testfiles/%d_%d/.raidfile-unreadable/%s", set, startDisc, mungefilename);
+		char stripe2munge[256];
+		sprintf(stripe2munge, "testfiles/%d_%d/.raidfile-unreadable/%s", set, (startDisc + 1) % RAID_NUMBER_DISCS, mungefilename);
+		
+
+#ifdef TRF_CAN_INTERCEPT
+		// Test I/O errors on opening
+		// stripe 1
+		intercept_setup_error(stripe1fn, 0, EIO, SYS_open);
+		testReadingFileContents(set, filename, data, datasize, false /* avoid recursion! */, UsageInBlocks);
+		TEST_THAT(intercept_triggered());
+		intercept_clear_setup();
+
+		// Check that the file was moved correctly.
+		TEST_THAT(TestFileExists(stripe1munge));
+		TEST_THAT(::rename(stripe1munge, stripe1fn) == 0);
+				
+		// Test error in reading stripe 2
+		intercept_setup_error(stripe2fn, 0, EIO, SYS_open);
+		testReadingFileContents(set, filename, data, datasize, false /* avoid recursion! */, UsageInBlocks);
+		TEST_THAT(intercept_triggered());
+		intercept_clear_setup();
+
+		// Check that the file was moved correctly.
+		TEST_THAT(TestFileExists(stripe2munge));
+		TEST_THAT(::rename(stripe2munge, stripe2fn) == 0);
+
+		// Test I/O errors on seeking
+		// stripe 1, if the file is bigger than the minimum thing that it'll get seeked for
+		if(datasize > 257)
+		{
+			intercept_setup_error(stripe1fn, 1, EIO, SYS_lseek);
+			testReadingFileContents(set, filename, data, datasize, false /* avoid recursion! */, UsageInBlocks);
+			TEST_THAT(intercept_triggered());
+			intercept_clear_setup();
+
+			// Check that the file was moved correctly.
+			TEST_THAT(TestFileExists(stripe1munge));
+			TEST_THAT(::rename(stripe1munge, stripe1fn) == 0);
+		}
+
+		// Stripe 2, only if the file is big enough to merit this
+		if(datasize > (RAID_BLOCK_SIZE + 4))
+		{
+			intercept_setup_error(stripe2fn, 1, EIO, SYS_lseek);
+			testReadingFileContents(set, filename, data, datasize, false /* avoid recursion! */, UsageInBlocks);
+			TEST_THAT(intercept_triggered());
+			intercept_clear_setup();
+
+			// Check that the file was moved correctly.
+			TEST_THAT(TestFileExists(stripe2munge));
+			TEST_THAT(::rename(stripe2munge, stripe2fn) == 0);
+		}
+
+		// Test I/O errors on read, but only if the file is size greater than 0
+		if(datasize > 0)
+		{
+			// Where shall we error after?
+			int errafter = datasize / 4;
+
+			// Test error in reading stripe 1
+			intercept_setup_error(stripe1fn, errafter, EIO, SYS_readv);
+			testReadingFileContents(set, filename, data, datasize, false /* avoid recursion! */, UsageInBlocks);
+			TEST_THAT(intercept_triggered());
+			intercept_clear_setup();
+
+			// Check that the file was moved correctly.
+			TEST_THAT(TestFileExists(stripe1munge));
+			TEST_THAT(::rename(stripe1munge, stripe1fn) == 0);
+
+			// Can only test error if file size > RAID_BLOCK_SIZE, as otherwise stripe2 has nothing in it
+			if(datasize > RAID_BLOCK_SIZE)
+			{
+				// Test error in reading stripe 2
+				intercept_setup_error(stripe2fn, errafter, EIO, SYS_readv);
+				testReadingFileContents(set, filename, data, datasize, false /* avoid recursion! */, UsageInBlocks);
+				TEST_THAT(intercept_triggered());
+				intercept_clear_setup();
+
+				// Check that the file was moved correctly.
+				TEST_THAT(TestFileExists(stripe2munge));
+				TEST_THAT(::rename(stripe2munge, stripe2fn) == 0);
+			}
+		}
+#endif // TRF_CAN_INTERCEPT
+	}
+}
+
+
+void testReadWriteFileDo(int set, const char *filename, void *data, int datasize, bool DoTransform)
+{
+	// Work out which disc is the "start" disc.
+	int h = 0;
+	int n = 0;
+	while(filename[n] != 0)
+	{
+		h += filename[n];
+		n++;
+	}
+	int startDisc = h % RAID_NUMBER_DISCS;
+
+	// Another to test the transform works OK...
+	RaidFileWrite write4(set, filename);
+	write4.Open();
+	write4.Write(data, datasize);
+	// This time, don't discard and transform it to a RAID File
+	char writefnPre[256];
+	sprintf(writefnPre, "testfiles/%d_%d/%s.rfwX", set, startDisc, filename);
+	TEST_THAT(TestFileExists(writefnPre));
+	char writefn[256];
+	sprintf(writefn, "testfiles/%d_%d/%s.rfw", set, startDisc, filename);
+	int usageInBlocks = write4.GetDiscUsageInBlocks();
+	write4.Commit(DoTransform);
+	// Check that files are nicely done...
+	if(!DoTransform)
+	{
+		TEST_THAT(TestFileExists(writefn));
+		TEST_THAT(!TestFileExists(writefnPre));
+	}
+	else
+	{
+		TEST_THAT(!TestFileExists(writefn));
+		TEST_THAT(!TestFileExists(writefnPre));
+		// Stripe file sizes
+		int fullblocks = datasize / RAID_BLOCK_SIZE;
+		int leftover = datasize - (fullblocks * RAID_BLOCK_SIZE);
+		int fs1 = -2;
+		if((fullblocks & 1) == 0)
+		{
+			// last block of data will be on the first stripe
+			fs1 = ((fullblocks / 2) * RAID_BLOCK_SIZE) + leftover;
+		}
+		else
+		{
+			// last block is on second stripe
+			fs1 = ((fullblocks / 2)+1) * RAID_BLOCK_SIZE;
+		}
+		char stripe1fn[256];
+		sprintf(stripe1fn, "testfiles/%d_%d/%s.rf", set, startDisc, filename);
+		TEST_THAT(TestGetFileSize(stripe1fn) == fs1);
+		char stripe2fn[256];
+		sprintf(stripe2fn, "testfiles/%d_%d/%s.rf", set, (startDisc + 1) % RAID_NUMBER_DISCS, filename);
+		TEST_THAT(TestGetFileSize(stripe2fn) == (int)(datasize - fs1));
+		// Parity file size
+		char parityfn[256];
+		sprintf(parityfn, "testfiles/%d_%d/%s.rf", set, (startDisc + 2) % RAID_NUMBER_DISCS, filename);
+		// Mildly complex calculation
+		unsigned int blocks = datasize / RAID_BLOCK_SIZE;
+		unsigned int bytesOver = datasize % RAID_BLOCK_SIZE;
+		int paritysize = (blocks / 2) * RAID_BLOCK_SIZE;
+		// Then add in stuff for the last couple of blocks
+		if((blocks & 1) == 0)
+		{
+			if(bytesOver == 0)
+			{
+				paritysize += sizeof(RaidFileRead::FileSizeType);
+			}
+			else
+			{
+				paritysize += (bytesOver == sizeof(RaidFileRead::FileSizeType))?(RAID_BLOCK_SIZE+sizeof(RaidFileRead::FileSizeType)):bytesOver;
+			}
+		}
+		else
+		{
+			paritysize += RAID_BLOCK_SIZE;
+			if(bytesOver == 0 || bytesOver >= (RAID_BLOCK_SIZE-sizeof(RaidFileRead::FileSizeType)))
+			{
+				paritysize += sizeof(RaidFileRead::FileSizeType);
+			}
+		}
+		//printf("datasize = %d, calc paritysize = %d, actual size of file = %d\n", datasize, paritysize, TestGetFileSize(parityfn));
+		TEST_THAT(TestGetFileSize(parityfn) == paritysize);
+		//printf("stripe1 size = %d, stripe2 size = %d, parity size = %d\n", TestGetFileSize(stripe1fn), TestGetFileSize(stripe2fn), TestGetFileSize(parityfn));
+	
+		// Check that block calculation is correct
+		//printf("filesize = %d\n", datasize);
+		#define TO_BLOCKS_ROUND_UP(x) (((x) + (RAID_BLOCK_SIZE-1)) / RAID_BLOCK_SIZE)
+		TEST_THAT(usageInBlocks == (TO_BLOCKS_ROUND_UP(paritysize) + TO_BLOCKS_ROUND_UP(fs1) + TO_BLOCKS_ROUND_UP(datasize - fs1)));
+	
+		// See about whether or not the files look correct
+		char testblock[1024];	// compiler bug? This can't go in the block below without corrupting stripe2fn...
+		if(datasize > (3*1024))
+		{
+			int f;
+			TEST_THAT((f = ::open(stripe1fn, O_RDONLY, 0)) != -1);
+			TEST_THAT(sizeof(testblock) == ::read(f, testblock, sizeof(testblock)));
+			for(unsigned int q = 0; q < sizeof(testblock); ++q)
+			{
+				TEST_THAT(testblock[q] == ((char*)data)[q]);
+			}
+			::close(f);
+			TEST_THAT((f = ::open(stripe2fn, O_RDONLY, 0)) != -1);
+			TEST_THAT(sizeof(testblock) == ::read(f, testblock, sizeof(testblock)));
+			for(unsigned int q = 0; q < sizeof(testblock); ++q)
+			{
+				TEST_THAT(testblock[q] == ((char*)data)[q+RAID_BLOCK_SIZE]);
+			}
+			::close(f);
+		}
+	}
+	
+	// See if the contents look right
+	testReadingFileContents(set, filename, data, datasize, DoTransform /* only test RAID stuff if it has been transformed to RAID */, usageInBlocks);
+}
+
+void testReadWriteFile(int set, const char *filename, void *data, int datasize)
+{
+	// Test once, transforming it...
+	testReadWriteFileDo(set, filename, data, datasize, true);
+	
+	// And then again, not transforming it
+	std::string fn(filename);
+	fn += "NT";
+	testReadWriteFileDo(set, fn.c_str(), data, datasize, false);	
+}
+
+bool list_matches(const std::vector<std::string> &rList, const char *compareto[])
+{
+	// count in compare to
+	int count = 0;
+	while(compareto[count] != 0)
+		count++;
+	
+	if((int)rList.size() != count)
+	{
+		return false;
+	}
+	
+	// Space for bools
+	bool *found = new bool[count];
+	
+	for(int c = 0; c < count; ++c)
+	{
+		found[c] = false;
+	}
+	
+	for(int c = 0; c < count; ++c)
+	{
+		bool f = false;
+		for(int l = 0; l < (int)rList.size(); ++l)
+		{
+			if(rList[l] == compareto[c])
+			{
+				f = true;
+				break;
+			}
+		}
+		found[c] = f;
+	}
+
+	bool ret = true;
+	for(int c = 0; c < count; ++c)
+	{
+		if(found[c] == false)
+		{
+			ret = false;
+		}
+	}
+
+	delete [] found;
+	
+	return ret;
+}
+
+void test_overwrites()
+{
+	// Opening twice is bad
+	{
+		RaidFileWrite writeA(0, "overwrite_A");
+		writeA.Open();
+		writeA.Write("TESTTEST", 8);
+	
+		{
+			RaidFileWrite writeA2(0, "overwrite_A");
+			TEST_CHECK_THROWS(writeA2.Open(), RaidFileException, FileIsCurrentlyOpenForWriting);
+		}
+	}
+	
+	// But opening a file which has previously been open, but isn't now, is OK.
+	
+	// Generate a random pre-existing write file (and ensure that it doesn't exist already)
+	int f;
+	TEST_THAT((f = ::open("testfiles/0_2/overwrite_B.rfwX", O_WRONLY | O_CREAT | O_EXCL, 0755)) != -1);
+	TEST_THAT(::write(f, "TESTTEST", 8) == 8);
+	::close(f);
+
+	// Attempt to overwrite it, which should work nicely.
+	RaidFileWrite writeB(0, "overwrite_B");
+	writeB.Open();
+	writeB.Write("TEST", 4);
+	TEST_THAT(writeB.GetFileSize() == 4);
+	writeB.Commit();
+}
+
+
+int test(int argc, const char *argv[])
+{
+	#ifndef TRF_CAN_INTERCEPT
+		printf("NOTE: Skipping intercept based tests on this platform.\n\n");
+	#endif
+
+	// Initialise the controller
+	RaidFileController &rcontroller = RaidFileController::GetController();
+	rcontroller.Initialise("testfiles/raidfile.conf");
+
+	// some data
+	char data[TEST_DATA_SIZE];
+	R250 random(619);
+	for(unsigned int l = 0; l < sizeof(data); ++l)
+	{
+		data[l] = random.next() & 0xff;
+	}
+	char data2[57];
+	for(unsigned int l = 0; l < sizeof(data2); ++l)
+	{
+		data2[l] = l;
+	}
+	
+	// Try creating a directory
+	RaidFileWrite::CreateDirectory(0, "test-dir");
+	TEST_THAT(TestDirExists("testfiles/0_0/test-dir"));
+	TEST_THAT(TestDirExists("testfiles/0_1/test-dir"));
+	TEST_THAT(TestDirExists("testfiles/0_2/test-dir"));
+	TEST_THAT(RaidFileRead::DirectoryExists(0, "test-dir"));
+	TEST_THAT(!RaidFileRead::DirectoryExists(0, "test-dir-not"));
+
+
+	// Test converting to disc set names
+	{
+		std::string n1(RaidFileController::DiscSetPathToFileSystemPath(0, "testX", 0));
+		std::string n2(RaidFileController::DiscSetPathToFileSystemPath(0, "testX", 1));
+		std::string n3(RaidFileController::DiscSetPathToFileSystemPath(0, "testX", 2));
+		std::string n4(RaidFileController::DiscSetPathToFileSystemPath(0, "testX", 3));
+		TEST_THAT(n1 != n2);
+		TEST_THAT(n2 != n3);
+		TEST_THAT(n1 != n3);
+		TEST_THAT(n1 == n4);		// ie wraps around
+		TRACE3("Gen paths= '%s','%s',%s'\n", n1.c_str(), n2.c_str(), n3.c_str());
+	}
+
+	// Create a RaidFile
+	RaidFileWrite write1(0, "test1");
+	IOStream &write1stream = write1;	// use the stream interface where possible
+	write1.Open();
+	write1stream.Write(data, sizeof(data));
+	write1stream.Seek(1024, IOStream::SeekType_Absolute);
+	write1stream.Write(data2, sizeof(data2));
+	write1stream.Seek(1024, IOStream::SeekType_Relative);
+	write1stream.Write(data2, sizeof(data2));
+	write1stream.Seek(0, IOStream::SeekType_End);
+	write1stream.Write(data, sizeof(data));
+	
+	// Before it's deleted, check to see the contents are as expected
+	int f;
+	TEST_THAT((f = ::open("testfiles/0_2/test1.rfwX", O_RDONLY, 0)) >= 0);
+	char buffer[sizeof(data)];
+	TEST_THAT(::read(f, buffer, sizeof(buffer)) == sizeof(buffer));
+	for(unsigned int l = 0; l < 1024; ++l)
+	{
+		TEST_THAT(buffer[l] == data[l]);
+	}
+	for(unsigned int l = 0; l < sizeof(data2); ++l)
+	{
+		TEST_THAT(buffer[l+1024] == data2[l]);
+	}
+	for(unsigned int l = 0; l < sizeof(data2); ++l)
+	{
+		TEST_THAT(buffer[l+2048+sizeof(data2)] == data2[l]);
+	}
+	TEST_THAT(::lseek(f, sizeof(data), SEEK_SET) == sizeof(buffer));
+	TEST_THAT(::read(f, buffer, sizeof(buffer)) == sizeof(buffer));
+	for(unsigned int l = 0; l < 1024; ++l)
+	{
+		TEST_THAT(buffer[l] == data[l]);
+	}
+	// make sure that's the end of the file
+	TEST_THAT(::read(f, buffer, sizeof(buffer)) == 0);
+	::close(f);
+	
+	// Commit the data
+	write1.Commit();
+	TEST_THAT((f = ::open("testfiles/0_2/test1.rfw", O_RDONLY, 0)) >= 0);
+	::close(f);
+
+	// Now try and read it
+	{
+		std::auto_ptr<RaidFileRead> pread = RaidFileRead::Open(0, "test1");
+		TEST_THAT(pread->GetFileSize() == sizeof(buffer)*2);
+		
+		char buffer[sizeof(data)];
+		TEST_THAT(pread->Read(buffer, sizeof(buffer)) == sizeof(buffer));
+		for(unsigned int l = 0; l < 1024; ++l)
+		{
+			TEST_THAT(buffer[l] == data[l]);
+		}
+		for(unsigned int l = 0; l < sizeof(data2); ++l)
+		{
+			TEST_THAT(buffer[l+1024] == data2[l]);
+		}
+		for(unsigned int l = 0; l < sizeof(data2); ++l)
+		{
+			TEST_THAT(buffer[l+2048+sizeof(data2)] == data2[l]);
+		}
+		pread->Seek(sizeof(data), IOStream::SeekType_Absolute);
+		TEST_THAT(pread->Read(buffer, sizeof(buffer)) == sizeof(buffer));
+		for(unsigned int l = 0; l < 1024; ++l)
+		{
+			TEST_THAT(buffer[l] == data[l]);
+		}
+		// make sure that's the end of the file
+		TEST_THAT(pread->Read(buffer, sizeof(buffer)) == 0);
+		// Seek backwards a bit
+		pread->Seek(-1024, IOStream::SeekType_Relative);
+		TEST_THAT(pread->Read(buffer, 1024) == 1024);
+		// make sure that's the end of the file
+		TEST_THAT(pread->Read(buffer, sizeof(buffer)) == 0);
+		// Test seeking to end works
+		pread->Seek(-1024, IOStream::SeekType_Relative);
+		TEST_THAT(pread->Read(buffer, 512) == 512);
+		pread->Seek(0, IOStream::SeekType_End);
+		TEST_THAT(pread->Read(buffer, sizeof(buffer)) == 0);
+	}
+	
+	// Delete it
+	RaidFileWrite writeDel(0, "test1");
+	writeDel.Delete();
+
+	// And again...
+	RaidFileWrite write2(0, "test1");
+	write2.Open();
+	write2.Write(data, sizeof(data));
+	// This time, discard it
+	write2.Discard();
+	TEST_THAT((f = ::open("testfiles/0_2/test1.rfw", O_RDONLY, 0)) == -1);
+	
+	// And leaving it there...
+	RaidFileWrite writeLeave(0, "test1");
+	writeLeave.Open();
+	writeLeave.Write(data, sizeof(data));
+	// This time, commit it
+	writeLeave.Commit();
+	TEST_THAT((f = ::open("testfiles/0_2/test1.rfw", O_RDONLY, 0)) != -1);
+	::close(f);
+	
+	// Then check that the thing will refuse to open it again.
+	RaidFileWrite write3(0, "test1");
+	TEST_CHECK_THROWS(write3.Open(), RaidFileException, CannotOverwriteExistingFile);
+	
+	// Test overwrite behaviour
+	test_overwrites();
+
+	// Then... open it again allowing overwrites
+	RaidFileWrite write3b(0, "test1");
+	write3b.Open(true);
+	// Write something
+	write3b.Write(data + 3, sizeof(data) - 3);
+	write3b.Commit();
+	// Test it
+	testReadingFileContents(0, "test1", data+3, sizeof(data) - 3, false /*TestRAIDProperties*/);
+
+	// And once again, but this time making it a raid file
+	RaidFileWrite write3c(0, "test1");
+	write3c.Open(true);
+	// Write something
+	write3c.Write(data + 7, sizeof(data) - 7);
+	write3c.Commit(true);	// make RAID
+	// Test it
+	testReadingFileContents(0, "test1", data+7, sizeof(data) - 7, false /*TestRAIDProperties*/);
+
+	// Test opening a file which doesn't exist
+	TEST_CHECK_THROWS(
+		std::auto_ptr<RaidFileRead> preadnotexist = RaidFileRead::Open(1, "doesnt-exist"),
+		RaidFileException, RaidFileDoesntExist);
+
+	{
+		// Test unrecoverable damage
+		RaidFileWrite w(0, "damage");
+		w.Open();
+		w.Write(data, sizeof(data));
+		w.Commit(true);
+
+		// Try removing the parity file
+		TEST_THAT(::rename("testfiles/0_0/damage.rf", "testfiles/0_0/damage.rf-NT") == 0);
+		{
+			std::auto_ptr<RaidFileRead> pr0 = RaidFileRead::Open(0, "damage");
+			pr0->Read(buffer, sizeof(data));
+		}		
+		TEST_THAT(::rename("testfiles/0_0/damage.rf-NT", "testfiles/0_0/damage.rf") == 0);
+		
+		// Delete one of the files
+		TEST_THAT(::unlink("testfiles/0_1/damage.rf") == 0);	// stripe 1
+		
+#ifdef TRF_CAN_INTERCEPT
+		// Open it and read...
+		{
+			intercept_setup_error("testfiles/0_2/damage.rf", 0, EIO, SYS_read);	// stripe 2
+			std::auto_ptr<RaidFileRead> pr1 = RaidFileRead::Open(0, "damage");
+			TEST_CHECK_THROWS(
+				pr1->Read(buffer, sizeof(data)),
+				RaidFileException, OSError);
+
+			TEST_THAT(intercept_triggered());
+			intercept_clear_setup();
+		}
+#endif //TRF_CAN_INTERCEPT
+
+		// Delete another
+		TEST_THAT(::unlink("testfiles/0_0/damage.rf") == 0);		// parity
+		
+		TEST_CHECK_THROWS(
+			std::auto_ptr<RaidFileRead> pread2 = RaidFileRead::Open(0, "damage"),
+			RaidFileException, FileIsDamagedNotRecoverable);
+	}
+
+	// Test reading a directory
+	{
+		RaidFileWrite::CreateDirectory(0, "dirread");
+		// Make some contents
+		RaidFileWrite::CreateDirectory(0, "dirread/dfsdf1");
+		RaidFileWrite::CreateDirectory(0, "dirread/ponwq2");
+		{
+			RaidFileWrite w(0, "dirread/sdf9873241");
+			w.Open();
+			w.Write(data, sizeof(data));
+			w.Commit(true);
+		}
+		{
+			RaidFileWrite w(0, "dirread/fsdcxjni3242");
+			w.Open();
+			w.Write(data, sizeof(data));
+			w.Commit(true);
+		}
+		{
+			RaidFileWrite w(0, "dirread/cskjnds3");
+			w.Open();
+			w.Write(data, sizeof(data));
+			w.Commit(false);
+		}
+		
+		const static char *dir_list1[] = {"dfsdf1", "ponwq2", 0};
+		const static char *file_list1[] = {"sdf9873241", "fsdcxjni3242", "cskjnds3", 0};
+		const static char *file_list2[] = {"fsdcxjni3242", "cskjnds3", 0};
+		
+		std::vector<std::string> names;
+		TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names));
+		TEST_THAT(list_matches(names, file_list1));
+		TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_DirsOnly, names));
+		TEST_THAT(list_matches(names, dir_list1));
+		// Delete things
+		TEST_THAT(::unlink("testfiles/0_0/dirread/sdf9873241.rf") == 0);
+		TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names));
+		TEST_THAT(list_matches(names, file_list1));
+		// Delete something else so that it's not recoverable
+		TEST_THAT(::unlink("testfiles/0_1/dirread/sdf9873241.rf") == 0);
+		TEST_THAT(false == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names));
+		TEST_THAT(list_matches(names, file_list1));
+		// And finally...
+		TEST_THAT(::unlink("testfiles/0_2/dirread/sdf9873241.rf") == 0);
+		TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names));
+		TEST_THAT(list_matches(names, file_list2));
+	}
+
+	// Check that sizes are reported correctly for non-raid discs
+	{
+		int sizeInBlocks = (sizeof(data) + RAID_BLOCK_SIZE - 1) / RAID_BLOCK_SIZE;
+		// for writing
+		{
+			RaidFileWrite write(2, "testS");
+			write.Open();
+			write.Write(data, sizeof(data));
+			TEST_THAT(write.GetDiscUsageInBlocks() == sizeInBlocks);
+			write.Commit();
+		}
+		// for reading
+		{
+			std::auto_ptr<RaidFileRead> pread(RaidFileRead::Open(2, "testS"));
+			TEST_THAT(pread->GetDiscUsageInBlocks() == sizeInBlocks);
+		}
+	}
+	
+//printf("SKIPPING tests ------------------\n");
+//return 0;
+
+	// Test a load of transformed things
+	#define BIG_BLOCK_SIZE (25*1024 + 19)
+	MemoryBlockGuard<void*> bigblock(BIG_BLOCK_SIZE);
+	R250 randomX2(2165);
+	for(unsigned int l = 0; l < BIG_BLOCK_SIZE; ++l)
+	{
+		((char*)(void*)bigblock)[l] = randomX2.next() & 0xff;
+	}
+	
+	// First on one size of data, on different discs
+	testReadWriteFile(0, "testdd", data, sizeof(data));
+	testReadWriteFile(0, "test2", bigblock, BIG_BLOCK_SIZE);
+	testReadWriteFile(1, "testThree", bigblock, BIG_BLOCK_SIZE - 2048);
+	testReadWriteFile(1, "testX", bigblock, BIG_BLOCK_SIZE - 2289);
+	testReadWriteFile(1, "testSmall0", data, 0);
+	testReadWriteFile(1, "testSmall1", data, 1);
+	testReadWriteFile(1, "testSmall2", data, 2);
+	testReadWriteFile(1, "testSmall3", data, 3);
+	testReadWriteFile(1, "testSmall4", data, 4);
+	testReadWriteFile(0, "testSmall5", data, 5);
+	testReadWriteFile(0, "testSmall6", data, 6);
+	testReadWriteFile(1, "testSmall7", data, 7);
+	testReadWriteFile(1, "testSmall8", data, 8);
+	testReadWriteFile(1, "testSmall9", data, 9);
+	testReadWriteFile(1, "testSmall10", data, 10);
+	// See about a file which is one block bigger than the previous tests
+	{
+		char dataonemoreblock[TEST_DATA_SIZE + RAID_BLOCK_SIZE];
+		R250 random(715);
+		for(unsigned int l = 0; l < sizeof(dataonemoreblock); ++l)
+		{
+			dataonemoreblock[l] = random.next() & 0xff;
+		}
+		testReadWriteFile(0, "testfour", dataonemoreblock, sizeof(dataonemoreblock));
+	}
+	
+	// Some more nasty sizes
+	static int nastysize[] = {0, 1, 2, 7, 8, 9, (RAID_BLOCK_SIZE/2)+3,
+			RAID_BLOCK_SIZE-9, RAID_BLOCK_SIZE-8, RAID_BLOCK_SIZE-7, RAID_BLOCK_SIZE-6, RAID_BLOCK_SIZE-5,
+			RAID_BLOCK_SIZE-4, RAID_BLOCK_SIZE-3, RAID_BLOCK_SIZE-2, RAID_BLOCK_SIZE-1};
+	for(int o = 0; o <= 5; ++o)
+	{
+		for(unsigned int n = 0; n < (sizeof(nastysize)/sizeof(nastysize[0])); ++n)
+		{
+			int s = (o*RAID_BLOCK_SIZE)+nastysize[n];
+			char fn[64];
+			sprintf(fn, "testN%d", s);
+			testReadWriteFile(n&1, fn, bigblock, s);
+		}
+	}
+	
+	// Finally, a mega test (not necessary for every run, I would have thought)
+/*	unsigned int megamax = (1024*128) + 9;
+	MemoryBlockGuard<void*> megablock(megamax);
+	R250 randomX3(183);
+	for(unsigned int l = 0; l < megamax; ++l)
+	{
+		((char*)(void*)megablock)[l] = randomX3.next() & 0xff;
+	}
+	for(unsigned int s = 0; s < megamax; ++s)
+	{
+		testReadWriteFile(s & 1, "mega", megablock, s);
+		RaidFileWrite deleter(s & 1, "mega");
+		deleter.Delete();
+		RaidFileWrite deleter2(s & 1, "megaNT");
+		deleter2.Delete();
+	}*/
+	
+	return 0;
+}