12 files changed, 3411 insertions, 0 deletions

diff --git a/lib/raidfile/Makefile.extra b/lib/raidfile/Makefile.extra
new file mode 100755
index 00000000..8d036633
--- /dev/null
+++ b/lib/raidfile/Makefile.extra
@@ -0,0 +1,7 @@
+
+MAKEEXCEPTION = ../../lib/common/makeexception.pl
+
+# AUTOGEN SEEDING
+autogen_RaidFileException.h autogen_RaidFileException.cpp:	$(MAKEEXCEPTION) RaidFileException.txt
+	perl $(MAKEEXCEPTION) RaidFileException.txt
+
diff --git a/lib/raidfile/RaidFileController.cpp b/lib/raidfile/RaidFileController.cpp
new file mode 100755
index 00000000..89ae6791
--- /dev/null
+++ b/lib/raidfile/RaidFileController.cpp
@@ -0,0 +1,217 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileController.cpp
+//		Purpose: Controls config and daemon comms for RaidFile classes
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "RaidFileController.h"
+#include "RaidFileException.h"
+#include "Configuration.h"
+
+#include "MemLeakFindOn.h"
+
+RaidFileController RaidFileController::mController;
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::RaidFileController()
+//		Purpose: Constructor
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+RaidFileController::RaidFileController()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::~RaidFileController()
+//		Purpose: Destructor
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+RaidFileController::~RaidFileController()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::RaidFileController()
+//		Purpose: Copy constructor
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+RaidFileController::RaidFileController(const RaidFileController &rController)
+{
+	THROW_EXCEPTION(RaidFileException, Internal)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::Initialise(const char *)
+//		Purpose: Initialises the system, loading the configuration file.
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+void RaidFileController::Initialise(const char *ConfigFilename)
+{
+	static const ConfigurationVerifyKey verifykeys[] =
+	{
+		{"SetNumber",	0,	ConfigTest_Exists | ConfigTest_IsInt, 0},
+		{"BlockSize",	0,	ConfigTest_Exists | ConfigTest_IsInt, 0},
+		{"Dir0", 		0,	ConfigTest_Exists, 0},
+		{"Dir1", 		0,	ConfigTest_Exists, 0},
+		{"Dir2", 		0,	ConfigTest_Exists | ConfigTest_LastEntry, 0}
+	};
+	
+	static const ConfigurationVerify subverify = 
+	{
+		"*",
+		0,
+		verifykeys,
+		ConfigTest_LastEntry,
+		0
+	};
+	
+	static const ConfigurationVerify verify = 
+	{
+		"RAID FILE CONFIG",
+		&subverify,
+		0,
+		ConfigTest_LastEntry,
+		0
+	};
+	
+	// Load the configuration
+	std::string err;
+	std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(ConfigFilename, &verify, err);
+	
+	if(pconfig.get() == 0 || !err.empty())
+	{
+		fprintf(stderr, "RaidFile configuation file errors:\n%s", err.c_str());
+		THROW_EXCEPTION(RaidFileException, BadConfigFile)
+	}
+	
+	// Use the values
+	int expectedSetNum = 0;
+	std::vector<std::string> confdiscs(pconfig->GetSubConfigurationNames());
+	for(std::vector<std::string>::const_iterator i(confdiscs.begin()); i != confdiscs.end(); ++i)
+	{
+		const Configuration &disc(pconfig->GetSubConfiguration((*i).c_str()));
+		
+		int setNum = disc.GetKeyValueInt("SetNumber");
+		if(setNum != expectedSetNum)
+		{
+			THROW_EXCEPTION(RaidFileException, BadConfigFile)			
+		}
+		RaidFileDiscSet set(setNum, (unsigned int)disc.GetKeyValueInt("BlockSize"));
+		// Get the values of the directory keys
+		std::string d0(disc.GetKeyValue("Dir0"));
+		std::string d1(disc.GetKeyValue("Dir1"));
+		std::string d2(disc.GetKeyValue("Dir2"));
+		// Are they all different (using RAID) or all the same (not using RAID)
+		if(d0 != d1 && d1 != d2 && d0 != d2)
+		{
+			set.push_back(d0);
+			set.push_back(d1);
+			set.push_back(d2);
+		}
+		else if(d0 == d1 && d0 == d2)
+		{
+			// Just push the first one, which is the non-RAID place to store files
+			set.push_back(d0);
+		}
+		else
+		{
+			// One must be the same as another! Which is bad.
+			THROW_EXCEPTION(RaidFileException, BadConfigFile)			
+		}
+		mSetList.push_back(set);
+		expectedSetNum++;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::GetDiscSet(int)
+//		Purpose: Returns the numbered disc set
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+RaidFileDiscSet &RaidFileController::GetDiscSet(unsigned int DiscSetNum)
+{
+	if(DiscSetNum < 0 || DiscSetNum >= mSetList.size())
+	{
+		THROW_EXCEPTION(RaidFileException, NoSuchDiscSet)
+	}
+
+	return mSetList[DiscSetNum];
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileDiscSet::GetSetNumForWriteFiles(const std::string &)
+//		Purpose: Returns the set number the 'temporary' written files should
+//				 be stored on, given a filename.
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+int RaidFileDiscSet::GetSetNumForWriteFiles(const std::string &rFilename) const
+{
+	// Simple hash function, add up the ASCII values of all the characters,
+	// and get modulo number of partitions in the set.
+	std::string::const_iterator i(rFilename.begin());
+	int h = 0;
+	for(; i != rFilename.end(); ++i)
+	{
+		h += (*i);
+	}
+	return h % size();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileController::DiscSetPathToFileSystemPath(unsigned int, const std::string &, int)
+//		Purpose: Given a Raid File style file name, return a filename for the physical filing system.
+//				 DiscOffset is effectively the disc number (but remember files are rotated around the
+//				 discs in a disc set)
+//		Created: 19/1/04
+//
+// --------------------------------------------------------------------------
+std::string RaidFileController::DiscSetPathToFileSystemPath(unsigned int DiscSetNum, const std::string &rFilename, int DiscOffset)
+{
+	if(DiscSetNum < 0 || DiscSetNum >= mController.mSetList.size())
+	{
+		THROW_EXCEPTION(RaidFileException, NoSuchDiscSet)
+	}
+
+	// Work out which disc it's to be on
+	int disc = (mController.mSetList[DiscSetNum].GetSetNumForWriteFiles(rFilename) + DiscOffset)
+						% mController.mSetList[DiscSetNum].size();
+	
+	// Make the string
+	std::string r((mController.mSetList[DiscSetNum])[disc]);
+	r += DIRECTORY_SEPARATOR_ASCHAR;
+	r += rFilename;
+	return r;
+}
+
+
+
diff --git a/lib/raidfile/RaidFileController.h b/lib/raidfile/RaidFileController.h
new file mode 100755
index 00000000..4962d236
--- /dev/null
+++ b/lib/raidfile/RaidFileController.h
@@ -0,0 +1,107 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileController.h
+//		Purpose: Controls config and daemon comms for RaidFile classes
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+/*  NOTE: will log to local5: include a line like
+	local5.info                                             /var/log/raidfile
+	in /etc/syslog.conf
+*/
+
+#ifndef RAIDFILECONTROLLER__H
+#define RAIDFILECONTROLLER__H
+
+#include <string>
+#include <vector>
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileDiscSet
+//		Purpose: Describes a set of paritions for RAID like files.
+//				 Use as list of directories containing the files.
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+class RaidFileDiscSet : public std::vector<std::string>
+{
+public:
+	RaidFileDiscSet(int SetID, unsigned int BlockSize)
+		: mSetID(SetID),
+		  mBlockSize(BlockSize)
+	{
+	}
+	RaidFileDiscSet(const RaidFileDiscSet &rToCopy)
+		: std::vector<std::string>(rToCopy),
+		  mSetID(rToCopy.mSetID),
+		  mBlockSize(rToCopy.mBlockSize)
+	{
+	}
+	
+	~RaidFileDiscSet()
+	{
+	}
+
+	int GetSetID() const {return mSetID;}
+	
+	int GetSetNumForWriteFiles(const std::string &rFilename) const;
+	
+	unsigned int GetBlockSize() const {return mBlockSize;}
+
+	// Is this disc set a non-RAID disc set? (ie files never get transformed to raid storage)
+	bool IsNonRaidSet() const {return 1 == size();}
+
+private:
+	int mSetID;
+	unsigned int mBlockSize;
+};
+
+class _RaidFileController;	// compiler warning avoidance
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileController
+//		Purpose: Manages the configuration of the RaidFile system, handles
+//				 communication with the daemon.
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+class RaidFileController
+{
+	friend class _RaidFileController;	// to avoid compiler warning
+private:
+	RaidFileController();
+	RaidFileController(const RaidFileController &rController);
+public:
+	~RaidFileController();
+	
+public:
+	void Initialise(const char *ConfigFilename = "/etc/box/raidfile.conf");
+	int GetNumDiscSets() {return mSetList.size();}
+
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    RaidFileController::GetController()
+	//		Purpose: Gets the one and only controller object.
+	//		Created: 2003/07/08
+	//
+	// --------------------------------------------------------------------------	
+	static RaidFileController &GetController() {return mController;}
+	RaidFileDiscSet &GetDiscSet(unsigned int DiscSetNum);
+
+	static std::string DiscSetPathToFileSystemPath(unsigned int DiscSetNum, const std::string &rFilename, int DiscOffset);
+	
+private:
+	std::vector<RaidFileDiscSet> mSetList;
+	
+	static RaidFileController mController;
+};
+
+#endif // RAIDFILECONTROLLER__H
+
diff --git a/lib/raidfile/RaidFileException.h b/lib/raidfile/RaidFileException.h
new file mode 100755
index 00000000..809d4d5a
--- /dev/null
+++ b/lib/raidfile/RaidFileException.h
@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileException.h
+//		Purpose: Exception
+//		Created: 2003/07/08
+//
+// --------------------------------------------------------------------------
+
+#ifndef RAIDFILEEXCEPTION__H
+#define RAIDFILEEXCEPTION__H
+
+// Compatibility
+#include "autogen_RaidFileException.h"
+
+#endif // RAIDFILEEXCEPTION__H
+
diff --git a/lib/raidfile/RaidFileException.txt b/lib/raidfile/RaidFileException.txt
new file mode 100644
index 00000000..6ad74563
--- /dev/null
+++ b/lib/raidfile/RaidFileException.txt
@@ -0,0 +1,25 @@
+EXCEPTION RaidFile 2
+
+Internal						0
+CantOpenConfigFile				1	The raidfile.conf file is not accessible. Check that it is present in the default location or daemon configuration files point to the correct location.
+BadConfigFile					2
+NoSuchDiscSet					3
+CannotOverwriteExistingFile		4
+AlreadyOpen						5
+ErrorOpeningWriteFile			6
+NotOpen							7
+OSError							8	Error when accessing an underlying file. Check file permissions allow files to be read and written in the configured raid directories.
+WriteFileOpenOnTransform		9
+WrongNumberOfDiscsInSet			10	There should be three directories in each disc set.
+RaidFileDoesntExist				11
+ErrorOpeningFileForRead			12
+FileIsDamagedNotRecoverable		13
+InvalidRaidFile					14
+DirectoryIncomplete				15
+UnexpectedFileInDirPlace		16
+FileExistsInDirectoryCreation	17
+UnsupportedReadWriteOrClose		18
+CanOnlyGetUsageBeforeCommit		19
+CanOnlyGetFileSizeBeforeCommit	20
+ErrorOpeningWriteFileOnTruncate	21
+FileIsCurrentlyOpenForWriting	22
diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp
new file mode 100755
index 00000000..fed22ac0
--- /dev/null
+++ b/lib/raidfile/RaidFileRead.cpp
@@ -0,0 +1,1701 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileRead.cpp
+//		Purpose: Read Raid like Files
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/uio.h>
+#include <syslog.h>
+#include <stdarg.h>
+#include <dirent.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <memory>
+#include <map>
+
+#include "RaidFileRead.h"
+#include "RaidFileException.h"
+#include "RaidFileController.h"
+#include "RaidFileUtil.h"
+
+#ifdef PLATFORM_LINUX
+	#include "LinuxWorkaround.h"
+#endif
+
+#include "MemLeakFindOn.h"
+
+#define READ_NUMBER_DISCS_REQUIRED	3
+#define READV_MAX_BLOCKS			64
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileRead_NonRaid
+//		Purpose: Internal class for reading RaidFiles which haven't been transformed
+//				 into the RAID like form yet.
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+class RaidFileRead_NonRaid : public RaidFileRead
+{
+public:
+	RaidFileRead_NonRaid(int SetNumber, const std::string &Filename, int OSFileHandle);
+	virtual ~RaidFileRead_NonRaid();
+private:
+	RaidFileRead_NonRaid(const RaidFileRead_NonRaid &rToCopy);
+
+public:
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type GetPosition() const;
+	virtual void Seek(IOStream::pos_type Offset, int SeekType);
+	virtual void Close();
+	virtual pos_type GetFileSize() const;
+	virtual bool StreamDataLeft();
+
+private:
+	int mOSFileHandle;
+	bool mEOF;
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid(int, const std::string &, const std::string &)
+//		Purpose: Constructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead_NonRaid::RaidFileRead_NonRaid(int SetNumber, const std::string &Filename, int OSFileHandle)
+	: RaidFileRead(SetNumber, Filename),
+	  mOSFileHandle(OSFileHandle),
+	  mEOF(false)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::~RaidFileRead_NonRaid()
+//		Purpose: Destructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead_NonRaid::~RaidFileRead_NonRaid()
+{
+	if(mOSFileHandle != -1)
+	{
+		Close();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::Read(const void *, int)
+//		Purpose: Reads bytes from the file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+int RaidFileRead_NonRaid::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Read data
+	int bytesRead = ::read(mOSFileHandle, pBuffer, NBytes);
+	if(bytesRead == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	// Check for EOF
+	if(bytesRead == 0)
+	{
+		mEOF = true;
+	}
+
+	return bytesRead;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::GetPosition()
+//		Purpose: Returns current position
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead::pos_type RaidFileRead_NonRaid::GetPosition() const
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Use lseek to find the current file position
+	off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR);
+	if(p == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	return p;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::Seek(pos_type, int)
+//		Purpose: Seek within the file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_NonRaid::Seek(IOStream::pos_type Offset, int SeekType)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Seek...
+	if(::lseek(mOSFileHandle, Offset, ConvertSeekTypeToOSWhence(SeekType)) == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	// Not EOF any more
+	mEOF = false;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::Close()
+//		Purpose: Close the file (automatically done by destructor)
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_NonRaid::Close()
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Close file...
+	if(::close(mOSFileHandle) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	mOSFileHandle = -1;
+	mEOF = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::GetFileSize()
+//		Purpose: Returns file size.
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+RaidFileRead::pos_type RaidFileRead_NonRaid::GetFileSize() const
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// stat the file
+	struct stat st;
+	if(::fstat(mOSFileHandle, &st) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	return st.st_size;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::StreamDataLeft()
+//		Purpose: Any data left?
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead_NonRaid::StreamDataLeft()
+{
+	return !mEOF;
+}
+
+// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileRead_Raid
+//		Purpose: Internal class for reading RaidFiles have been transformed.
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+class RaidFileRead_Raid : public RaidFileRead
+{
+public:
+	friend class RaidFileRead;
+	RaidFileRead_Raid(int SetNumber, const std::string &Filename, int Stripe1Handle,
+		int Stripe2Handle, int ParityHandle, pos_type FileSize, unsigned int BlockSize,
+		bool LastBlockHasSize);
+	virtual ~RaidFileRead_Raid();
+private:
+	RaidFileRead_Raid(const RaidFileRead_Raid &rToCopy);
+
+public:
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);
+	virtual pos_type GetPosition() const;
+	virtual void Seek(IOStream::pos_type Offset, int SeekType);
+	virtual void Close();
+	virtual pos_type GetFileSize() const;
+	virtual bool StreamDataLeft();
+
+private:
+	int ReadRecovered(void *pBuffer, int NBytes);
+	void AttemptToRecoverFromIOError(bool Stripe1);
+	void SetPosition(pos_type FilePosition);
+	static void MoveDamagedFileAlertDaemon(int SetNumber, const std::string &Filename, bool Stripe1);
+
+private:
+	int mStripe1Handle;
+	int mStripe2Handle;
+	int mParityHandle;
+	pos_type mFileSize;
+	unsigned int mBlockSize;
+	pos_type mCurrentPosition;
+	char *mRecoveryBuffer;
+	pos_type mRecoveryBufferStart;
+	bool mLastBlockHasSize;
+	bool mEOF;
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid(int, const std::string &, const std::string &)
+//		Purpose: Constructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead_Raid::RaidFileRead_Raid(int SetNumber, const std::string &Filename, int Stripe1Handle, int Stripe2Handle, int ParityHandle, pos_type FileSize, unsigned int BlockSize, bool LastBlockHasSize)
+	: RaidFileRead(SetNumber, Filename),
+	  mStripe1Handle(Stripe1Handle),
+	  mStripe2Handle(Stripe2Handle),
+	  mParityHandle(ParityHandle),
+	  mFileSize(FileSize),
+	  mBlockSize(BlockSize),
+	  mCurrentPosition(0),
+	  mRecoveryBuffer(0),
+	  mRecoveryBufferStart(-1),
+	  mLastBlockHasSize(LastBlockHasSize),
+	  mEOF(false)
+{
+	// Make sure size of the IOStream::pos_type matches the pos_type used
+#ifdef PLATFORM_LINUX
+	ASSERT(sizeof(pos_type) >= sizeof(off_t));
+#else
+	ASSERT(sizeof(pos_type) == sizeof(off_t));
+#endif
+	
+	// Sanity check handles
+	if(mStripe1Handle != -1 && mStripe2Handle != -1)
+	{
+		// Everything is lovely, got two perfect files
+	}
+	else
+	{
+		// Check we have at least one stripe and a parity file
+		if((mStripe1Handle == -1 && mStripe2Handle == -1) || mParityHandle == -1)
+		{
+			// Should never have got this far
+			THROW_EXCEPTION(RaidFileException, Internal)
+		}
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::~RaidFileRead_Raid()
+//		Purpose: Destructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead_Raid::~RaidFileRead_Raid()
+{
+	Close();
+	if(mRecoveryBuffer != 0)
+	{
+		::free(mRecoveryBuffer);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::Read(const void *, int)
+//		Purpose: Reads bytes from the file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+int RaidFileRead_Raid::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	// How many more bytes could we read?
+	unsigned int maxRead = mFileSize - mCurrentPosition;
+	if((unsigned int)NBytes > maxRead)
+	{
+		NBytes = maxRead;
+	}
+	
+	// Return immediately if there's nothing to read, and set EOF
+	if(NBytes == 0)
+	{
+		mEOF = true;
+		return 0;
+	}
+	
+	// Can we use the normal file reading routine?
+	if(mStripe1Handle == -1 || mStripe2Handle == -1)
+	{
+		// File is damaged, try a the recovery read function
+		return ReadRecovered(pBuffer, NBytes);
+	}
+	
+	// Vectors for reading stuff from the files
+	struct iovec stripe1Reads[READV_MAX_BLOCKS];
+	struct iovec stripe2Reads[READV_MAX_BLOCKS];
+	struct iovec *stripeReads[2] = {stripe1Reads, stripe2Reads};
+	unsigned int stripeReadsDataSize[2] = {0, 0};
+	unsigned int stripeReadsSize[2] = {0, 0};
+	int stripeHandles[2] = {mStripe1Handle, mStripe2Handle};
+	
+	// Which block are we doing?
+	unsigned int currentBlock = mCurrentPosition / mBlockSize;
+	unsigned int bytesLeftInCurrentBlock = mBlockSize - (mCurrentPosition % mBlockSize);
+	ASSERT(bytesLeftInCurrentBlock > 0)
+	unsigned int leftToRead = NBytes;
+	char *bufferPtr = (char*)pBuffer;
+	
+	// Now... add some whole block entries in...
+	try
+	{
+		while(leftToRead > 0)
+		{
+			int whichStripe = (currentBlock & 1);
+			size_t rlen = mBlockSize;
+			// Adjust if it's the first block
+			if(bytesLeftInCurrentBlock != 0)
+			{
+				rlen = bytesLeftInCurrentBlock;
+				bytesLeftInCurrentBlock = 0;
+			}
+			// Adjust if we're out of bytes
+			if(rlen > leftToRead)
+			{
+				rlen = leftToRead;
+			}
+			stripeReads[whichStripe][stripeReadsSize[whichStripe]].iov_base = bufferPtr;
+			stripeReads[whichStripe][stripeReadsSize[whichStripe]].iov_len = rlen;
+			stripeReadsSize[whichStripe]++;
+			stripeReadsDataSize[whichStripe] += rlen;
+			leftToRead -= rlen;
+			bufferPtr += rlen;
+			currentBlock++;
+
+			// Read data?
+			for(int s = 0; s < 2; ++s)
+			{
+				if((leftToRead == 0 || stripeReadsSize[s] >= READV_MAX_BLOCKS) && stripeReadsSize[s] > 0)
+				{
+					int r = ::readv(stripeHandles[s], stripeReads[s], stripeReadsSize[s]);
+					if(r == -1)
+					{
+						// Bad news... IO error?
+						if(errno == EIO)
+						{
+							// Attempt to recover from this failure
+							AttemptToRecoverFromIOError((s == 0) /* is stripe 1 */);
+							// Retry
+							return Read(pBuffer, NBytes, Timeout);
+						}
+						else
+						{
+							// Can't do anything, throw
+							THROW_EXCEPTION(RaidFileException, OSError)
+						}
+					}
+					else if(r != (int)stripeReadsDataSize[s])
+					{
+						// Got the file sizes wrong/logic error!
+						THROW_EXCEPTION(RaidFileException, Internal)
+					}
+					stripeReadsSize[s] = 0;
+					stripeReadsDataSize[s] = 0;
+				}
+			}
+		}
+	}
+	catch(...)
+	{
+		// Get file pointers to right place (to meet exception safe stuff)
+		SetPosition(mCurrentPosition);
+		
+		throw;
+	}
+	
+	// adjust current position
+	mCurrentPosition += NBytes;
+
+	return NBytes;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::MoveDamagedFileAlertDaemon(bool)
+//		Purpose: Moves a file into the damaged directory, and alerts the Daemon to recover it properly later.
+//		Created: 2003/07/22
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::MoveDamagedFileAlertDaemon(int SetNumber, const std::string &Filename, bool Stripe1)
+{
+	// Move the dodgy file away
+	// Get the controller and the disc set we're on
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+	if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size())
+	{
+		THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet)
+	}
+	// Start disc
+	int startDisc = rdiscSet.GetSetNumForWriteFiles(Filename);
+	int errOnDisc = (startDisc + (Stripe1?0:1)) % READ_NUMBER_DISCS_REQUIRED;
+	
+	// Make a munged filename for renaming
+	std::string mungeFn(Filename + RAIDFILE_EXTENSION);
+	std::string awayName;
+	for(std::string::const_iterator i = mungeFn.begin(); i != mungeFn.end(); ++i)
+	{
+		char c = (*i);
+		if(c == DIRECTORY_SEPARATOR_ASCHAR)
+		{
+			awayName += '_';
+		}
+		else if(c == '_')
+		{
+			awayName += "__";
+		}
+		else
+		{
+			awayName += c;
+		}
+	}
+	// Make sure the error files directory exists
+	std::string dirname(rdiscSet[errOnDisc] + DIRECTORY_SEPARATOR ".raidfile-unreadable");
+	int mdr = ::mkdir(dirname.c_str(), 0750);
+	if(mdr != 0 && errno != EEXIST)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	// Attempt to rename the file there -- ignore any return code here, as it's dubious anyway
+	std::string errorFile(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, errOnDisc));
+	::rename(errorFile.c_str(), (dirname + DIRECTORY_SEPARATOR_ASCHAR + awayName).c_str());
+
+	// TODO: Inform the recovery daemon
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::AttemptToRecoverFromIOError(bool)
+//		Purpose: Attempt to recover from an IO error, setting up to read from parity instead.
+//				 Will exception if this isn't possible.
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::AttemptToRecoverFromIOError(bool Stripe1)
+{
+	TRACE3("Attempting to recover from I/O error: %d %s, on stripe %d\n", mSetNumber, mFilename.c_str(), Stripe1?1:2);
+	::syslog(LOG_ERR | LOG_LOCAL5, "Attempting to recover from I/O error: %d %s, on stripe %d\n", mSetNumber, mFilename.c_str(), Stripe1?1:2);
+
+	// Close offending file
+	if(Stripe1)
+	{
+		if(mStripe1Handle != -1)
+		{
+			::close(mStripe1Handle);
+			mStripe1Handle = -1;
+		}
+	}
+	else
+	{
+		if(mStripe2Handle != -1)
+		{
+			::close(mStripe2Handle);
+			mStripe2Handle = -1;
+		}
+	}
+
+	// Check...
+	ASSERT((Stripe1?mStripe2Handle:mStripe1Handle) != -1);
+
+	// Get rid of the damaged file
+	MoveDamagedFileAlertDaemon(mSetNumber, mFilename, Stripe1);
+
+	// Get the controller and the disc set we're on
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size())
+	{
+		THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet)
+	}
+	// Start disc
+	int startDisc = rdiscSet.GetSetNumForWriteFiles(mFilename);
+
+	// Mark as nothing in recovery buffer
+	mRecoveryBufferStart = -1;
+	
+	// Seek to zero on the remaining file -- get to nice state
+	if(::lseek(Stripe1?mStripe2Handle:mStripe1Handle, 0, SEEK_SET) == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	// Open the parity file
+	std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (2 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+	mParityHandle = ::open(parityFilename.c_str(), O_RDONLY, 0555);
+	if(mParityHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	// Work out whether or not there's a size XORed into the last block
+	unsigned int bytesInLastTwoBlocks = mFileSize % (mBlockSize * 2);
+	if(bytesInLastTwoBlocks > mBlockSize && bytesInLastTwoBlocks < ((mBlockSize * 2) - sizeof(FileSizeType)))
+	{
+		// Yes, there's something to XOR in the last block
+		mLastBlockHasSize = true;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::ReadRecovered(const void *, int)
+//		Purpose: Reads data recreating from the parity stripe
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+int RaidFileRead_Raid::ReadRecovered(void *pBuffer, int NBytes)
+{
+	// Note: NBytes has been adjusted to definately be a range
+	// inside the given file length.
+
+	// Make sure a buffer is allocated
+	if(mRecoveryBuffer == 0)
+	{
+		mRecoveryBuffer = (char*)::malloc(mBlockSize * 2);
+		if(mRecoveryBuffer == 0)
+		{
+			throw std::bad_alloc();
+		}
+	}
+	
+	// Which stripe?
+	int stripe = (mStripe1Handle != -1)?mStripe1Handle:mStripe2Handle;
+	if(stripe == -1)
+	{
+		// Not enough file handles around
+		THROW_EXCEPTION(RaidFileException, FileIsDamagedNotRecoverable)
+	}
+	
+	char *outptr = (char*)pBuffer;
+	int bytesToGo = NBytes;
+	
+	pos_type preservedCurrentPosition = mCurrentPosition;
+	
+	try
+	{
+		// Start offset within buffer
+		int offset = (mCurrentPosition - mRecoveryBufferStart);
+		// Let's go!
+		while(bytesToGo > 0)
+		{
+			int bytesLeftInBuffer = 0;
+			if(mRecoveryBufferStart != -1)
+			{
+				bytesLeftInBuffer = (mRecoveryBufferStart + (mBlockSize*2)) - mCurrentPosition;
+				ASSERT(bytesLeftInBuffer >= 0);
+			}
+			
+			// How many bytes can be copied out?
+			int toCopy = bytesLeftInBuffer;
+			if(toCopy > bytesToGo) toCopy = bytesToGo;
+			//printf("offset = %d, tocopy = %d, bytestogo = %d, leftinbuffer = %d\n", (int)offset, toCopy, bytesToGo, bytesLeftInBuffer);
+			if(toCopy > 0)
+			{
+				for(int l = 0; l < toCopy; ++l)
+				{
+					*(outptr++) = mRecoveryBuffer[offset++];
+				}
+				bytesToGo -= toCopy;
+				mCurrentPosition += toCopy;
+			}
+			
+			// Load in the next buffer?
+			if(bytesToGo > 0)
+			{
+				// Calculate the blocks within the file that are needed to be loaded.
+				pos_type fileBlock = mCurrentPosition / (mBlockSize * 2);
+				// Is this the last block
+				bool isLastBlock = (fileBlock == (mFileSize / (mBlockSize * 2)));
+				
+				// Need to reposition file pointers?
+				if(mRecoveryBufferStart == -1)
+				{
+					// Yes!
+					// And the offset from which to read it
+					pos_type filePos = fileBlock * mBlockSize;
+					// Then seek
+					if(::lseek(stripe, filePos, SEEK_SET) == -1
+						|| ::lseek(mParityHandle, filePos, SEEK_SET) == -1)
+					{
+						THROW_EXCEPTION(RaidFileException, OSError)
+					}
+				}
+				
+				// Load a block from each file, getting the ordering the right way round
+				int r1 = ::read((mStripe1Handle != -1)?stripe:mParityHandle, mRecoveryBuffer, mBlockSize);
+				int r2 = ::read((mStripe1Handle != -1)?mParityHandle:stripe, mRecoveryBuffer + mBlockSize, mBlockSize);				
+				if(r1 == -1 || r2 == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+
+				// error checking and manipulation
+				if(isLastBlock)
+				{
+					// Allow not full reads, and append zeros if necessary to fill the space.
+					int r1zeros = mBlockSize - r1;
+					if(r1zeros > 0)
+					{
+						::memset(mRecoveryBuffer + r1, 0, r1zeros);
+					}
+					int r2zeros = mBlockSize - r2;
+					if(r2zeros > 0)
+					{
+						::memset(mRecoveryBuffer + mBlockSize + r2, 0, r2zeros);
+					}
+					
+					// if it's got the file size in it, XOR it off
+					if(mLastBlockHasSize)
+					{
+						int sizeXorOffset = (mBlockSize - sizeof(FileSizeType)) + ((mStripe1Handle != -1)?mBlockSize:0);
+						*((FileSizeType*)(mRecoveryBuffer + sizeXorOffset)) ^= ntoh64(mFileSize);
+					}
+				}
+				else
+				{
+					// Must have got a full block, otherwise things are a bit bad here.
+					if(r1 != (int)mBlockSize || r2 != (int)mBlockSize)
+					{
+						THROW_EXCEPTION(RaidFileException, InvalidRaidFile)
+					}
+				}
+				
+				// Go XORing!
+				unsigned int *b1 = (unsigned int*)mRecoveryBuffer;
+				unsigned int *b2 = (unsigned int *)(mRecoveryBuffer + mBlockSize);
+				if((mStripe1Handle == -1))
+				{
+					b1 = b2;
+					b2 = (unsigned int*)mRecoveryBuffer;
+				}
+				for(int x = ((mBlockSize/sizeof(unsigned int)) - 1); x >= 0; --x)
+				{
+					*b2 = (*b1) ^ (*b2);
+					++b1;
+					++b2;
+				}
+				
+				// New block location
+				mRecoveryBufferStart = fileBlock * (mBlockSize * 2);
+				
+				// New offset withing block
+				offset = (mCurrentPosition - mRecoveryBufferStart);
+				ASSERT(offset >= 0);
+			}
+		}
+	}
+	catch(...)
+	{
+		// Change variables so 1) buffer is invalidated and 2) the file will be seeked properly the next time round
+		mRecoveryBufferStart = -1;
+		mCurrentPosition = preservedCurrentPosition;
+		throw;
+	}
+	
+	return NBytes;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::GetPosition()
+//		Purpose: Returns current position
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileRead_Raid::GetPosition() const
+{
+	return mCurrentPosition;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::Seek(RaidFileRead::pos_type, bool)
+//		Purpose: Seek within the file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::Seek(IOStream::pos_type Offset, int SeekType)
+{
+	pos_type newpos = mCurrentPosition;
+	switch(SeekType)
+	{
+	case IOStream::SeekType_Absolute:
+		newpos = Offset;
+		break;
+		
+	case IOStream::SeekType_Relative:
+		newpos += Offset;
+		break;
+		
+	case IOStream::SeekType_End:
+		newpos = mFileSize + Offset;
+		break;
+		
+	default:
+		THROW_EXCEPTION(CommonException, IOStreamBadSeekType)
+	}
+	
+	if(newpos != mCurrentPosition)
+	{
+		SetPosition(newpos);
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::SetPosition(pos_type)
+//		Purpose: Move the file pointers
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::SetPosition(pos_type FilePosition)
+{
+	if(FilePosition > mFileSize)
+	{
+		FilePosition = mFileSize;
+	}
+
+	if(mStripe1Handle != -1 && mStripe2Handle != -1)
+	{
+		// right then... which block is it in?
+		pos_type block = FilePosition / mBlockSize;
+		pos_type offset = FilePosition % mBlockSize;
+		
+		// Calculate offsets for each file
+		pos_type basepos = (block / 2) * mBlockSize;
+		pos_type s1p, s2p;
+		if((block & 1) == 0)
+		{
+			s1p = basepos + offset;
+			s2p = basepos;
+		}
+		else
+		{
+			s1p = basepos + mBlockSize;
+			s2p = basepos + offset;
+		}
+		// Note: lseek isn't in the man pages to return EIO, but assuming that it can return this,
+		// as it calls various OS bits and returns their error codes, and those fns look like they might.
+		if(::lseek(mStripe1Handle, s1p, SEEK_SET) == -1)
+		{
+			if(errno == EIO)
+			{
+				TRACE3("I/O error when seeking in %d %s (to %d), stripe 1\n", mSetNumber, mFilename.c_str(), (int)FilePosition);
+				::syslog(LOG_ERR | LOG_LOCAL5, "I/O error when seeking in %d %s (to %d), stripe 1\n", mSetNumber, mFilename.c_str(), (int)FilePosition);
+				// Attempt to recover
+				AttemptToRecoverFromIOError(true /* is stripe 1 */);
+				ASSERT(mStripe1Handle == -1);
+				// Retry
+				SetPosition(FilePosition);
+				return;
+			}
+			else
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+		if(::lseek(mStripe2Handle, s2p, SEEK_SET) == -1)
+		{
+			if(errno == EIO)
+			{
+				TRACE3("I/O error when seeking in %d %s (to %d), stripe 2\n", mSetNumber, mFilename.c_str(), (int)FilePosition);
+				::syslog(LOG_ERR | LOG_LOCAL5, "I/O error when seeking in %d %s (to %d), stripe 2\n", mSetNumber, mFilename.c_str(), (int)FilePosition);
+				// Attempt to recover
+				AttemptToRecoverFromIOError(false /* is stripe 2 */);
+				ASSERT(mStripe2Handle == -1);
+				// Retry
+				SetPosition(FilePosition);
+				return;
+			}
+			else
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+		
+		// Store position
+		mCurrentPosition = FilePosition;
+	}
+	else
+	{
+		// Simply store, and mark the recovery buffer invalid
+		mCurrentPosition = FilePosition;
+		mRecoveryBufferStart = -1;
+	}
+
+	// not EOF any more
+	mEOF = false;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::Close()
+//		Purpose: Close the file (automatically done by destructor)
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileRead_Raid::Close()
+{
+	if(mStripe1Handle != -1)
+	{
+		::close(mStripe1Handle);
+		mStripe1Handle = -1;
+	}
+	if(mStripe2Handle != -1)
+	{
+		::close(mStripe2Handle);
+		mStripe2Handle = -1;
+	}
+	if(mParityHandle != -1)
+	{
+		::close(mParityHandle);
+		mParityHandle = -1;
+	}
+	
+	mEOF = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_NonRaid::StreamDataLeft()
+//		Purpose: Any data left?
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead_Raid::StreamDataLeft()
+{
+	return !mEOF;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead_Raid::GetFileSize()
+//		Purpose: Returns file size.
+//		Created: 2003/07/14
+//
+// --------------------------------------------------------------------------
+RaidFileRead::pos_type RaidFileRead_Raid::GetFileSize() const
+{
+	return mFileSize;
+}
+
+
+// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::RaidFileRead(int, const std::string &)
+//		Purpose: Constructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead::RaidFileRead(int SetNumber, const std::string &Filename)
+	: mSetNumber(SetNumber),
+	  mFilename(Filename)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::~RaidFileRead()
+//		Purpose: Destructor
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+RaidFileRead::~RaidFileRead()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::Open(int, const std::string &, int)
+//		Purpose: Opens a RaidFile for reading.
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string &Filename, int64_t *pRevisionID, int BufferSizeHint)
+{
+	// See what's available...
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+	if(READ_NUMBER_DISCS_REQUIRED != rdiscSet.size() && 1 != rdiscSet.size()) // allow non-RAID configurations
+	{
+		THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet)
+	}
+
+	// See if the file exists
+	int startDisc = 0, existingFiles = 0;
+	RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, Filename, &startDisc, &existingFiles, pRevisionID);
+	if(existance == RaidFileUtil::NoFile)
+	{
+		THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist)
+	}
+	else if(existance == RaidFileUtil::NonRaid)
+	{
+		// Simple non-RAID file so far...
+	
+		// Get the filename for the write file
+		std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, Filename));
+
+		// Attempt to open
+		int osFileHandle = ::open(writeFilename.c_str(), O_RDONLY, 0);
+		if(osFileHandle == -1)
+		{
+			THROW_EXCEPTION(RaidFileException, ErrorOpeningFileForRead)
+		}
+		
+		// Return a read object for this file
+		try
+		{
+			return std::auto_ptr<RaidFileRead>(new RaidFileRead_NonRaid(SetNumber, Filename, osFileHandle));
+		}
+		catch(...)
+		{
+			::close(osFileHandle);
+			throw;
+		}
+	}
+	else if(existance == RaidFileUtil::AsRaid
+		|| ((existingFiles & RaidFileUtil::Stripe1Exists) && (existingFiles & RaidFileUtil::Stripe2Exists)))
+	{
+		if(existance != RaidFileUtil::AsRaid)
+		{
+			TRACE2("Opening %d %s in normal mode, but parity file doesn't exist\n", SetNumber, Filename.c_str());
+			::syslog(LOG_ERR | LOG_LOCAL5, "Opening %d %s in normal mode, but parity file doesn't exist\n", SetNumber, Filename.c_str());
+			// TODO: Alert recovery daemon
+		}
+	
+		// Open the two stripe files
+		std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (0 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+		std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (1 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+		int stripe1 = -1;
+		int stripe1errno = 0;
+		int stripe2 = -1;
+		int stripe2errno = 0;
+		
+		try
+		{
+			// Open stripe1
+			stripe1 = ::open(stripe1Filename.c_str(), O_RDONLY, 0555);
+			if(stripe1 == -1)
+			{
+				stripe1errno = errno;
+			}
+			// Open stripe2
+			stripe2 = ::open(stripe2Filename.c_str(), O_RDONLY, 0555);
+			if(stripe2 == -1)
+			{
+				stripe2errno = errno;
+			}
+			if(stripe1errno != 0 || stripe2errno != 0)
+			{
+				THROW_EXCEPTION(RaidFileException, ErrorOpeningFileForRead)
+			}
+			
+			// stat stripe 1 to find ('half' of) length...
+			struct stat st;
+			if(::fstat(stripe1, &st) != 0)
+			{
+				stripe1errno = errno;
+			}
+			pos_type length = st.st_size;
+			
+			// stat stripe2 to find (other 'half' of) length...
+			if(::fstat(stripe2, &st) != 0)
+			{
+				stripe2errno = errno;
+			}
+			length += st.st_size;
+			
+			// Handle errors
+			if(stripe1errno != 0 || stripe2errno != 0)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			
+			// Make a nice object to represent this file
+			return std::auto_ptr<RaidFileRead>(new RaidFileRead_Raid(SetNumber, Filename, stripe1, stripe2, -1, length, rdiscSet.GetBlockSize(), false /* actually we don't know */));
+		}
+		catch(...)
+		{
+			// Close open files
+			if(stripe1 != -1)
+			{
+				::close(stripe1);
+				stripe1 = -1;
+			}
+			if(stripe2 != -1)
+			{
+				::close(stripe2);
+				stripe2 = -1;
+			}
+			
+			// Now... maybe we can try again with one less file?
+			bool oktotryagain = true;
+			if(stripe1errno == EIO)
+			{
+				TRACE2("I/O error on opening %d %s stripe 1, trying recovery mode\n", SetNumber, Filename.c_str());
+				::syslog(LOG_ERR | LOG_LOCAL5, "I/O error on opening %d %s stripe 1, trying recovery mode\n", SetNumber, Filename.c_str());
+				RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, true /* is stripe 1 */);
+
+				existingFiles = existingFiles & ~RaidFileUtil::Stripe1Exists;
+				existance = (existance == RaidFileUtil::AsRaidWithMissingReadable)
+					?RaidFileUtil::AsRaidWithMissingNotRecoverable
+					:RaidFileUtil::AsRaidWithMissingReadable;
+			}
+			else if(stripe1errno != 0)
+			{
+				oktotryagain = false;
+			}
+			
+			if(stripe2errno == EIO)
+			{
+				TRACE2("I/O error on opening %d %s stripe 2, trying recovery mode\n", SetNumber, Filename.c_str());
+				::syslog(LOG_ERR | LOG_LOCAL5, "I/O error on opening %d %s stripe 2, trying recovery mode\n", SetNumber, Filename.c_str());
+				RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, false /* is stripe 2 */);
+
+				existingFiles = existingFiles & ~RaidFileUtil::Stripe2Exists;
+				existance = (existance == RaidFileUtil::AsRaidWithMissingReadable)
+					?RaidFileUtil::AsRaidWithMissingNotRecoverable
+					:RaidFileUtil::AsRaidWithMissingReadable;
+			}
+			else if(stripe2errno != 0)
+			{
+				oktotryagain = false;
+			}
+			
+			if(!oktotryagain)
+			{
+				throw;
+			}
+		}
+	}
+
+	if(existance == RaidFileUtil::AsRaidWithMissingReadable)
+	{
+		TRACE3("Attempting to open RAID file %d %s in recovery mode (stripe %d present)\n", SetNumber, Filename.c_str(), (existingFiles & RaidFileUtil::Stripe1Exists)?1:2);
+		::syslog(LOG_ERR | LOG_LOCAL5, "Attempting to open RAID file %d %s in recovery mode (stripe %d present)\n", SetNumber, Filename.c_str(), (existingFiles & RaidFileUtil::Stripe1Exists)?1:2);
+	
+		// Generate the filenames of all the lovely files
+		std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (0 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+		std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (1 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+		std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (2 + startDisc) % READ_NUMBER_DISCS_REQUIRED));
+
+		int stripe1 = -1;
+		int stripe2 = -1;
+		int parity = -1;
+
+		try
+		{
+			// Open stripe1?
+			if(existingFiles & RaidFileUtil::Stripe1Exists)
+			{
+				stripe1 = ::open(stripe1Filename.c_str(), O_RDONLY, 0555);
+				if(stripe1 == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+			}
+			// Open stripe2?
+			if(existingFiles & RaidFileUtil::Stripe2Exists)
+			{
+				stripe2 = ::open(stripe2Filename.c_str(), O_RDONLY, 0555);
+				if(stripe2 == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+			}
+			// Open parity
+			parity = ::open(parityFilename.c_str(), O_RDONLY, 0555);
+			if(parity == -1)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			
+			// Find the length. This is slightly complex.
+			unsigned int blockSize = rdiscSet.GetBlockSize();
+			pos_type length = 0;
+			
+			// The easy one... if the parity file is of an integral block size + sizeof(FileSizeType)
+			// then it's stored at the end of the parity file
+			struct stat st;
+			if(::fstat(parity, &st) != 0)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			pos_type paritySize = st.st_size;
+			FileSizeType parityLastData = 0;
+			bool parityIntegralPlusOffT = ((paritySize % blockSize) == sizeof(FileSizeType));
+			if(paritySize >= static_cast<pos_type>(sizeof(parityLastData)) && (parityIntegralPlusOffT || stripe1 != -1))
+			{
+				// Seek to near the end
+				ASSERT(sizeof(FileSizeType) == 8); // compiler bug (I think) prevents from using 0 - sizeof(FileSizeType)...
+				if(::lseek(parity, -8 /*(0 - sizeof(FileSizeType))*/, SEEK_END) == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+				// Read it in
+				if(::read(parity, &parityLastData, sizeof(parityLastData)) != sizeof(parityLastData))
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+				// Set back to beginning of file
+				if(::lseek(parity, 0, SEEK_SET) == -1)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+			}
+			
+			bool lastBlockHasSize = false;
+			if(parityIntegralPlusOffT)
+			{
+				// Wonderful! Have the value
+				length = ntoh64(parityLastData);
+			}
+			else
+			{
+				// Have to resort to more devious means.
+				if(existingFiles & RaidFileUtil::Stripe1Exists)
+				{
+					// Procedure for stripe 1 existence...
+					// 	Get size of stripe1.
+					//  If this is not an integral block size, then size can use this
+					//  to work out the size of the file.
+					//  Otherwise, read in the end of the last block, and use a bit of XORing
+					//  to get the size from the FileSizeType value at end of the file.
+					if(::fstat(stripe1, &st) != 0)
+					{
+						THROW_EXCEPTION(RaidFileException, OSError)
+					}
+					pos_type stripe1Size = st.st_size;
+					// Is size integral?
+					if((stripe1Size % ((pos_type)blockSize)) != 0)
+					{
+						// No, so know the size.
+						length = stripe1Size + ((stripe1Size / blockSize) * blockSize);
+					}
+					else
+					{
+						// Must read the last bit of data from the block and XOR.
+						FileSizeType stripe1LastData = 0;	// initialise to zero, as we may not read everything from it
+						
+						// Work out how many bytes to read
+						int btr = 0;			// bytes to read off end
+						unsigned int lbs = stripe1Size % blockSize;
+						if(lbs == 0 && stripe1Size > 0)
+						{
+							// integral size, need the entire bit
+							btr = sizeof(FileSizeType);
+						}
+						else if(lbs > (blockSize - sizeof(FileSizeType)))
+						{
+							btr = lbs - (blockSize - sizeof(FileSizeType));
+						}
+						
+						// Seek to near the end
+						if(btr > 0)
+						{
+							ASSERT(sizeof(FileSizeType) == 8); // compiler bug (I think) prevents from using 0 - sizeof(FileSizeType)...
+							ASSERT(btr <= (int)sizeof(FileSizeType));
+							if(::lseek(stripe1, 0 - btr, SEEK_END) == -1)
+							{
+								THROW_EXCEPTION(RaidFileException, OSError)
+							}
+							// Read it in
+							if(::read(stripe1, &stripe1LastData, btr) != btr)
+							{
+								THROW_EXCEPTION(RaidFileException, OSError)
+							}
+							// Set back to beginning of file
+							if(::lseek(stripe1, 0, SEEK_SET) == -1)
+							{
+								THROW_EXCEPTION(RaidFileException, OSError)
+							}
+						}
+						// Lovely!
+						length = stripe1LastData ^ parityLastData;
+						// Convert to host byte order
+						length = ntoh64(length);
+						ASSERT(length <= (paritySize + stripe1Size));
+						// Mark is as having this to aid code later
+						lastBlockHasSize = true;
+					}
+				}
+				else
+				{
+					ASSERT(existingFiles & RaidFileUtil::Stripe2Exists);
+				}
+
+				if(existingFiles & RaidFileUtil::Stripe2Exists)
+				{
+					// Get size of stripe2 file
+					if(::fstat(stripe2, &st) != 0)
+					{
+						THROW_EXCEPTION(RaidFileException, OSError)
+					}
+					pos_type stripe2Size = st.st_size;
+					
+					// Is it an integral size?
+					if(stripe2Size % blockSize != 0)
+					{
+						// No. Working out the size is easy.
+						length = stripe2Size + (((stripe2Size / blockSize)+1) * blockSize);
+						// Got last block size in there?
+						if((stripe2Size % blockSize) <= static_cast<pos_type>((blockSize - sizeof(pos_type))))
+						{
+							// Yes...
+							lastBlockHasSize = true;
+						}
+					}
+					else
+					{
+						// Yes. So we need to compare with the parity file to get a clue...
+						pos_type stripe2Blocks = stripe2Size / blockSize;
+						pos_type parityBlocks = paritySize / blockSize;
+						if(stripe2Blocks == parityBlocks)
+						{
+							// Same size, so stripe1 must be the same size
+							length = (stripe2Blocks * 2) * blockSize;
+						}
+						else
+						{
+							// Different size, so stripe1 must be one block bigger
+							ASSERT(stripe2Blocks < parityBlocks);
+							length = ((stripe2Blocks * 2)+1) * blockSize;
+						}
+						
+						// Then... add in the extra bit of the parity length
+						unsigned int lastBlockSize = paritySize % blockSize;
+						length += lastBlockSize;
+					}
+				}
+				else
+				{
+					ASSERT(existingFiles & RaidFileUtil::Stripe1Exists);
+				}
+			}
+
+			// Create a lovely object to return
+			return std::auto_ptr<RaidFileRead>(new RaidFileRead_Raid(SetNumber, Filename, stripe1, stripe2, parity, length, blockSize, lastBlockHasSize));
+		}
+		catch(...)
+		{
+			// Close open files
+			if(stripe1 != -1)
+			{
+				::close(stripe1);
+				stripe1 = -1;
+			}
+			if(stripe2 != -1)
+			{
+				::close(stripe2);
+				stripe2 = -1;
+			}
+			if(parity != -1)
+			{
+				::close(parity);
+				parity = -1;
+			}
+			throw;
+		}
+	}
+	
+	THROW_EXCEPTION(RaidFileException, FileIsDamagedNotRecoverable)
+	
+	// Avoid compiler warning -- it'll never get here...
+	return std::auto_ptr<RaidFileRead>();
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::DirectoryExists(int, const std::string &)
+//		Purpose: Returns true if the directory exists. Throws exception if it's partially in existence.
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::DirectoryExists(int SetNumber, const std::string &rDirName)
+{
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+
+	return DirectoryExists(rdiscSet, rDirName);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::DirectoryExists(const RaidFileDiscSet &, const std::string &)
+//		Purpose: Returns true if the directory exists. Throws exception if it's partially in existence.
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::string &rDirName)
+{
+	// For each directory, test to see if it exists
+	unsigned int nexist = 0;
+	for(unsigned int l = 0; l < rSet.size(); ++l)
+	{
+		// build name
+		std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName);
+		
+		// check for existence
+		struct stat st;
+		if(::stat(dn.c_str(), &st) == 0)
+		{
+			// Directory?
+			if(st.st_mode & S_IFDIR)
+			{
+				// yes
+				nexist++;
+			}
+			else
+			{
+				// No. It's a file. Bad!
+				THROW_EXCEPTION(RaidFileException, UnexpectedFileInDirPlace)
+			}
+		}
+		else
+		{
+			// Was it a non-exist error?
+			if(errno != ENOENT)
+			{
+				// No. Bad things.
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+	}
+	
+	// Were all of them found?
+	if(nexist == 0)
+	{
+		// None.
+		return false;
+	}
+	else if(nexist == rSet.size())
+	{
+		// All
+		return true;
+	}
+
+	// Some exist. We don't like this -- it shows something bad happened before
+	// TODO: notify recovery daemon
+	THROW_EXCEPTION(RaidFileException, DirectoryIncomplete)
+	return false;	// avoid compiler warning
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::FileExists(int, const std::string &, int64_t *)
+//		Purpose: Does a Raid file exist? Optionally return a revision number, which is
+//				 unique to this saving of the file. (revision number may change
+//				 after transformation to RAID -- so only use for cache control,
+//				 not detecting changes to content).
+//		Created: 2003/09/02
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::FileExists(int SetNumber, const std::string &rFilename, int64_t *pRevisionID)
+{
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+
+	return RaidFileUtil::RaidFileExists(rdiscSet, rFilename, 0, 0, pRevisionID) != RaidFileUtil::NoFile;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    ReadDirectoryContents(int, const std::string &, int, std::vector<std::string> &)
+//		Purpose: Read directory contents, returning whether or not all entries are likely to be readable or not
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput)
+{
+	// Remove anything in the vector to begin with.
+	rOutput.clear();
+	
+	// Controller and set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+
+	// Collect the directory listings
+	std::map<std::string, unsigned int> counts;
+	
+	unsigned int numDiscs = rdiscSet.size();
+	
+	for(unsigned int l = 0; l < numDiscs; ++l)
+	{
+		// build name
+		std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName);
+		
+		// read the contents...
+		DIR *dirHandle = 0;
+		try
+		{
+			dirHandle = ::opendir(dn.c_str());
+			if(dirHandle == 0)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			
+			struct dirent *en = 0;
+			while((en = ::readdir(dirHandle)) != 0)
+			{
+#ifdef PLATFORM_LINUX
+			LinuxWorkaround_FinishDirentStruct(en, dn.c_str());
+#endif
+
+				if(en->d_name[0] == '.' && 
+					(en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0')))
+				{
+					// ignore, it's . or ..
+					continue;
+				}
+				
+				// stat the file to find out what type it is
+/*				struct stat st;
+				std::string fullName(dn + DIRECTORY_SEPARATOR + en->d_name);
+				if(::stat(fullName.c_str(), &st) != 0)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}*/
+				
+				// Entry...
+				std::string name;
+				unsigned int countToAdd = 1;
+				if(DirReadType == DirReadType_FilesOnly && en->d_type == DT_REG) // (st.st_mode & S_IFDIR) == 0)
+				{
+					// File. Complex, need to check the extension
+					int dot = -1;
+					int p = 0;
+					while(en->d_name[p] != '\0')
+					{
+						if(en->d_name[p] == '.')
+						{
+							// store location of dot
+							dot = p;
+						}
+						++p;
+					}
+					// p is length of string
+					if(dot != -1 && ((p - dot) == 3 || (p - dot) == 4)
+						&& en->d_name[dot+1] == 'r' && en->d_name[dot+2] == 'f'
+						&& (en->d_name[dot+3] == 'w' || en->d_name[dot+3] == '\0'))
+					{
+						// so has right extension
+						name.assign(en->d_name, dot);	/* get name up to last . */
+						// Was it a write file (which counts as everything)
+						if(en->d_name[dot+3] == 'w')
+						{
+							countToAdd = numDiscs;
+						}
+					}
+				}
+				if(DirReadType == DirReadType_DirsOnly && en->d_type == DT_DIR) // (st.st_mode & S_IFDIR))
+				{
+					// Directory, and we want directories
+					name = en->d_name;
+				}
+				// Eligable for entry?
+				if(!name.empty())
+				{
+					// add to map...
+					std::map<std::string, unsigned int>::iterator i = counts.find(name);
+					if(i != counts.end())
+					{
+						// add to count
+						i->second += countToAdd;
+					}
+					else
+					{
+						// insert into map
+						counts[name] = countToAdd;
+					}
+				}
+			}
+			
+			if(::closedir(dirHandle) != 0)
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+			dirHandle = 0;
+		}
+		catch(...)
+		{
+			if(dirHandle != 0)
+			{
+				::closedir(dirHandle);
+			}
+			throw;
+		}
+	}
+	
+	// Now go through the map, adding in entries
+	bool everythingReadable = true;
+	
+	for(std::map<std::string, unsigned int>::const_iterator i = counts.begin(); i != counts.end(); ++i)
+	{
+		if(i->second < (numDiscs - 1))
+		{
+			// Too few discs to be confident of reading everything
+			everythingReadable = false;
+		}
+		
+		// Add name to vector
+		rOutput.push_back(i->first);
+	}
+	
+	return everythingReadable;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::Write(const void *, int)
+//		Purpose: Not support, throws exception
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void RaidFileRead::Write(const void *pBuffer, int NBytes)
+{
+	THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::StreamClosed()
+//		Purpose: Never any data to write
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileRead::StreamClosed()
+{
+	return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::BytesLeftToRead()
+//		Purpose: Can tell how many bytes there are to go
+//		Created: 2003/08/26
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileRead::BytesLeftToRead()
+{
+	return GetFileSize() - GetPosition();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileRead::GetDiscUsageInBlocks()
+//		Purpose: Return how many blocks are used.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileRead::GetDiscUsageInBlocks()
+{
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	return RaidFileUtil::DiscUsageInBlocks(GetFileSize(), rdiscSet);
+}
+
+
+
+
diff --git a/lib/raidfile/RaidFileRead.h b/lib/raidfile/RaidFileRead.h
new file mode 100755
index 00000000..93bf7388
--- /dev/null
+++ b/lib/raidfile/RaidFileRead.h
@@ -0,0 +1,72 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileRead.h
+//		Purpose: Read Raid like Files
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+
+#ifndef RAIDFILEREAD__H
+#define RAIDFILEREAD__H
+
+#include <string>
+#include <memory>
+#include <vector>
+
+#include "IOStream.h"
+
+class RaidFileDiscSet;
+
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileRead
+//		Purpose: Read RAID like files
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+class RaidFileRead : public IOStream
+{
+protected:
+	RaidFileRead(int SetNumber, const std::string &Filename);
+public:
+	virtual ~RaidFileRead();
+private:
+	RaidFileRead(const RaidFileRead &rToCopy);
+	
+public:
+	// Open a raid file
+	static std::auto_ptr<RaidFileRead> Open(int SetNumber, const std::string &Filename, int64_t *pRevisionID = 0, int BufferSizeHint = 4096);
+
+	// Extra info
+	virtual pos_type GetFileSize() const = 0;
+
+	// Utility functions
+	static bool FileExists(int SetNumber, const std::string &rFilename, int64_t *pRevisionID = 0);
+	static bool DirectoryExists(const RaidFileDiscSet &rSet, const std::string &rDirName);
+	static bool DirectoryExists(int SetNumber, const std::string &rDirName);
+	enum
+	{
+		DirReadType_FilesOnly = 0,
+		DirReadType_DirsOnly = 1
+	};
+	static bool ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput);
+
+	// Common IOStream interface implementation
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual bool StreamClosed();
+	virtual pos_type BytesLeftToRead();
+
+	pos_type GetDiscUsageInBlocks();
+
+	typedef int64_t FileSizeType;
+
+protected:
+	int mSetNumber;
+	std::string mFilename;
+};
+
+#endif // RAIDFILEREAD__H
+
diff --git a/lib/raidfile/RaidFileUtil.cpp b/lib/raidfile/RaidFileUtil.cpp
new file mode 100755
index 00000000..9177c8ba
--- /dev/null
+++ b/lib/raidfile/RaidFileUtil.cpp
@@ -0,0 +1,188 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileUtil.cpp
+//		Purpose: Utilities for raid files
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "RaidFileUtil.h"
+#include "FileModificationTime.h"
+#include "RaidFileRead.h"	// for type definition
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileUtil::RaidFileExists(RaidFileDiscSet &, const std::string &)
+//		Purpose: Check to see the state of a RaidFile on disc (doesn't look at contents,
+//				 just at existense of files)
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pStartDisc, int *pExisitingFiles, int64_t *pRevisionID)
+{
+	if(pExisitingFiles)
+	{
+		*pExisitingFiles = 0;
+	}
+	
+	// For stat call, although the results are not examined
+	struct stat st;
+
+	// check various files
+	int startDisc = 0;
+	{
+		std::string writeFile(RaidFileUtil::MakeWriteFileName(rDiscSet, rFilename, &startDisc));
+		if(pStartDisc)
+		{
+			*pStartDisc = startDisc;
+		}
+		if(::stat(writeFile.c_str(), &st) == 0)
+		{
+			// write file exists, use that
+			
+			// Get unique ID
+			if(pRevisionID != 0)
+			{
+				(*pRevisionID) = FileModificationTime(st);
+#ifdef PLATFORM_LINUX
+				// On linux, the time resolution is very low for modification times.
+				// So add the size to it to give a bit more chance of it changing.
+				// TODO: Make this better.
+				(*pRevisionID) += st.st_size;
+#endif
+			}
+			
+			// return non-raid file
+			return NonRaid;
+		}
+	}
+	
+	// Now see how many of the raid components exist
+	int64_t revisionID = 0;
+	int setSize = rDiscSet.size();
+	int rfCount = 0;
+#ifdef PLATFORM_LINUX
+	// TODO: replace this with better linux revision ID detection
+	int64_t revisionIDplus = 0;
+#endif
+	for(int f = 0; f < setSize; ++f)
+	{
+		std::string componentFile(RaidFileUtil::MakeRaidComponentName(rDiscSet, rFilename, (f + startDisc) % setSize));
+		if(::stat(componentFile.c_str(), &st) == 0)
+		{
+			// Component file exists, add to count
+			rfCount++;
+			// Set flags for existance?
+			if(pExisitingFiles)
+			{
+				(*pExisitingFiles) |= (1 << f);
+			}
+			// Revision ID
+			if(pRevisionID != 0)
+			{
+				int64_t rid = FileModificationTime(st);
+				if(rid > revisionID) revisionID = rid;
+#ifdef PLATFORM_LINUX
+				revisionIDplus += st.st_size;
+#endif
+			}
+		}
+	}
+	if(pRevisionID != 0)
+	{
+		(*pRevisionID) = revisionID;
+#ifdef PLATFORM_LINUX
+		(*pRevisionID) += revisionIDplus;
+#endif
+	}
+	
+	// Return a status based on how many parts are available
+	if(rfCount == setSize)
+	{
+		return AsRaid;
+	}
+	else if((setSize > 1) && rfCount == (setSize - 1))
+	{
+		return AsRaidWithMissingReadable;
+	}
+	else if(rfCount > 0)
+	{
+		return AsRaidWithMissingNotRecoverable;
+	}
+	
+	return NoFile;	// Obviously doesn't exist
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileUtil::DiscUsageInBlocks(int64_t, const RaidFileDiscSet &)
+//		Purpose: Returns the size of the file in blocks, given the file size and disc set
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+int64_t RaidFileUtil::DiscUsageInBlocks(int64_t FileSize, const RaidFileDiscSet &rDiscSet)
+{
+	// Get block size
+	int blockSize = rDiscSet.GetBlockSize();
+
+	// OK... so as the size of the file is always sizes of stripe1 + stripe2, we can
+	// do a very simple calculation for the main data.
+	int64_t blocks = (FileSize + (((int64_t)blockSize) - 1)) / ((int64_t)blockSize);
+	
+	// It's just that simple calculation for non-RAID disc sets
+	if(rDiscSet.IsNonRaidSet())
+	{
+		return blocks;
+	}
+
+	// It's the parity which is mildly complex.
+	// First of all, add in size for all but the last two blocks.
+	int64_t parityblocks = (FileSize / ((int64_t)blockSize)) / 2;
+	blocks += parityblocks;
+	
+	// Work out how many bytes are left
+	int bytesOver = (int)(FileSize - (parityblocks * ((int64_t)(blockSize*2))));
+	
+	// Then... (let compiler optimise this out)
+	if(bytesOver == 0)
+	{
+		// Extra block for the size info
+		blocks++;
+	}
+	else if(bytesOver == sizeof(RaidFileRead::FileSizeType))
+	{
+		// For last block of parity, plus the size info
+		blocks += 2;
+	}
+	else if(bytesOver < blockSize)
+	{
+		// Just want the parity block
+		blocks += 1;
+	}
+	else if(bytesOver == blockSize || bytesOver >= ((blockSize*2)-((int)sizeof(RaidFileRead::FileSizeType))))
+	{
+		// Last block, plus size info
+		blocks += 2;
+	}
+	else
+	{
+		// Just want parity block
+		blocks += 1;
+	}
+	
+	return blocks;
+}
+
+
diff --git a/lib/raidfile/RaidFileUtil.h b/lib/raidfile/RaidFileUtil.h
new file mode 100755
index 00000000..16670bf1
--- /dev/null
+++ b/lib/raidfile/RaidFileUtil.h
@@ -0,0 +1,97 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileUtil.h
+//		Purpose: Utilities for the raid file classes
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+
+#ifndef RAIDFILEUTIL__H
+#define RAIDFILEUTIL__H
+
+#include "RaidFileController.h"
+#include "RaidFileException.h"
+
+// note: these are hardcoded into the directory searching code
+#define RAIDFILE_EXTENSION			".rf"
+#define RAIDFILE_WRITE_EXTENSION	".rfw"
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileUtil
+//		Purpose: Utility functions for RaidFile classes
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+class RaidFileUtil
+{
+public:
+	typedef enum 
+	{
+		NoFile = 0,
+		NonRaid = 1,
+		AsRaid = 2,
+		AsRaidWithMissingReadable = 3,
+		AsRaidWithMissingNotRecoverable = 4
+	} ExistType;
+	
+	typedef enum
+	{
+		Stripe1Exists = 1,
+		Stripe2Exists = 2,
+		ParityExists = 4
+	};
+	
+	static ExistType RaidFileExists(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pStartDisc = 0, int *pExisitingFiles = 0, int64_t *pRevisionID = 0);
+	
+	static int64_t DiscUsageInBlocks(int64_t FileSize, const RaidFileDiscSet &rDiscSet);
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    std::string MakeRaidComponentName(RaidFileDiscSet &, const std::string &, int)
+	//		Purpose: Returns the OS filename for a file of part of a disc set
+	//		Created: 2003/07/11
+	//
+	// --------------------------------------------------------------------------	
+	static inline std::string MakeRaidComponentName(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int Disc)
+	{
+		if(Disc < 0 || Disc >= (int)rDiscSet.size())
+		{
+			THROW_EXCEPTION(RaidFileException, NoSuchDiscSet)
+		}
+		std::string r(rDiscSet[Disc]);
+		r += DIRECTORY_SEPARATOR_ASCHAR;
+		r += rFilename;
+		r += RAIDFILE_EXTENSION;
+		return r;
+	}
+	
+	// --------------------------------------------------------------------------
+	//
+	// Function
+	//		Name:    std::string MakeWriteFileName(RaidFileDiscSet &, const std::string &)
+	//		Purpose: Returns the OS filename for the temporary write file
+	//		Created: 2003/07/11
+	//
+	// --------------------------------------------------------------------------	
+	static inline std::string MakeWriteFileName(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pOnDiscSet = 0)
+	{
+		int livesOnSet = rDiscSet.GetSetNumForWriteFiles(rFilename);
+		
+		// does the caller want to know which set it's on?
+		if(pOnDiscSet) *pOnDiscSet = livesOnSet;
+		
+		// Make the string
+		std::string r(rDiscSet[livesOnSet]);
+		r += DIRECTORY_SEPARATOR_ASCHAR;
+		r += rFilename;
+		r += RAIDFILE_WRITE_EXTENSION;
+		return r;
+	}
+};
+
+#endif // RAIDFILEUTIL__H
+
diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp
new file mode 100755
index 00000000..414f24e6
--- /dev/null
+++ b/lib/raidfile/RaidFileWrite.cpp
@@ -0,0 +1,817 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileWrite.cpp
+//		Purpose: Writing RAID like files
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/file.h>
+
+#include <stdio.h>
+#include <string.h>
+
+#include "Guards.h"
+#include "RaidFileWrite.h"
+#include "RaidFileController.h"
+#include "RaidFileException.h"
+#include "RaidFileUtil.h"
+#include "Utils.h"
+// For DirectoryExists fn
+#include "RaidFileRead.h"
+
+#include "MemLeakFindOn.h"
+
+// should be a multiple of 2
+#define TRANSFORM_BLOCKS_TO_LOAD		4
+// Must have this number of discs in the set
+#define TRANSFORM_NUMBER_DISCS_REQUIRED	3
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::RaidFileWrite(int, const std::string &)
+//		Purpose: Construtor, just stores requried details
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+RaidFileWrite::RaidFileWrite(int SetNumber, const std::string &Filename)
+	: mSetNumber(SetNumber),
+	  mFilename(Filename),
+	  mOSFileHandle(-1)		// not valid file handle
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::~RaidFileWrite()
+//		Purpose: Destructor (will discard written file if not commited)
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+RaidFileWrite::~RaidFileWrite()
+{
+	if(mOSFileHandle != -1)
+	{
+		Discard();
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Open()
+//		Purpose: Opens the file for writing
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Open(bool AllowOverwrite)
+{
+	if(mOSFileHandle != -1)
+	{
+		THROW_EXCEPTION(RaidFileException, AlreadyOpen)
+	}
+	
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+
+	// Check for overwriting? (step 1)
+	if(!AllowOverwrite)
+	{
+		// See if the file exists already -- can't overwrite existing files
+		RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename);
+		if(existance != RaidFileUtil::NoFile)
+		{
+			TRACE2("Trying to overwrite raidfile %d %s\n", mSetNumber, mFilename.c_str());
+			THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile)
+		}
+	}
+
+	// Get the filename for the write file
+	std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename));
+	// Add on a temporary extension
+	writeFilename += 'X';
+
+	// Attempt to open
+	mOSFileHandle = ::open(writeFilename.c_str(), O_WRONLY | O_CREAT,
+		S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFile)
+	}
+	
+	// Get a lock on the write file
+	if(::flock(mOSFileHandle, LOCK_EX | LOCK_NB) != 0)
+	{
+		// Lock was not obtained.
+		bool wasLocked = (errno == EWOULDBLOCK);
+		// Close the file
+		::close(mOSFileHandle);
+		mOSFileHandle = -1;
+		// Report an exception?
+		if(wasLocked)
+		{
+			THROW_EXCEPTION(RaidFileException, FileIsCurrentlyOpenForWriting)
+		}
+		else
+		{
+			// Random error occured
+			THROW_EXCEPTION(RaidFileException, OSError)
+		}
+	}
+	
+	// Truncate it to size zero
+	if(::ftruncate(mOSFileHandle, 0) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFileOnTruncate)
+	}
+	
+	// Done!
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Write(const void *, int)
+//		Purpose: Writes a block of data
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Write(const void *pBuffer, int Length)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Write data
+	int written = ::write(mOSFileHandle, pBuffer, Length);
+	if(written != Length)
+	{
+		TRACE3("RaidFileWrite::Write: Write failure, Length = %d, written = %d, errno = %d\n", Length, written, errno);
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::GetPosition()
+//		Purpose: Returns current position in file
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileWrite::GetPosition() const
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Use lseek to find the current file position
+	off_t p = ::lseek(mOSFileHandle, 0, SEEK_CUR);
+	if(p == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+
+	return p;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Seek(RaidFileWrite::pos_type, bool)
+//		Purpose: Seeks in the file, relative to current position if Relative is true.
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Seek(IOStream::pos_type SeekTo, int SeekType)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Seek...
+	if(::lseek(mOSFileHandle, SeekTo, ConvertSeekTypeToOSWhence(SeekType)) == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Commit(bool)
+//		Purpose: Closes, and commits the written file
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Commit(bool ConvertToRaidNow)
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+	
+	// Rename it into place -- BEFORE it's closed so lock remains
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	// Get the filename for the write file
+	std::string renameTo(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename));
+	// And the current name
+	std::string renameFrom(renameTo + 'X');
+	if(::rename(renameFrom.c_str(), renameTo.c_str()) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	// Close file...
+	if(::close(mOSFileHandle) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	mOSFileHandle = -1;
+	
+	// Raid it?
+	if(ConvertToRaidNow)
+	{
+		TransformToRaidStorage();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Discard()
+//		Purpose: Closes, discarding the data written.
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Discard()
+{
+	// open?
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, NotOpen)
+	}
+
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+
+	// Get the filename for the write file (temporary)
+	std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename));
+	writeFilename += 'X';
+	
+	// Unlink and close it
+	if((::unlink(writeFilename.c_str()) != 0)
+		|| (::close(mOSFileHandle) != 0))
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	// reset file handle
+	mOSFileHandle = -1;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::TransformToRaidStorage()
+//		Purpose: Turns the file into the RAID storage form
+//		Created: 2003/07/11
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::TransformToRaidStorage()
+{
+	// open?
+	if(mOSFileHandle != -1)
+	{
+		THROW_EXCEPTION(RaidFileException, WriteFileOpenOnTransform)
+	}
+
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	if(rdiscSet.IsNonRaidSet())
+	{
+		// Not in RAID mode -- do nothing
+		return;
+	}
+	// Otherwise check that it's the right sized set
+	if(TRANSFORM_NUMBER_DISCS_REQUIRED != rdiscSet.size())
+	{
+		THROW_EXCEPTION(RaidFileException, WrongNumberOfDiscsInSet)
+	}
+	unsigned int blockSize = rdiscSet.GetBlockSize();
+
+	// Get the filename for the write file (and get the disc set name for the start disc)
+	int startDisc = 0;
+	std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename, &startDisc));
+	
+	// Open it
+	FileHandleGuard<> writeFile(writeFilename.c_str());
+
+	// Get file information for write file	
+	struct stat writeFileStat;
+	if(::fstat(writeFile, &writeFileStat) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+//	// DEBUG MODE -- check file system size block size is same as block size for files
+//	// doesn't really apply, as space benefits of using fragment size are worth efficiency,
+//	// and anyway, it'll be buffered eventually so it won't matter.
+//	#ifndef NDEBUG
+//	{
+//		if(writeFileStat.st_blksize != blockSize)
+//		{
+//			TRACE2("TransformToRaidStorage: optimal block size of file = %d, of set = %d, MISMATCH\n",
+//					writeFileStat.st_blksize, blockSize);
+//		}
+//	}
+//	#endif
+	
+	// How many blocks is the file? (rounding up)
+	int writeFileSizeInBlocks = (writeFileStat.st_size + (blockSize - 1)) / blockSize;
+	// And how big should the buffer be? (round up to multiple of 2, and no bigger than the preset limit)
+	int bufferSizeBlocks = (writeFileSizeInBlocks + 1) & ~1;
+	if(bufferSizeBlocks > TRANSFORM_BLOCKS_TO_LOAD) bufferSizeBlocks = TRANSFORM_BLOCKS_TO_LOAD;
+	// How big should the buffer be?
+	int bufferSize = (TRANSFORM_BLOCKS_TO_LOAD * blockSize);
+	
+	// Allocate buffer...
+	MemoryBlockGuard<char*> buffer(bufferSize);
+	
+	// Allocate buffer for parity file
+	MemoryBlockGuard<char*> parityBuffer(blockSize);
+	
+	// Get filenames of eventual files
+	std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 0) % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 1) % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, (startDisc + 2) % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	// Make write equivalents
+	std::string stripe1FilenameW(stripe1Filename + 'P');
+	std::string stripe2FilenameW(stripe2Filename + 'P');
+	std::string parityFilenameW(parityFilename + 'P');
+	
+	
+	// Then open them all for writing (in strict order)
+	try
+	{
+#ifdef PLATFORM_LINUX
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> stripe1(stripe1FilenameW.c_str());
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> stripe2(stripe2FilenameW.c_str());
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> parity(parityFilenameW.c_str());
+#else
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> stripe1(stripe1FilenameW.c_str());
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> stripe2(stripe2FilenameW.c_str());
+		FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> parity(parityFilenameW.c_str());
+#endif
+
+		// Then... read in data...
+		int bytesRead = -1;
+		bool sizeRecordRequired = false;
+		int blocksDone = 0;
+		while((bytesRead = ::read(writeFile, buffer, bufferSize)) > 0)
+		{
+			// Blocks to do...
+			int blocksToDo = (bytesRead + (blockSize - 1)) / blockSize;
+
+			// Need to add zeros to end?
+			int blocksRoundUp = (blocksToDo + 1) & ~1;
+			int zerosEnd = (blocksRoundUp * blockSize);
+			if(bytesRead != zerosEnd)
+			{
+				// Set the end of the blocks to zero
+				::memset(buffer + bytesRead, 0, zerosEnd - bytesRead);
+			}
+
+			// number of int's to XOR
+			unsigned int num = blockSize / sizeof(unsigned int);
+
+			// Then... calculate and write parity data
+			for(int b = 0; b < blocksToDo; b += 2)
+			{
+				// Calculate int pointers
+				unsigned int *pstripe1 = (unsigned int *)(buffer + (b * blockSize));
+				unsigned int *pstripe2 = (unsigned int *)(buffer + ((b+1) * blockSize));
+				unsigned int *pparity = (unsigned int *)((char*)parityBuffer);
+
+				// Do XOR
+				for(unsigned int n = 0; n < num; ++n)
+				{
+					pparity[n] = pstripe1[n] ^ pstripe2[n];
+				}
+				
+				// Size of parity to write...
+				int parityWriteSize = blockSize;
+				
+				// Adjust if it's the last block
+				if((blocksDone + (b + 2)) >= writeFileSizeInBlocks)
+				{
+					// Yes...
+					unsigned int bytesInLastTwoBlocks = bytesRead - (b * blockSize);
+					
+					// Some special cases...
+					// Zero will never happen... but in the (imaginary) case it does, the file size will be appended
+					// by the test at the end.
+					if(bytesInLastTwoBlocks == sizeof(RaidFileRead::FileSizeType)
+						|| bytesInLastTwoBlocks == blockSize)
+					{
+						// Write the entire block, and put the file size at end
+						sizeRecordRequired = true;
+					}
+					else if(bytesInLastTwoBlocks < blockSize)
+					{
+						// write only these bits
+						parityWriteSize = bytesInLastTwoBlocks;
+					}
+					else if(bytesInLastTwoBlocks < ((blockSize * 2) - sizeof(RaidFileRead::FileSizeType)))
+					{
+						// XOR in the size at the end of the parity block
+						ASSERT(sizeof(RaidFileRead::FileSizeType) == (2*sizeof(unsigned int)));
+#ifdef PLATFORM_LINUX
+						ASSERT(sizeof(RaidFileRead::FileSizeType) >= sizeof(off_t));
+#else
+						ASSERT(sizeof(RaidFileRead::FileSizeType) == sizeof(off_t));
+#endif
+						int sizePos = (blockSize/sizeof(unsigned int)) - 2;
+						RaidFileRead::FileSizeType sw = hton64(writeFileStat.st_size);
+						unsigned int *psize = (unsigned int *)(&sw);
+						pparity[sizePos+0] = pstripe1[sizePos+0] ^ psize[0];
+						pparity[sizePos+1] = pstripe1[sizePos+1] ^ psize[1];
+					}
+					else
+					{
+						// Write the entire block, and put the file size at end
+						sizeRecordRequired = true;
+					}
+				}
+
+				// Write block
+				if(::write(parity, parityBuffer, parityWriteSize) != parityWriteSize)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}
+			}
+
+			// Write stripes
+			char *writeFrom = buffer;
+			for(int l = 0; l < blocksToDo; ++l)
+			{
+				// Write the block
+				int toWrite = (l == (blocksToDo - 1))
+								?(bytesRead - ((blocksToDo-1)*blockSize))
+								:blockSize;
+				if(::write(((l&1)==0)?stripe1:stripe2, writeFrom, toWrite) != toWrite)
+				{
+					THROW_EXCEPTION(RaidFileException, OSError)
+				}			
+
+				// Next block
+				writeFrom += blockSize;
+			}
+			
+			// Count of blocks done
+			blocksDone += blocksToDo;
+		}
+		// Error on read?
+		if(bytesRead == -1)
+		{
+			THROW_EXCEPTION(RaidFileException, OSError)
+		}
+		
+		// Special case for zero length files
+		if(writeFileStat.st_size == 0)
+		{
+			sizeRecordRequired = true;
+		}
+
+		// Might need to write the file size to the end of the parity file
+		// if it can't be worked out some other means -- size is required to rebuild the file if one of the stripe files is missing
+		if(sizeRecordRequired)
+		{
+#ifdef PLATFORM_LINUX
+			ASSERT(sizeof(writeFileStat.st_size) <= sizeof(RaidFileRead::FileSizeType));
+#else
+			ASSERT(sizeof(writeFileStat.st_size) == sizeof(RaidFileRead::FileSizeType));
+#endif
+			RaidFileRead::FileSizeType sw = hton64(writeFileStat.st_size);
+			ASSERT((::lseek(parity, 0, SEEK_CUR) % blockSize) == 0);
+			if(::write(parity, &sw, sizeof(sw)) != sizeof(sw))
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+
+		// Then close the written files (note in reverse order of opening)
+		parity.Close();
+		stripe2.Close();
+		stripe1.Close();
+		
+		// Rename them into place
+		if(::rename(stripe1FilenameW.c_str(), stripe1Filename.c_str()) != 0
+			|| ::rename(stripe2FilenameW.c_str(), stripe2Filename.c_str()) != 0
+			|| ::rename(parityFilenameW.c_str(), parityFilename.c_str()) != 0)
+		{
+			THROW_EXCEPTION(RaidFileException, OSError)
+		}
+
+		// Close the write file
+		writeFile.Close();
+
+		// Finally delete the write file
+		if(::unlink(writeFilename.c_str()) != 0)
+		{
+			THROW_EXCEPTION(RaidFileException, OSError)
+		}
+	}
+	catch(...)
+	{
+		// Unlink all the dodgy files
+		::unlink(stripe1Filename.c_str());
+		::unlink(stripe2Filename.c_str());
+		::unlink(parityFilename.c_str());
+		::unlink(stripe1FilenameW.c_str());
+		::unlink(stripe2FilenameW.c_str());
+		::unlink(parityFilenameW.c_str());
+		
+		// and send the error on its way
+		throw;
+	}
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Delete()
+//		Purpose: Deletes a RAID file
+//		Created: 2003/07/13
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Delete()
+{
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+
+	// See if the file exists already -- can't delete files which don't exist
+	RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename);
+	if(existance == RaidFileUtil::NoFile)
+	{
+		THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist)
+	}
+
+	// Get the filename for the write file
+	std::string writeFilename(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename));
+
+	// Attempt to delete it
+	bool deletedSomething = false;
+	if(::unlink(writeFilename.c_str()) == 0)
+	{
+		deletedSomething = true;
+	}
+	
+	// If we're not running in RAID mode, stop now
+	if(rdiscSet.size() == 1)
+	{
+		return;
+	}
+	
+	// Now the other files
+	std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 0 % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	std::string stripe2Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 1 % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	std::string parityFilename(RaidFileUtil::MakeRaidComponentName(rdiscSet, mFilename, 2 % TRANSFORM_NUMBER_DISCS_REQUIRED));
+	if(::unlink(stripe1Filename.c_str()) == 0)
+	{
+		deletedSomething = true;
+	}
+	if(::unlink(stripe2Filename.c_str()) == 0)
+	{
+		deletedSomething = true;
+	}
+	if(::unlink(parityFilename.c_str()) == 0)
+	{
+		deletedSomething = true;
+	}
+	
+	// Check something happened
+	if(!deletedSomething)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::CreateDirectory(int, const std::string &, bool, int)
+//		Purpose: Creates a directory within the raid file directories with the given name.
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive, int mode)
+{
+	// Get disc set
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(SetNumber));
+	// Pass on...
+	CreateDirectory(rdiscSet, rDirName, Recursive, mode);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::CreateDirectory(const RaidFileDiscSet &, const std::string &, bool, int)
+//		Purpose: Creates a directory within the raid file directories with the given name.
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive, int mode)
+{
+	if(Recursive)
+	{
+		// split up string
+		std::vector<std::string> elements;
+		SplitString(rDirName, DIRECTORY_SEPARATOR_ASCHAR, elements);
+		
+		// Do each element in turn
+		std::string pn;
+		for(unsigned int e = 0; e < elements.size(); ++e)
+		{
+			// Only do this if the element has some text in it
+			if(elements[e].size() > 0)
+			{
+				pn += elements[e];
+				if(!RaidFileRead::DirectoryExists(rSet, pn))
+				{
+					CreateDirectory(rSet, pn, false, mode);
+				}
+
+				// add separator
+				pn += DIRECTORY_SEPARATOR_ASCHAR;
+			}
+		}
+	
+		return;
+	}
+	
+	// Create a directory in every disc of the set
+	for(unsigned int l = 0; l < rSet.size(); ++l)
+	{
+		// build name
+		std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName);
+	
+		// attempt to create
+		if(::mkdir(dn.c_str(), mode) != 0)
+		{
+			if(errno == EEXIST)
+			{
+				// No. Bad things.
+				THROW_EXCEPTION(RaidFileException, FileExistsInDirectoryCreation)
+			}
+			else
+			{
+				THROW_EXCEPTION(RaidFileException, OSError)
+			}
+		}
+	}
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Read(void *, int, int)
+//		Purpose: Unsupported, will exception
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+int RaidFileWrite::Read(void *pBuffer, int NBytes, int Timeout)
+{
+	THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose)
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::Close()
+//		Purpose: Close, discarding file.
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void RaidFileWrite::Close()
+{
+	TRACE0("Warning: RaidFileWrite::Close() called, discarding file\n");
+	if(mOSFileHandle != -1)
+	{
+		Discard();
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::StreamDataLeft()
+//		Purpose: Never any data left to read!
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileWrite::StreamDataLeft()
+{
+	return false;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::StreamClosed()
+//		Purpose: Is stream closed for writing?
+//		Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool RaidFileWrite::StreamClosed()
+{
+	return mOSFileHandle == -1;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::GetFileSize()
+//		Purpose: Returns the size of the file written.
+//				 Can only be used before the file is commited.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileWrite::GetFileSize()
+{
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, CanOnlyGetFileSizeBeforeCommit)
+	}
+	
+	// Stat to get size
+	struct stat st;
+	if(fstat(mOSFileHandle, &st) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	return st.st_size;
+}
+
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    RaidFileWrite::GetDiscUsageInBlocks()
+//		Purpose: Returns the amount of disc space used, in blocks.
+//				 Can only be used before the file is commited.
+//		Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+IOStream::pos_type RaidFileWrite::GetDiscUsageInBlocks()
+{
+	if(mOSFileHandle == -1)
+	{
+		THROW_EXCEPTION(RaidFileException, CanOnlyGetUsageBeforeCommit)
+	}
+	
+	// Stat to get size
+	struct stat st;
+	if(fstat(mOSFileHandle, &st) != 0)
+	{
+		THROW_EXCEPTION(RaidFileException, OSError)
+	}
+	
+	// Then return calculation
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber));
+	return RaidFileUtil::DiscUsageInBlocks(st.st_size, rdiscSet);
+}
+
+
diff --git a/lib/raidfile/RaidFileWrite.h b/lib/raidfile/RaidFileWrite.h
new file mode 100755
index 00000000..d7e51f21
--- /dev/null
+++ b/lib/raidfile/RaidFileWrite.h
@@ -0,0 +1,66 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    RaidFileWrite.h
+//		Purpose: Writing RAID like files
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+
+#ifndef RAIDFILEWRITE__H
+#define RAIDFILEWRITE__H
+
+#include <string>
+
+#include "IOStream.h"
+
+class RaidFileDiscSet;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    RaidFileWrite
+//		Purpose: Writing RAID like files
+//		Created: 2003/07/10
+//
+// --------------------------------------------------------------------------
+class RaidFileWrite : public IOStream
+{
+public:
+	RaidFileWrite(int SetNumber, const std::string &Filename);
+	~RaidFileWrite();
+private:
+	RaidFileWrite(const RaidFileWrite &rToCopy);
+
+public:
+	// IOStream interface
+	virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite);	// will exception
+	virtual void Write(const void *pBuffer, int NBytes);
+	virtual pos_type GetPosition() const;
+	virtual void Seek(pos_type Offset, int SeekType);
+	virtual void Close();		// will discard the file! Use commit instead.
+	virtual bool StreamDataLeft();
+	virtual bool StreamClosed();
+
+	// Extra bits
+	void Open(bool AllowOverwrite = false);
+	void Commit(bool ConvertToRaidNow = false);
+	void Discard();
+	void TransformToRaidStorage();
+	void Delete();
+	pos_type GetFileSize();
+	pos_type GetDiscUsageInBlocks();
+	
+	static void CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive = false, int mode = 0777);
+	static void CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive = false, int mode = 0777);
+	
+private:
+
+private:
+	int mSetNumber;
+	std::string mFilename;
+	int mOSFileHandle;
+};
+
+#endif // RAIDFILEWRITE__H
+
diff --git a/lib/raidfile/raidfile-config b/lib/raidfile/raidfile-config
new file mode 100755
index 00000000..b3f31077
--- /dev/null
+++ b/lib/raidfile/raidfile-config
@@ -0,0 +1,97 @@
+#!/usr/bin/perl
+use strict;
+
+# should be running as root
+if($> != 0)
+{
+	printf "\nWARNING: this should be run as root\n\n"
+}
+
+# check and get command line parameters
+if($#ARGV != 4 && $#ARGV != 2)
+{
+	print <<__E;
+
+Setup raidfile config utility.
+
+Bad command line parameters.
+Usage:
+	raidfile-config config-dir block-size dir0 [dir1 dir2]
+
+config-dir usually /etc/box
+block-size must be a power of two, and usually the block or fragment size of your filing system
+dir0, dir1, dir2 are the directories used as the root of the raid file system
+
+If only one directory is specified, then userland RAID is disabled. Specifying three directories
+enables it.
+
+__E
+	exit(1);
+}
+
+my ($config_dir,$block_size,@dirs) = @ARGV;
+
+my $conf = $config_dir . '/raidfile.conf';
+
+# check dirs are unique, and exist
+my %d;
+for(@dirs)
+{
+	die "$_ is used twice" if exists $d{$_};
+	die "$_ is not a directory" unless -d $_;
+	die "$_ should be an absolute path" unless m/\A\//;
+	$d{$_} = 1;
+}
+
+# check block size is OK
+$block_size = int($block_size);
+die "Bad block size" if $block_size <= 0;
+my $c = 1;
+while(1)
+{
+	last if $c == $block_size;
+	die "Block size $block_size is not a power of two" if $c > $block_size;
+	$c = $c * 2;
+}
+
+# check that it doesn't already exist
+if(-f $conf)
+{
+	die "$conf already exists. Delete and try again"
+}
+
+# create directory
+if(!-d $config_dir)
+{
+	print "Creating $config_dir...\n";
+	mkdir $config_dir,0755 or die "Can't create $config_dir";
+}
+
+# adjust if userland RAID is disabled
+if($#dirs == 0)
+{
+	$dirs[1] = $dirs[0];
+	$dirs[2] = $dirs[0];
+	print "WARNING: userland RAID is disabled.\n"
+}
+
+# write the file
+open CONFIG,">$conf" or die "Can't open $conf for writing";
+
+print CONFIG <<__E;
+
+disc0
+{
+	SetNumber = 0
+	BlockSize = $block_size
+	Dir0 = $dirs[0]
+	Dir1 = $dirs[1]
+	Dir2 = $dirs[2]
+}
+
+__E
+
+close CONFIG;
+
+print "Config file written.\n";
+