From d94983dfbefd935a592c070c59cd05a09c0ccab4 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 13 Sep 2010 17:58:18 +0000 Subject: Implement write buffering on directories. --- bin/bbstored/BackupStoreContext.cpp | 22 +++-- lib/common/BufferedStream.cpp | 2 +- lib/common/BufferedStream.h | 2 +- lib/common/BufferedWriteStream.cpp | 181 ++++++++++++++++++++++++++++++++++++ lib/common/BufferedWriteStream.h | 44 +++++++++ lib/raidfile/RaidFileWrite.cpp | 2 +- 6 files changed, 241 insertions(+), 12 deletions(-) create mode 100644 lib/common/BufferedWriteStream.cpp create mode 100644 lib/common/BufferedWriteStream.h diff --git a/bin/bbstored/BackupStoreContext.cpp b/bin/bbstored/BackupStoreContext.cpp index 6d955a00..9a98decf 100644 --- a/bin/bbstored/BackupStoreContext.cpp +++ b/bin/bbstored/BackupStoreContext.cpp @@ -11,21 +11,22 @@ #include +#include "BackupConstants.h" #include "BackupStoreContext.h" -#include "RaidFileWrite.h" -#include "RaidFileRead.h" +#include "BackupStoreDaemon.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" -#include "BackupStoreInfo.h" -#include "BackupConstants.h" #include "BackupStoreFile.h" +#include "BackupStoreInfo.h" #include "BackupStoreObjectMagic.h" -#include "StoreStructure.h" -#include "BackupStoreDaemon.h" -#include "RaidFileController.h" +#include "BufferedStream.h" +#include "BufferedWriteStream.h" #include "FileStream.h" #include "InvisibleTempFileStream.h" -#include "BufferedStream.h" +#include "RaidFileController.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "StoreStructure.h" #include "MemLeakFindOn.h" @@ -942,7 +943,10 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec { RaidFileWrite writeDir(mStoreDiscSet, dirfn); writeDir.Open(true /* allow overwriting */); - rDir.WriteToStream(writeDir); + + BufferedWriteStream buffer(writeDir); + rDir.WriteToStream(buffer); + buffer.Flush(); // get the disc usage (must do this before commiting it) int64_t dirSize = writeDir.GetDiscUsageInBlocks(); diff --git a/lib/common/BufferedStream.cpp b/lib/common/BufferedStream.cpp index 288e1ed1..b58253f3 100644 --- a/lib/common/BufferedStream.cpp +++ b/lib/common/BufferedStream.cpp @@ -2,7 +2,7 @@ // // File // Name: BufferedStream.cpp -// Purpose: Buffering wrapper around IOStreams +// Purpose: Buffering read-only wrapper around IOStreams // Created: 2007/01/16 // // -------------------------------------------------------------------------- diff --git a/lib/common/BufferedStream.h b/lib/common/BufferedStream.h index 234061c4..079c482a 100644 --- a/lib/common/BufferedStream.h +++ b/lib/common/BufferedStream.h @@ -2,7 +2,7 @@ // // File // Name: BufferedStream.h -// Purpose: Buffering wrapper around IOStreams +// Purpose: Buffering read-only wrapper around IOStreams // Created: 2007/01/16 // // -------------------------------------------------------------------------- diff --git a/lib/common/BufferedWriteStream.cpp b/lib/common/BufferedWriteStream.cpp new file mode 100644 index 00000000..e4d740a0 --- /dev/null +++ b/lib/common/BufferedWriteStream.cpp @@ -0,0 +1,181 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BufferedWriteStream.cpp +// Purpose: Buffering write-only wrapper around IOStreams +// Created: 2010/09/13 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BufferedWriteStream.h" +#include "CommonException.h" + +#include + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::BufferedWriteStream(const char *, int, int) +// Purpose: Constructor, set up buffer +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +BufferedWriteStream::BufferedWriteStream(IOStream& rSink) +: mrSink(rSink), mBufferPosition(0) +{ } + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Read(void *, int) +// Purpose: Reads bytes from the file - throws exception +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +int BufferedWriteStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +IOStream::pos_type BufferedWriteStream::BytesLeftToRead() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Write(void *, int) +// Purpose: Writes bytes to the underlying stream (not supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedWriteStream::Write(const void *pBuffer, int NBytes) +{ + int numBytesRemain = NBytes; + + do + { + int maxWritable = sizeof(mBuffer) - mBufferPosition; + int numBytesToWrite = (NBytes < maxWritable) ? NBytes : + maxWritable; + + if(numBytesToWrite > 0) + { + memcpy(mBuffer + mBufferPosition, pBuffer, + numBytesToWrite); + mBufferPosition += numBytesToWrite; + pBuffer = ((const char *)pBuffer) + numBytesToWrite; + numBytesRemain -= numBytesToWrite; + } + + if(numBytesRemain > 0) + { + Flush(); + } + } + while(numBytesRemain > 0); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type BufferedWriteStream::GetPosition() const +{ + return mrSink.GetPosition() + mBufferPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek, invalidate buffer +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedWriteStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + // Always flush the buffer before seeking + Flush(); + + mrSink.Seek(Offset, SeekType); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Flush(); +// Purpose: Write out current buffer contents and invalidate +// Created: 2010/09/13 +// +// -------------------------------------------------------------------------- +void BufferedWriteStream::Flush(int Timeout) +{ + if(mBufferPosition > 0) + { + mrSink.Write(mBuffer, mBufferPosition); + } + + mBufferPosition = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::Close() +// Purpose: Closes the underlying stream (not needed) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedWriteStream::Close() +{ + Flush(); + mrSink.Close(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool BufferedWriteStream::StreamDataLeft() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedWriteStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool BufferedWriteStream::StreamClosed() +{ + return mrSink.StreamClosed(); +} + diff --git a/lib/common/BufferedWriteStream.h b/lib/common/BufferedWriteStream.h new file mode 100644 index 00000000..7a1c8c17 --- /dev/null +++ b/lib/common/BufferedWriteStream.h @@ -0,0 +1,44 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BufferedWriteStream.h +// Purpose: Buffering write-only wrapper around IOStreams +// Created: 2010/09/13 +// +// -------------------------------------------------------------------------- + +#ifndef BUFFEREDWRITESTREAM__H +#define BUFFEREDWRITESTREAM__H + +#include "IOStream.h" + +class BufferedWriteStream : public IOStream +{ +private: + IOStream& mrSink; + char mBuffer[4096]; + int mBufferPosition; + +public: + BufferedWriteStream(IOStream& rSource); + virtual ~BufferedWriteStream() { Close(); } + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Flush(int Timeout = IOStream::TimeOutInfinite); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + BufferedWriteStream(const BufferedWriteStream &rToCopy) + : mrSink(rToCopy.mrSink) { /* do not call */ } +}; + +#endif // BUFFEREDWRITESTREAM__H + + diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp index 2d852f86..f24c2422 100644 --- a/lib/raidfile/RaidFileWrite.cpp +++ b/lib/raidfile/RaidFileWrite.cpp @@ -836,9 +836,9 @@ int RaidFileWrite::Read(void *pBuffer, int NBytes, int Timeout) // -------------------------------------------------------------------------- void RaidFileWrite::Close() { - BOX_WARNING("RaidFileWrite::Close() called, discarding file"); if(mOSFileHandle != -1) { + BOX_WARNING("RaidFileWrite::Close() called, discarding file"); Discard(); } } -- cgit v1.2.3