Refactor GetLines to share common code, fix whitespace removal at end of line.

6 files changed, 317 insertions, 298 deletions

diff --git a/lib/common/FdGetLine.cpp b/lib/common/FdGetLine.cpp
index 9add2bd1..8b6ff974 100644
--- a/lib/common/FdGetLine.cpp
+++ b/lib/common/FdGetLine.cpp
@@ -20,13 +20,6 @@
 
 #include "MemLeakFindOn.h"
 
-// utility whitespace function
-inline bool iw(int c)
-{
-	return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded
-}
-
-
 // --------------------------------------------------------------------------
 //
 // Function
@@ -36,12 +29,7 @@ inline bool iw(int c)
 //
 // --------------------------------------------------------------------------
 FdGetLine::FdGetLine(int fd)
-	: mFileHandle(fd),
-	  mLineNumber(0),
-	  mBufferBegin(0),
-	  mBytesInBuffer(0),
-	  mPendingEOF(false),
-	  mEOF(false)
+: mFileHandle(fd)
 {
 	if(mFileHandle < 0) {THROW_EXCEPTION(CommonException, BadArguments)}
 	//printf("FdGetLine buffer size = %d\n", sizeof(mBuffer));
@@ -74,130 +62,50 @@ FdGetLine::~FdGetLine()
 std::string FdGetLine::GetLine(bool Preprocess)
 {
 	if(mFileHandle == -1) {THROW_EXCEPTION(CommonException, GetLineNoHandle)}
-
-	// EOF?
-	if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)}
 	
 	std::string r;
+	bool result = GetLineInternal(r, Preprocess);
 
-	bool foundLineEnd = false;
-
-	while(!foundLineEnd && !mEOF)
+	if(!result)
 	{
-		// Use any bytes left in the buffer
-		while(mBufferBegin < mBytesInBuffer)
-		{
-			int c = mBuffer[mBufferBegin++];
-			if(c == '\r')
-			{
-				// Ignore nasty Windows line ending extra chars
-			}
-			else if(c == '\n')
-			{
-				// Line end!
-				foundLineEnd = true;
-				break;
-			}
-			else
-			{
-				// Add to string
-				r += c;
-			}
-			
-			// Implicit line ending at EOF
-			if(mBufferBegin >= mBytesInBuffer && mPendingEOF)
-			{
-				foundLineEnd = true;
-			}
-		}
-		
-		// Check size
-		if(r.size() > FDGETLINE_MAX_LINE_SIZE)
-		{
-			THROW_EXCEPTION(CommonException, GetLineTooLarge)
-		}
-		
-		// Read more in?
-		if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF)
-		{
-#ifdef WIN32
-			int bytes;
-
-			if (mFileHandle == _fileno(stdin))
-			{
-				bytes = console_read(mBuffer, sizeof(mBuffer));
-			}
-			else
-			{
-				bytes = ::read(mFileHandle, mBuffer, 
-					sizeof(mBuffer));
-			}
-#else // !WIN32
-			int bytes = ::read(mFileHandle, mBuffer, sizeof(mBuffer));
-#endif // WIN32
-			
-			// Error?
-			if(bytes == -1)
-			{
-				THROW_EXCEPTION(CommonException, OSFileError)
-			}
-			
-			// Adjust buffer info
-			mBytesInBuffer = bytes;
-			mBufferBegin = 0;
-			
-			// EOF / closed?
-			if(bytes == 0)
-			{
-				mPendingEOF = true;
-			}
-		}
-		
-		// EOF?
-		if(mPendingEOF && mBufferBegin >= mBytesInBuffer)
-		{
-			// File is EOF, and now we've depleted the buffer completely, so tell caller as well.
-			mEOF = true;
-		}
+		// should never fail for FdGetLine
+		THROW_EXCEPTION(CommonException, Internal);
 	}
+	
+	return r;
+}
 
-	if(!Preprocess)
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    FdGetLine::ReadMore()
+//		Purpose: Read more bytes from the handle, possible the
+//			 console, into mBuffer and return the number of
+//			 bytes read, 0 on EOF or -1 on error.
+//		Created: 2011/04/22
+//
+// --------------------------------------------------------------------------
+int FdGetLine::ReadMore(int Timeout)
+{
+	int bytes;
+	
+#ifdef WIN32
+	if (mFileHandle == _fileno(stdin))
 	{
-		return r;
+		bytes = console_read(mBuffer, sizeof(mBuffer));
 	}
 	else
 	{
-		// Check for comment char, but char before must be whitespace
-		int end = 0;
-		int size = r.size();
-		while(end < size)
-		{
-			if(r[end] == '#' && (end == 0 || (iw(r[end-1]))))
-			{
-				break;
-			}
-			end++;
-		}
-		
-		// Remove whitespace
-		int begin = 0;
-		while(begin < size && iw(r[begin]))
-		{
-			begin++;
-		}
-
-		if(end < size && !iw(r[end]))
-		{
-			end--;
-		}
+		bytes = ::read(mFileHandle, mBuffer, sizeof(mBuffer));
+	}
+#else // !WIN32
+	bytes = ::read(mFileHandle, mBuffer, sizeof(mBuffer));
+#endif // WIN32
 
-		while(end > begin && end < size && iw(r[end]))
-		{
-			end--;
-		}
-		
-		// Return a sub string
-		return r.substr(begin, end - begin + 1);
+	if(bytes == 0)
+	{
+		mPendingEOF = true;
 	}
 }
 
@@ -207,7 +115,7 @@ std::string FdGetLine::GetLine(bool Preprocess)
 // Function
 //		Name:    FdGetLine::DetachFile()
 //		Purpose: Detaches the file handle, setting the file pointer correctly.
-//				 Probably not good for sockets...
+//			 Probably not good for sockets...
 //		Created: 2003/07/24
 //
 // --------------------------------------------------------------------------
@@ -230,4 +138,3 @@ void FdGetLine::DetachFile()
 	mFileHandle = -1;
 }
 
-
diff --git a/lib/common/FdGetLine.h b/lib/common/FdGetLine.h
index df43c3c9..5ad82c64 100644
--- a/lib/common/FdGetLine.h
+++ b/lib/common/FdGetLine.h
@@ -12,19 +12,7 @@
 
 #include <string>
 
-#ifdef BOX_RELEASE_BUILD
-	#define FDGETLINE_BUFFER_SIZE		1024
-#elif defined WIN32
-	// need enough space for at least one unicode character 
-	// in UTF-8 when calling console_read() from bbackupquery
-	#define FDGETLINE_BUFFER_SIZE		5
-#else
-	#define FDGETLINE_BUFFER_SIZE		4
-#endif
-
-// Just a very large upper bound for line size to avoid
-// people sending lots of data over sockets and causing memory problems.
-#define FDGETLINE_MAX_LINE_SIZE			(1024*256)
+#include "GetLine.h"
 
 // --------------------------------------------------------------------------
 //
@@ -34,7 +22,7 @@
 //		Created: 2003/07/24
 //
 // --------------------------------------------------------------------------
-class FdGetLine
+class FdGetLine : public GetLine
 {
 public:
 	FdGetLine(int fd);
@@ -43,22 +31,19 @@ private:
 	FdGetLine(const FdGetLine &rToCopy);
 
 public:
-	std::string GetLine(bool Preprocess = false);
-	bool IsEOF() {return mEOF;}
-	int GetLineNumber() {return mLineNumber;}
-	
+	virtual std::string GetLine(bool Preprocess = false);
 	// Call to detach, setting file pointer correctly to last bit read.
 	// Only works for lseek-able file descriptors.
 	void DetachFile();
-	
+	// if we read 0 bytes from an fd, it must be end of stream,
+	// because we don't support timeouts
+	virtual bool IsStreamDataLeft() { return false; }
+
+protected:
+	int ReadMore(int Timeout = IOStream::TimeOutInfinite);
+
 private:
-	char mBuffer[FDGETLINE_BUFFER_SIZE];
 	int mFileHandle;
-	int mLineNumber;
-	int mBufferBegin;
-	int mBytesInBuffer;
-	bool mPendingEOF;
-	bool mEOF;
 };
 
 #endif // FDGETLINE__H
diff --git a/lib/common/GetLine.cpp b/lib/common/GetLine.cpp
new file mode 100644
index 00000000..e6b26c8a
--- /dev/null
+++ b/lib/common/GetLine.cpp
@@ -0,0 +1,176 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    GetLine.cpp
+//		Purpose: Common base class for line based file descriptor reading
+//		Created: 2011/04/22
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <sys/types.h>
+
+#ifdef HAVE_UNISTD_H
+	#include <unistd.h>
+#endif
+
+#include "GetLine.h"
+#include "CommonException.h"
+
+#include "MemLeakFindOn.h"
+
+// utility whitespace function
+inline bool iw(int c)
+{
+	return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    GetLine::GetLine(int)
+//		Purpose: Constructor, taking file descriptor
+//		Created: 2011/04/22
+//
+// --------------------------------------------------------------------------
+GetLine::GetLine()
+: mLineNumber(0),
+  mBufferBegin(0),
+  mBytesInBuffer(0),
+  mPendingEOF(false),
+  mEOF(false)
+{ }
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    GetLine::GetLineInternal(std::string &, bool, int)
+//		Purpose: Gets a line from the file, returning it in rOutput.
+//			 If Preprocess is true, leading and trailing
+//			 whitespace is removed, and comments (after #)  are
+//			 deleted. Returns true if a line is available now,
+//			 false if retrying may get a line (eg timeout,
+//			 signal), and exceptions if it's EOF.
+//		Created: 2011/04/22
+//
+// --------------------------------------------------------------------------
+bool GetLine::GetLineInternal(std::string &rOutput, bool Preprocess,
+	int Timeout)
+{
+	// EOF?
+	if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)}
+	
+	// Initialise string to stored into
+	rOutput = mPendingString;
+	mPendingString.erase();
+
+	bool foundLineEnd = false;
+
+	while(!foundLineEnd && !mEOF)
+	{
+		// Use any bytes left in the buffer
+		while(mBufferBegin < mBytesInBuffer)
+		{
+			int c = mBuffer[mBufferBegin++];
+			if(c == '\r')
+			{
+				// Ignore nasty Windows line ending extra chars
+			}
+			else if(c == '\n')
+			{
+				// Line end!
+				foundLineEnd = true;
+				break;
+			}
+			else
+			{
+				// Add to string
+				rOutput += c;
+			}
+			
+			// Implicit line ending at EOF
+			if(mBufferBegin >= mBytesInBuffer && mPendingEOF)
+			{
+				foundLineEnd = true;
+			}
+		}
+		
+		// Check size
+		if(rOutput.size() > GETLINE_MAX_LINE_SIZE)
+		{
+			THROW_EXCEPTION(CommonException, GetLineTooLarge)
+		}
+		
+		// Read more in?
+		if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF)
+		{
+			int bytes = ReadMore(Timeout);
+			
+			// Error?
+			if(bytes == -1)
+			{
+				THROW_EXCEPTION(CommonException, OSFileError)
+			}
+			
+			// Adjust buffer info
+			mBytesInBuffer = bytes;
+			mBufferBegin = 0;
+			
+			// No data returned?
+			if(bytes == 0 && IsStreamDataLeft())
+			{
+			       // store string away
+			       mPendingString = rOutput;
+			       // Return false;
+			       return false;
+			}
+		}
+		
+		// EOF?
+		if(mPendingEOF && mBufferBegin >= mBytesInBuffer)
+		{
+			// File is EOF, and now we've depleted the buffer completely, so tell caller as well.
+			mEOF = true;
+		}
+	}
+
+	if(Preprocess)
+	{
+		// Check for comment char, but char before must be whitespace
+		// end points to a gap between characters, may equal start if
+		// the string to be extracted has zero length, and indexes the
+		// first character not in the string (== length, or a # mark
+		// or whitespace)
+		int end = 0;
+		int size = rOutput.size();
+		while(end < size)
+		{
+			if(rOutput[end] == '#' && (end == 0 || (iw(rOutput[end-1]))))
+			{
+				break;
+			}
+			end++;
+		}
+		
+		// Remove whitespace
+		int begin = 0;
+		while(begin < size && iw(rOutput[begin]))
+		{
+			begin++;
+		}
+
+		while(end > begin && end <= size && iw(rOutput[end-1]))
+		{
+			end--;
+		}
+		
+		// Return a sub string
+		rOutput = rOutput.substr(begin, end - begin);
+	}
+
+	return true;
+}
+
+
diff --git a/lib/common/GetLine.h b/lib/common/GetLine.h
new file mode 100644
index 00000000..6f24c7e8
--- /dev/null
+++ b/lib/common/GetLine.h
@@ -0,0 +1,66 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    GetLine.h
+//		Purpose: Common base class for line based file descriptor reading
+//		Created: 2011/04/22
+//
+// --------------------------------------------------------------------------
+
+#ifndef GETLINE__H
+#define GETLINE__H
+
+#include <string>
+
+#ifdef BOX_RELEASE_BUILD
+	#define GETLINE_BUFFER_SIZE		1024
+#elif defined WIN32
+	// need enough space for at least one unicode character 
+	// in UTF-8 when calling console_read() from bbackupquery
+	#define GETLINE_BUFFER_SIZE		5
+#else
+	#define GETLINE_BUFFER_SIZE		4
+#endif
+
+// Just a very large upper bound for line size to avoid
+// people sending lots of data over sockets and causing memory problems.
+#define GETLINE_MAX_LINE_SIZE			(1024*256)
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    GetLine
+//		Purpose: Common base class for line based file descriptor reading
+//		Created: 2011/04/22
+//
+// --------------------------------------------------------------------------
+class GetLine
+{
+protected:
+	GetLine();
+
+private:
+	GetLine(const GetLine &rToCopy);
+
+public:
+	virtual bool IsEOF() {return mEOF;}
+	int GetLineNumber() {return mLineNumber;}
+	
+protected:
+	bool GetLineInternal(std::string &rOutput,
+		bool Preprocess = false,
+		int Timeout = IOStream::TimeOutInfinite);
+	virtual int ReadMore(int Timeout = IOStream::TimeOutInfinite) = 0;
+	virtual bool IsStreamDataLeft() = 0;
+
+	char mBuffer[GETLINE_BUFFER_SIZE];
+	int mLineNumber;
+	int mBufferBegin;
+	int mBytesInBuffer;
+	bool mPendingEOF;
+	std::string mPendingString;
+	bool mEOF;
+};
+
+#endif // GETLINE__H
+
diff --git a/lib/common/IOStreamGetLine.cpp b/lib/common/IOStreamGetLine.cpp
index 3e6612cb..ef8930b8 100644
--- a/lib/common/IOStreamGetLine.cpp
+++ b/lib/common/IOStreamGetLine.cpp
@@ -13,13 +13,6 @@
 
 #include "MemLeakFindOn.h"
 
-// utility whitespace function
-inline bool iw(int c)
-{
-	return (c == ' ' || c == '\t' || c == '\v' || c == '\f'); // \r, \n are already excluded
-}
-
-
 // --------------------------------------------------------------------------
 //
 // Function
@@ -29,12 +22,7 @@ inline bool iw(int c)
 //
 // --------------------------------------------------------------------------
 IOStreamGetLine::IOStreamGetLine(IOStream &Stream)
-	: mrStream(Stream),
-	  mLineNumber(0),
-	  mBufferBegin(0),
-	  mBytesInBuffer(0),
-	  mPendingEOF(false),
-	  mEOF(false)
+: mrStream(Stream)
 {
 }
 
@@ -66,123 +54,30 @@ IOStreamGetLine::~IOStreamGetLine()
 // --------------------------------------------------------------------------
 bool IOStreamGetLine::GetLine(std::string &rOutput, bool Preprocess, int Timeout)
 {
-	// EOF?
-	if(mEOF) {THROW_EXCEPTION(CommonException, GetLineEOF)}
-	
-	// Initialise string to stored into
-	std::string r(mPendingString);
-	mPendingString.erase();
+	return GetLineInternal(rOutput, Preprocess, Timeout);
+}
 
-	bool foundLineEnd = false;
 
-	while(!foundLineEnd && !mEOF)
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    IOStreamGetLine::ReadMore()
+//		Purpose: Read more bytes from the handle, possible the
+//			 console, into mBuffer and return the number of
+//			 bytes read, 0 on EOF or -1 on error.
+//		Created: 2011/04/22
+//
+// --------------------------------------------------------------------------
+int IOStreamGetLine::ReadMore(int Timeout)
+{
+	int bytes = mrStream.Read(mBuffer, sizeof(mBuffer), Timeout);
+	
+	if(!mrStream.StreamDataLeft())
 	{
-		// Use any bytes left in the buffer
-		while(mBufferBegin < mBytesInBuffer)
-		{
-			int c = mBuffer[mBufferBegin++];
-			if(c == '\r')
-			{
-				// Ignore nasty Windows line ending extra chars
-			}
-			else if(c == '\n')
-			{
-				// Line end!
-				foundLineEnd = true;
-				break;
-			}
-			else
-			{
-				// Add to string
-				r += c;
-			}
-			
-			// Implicit line ending at EOF
-			if(mBufferBegin >= mBytesInBuffer && mPendingEOF)
-			{
-				foundLineEnd = true;
-			}
-		}
-		
-		// Check size
-		if(r.size() > IOSTREAMGETLINE_MAX_LINE_SIZE)
-		{
-			THROW_EXCEPTION(CommonException, GetLineTooLarge)
-		}
-		
-		// Read more in?
-		if(!foundLineEnd && mBufferBegin >= mBytesInBuffer && !mPendingEOF)
-		{
-			int bytes = mrStream.Read(mBuffer, sizeof(mBuffer), Timeout);
-			
-			// Adjust buffer info
-			mBytesInBuffer = bytes;
-			mBufferBegin = 0;
-			
-			// EOF / closed?
-			if(!mrStream.StreamDataLeft())
-			{
-				mPendingEOF = true;
-			}
-			
-			// No data returned?
-			if(bytes == 0 && mrStream.StreamDataLeft())
-			{
-				// store string away
-				mPendingString = r;
-				// Return false;
-				return false;
-			}
-		}
-		
-		// EOF?
-		if(mPendingEOF && mBufferBegin >= mBytesInBuffer)
-		{
-			// File is EOF, and now we've depleted the buffer completely, so tell caller as well.
-			mEOF = true;
-		}
+		mPendingEOF = true;
 	}
 
-	if(!Preprocess)
-	{
-		rOutput = r;
-		return true;
-	}
-	else
-	{
-		// Check for comment char, but char before must be whitespace
-		int end = 0;
-		int size = r.size();
-		while(end < size)
-		{
-			if(r[end] == '#' && (end == 0 || (iw(r[end-1]))))
-			{
-				break;
-			}
-			end++;
-		}
-		
-		// Remove whitespace
-		int begin = 0;
-		while(begin < size && iw(r[begin]))
-		{
-			begin++;
-		}
-
-		if(end < size && !iw(r[end]))
-		{
-			end--;
-		}
-
-		while(end > begin && end < size && iw(r[end]))
-		{
-			end--;
-		}
-		
-		// Return a sub string
-		rOutput = r.substr(begin, end - begin + 1);
-		return true;
-	}
+	return bytes;	
 }
 
 
diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h
index 9a5d1818..8e047955 100644
--- a/lib/common/IOStreamGetLine.h
+++ b/lib/common/IOStreamGetLine.h
@@ -12,18 +12,9 @@
 
 #include <string>
 
+#include "GetLine.h"
 #include "IOStream.h"
 
-#ifdef BOX_RELEASE_BUILD
-	#define IOSTREAMGETLINE_BUFFER_SIZE		1024
-#else
-	#define IOSTREAMGETLINE_BUFFER_SIZE		4
-#endif
-
-// Just a very large upper bound for line size to avoid
-// people sending lots of data over sockets and causing memory problems.
-#define IOSTREAMGETLINE_MAX_LINE_SIZE			(1024*256)
-
 // --------------------------------------------------------------------------
 //
 // Class
@@ -32,7 +23,7 @@
 //		Created: 2003/07/24
 //
 // --------------------------------------------------------------------------
-class IOStreamGetLine
+class IOStreamGetLine : public GetLine
 {
 public:
 	IOStreamGetLine(IOStream &Stream);
@@ -42,29 +33,28 @@ private:
 
 public:
 	bool GetLine(std::string &rOutput, bool Preprocess = false, int Timeout = IOStream::TimeOutInfinite);
-	bool IsEOF() {return mEOF;}
-	int GetLineNumber() {return mLineNumber;}
 	
 	// Call to detach, setting file pointer correctly to last bit read.
 	// Only works for lseek-able file descriptors.
 	void DetachFile();
 	
+	virtual bool IsStreamDataLeft()
+	{
+		return mrStream.StreamDataLeft();
+	}
+	
 	// For doing interesting stuff with the remaining data...
 	// Be careful with this!
 	const void *GetBufferedData() const {return mBuffer + mBufferBegin;}
 	int GetSizeOfBufferedData() const {return mBytesInBuffer - mBufferBegin;}
 	void IgnoreBufferedData(int BytesToIgnore);
 	IOStream &GetUnderlyingStream() {return mrStream;}
+
+protected:
+	int ReadMore(int Timeout = IOStream::TimeOutInfinite);
 	
 private:
-	char mBuffer[IOSTREAMGETLINE_BUFFER_SIZE];
 	IOStream &mrStream;
-	int mLineNumber;
-	int mBufferBegin;
-	int mBytesInBuffer;
-	bool mPendingEOF;
-	bool mEOF;
-	std::string mPendingString;
 };
 
 #endif // IOSTREAMGETLINE__H