From dc4ba0969146c6c4a82b1dace6bd9b11b818bcc9 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 9 Apr 2014 22:15:57 +0000 Subject: Poll command socket regularly during file uploads. Makes the daemon responsive to polling and commands, even during a file upload operation. --- lib/backupstore/BackupStoreFileEncodeStream.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib/backupstore/BackupStoreFileEncodeStream.h') diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h index a169e036..80bdff58 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.h +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -79,7 +79,8 @@ public: const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL); + RunStatusProvider* pRunStatusProvider = NULL, + BackgroundTask* pBackgroundTask = NULL); virtual int Read(void *pBuffer, int NBytes, int Timeout); virtual void Write(const void *pBuffer, int NBytes); @@ -109,14 +110,18 @@ private: CollectInBufferStream mData; // buffer for header and index entries IOStream *mpLogging; RunStatusProvider* mpRunStatusProvider; + BackgroundTask* mpBackgroundTask; int mStatus; bool mSendData; // true if there's file data to send (ie not a symlink) int64_t mTotalBlocks; // Total number of blocks in the file + int64_t mBytesToUpload; // Total number of clear bytes to encode and upload + int64_t mBytesUploaded; // Total number of clear bytes already encoded + // excluding reused blocks already on the server. int64_t mAbsoluteBlockNumber; // The absolute block number currently being output // Instruction number int64_t mInstructionNumber; // All the below are within the current instruction - int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases + int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases int64_t mCurrentBlock; int32_t mCurrentBlockEncodedSize; int32_t mPositionInCurrentBlock; // for reading out -- cgit v1.2.3 From 4458bf17916973aeb9e99e9166070f645fb3295e Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 15 Aug 2014 22:47:44 +0000 Subject: Fix deadlock waiting for read or write on closed connection. If the system is suspended then it may not realise that a TCP connection has been closed, while waiting for data to arrive on it. We didn't used to apply a timeout to this read operation. Now we use the connection's default timeout on all read and write operations. Network operations that don't pass a timeout will be logged with a backtrace, so that they can be fixed. --- lib/backupstore/BackupStoreFileEncodeStream.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/backupstore/BackupStoreFileEncodeStream.h') diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h index 80bdff58..13648d47 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.h +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -83,7 +83,8 @@ public: BackgroundTask* pBackgroundTask = NULL); virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); int64_t GetTotalBytesSent() { return mTotalBytesSent; } -- cgit v1.2.3 From 6b7001ee308fe0550d5499a1a668d5071668c26a Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 4 Sep 2014 01:36:34 +0000 Subject: Add EstimatedBytesToUpload to NotifyFileUploadingPatch. Merged back changes from the test refactor branch to reduce diffs. --- lib/backupstore/BackupStoreFileEncodeStream.h | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/backupstore/BackupStoreFileEncodeStream.h') diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h index 13648d47..6db09182 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.h +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -87,6 +87,7 @@ public: int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); + int64_t GetBytesToUpload() { return mBytesToUpload; } int64_t GetTotalBytesSent() { return mTotalBytesSent; } private: -- cgit v1.2.3 From 2cbdea2166057c3c3996e733d78048d528fa694e Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 20 Sep 2015 21:31:23 +0100 Subject: Remove all timing dependency from test_ssl_keepalives(). This test has always been fragile, and usually fails on Travis for reasons unknown, so replace the log parsing and intercepts with simple mocking code. This doesn't cover 100% of what the old test did, including diff timer aborts and the number of blocks used by the uploaded files, but it should be completely robust. --- lib/backupstore/BackupStoreFileEncodeStream.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib/backupstore/BackupStoreFileEncodeStream.h') diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h index 6db09182..5b9b4a61 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.h +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -90,6 +90,9 @@ public: int64_t GetBytesToUpload() { return mBytesToUpload; } int64_t GetTotalBytesSent() { return mTotalBytesSent; } + static void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, + int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); + private: enum { @@ -98,15 +101,12 @@ private: Status_BlockListing = 2, Status_Finished = 3 }; - -private: + void EncodeCurrentBlock(); - void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); void SkipPreviousBlocksInInstruction(); void SetForInstruction(); void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); -private: Recipe *mpRecipe; IOStream *mpFile; // source file CollectInBufferStream mData; // buffer for header and index entries -- cgit v1.2.3