diff options
author | Chris Wilson <chris+github@qwirx.com> | 2014-04-09 22:15:57 +0000 |
---|---|---|
committer | Chris Wilson <chris+github@qwirx.com> | 2014-04-09 22:15:57 +0000 |
commit | dc4ba0969146c6c4a82b1dace6bd9b11b818bcc9 (patch) | |
tree | f29918fdcab9a801685ebe6a6a7b5446c9cc69ba | |
parent | 06af9253bc5df053dbe2208cbc724853294020a3 (diff) |
Poll command socket regularly during file uploads.
Makes the daemon responsive to polling and commands, even during a file
upload operation.
-rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.cpp | 12 | ||||
-rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.h | 5 | ||||
-rw-r--r-- | bin/bbackupd/BackupDaemon.cpp | 53 | ||||
-rw-r--r-- | bin/bbackupd/BackupDaemon.h | 16 | ||||
-rw-r--r-- | bin/bbackupquery/BackupQueries.h | 2 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreException.txt | 1 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreFile.cpp | 6 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreFile.h | 7 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreFileDiff.cpp | 21 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreFileEncodeStream.cpp | 62 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreFileEncodeStream.h | 9 |
11 files changed, 155 insertions, 39 deletions
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index f2f1f455..4a7eb5a5 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -1707,7 +1707,8 @@ int64_t BackupClientDirectoryRecord::UploadFile( connection.GetTimeout(), &rContext, // DiffTimer implementation 0 /* not interested in the modification time */, - &isCompletelyDifferent); + &isCompletelyDifferent, + rParams.mpBackgroundTask); if(isCompletelyDifferent) { @@ -1727,7 +1728,8 @@ int64_t BackupClientDirectoryRecord::UploadFile( apStreamToUpload = BackupStoreFile::EncodeFile( rFilename, mObjectID, /* containing directory */ rStoreFilename, NULL, &rParams, - &(rParams.mrRunStatusProvider)); + &(rParams.mrRunStatusProvider), + rParams.mpBackgroundTask); } rContext.SetNiceMode(true); @@ -1842,7 +1844,8 @@ BackupClientDirectoryRecord::SyncParams::SyncParams( RunStatusProvider &rRunStatusProvider, SysadminNotifier &rSysadminNotifier, ProgressNotifier &rProgressNotifier, - BackupClientContext &rContext) + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask) : mSyncPeriodStart(0), mSyncPeriodEnd(0), mMaxUploadWait(0), @@ -1856,7 +1859,8 @@ BackupClientDirectoryRecord::SyncParams::SyncParams( mReadErrorsOnFilesystemObjects(false), mMaxUploadRate(0), mUploadAfterThisTimeInTheFuture(99999999999999999LL), - mHaveLoggedWarningAboutFutureFileTimes(false) + mHaveLoggedWarningAboutFutureFileTimes(false), + mpBackgroundTask(pBackgroundTask) { } diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h index 79eed587..ec0ec86b 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.h +++ b/bin/bbackupd/BackupClientDirectoryRecord.h @@ -14,6 +14,7 @@ #include <map> #include <memory> +#include "BackgroundTask.h" #include "BackupClientFileAttributes.h" #include "BackupDaemonInterface.h" #include "BackupStoreDirectory.h" @@ -76,7 +77,8 @@ public: RunStatusProvider &rRunStatusProvider, SysadminNotifier &rSysadminNotifier, ProgressNotifier &rProgressNotifier, - BackupClientContext &rContext); + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask); ~SyncParams(); private: // No copying @@ -91,6 +93,7 @@ public: box_time_t mMaxFileTimeInFuture; int32_t mFileTrackingSizeThreshold; int32_t mDiffingUploadSizeThreshold; + BackgroundTask *mpBackgroundTask; RunStatusProvider &mrRunStatusProvider; SysadminNotifier &mrSysadminNotifier; ProgressNotifier &mrProgressNotifier; diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp index dc494100..a31ae7cd 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/bin/bbackupd/BackupDaemon.cpp @@ -202,7 +202,8 @@ BackupDaemon::BackupDaemon() mpProgressNotifier(this), mpLocationResolver(this), mpRunStatusProvider(this), - mpSysadminNotifier(this) + mpSysadminNotifier(this), + mapCommandSocketPollTimer(NULL) #ifdef WIN32 , mInstallService(false), mRemoveService(false), @@ -442,6 +443,9 @@ void BackupDaemon::Run() // initialise global timer mechanism Timers::Init(); + mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, + "CommandSocketPollTimer")); + #ifndef WIN32 // Ignore SIGPIPE so that if a command connection is broken, // the daemon doesn't terminate. @@ -926,7 +930,7 @@ void BackupDaemon::RunSyncNow() // Set up the sync parameters BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider, - *mpSysadminNotifier, *mpProgressNotifier, clientContext); + *mpSysadminNotifier, *mpProgressNotifier, clientContext, this); params.mSyncPeriodStart = syncPeriodStart; params.mSyncPeriodEnd = syncPeriodEndExtended; // use potentially extended end time @@ -1844,6 +1848,51 @@ int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, // -------------------------------------------------------------------------- // // Function +// Name: BackupDaemon::RunBackgroundTask() +// Purpose: Checks for connections or commands on the command +// socket and handles them with minimal delay. Polled +// during lengthy operations such as file uploads. +// Created: 07/04/14 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum) +{ + BOX_TRACE("BackupDaemon::RunBackgroundTask: state = " << state << + ", progress = " << progress << "/" << maximum); + + if(mapCommandSocketPollTimer->HasExpired()) + { + mapCommandSocketPollTimer->Reset(COMMAND_SOCKET_POLL_INTERVAL); + } + else + { + // Do no more work right now + return true; + } + + if(mapCommandSocketInfo.get()) + { + BOX_TRACE("BackupDaemon::RunBackgroundTask: polling command socket"); + + bool sync_flag_out, sync_is_forced_out; + + WaitOnCommandSocket(0, // RequiredDelay + sync_flag_out, sync_is_forced_out); + + if(sync_flag_out) + { + BOX_WARNING("Ignoring request to sync while " + "already syncing."); + } + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function // Name: BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &) // Purpose: Waits on a the command socket for a time of UP TO the required time // but may be much less, and handles a command if necessary. diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h index 1d3c991e..5a811f7e 100644 --- a/bin/bbackupd/BackupDaemon.h +++ b/bin/bbackupd/BackupDaemon.h @@ -25,6 +25,7 @@ #include "TLSContext.h" #include "autogen_BackupProtocol.h" +#include "autogen_BackupStoreException.h" #ifdef WIN32 #include "WinNamedPipeListener.h" @@ -38,6 +39,8 @@ # include <VsBackup.h> #endif +#define COMMAND_SOCKET_POLL_INTERVAL 1000 + class BackupClientDirectoryRecord; class BackupClientContext; class Configuration; @@ -55,7 +58,7 @@ class Archive; // // -------------------------------------------------------------------------- class BackupDaemon : public Daemon, ProgressNotifier, LocationResolver, -RunStatusProvider, SysadminNotifier +RunStatusProvider, SysadminNotifier, BackgroundTask { public: BackupDaemon(); @@ -232,12 +235,15 @@ public: void SetLocationResolver (LocationResolver* p) { mpLocationResolver = p; } void SetRunStatusProvider(RunStatusProvider* p) { mpRunStatusProvider = p; } void SetSysadminNotifier (SysadminNotifier* p) { mpSysadminNotifier = p; } + virtual bool RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum); private: ProgressNotifier* mpProgressNotifier; LocationResolver* mpLocationResolver; RunStatusProvider* mpRunStatusProvider; SysadminNotifier* mpSysadminNotifier; + std::auto_ptr<Timer> mapCommandSocketPollTimer; /* ProgressNotifier implementation */ public: @@ -250,7 +256,13 @@ public: if (mLogAllFileAccess) { BOX_INFO("Scanning directory: " << rLocalPath); - } + } + + if (!RunBackgroundTask(BackgroundTask::Scanning_Dirs, 0, 0)) + { + THROW_EXCEPTION(BackupStoreException, + CancelledByBackgroundTask); + } } virtual void NotifyDirStatFailed( const BackupClientDirectoryRecord* pDirRecord, diff --git a/bin/bbackupquery/BackupQueries.h b/bin/bbackupquery/BackupQueries.h index 82fc7bed..bad1a39c 100644 --- a/bin/bbackupquery/BackupQueries.h +++ b/bin/bbackupquery/BackupQueries.h @@ -272,6 +272,7 @@ public: virtual void NotifyDirComparing(const std::string& rLocalPath, const std::string& rRemotePath) { + BOX_INFO("Comparing directory: " << rLocalPath); } virtual void NotifyDirCompared( @@ -303,6 +304,7 @@ public: virtual void NotifyFileComparing(const std::string& rLocalPath, const std::string& rRemotePath) { + BOX_TRACE("Comparing file: " << rLocalPath); } virtual void NotifyFileCompared(const std::string& rLocalPath, diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt index 9e21fc62..b49a9cf2 100644 --- a/lib/backupstore/BackupStoreException.txt +++ b/lib/backupstore/BackupStoreException.txt @@ -71,3 +71,4 @@ PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. R UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first CorruptReferenceCountDatabase 70 The account's refcount database is corrupt and must be rebuilt by housekeeping. +CancelledByBackgroundTask 71 The current task was cancelled on request by the background task. diff --git a/lib/backupstore/BackupStoreFile.cpp b/lib/backupstore/BackupStoreFile.cpp index e011f8f4..d4c73f8b 100644 --- a/lib/backupstore/BackupStoreFile.cpp +++ b/lib/backupstore/BackupStoreFile.cpp @@ -78,7 +78,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) + RunStatusProvider* pRunStatusProvider, + BackgroundTask* pBackgroundTask) { // Create the stream std::auto_ptr<BackupStoreFileEncodeStream> stream( @@ -86,7 +87,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( // Do the initial setup stream->Setup(Filename, 0 /* no recipe, just encode */, ContainerID, - rStoreFilename, pModificationTime, pLogger, pRunStatusProvider); + rStoreFilename, pModificationTime, pLogger, pRunStatusProvider, + pBackgroundTask); // Return the stream for the caller return stream; diff --git a/lib/backupstore/BackupStoreFile.h b/lib/backupstore/BackupStoreFile.h index 8e2cefb6..776e6d22 100644 --- a/lib/backupstore/BackupStoreFile.h +++ b/lib/backupstore/BackupStoreFile.h @@ -27,6 +27,7 @@ typedef struct int64_t mTotalFileStreamSize; } BackupStoreFileStats; +class BackgroundTask; class RunStatusProvider; // Uncomment to disable backwards compatibility @@ -128,7 +129,8 @@ public: int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0, ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL + RunStatusProvider* pRunStatusProvider = NULL, + BackgroundTask* pBackgroundTask = NULL ); static std::auto_ptr<BackupStoreFileEncodeStream> EncodeFileDiff ( @@ -138,7 +140,8 @@ public: int Timeout, DiffTimer *pDiffTimer, int64_t *pModificationTime = 0, - bool *pIsCompletelyDifferent = 0 + bool *pIsCompletelyDifferent = 0, + BackgroundTask* pBackgroundTask = NULL ); // Shortcut interface static int64_t QueryStoreFileDiff(BackupProtocolCallable& protocol, diff --git a/lib/backupstore/BackupStoreFileDiff.cpp b/lib/backupstore/BackupStoreFileDiff.cpp index fa8cb892..176d56e1 100644 --- a/lib/backupstore/BackupStoreFileDiff.cpp +++ b/lib/backupstore/BackupStoreFileDiff.cpp @@ -128,7 +128,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff const std::string& Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, - int64_t *pModificationTime, bool *pIsCompletelyDifferent) + int64_t *pModificationTime, bool *pIsCompletelyDifferent, + BackgroundTask* pBackgroundTask) { // Is it a symlink? { @@ -144,7 +145,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff { *pIsCompletelyDifferent = true; } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + return EncodeFile(Filename, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); // BackgroundTask } } @@ -162,7 +167,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff { *pIsCompletelyDifferent = true; } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + return EncodeFile(Filename, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); // BackgroundTask } // Pointer to recipe we're going to create @@ -210,7 +219,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff new BackupStoreFileEncodeStream); // Do the initial setup - stream->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime); + stream->Setup(Filename, precipe, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); precipe = 0; // Stream has taken ownership of this // Tell user about completely different status? diff --git a/lib/backupstore/BackupStoreFileEncodeStream.cpp b/lib/backupstore/BackupStoreFileEncodeStream.cpp index b53c4c26..b40bcd08 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.cpp +++ b/lib/backupstore/BackupStoreFileEncodeStream.cpp @@ -11,6 +11,7 @@ #include <string.h> +#include "BackgroundTask.h" #include "BackupClientFileAttributes.h" #include "BackupStoreConstants.h" #include "BackupStoreException.h" @@ -40,25 +41,28 @@ using namespace BackupStoreFileCryptVar; // // -------------------------------------------------------------------------- BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() - : mpRecipe(0), - mpFile(0), - mpLogging(0), - mpRunStatusProvider(NULL), - mStatus(Status_Header), - mSendData(true), - mTotalBlocks(0), - mAbsoluteBlockNumber(-1), - mInstructionNumber(-1), - mNumBlocks(0), - mCurrentBlock(-1), - mCurrentBlockEncodedSize(0), - mPositionInCurrentBlock(0), - mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), - mLastBlockSize(0), - mTotalBytesSent(0), - mpRawBuffer(0), - mAllocatedBufferSize(0), - mEntryIVBase(0) +: mpRecipe(0), + mpFile(0), + mpLogging(0), + mpRunStatusProvider(NULL), + mpBackgroundTask(NULL), + mStatus(Status_Header), + mSendData(true), + mTotalBlocks(0), + mBytesToUpload(0), + mBytesUploaded(0), + mAbsoluteBlockNumber(-1), + mInstructionNumber(-1), + mNumBlocks(0), + mCurrentBlock(-1), + mCurrentBlockEncodedSize(0), + mPositionInCurrentBlock(0), + mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), + mLastBlockSize(0), + mTotalBytesSent(0), + mpRawBuffer(0), + mAllocatedBufferSize(0), + mEntryIVBase(0) { } @@ -115,7 +119,8 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, BackupStoreFileEncodeStream::Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) + RunStatusProvider* pRunStatusProvider, + BackgroundTask* pBackgroundTask) { // Pointer to a blank recipe which we might create BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; @@ -162,6 +167,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); // Add to accumlated total mTotalBlocks += numBlocks; + mBytesToUpload += (*pRecipe)[inst].mSpaceBefore; // Update maximum clear size if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; @@ -278,6 +284,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, } mpRunStatusProvider = pRunStatusProvider; + mpBackgroundTask = pBackgroundTask; } @@ -341,6 +348,19 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) THROW_EXCEPTION(BackupStoreException, SignalReceived); } + if(mpBackgroundTask) + { + BackgroundTask::State state = (mpRecipe->at(0).mBlocks == 0) + ? BackgroundTask::Uploading_Full + : BackgroundTask::Uploading_Patch; + if(!mpBackgroundTask->RunBackgroundTask(state, mBytesUploaded, + mBytesToUpload)) + { + THROW_EXCEPTION(BackupStoreException, + CancelledByBackgroundTask); + } + } + int bytesToRead = NBytes; uint8_t *buffer = (uint8_t*)pBuffer; @@ -575,6 +595,8 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() // Encode it mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer); + + mBytesUploaded += blockRawSize; //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h index a169e036..80bdff58 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.h +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -79,7 +79,8 @@ public: const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL); + RunStatusProvider* pRunStatusProvider = NULL, + BackgroundTask* pBackgroundTask = NULL); virtual int Read(void *pBuffer, int NBytes, int Timeout); virtual void Write(const void *pBuffer, int NBytes); @@ -109,14 +110,18 @@ private: CollectInBufferStream mData; // buffer for header and index entries IOStream *mpLogging; RunStatusProvider* mpRunStatusProvider; + BackgroundTask* mpBackgroundTask; int mStatus; bool mSendData; // true if there's file data to send (ie not a symlink) int64_t mTotalBlocks; // Total number of blocks in the file + int64_t mBytesToUpload; // Total number of clear bytes to encode and upload + int64_t mBytesUploaded; // Total number of clear bytes already encoded + // excluding reused blocks already on the server. int64_t mAbsoluteBlockNumber; // The absolute block number currently being output // Instruction number int64_t mInstructionNumber; // All the below are within the current instruction - int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases + int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases int64_t mCurrentBlock; int32_t mCurrentBlockEncodedSize; int32_t mPositionInCurrentBlock; // for reading out |