summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Wilson <chris+github@qwirx.com>2014-04-09 22:15:57 +0000
committerChris Wilson <chris+github@qwirx.com>2014-04-09 22:15:57 +0000
commitdc4ba0969146c6c4a82b1dace6bd9b11b818bcc9 (patch)
treef29918fdcab9a801685ebe6a6a7b5446c9cc69ba
parent06af9253bc5df053dbe2208cbc724853294020a3 (diff)
Poll command socket regularly during file uploads.
Makes the daemon responsive to polling and commands, even during a file upload operation.
-rw-r--r--bin/bbackupd/BackupClientDirectoryRecord.cpp12
-rw-r--r--bin/bbackupd/BackupClientDirectoryRecord.h5
-rw-r--r--bin/bbackupd/BackupDaemon.cpp53
-rw-r--r--bin/bbackupd/BackupDaemon.h16
-rw-r--r--bin/bbackupquery/BackupQueries.h2
-rw-r--r--lib/backupstore/BackupStoreException.txt1
-rw-r--r--lib/backupstore/BackupStoreFile.cpp6
-rw-r--r--lib/backupstore/BackupStoreFile.h7
-rw-r--r--lib/backupstore/BackupStoreFileDiff.cpp21
-rw-r--r--lib/backupstore/BackupStoreFileEncodeStream.cpp62
-rw-r--r--lib/backupstore/BackupStoreFileEncodeStream.h9
11 files changed, 155 insertions, 39 deletions
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp
index f2f1f455..4a7eb5a5 100644
--- a/bin/bbackupd/BackupClientDirectoryRecord.cpp
+++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp
@@ -1707,7 +1707,8 @@ int64_t BackupClientDirectoryRecord::UploadFile(
connection.GetTimeout(),
&rContext, // DiffTimer implementation
0 /* not interested in the modification time */,
- &isCompletelyDifferent);
+ &isCompletelyDifferent,
+ rParams.mpBackgroundTask);
if(isCompletelyDifferent)
{
@@ -1727,7 +1728,8 @@ int64_t BackupClientDirectoryRecord::UploadFile(
apStreamToUpload = BackupStoreFile::EncodeFile(
rFilename, mObjectID, /* containing directory */
rStoreFilename, NULL, &rParams,
- &(rParams.mrRunStatusProvider));
+ &(rParams.mrRunStatusProvider),
+ rParams.mpBackgroundTask);
}
rContext.SetNiceMode(true);
@@ -1842,7 +1844,8 @@ BackupClientDirectoryRecord::SyncParams::SyncParams(
RunStatusProvider &rRunStatusProvider,
SysadminNotifier &rSysadminNotifier,
ProgressNotifier &rProgressNotifier,
- BackupClientContext &rContext)
+ BackupClientContext &rContext,
+ BackgroundTask *pBackgroundTask)
: mSyncPeriodStart(0),
mSyncPeriodEnd(0),
mMaxUploadWait(0),
@@ -1856,7 +1859,8 @@ BackupClientDirectoryRecord::SyncParams::SyncParams(
mReadErrorsOnFilesystemObjects(false),
mMaxUploadRate(0),
mUploadAfterThisTimeInTheFuture(99999999999999999LL),
- mHaveLoggedWarningAboutFutureFileTimes(false)
+ mHaveLoggedWarningAboutFutureFileTimes(false),
+ mpBackgroundTask(pBackgroundTask)
{
}
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h
index 79eed587..ec0ec86b 100644
--- a/bin/bbackupd/BackupClientDirectoryRecord.h
+++ b/bin/bbackupd/BackupClientDirectoryRecord.h
@@ -14,6 +14,7 @@
#include <map>
#include <memory>
+#include "BackgroundTask.h"
#include "BackupClientFileAttributes.h"
#include "BackupDaemonInterface.h"
#include "BackupStoreDirectory.h"
@@ -76,7 +77,8 @@ public:
RunStatusProvider &rRunStatusProvider,
SysadminNotifier &rSysadminNotifier,
ProgressNotifier &rProgressNotifier,
- BackupClientContext &rContext);
+ BackupClientContext &rContext,
+ BackgroundTask *pBackgroundTask);
~SyncParams();
private:
// No copying
@@ -91,6 +93,7 @@ public:
box_time_t mMaxFileTimeInFuture;
int32_t mFileTrackingSizeThreshold;
int32_t mDiffingUploadSizeThreshold;
+ BackgroundTask *mpBackgroundTask;
RunStatusProvider &mrRunStatusProvider;
SysadminNotifier &mrSysadminNotifier;
ProgressNotifier &mrProgressNotifier;
diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp
index dc494100..a31ae7cd 100644
--- a/bin/bbackupd/BackupDaemon.cpp
+++ b/bin/bbackupd/BackupDaemon.cpp
@@ -202,7 +202,8 @@ BackupDaemon::BackupDaemon()
mpProgressNotifier(this),
mpLocationResolver(this),
mpRunStatusProvider(this),
- mpSysadminNotifier(this)
+ mpSysadminNotifier(this),
+ mapCommandSocketPollTimer(NULL)
#ifdef WIN32
, mInstallService(false),
mRemoveService(false),
@@ -442,6 +443,9 @@ void BackupDaemon::Run()
// initialise global timer mechanism
Timers::Init();
+ mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL,
+ "CommandSocketPollTimer"));
+
#ifndef WIN32
// Ignore SIGPIPE so that if a command connection is broken,
// the daemon doesn't terminate.
@@ -926,7 +930,7 @@ void BackupDaemon::RunSyncNow()
// Set up the sync parameters
BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider,
- *mpSysadminNotifier, *mpProgressNotifier, clientContext);
+ *mpSysadminNotifier, *mpProgressNotifier, clientContext, this);
params.mSyncPeriodStart = syncPeriodStart;
params.mSyncPeriodEnd = syncPeriodEndExtended;
// use potentially extended end time
@@ -1844,6 +1848,51 @@ int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script,
// --------------------------------------------------------------------------
//
// Function
+// Name: BackupDaemon::RunBackgroundTask()
+// Purpose: Checks for connections or commands on the command
+// socket and handles them with minimal delay. Polled
+// during lengthy operations such as file uploads.
+// Created: 07/04/14
+//
+// --------------------------------------------------------------------------
+bool BackupDaemon::RunBackgroundTask(State state, uint64_t progress,
+ uint64_t maximum)
+{
+ BOX_TRACE("BackupDaemon::RunBackgroundTask: state = " << state <<
+ ", progress = " << progress << "/" << maximum);
+
+ if(mapCommandSocketPollTimer->HasExpired())
+ {
+ mapCommandSocketPollTimer->Reset(COMMAND_SOCKET_POLL_INTERVAL);
+ }
+ else
+ {
+ // Do no more work right now
+ return true;
+ }
+
+ if(mapCommandSocketInfo.get())
+ {
+ BOX_TRACE("BackupDaemon::RunBackgroundTask: polling command socket");
+
+ bool sync_flag_out, sync_is_forced_out;
+
+ WaitOnCommandSocket(0, // RequiredDelay
+ sync_flag_out, sync_is_forced_out);
+
+ if(sync_flag_out)
+ {
+ BOX_WARNING("Ignoring request to sync while "
+ "already syncing.");
+ }
+ }
+
+ return true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
// Name: BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &)
// Purpose: Waits on a the command socket for a time of UP TO the required time
// but may be much less, and handles a command if necessary.
diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h
index 1d3c991e..5a811f7e 100644
--- a/bin/bbackupd/BackupDaemon.h
+++ b/bin/bbackupd/BackupDaemon.h
@@ -25,6 +25,7 @@
#include "TLSContext.h"
#include "autogen_BackupProtocol.h"
+#include "autogen_BackupStoreException.h"
#ifdef WIN32
#include "WinNamedPipeListener.h"
@@ -38,6 +39,8 @@
# include <VsBackup.h>
#endif
+#define COMMAND_SOCKET_POLL_INTERVAL 1000
+
class BackupClientDirectoryRecord;
class BackupClientContext;
class Configuration;
@@ -55,7 +58,7 @@ class Archive;
//
// --------------------------------------------------------------------------
class BackupDaemon : public Daemon, ProgressNotifier, LocationResolver,
-RunStatusProvider, SysadminNotifier
+RunStatusProvider, SysadminNotifier, BackgroundTask
{
public:
BackupDaemon();
@@ -232,12 +235,15 @@ public:
void SetLocationResolver (LocationResolver* p) { mpLocationResolver = p; }
void SetRunStatusProvider(RunStatusProvider* p) { mpRunStatusProvider = p; }
void SetSysadminNotifier (SysadminNotifier* p) { mpSysadminNotifier = p; }
+ virtual bool RunBackgroundTask(State state, uint64_t progress,
+ uint64_t maximum);
private:
ProgressNotifier* mpProgressNotifier;
LocationResolver* mpLocationResolver;
RunStatusProvider* mpRunStatusProvider;
SysadminNotifier* mpSysadminNotifier;
+ std::auto_ptr<Timer> mapCommandSocketPollTimer;
/* ProgressNotifier implementation */
public:
@@ -250,7 +256,13 @@ public:
if (mLogAllFileAccess)
{
BOX_INFO("Scanning directory: " << rLocalPath);
- }
+ }
+
+ if (!RunBackgroundTask(BackgroundTask::Scanning_Dirs, 0, 0))
+ {
+ THROW_EXCEPTION(BackupStoreException,
+ CancelledByBackgroundTask);
+ }
}
virtual void NotifyDirStatFailed(
const BackupClientDirectoryRecord* pDirRecord,
diff --git a/bin/bbackupquery/BackupQueries.h b/bin/bbackupquery/BackupQueries.h
index 82fc7bed..bad1a39c 100644
--- a/bin/bbackupquery/BackupQueries.h
+++ b/bin/bbackupquery/BackupQueries.h
@@ -272,6 +272,7 @@ public:
virtual void NotifyDirComparing(const std::string& rLocalPath,
const std::string& rRemotePath)
{
+ BOX_INFO("Comparing directory: " << rLocalPath);
}
virtual void NotifyDirCompared(
@@ -303,6 +304,7 @@ public:
virtual void NotifyFileComparing(const std::string& rLocalPath,
const std::string& rRemotePath)
{
+ BOX_TRACE("Comparing file: " << rLocalPath);
}
virtual void NotifyFileCompared(const std::string& rLocalPath,
diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt
index 9e21fc62..b49a9cf2 100644
--- a/lib/backupstore/BackupStoreException.txt
+++ b/lib/backupstore/BackupStoreException.txt
@@ -71,3 +71,4 @@ PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. R
UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known.
MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first
CorruptReferenceCountDatabase 70 The account's refcount database is corrupt and must be rebuilt by housekeeping.
+CancelledByBackgroundTask 71 The current task was cancelled on request by the background task.
diff --git a/lib/backupstore/BackupStoreFile.cpp b/lib/backupstore/BackupStoreFile.cpp
index e011f8f4..d4c73f8b 100644
--- a/lib/backupstore/BackupStoreFile.cpp
+++ b/lib/backupstore/BackupStoreFile.cpp
@@ -78,7 +78,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile(
const BackupStoreFilename &rStoreFilename,
int64_t *pModificationTime,
ReadLoggingStream::Logger* pLogger,
- RunStatusProvider* pRunStatusProvider)
+ RunStatusProvider* pRunStatusProvider,
+ BackgroundTask* pBackgroundTask)
{
// Create the stream
std::auto_ptr<BackupStoreFileEncodeStream> stream(
@@ -86,7 +87,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile(
// Do the initial setup
stream->Setup(Filename, 0 /* no recipe, just encode */, ContainerID,
- rStoreFilename, pModificationTime, pLogger, pRunStatusProvider);
+ rStoreFilename, pModificationTime, pLogger, pRunStatusProvider,
+ pBackgroundTask);
// Return the stream for the caller
return stream;
diff --git a/lib/backupstore/BackupStoreFile.h b/lib/backupstore/BackupStoreFile.h
index 8e2cefb6..776e6d22 100644
--- a/lib/backupstore/BackupStoreFile.h
+++ b/lib/backupstore/BackupStoreFile.h
@@ -27,6 +27,7 @@ typedef struct
int64_t mTotalFileStreamSize;
} BackupStoreFileStats;
+class BackgroundTask;
class RunStatusProvider;
// Uncomment to disable backwards compatibility
@@ -128,7 +129,8 @@ public:
int64_t ContainerID, const BackupStoreFilename &rStoreFilename,
int64_t *pModificationTime = 0,
ReadLoggingStream::Logger* pLogger = NULL,
- RunStatusProvider* pRunStatusProvider = NULL
+ RunStatusProvider* pRunStatusProvider = NULL,
+ BackgroundTask* pBackgroundTask = NULL
);
static std::auto_ptr<BackupStoreFileEncodeStream> EncodeFileDiff
(
@@ -138,7 +140,8 @@ public:
int Timeout,
DiffTimer *pDiffTimer,
int64_t *pModificationTime = 0,
- bool *pIsCompletelyDifferent = 0
+ bool *pIsCompletelyDifferent = 0,
+ BackgroundTask* pBackgroundTask = NULL
);
// Shortcut interface
static int64_t QueryStoreFileDiff(BackupProtocolCallable& protocol,
diff --git a/lib/backupstore/BackupStoreFileDiff.cpp b/lib/backupstore/BackupStoreFileDiff.cpp
index fa8cb892..176d56e1 100644
--- a/lib/backupstore/BackupStoreFileDiff.cpp
+++ b/lib/backupstore/BackupStoreFileDiff.cpp
@@ -128,7 +128,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff
const std::string& Filename, int64_t ContainerID,
const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID,
IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer,
- int64_t *pModificationTime, bool *pIsCompletelyDifferent)
+ int64_t *pModificationTime, bool *pIsCompletelyDifferent,
+ BackgroundTask* pBackgroundTask)
{
// Is it a symlink?
{
@@ -144,7 +145,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff
{
*pIsCompletelyDifferent = true;
}
- return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+ return EncodeFile(Filename, ContainerID, rStoreFilename,
+ pModificationTime,
+ NULL, // ReadLoggingStream::Logger
+ NULL, // RunStatusProvider
+ pBackgroundTask); // BackgroundTask
}
}
@@ -162,7 +167,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff
{
*pIsCompletelyDifferent = true;
}
- return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime);
+ return EncodeFile(Filename, ContainerID, rStoreFilename,
+ pModificationTime,
+ NULL, // ReadLoggingStream::Logger
+ NULL, // RunStatusProvider
+ pBackgroundTask); // BackgroundTask
}
// Pointer to recipe we're going to create
@@ -210,7 +219,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff
new BackupStoreFileEncodeStream);
// Do the initial setup
- stream->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime);
+ stream->Setup(Filename, precipe, ContainerID, rStoreFilename,
+ pModificationTime,
+ NULL, // ReadLoggingStream::Logger
+ NULL, // RunStatusProvider
+ pBackgroundTask);
precipe = 0; // Stream has taken ownership of this
// Tell user about completely different status?
diff --git a/lib/backupstore/BackupStoreFileEncodeStream.cpp b/lib/backupstore/BackupStoreFileEncodeStream.cpp
index b53c4c26..b40bcd08 100644
--- a/lib/backupstore/BackupStoreFileEncodeStream.cpp
+++ b/lib/backupstore/BackupStoreFileEncodeStream.cpp
@@ -11,6 +11,7 @@
#include <string.h>
+#include "BackgroundTask.h"
#include "BackupClientFileAttributes.h"
#include "BackupStoreConstants.h"
#include "BackupStoreException.h"
@@ -40,25 +41,28 @@ using namespace BackupStoreFileCryptVar;
//
// --------------------------------------------------------------------------
BackupStoreFileEncodeStream::BackupStoreFileEncodeStream()
- : mpRecipe(0),
- mpFile(0),
- mpLogging(0),
- mpRunStatusProvider(NULL),
- mStatus(Status_Header),
- mSendData(true),
- mTotalBlocks(0),
- mAbsoluteBlockNumber(-1),
- mInstructionNumber(-1),
- mNumBlocks(0),
- mCurrentBlock(-1),
- mCurrentBlockEncodedSize(0),
- mPositionInCurrentBlock(0),
- mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
- mLastBlockSize(0),
- mTotalBytesSent(0),
- mpRawBuffer(0),
- mAllocatedBufferSize(0),
- mEntryIVBase(0)
+: mpRecipe(0),
+ mpFile(0),
+ mpLogging(0),
+ mpRunStatusProvider(NULL),
+ mpBackgroundTask(NULL),
+ mStatus(Status_Header),
+ mSendData(true),
+ mTotalBlocks(0),
+ mBytesToUpload(0),
+ mBytesUploaded(0),
+ mAbsoluteBlockNumber(-1),
+ mInstructionNumber(-1),
+ mNumBlocks(0),
+ mCurrentBlock(-1),
+ mCurrentBlockEncodedSize(0),
+ mPositionInCurrentBlock(0),
+ mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
+ mLastBlockSize(0),
+ mTotalBytesSent(0),
+ mpRawBuffer(0),
+ mAllocatedBufferSize(0),
+ mEntryIVBase(0)
{
}
@@ -115,7 +119,8 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename,
BackupStoreFileEncodeStream::Recipe *pRecipe,
int64_t ContainerID, const BackupStoreFilename &rStoreFilename,
int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger,
- RunStatusProvider* pRunStatusProvider)
+ RunStatusProvider* pRunStatusProvider,
+ BackgroundTask* pBackgroundTask)
{
// Pointer to a blank recipe which we might create
BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0;
@@ -162,6 +167,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename,
CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize);
// Add to accumlated total
mTotalBlocks += numBlocks;
+ mBytesToUpload += (*pRecipe)[inst].mSpaceBefore;
// Update maximum clear size
if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize;
if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize;
@@ -278,6 +284,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename,
}
mpRunStatusProvider = pRunStatusProvider;
+ mpBackgroundTask = pBackgroundTask;
}
@@ -341,6 +348,19 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout)
THROW_EXCEPTION(BackupStoreException, SignalReceived);
}
+ if(mpBackgroundTask)
+ {
+ BackgroundTask::State state = (mpRecipe->at(0).mBlocks == 0)
+ ? BackgroundTask::Uploading_Full
+ : BackgroundTask::Uploading_Patch;
+ if(!mpBackgroundTask->RunBackgroundTask(state, mBytesUploaded,
+ mBytesToUpload))
+ {
+ THROW_EXCEPTION(BackupStoreException,
+ CancelledByBackgroundTask);
+ }
+ }
+
int bytesToRead = NBytes;
uint8_t *buffer = (uint8_t*)pBuffer;
@@ -575,6 +595,8 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock()
// Encode it
mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer,
blockRawSize, mEncodedBuffer);
+
+ mBytesUploaded += blockRawSize;
//TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize);
diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h
index a169e036..80bdff58 100644
--- a/lib/backupstore/BackupStoreFileEncodeStream.h
+++ b/lib/backupstore/BackupStoreFileEncodeStream.h
@@ -79,7 +79,8 @@ public:
const BackupStoreFilename &rStoreFilename,
int64_t *pModificationTime,
ReadLoggingStream::Logger* pLogger = NULL,
- RunStatusProvider* pRunStatusProvider = NULL);
+ RunStatusProvider* pRunStatusProvider = NULL,
+ BackgroundTask* pBackgroundTask = NULL);
virtual int Read(void *pBuffer, int NBytes, int Timeout);
virtual void Write(const void *pBuffer, int NBytes);
@@ -109,14 +110,18 @@ private:
CollectInBufferStream mData; // buffer for header and index entries
IOStream *mpLogging;
RunStatusProvider* mpRunStatusProvider;
+ BackgroundTask* mpBackgroundTask;
int mStatus;
bool mSendData; // true if there's file data to send (ie not a symlink)
int64_t mTotalBlocks; // Total number of blocks in the file
+ int64_t mBytesToUpload; // Total number of clear bytes to encode and upload
+ int64_t mBytesUploaded; // Total number of clear bytes already encoded
+ // excluding reused blocks already on the server.
int64_t mAbsoluteBlockNumber; // The absolute block number currently being output
// Instruction number
int64_t mInstructionNumber;
// All the below are within the current instruction
- int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases
+ int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases
int64_t mCurrentBlock;
int32_t mCurrentBlockEncodedSize;
int32_t mPositionInCurrentBlock; // for reading out