From 9e025fd0ea42782e93363958804c6e2437b44a30 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 28 Jan 2006 00:13:44 +0000 Subject: * configure.ac * bin/bbstored/backupprotocol.txt * bin/bbstored/BackupCommands.cpp * bin/bbackupd/BackupClientContext.cpp * bin/bbackupd/BackupClientContext.h * bin/bbackupd/BackupClientDirectoryRecord.cpp * bin/bbackupd/BackupDaemon.cpp * lib/backupclient/BackupStoreFileDiff.cpp * lib/backupclient/BackupDaemonConfigVerify.cpp * lib/backupclient/BackupStoreFile.h * test/backupstore/testbackupstore.cpp * test/backupstorepatch/testbackupstorepatch.cpp - Applied changes from chris/diff-timeout-and-ssl-keepalive * test/backupdiff/testbackupdiff.cpp - Fixed test to match new prototype for EncodeFileDiff --- lib/backupclient/BackupDaemonConfigVerify.cpp | 1 + lib/backupclient/BackupStoreFile.h | 35 +++++- lib/backupclient/BackupStoreFileDiff.cpp | 156 +++++++++++++------------- 3 files changed, 106 insertions(+), 86 deletions(-) (limited to 'lib/backupclient') diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index eec2ceaf..f1d5bd16 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -84,6 +84,7 @@ static const ConfigurationVerifyKey verifyrootkeys[] = {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file {"CommandSocket", 0, 0, 0}, // not compulsory to have this + {"KeepAliveTime", 0, ConfigTest_IsInt, 0}, // optional {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h index 446e9422..27b2ef98 100644 --- a/lib/backupclient/BackupStoreFile.h +++ b/lib/backupclient/BackupStoreFile.h @@ -35,6 +35,24 @@ typedef struct // Have some memory allocation commands, note closing "Off" at end of file. #include "MemLeakFindOn.h" +// -------------------------------------------------------------------------- +// +// Class +// Name: DiffTimer +// Purpose: Interface for classes that can keep track of diffing time, +// and send SSL keepalive messages +// Created: 2006/01/19 +// +// -------------------------------------------------------------------------- +class DiffTimer +{ +public: + virtual void DoKeepAlive() = 0; + virtual time_t GetTimeMgmtEpoch() = 0; + virtual int GetMaximumDiffingTime() = 0; + virtual int GetKeepaliveTime() = 0; +}; + // -------------------------------------------------------------------------- // // Class @@ -95,9 +113,16 @@ public: // Main interface static std::auto_ptr EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0); - static std::auto_ptr EncodeFileDiff(const char *Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, - int Timeout, int64_t *pModificationTime = 0, bool *pIsCompletelyDifferent = 0); + static std::auto_ptr EncodeFileDiff + ( + const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, + int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, + int Timeout, + DiffTimer *pDiffTimer, + int64_t *pModificationTime = 0, + bool *pIsCompletelyDifferent = 0 + ); static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); @@ -144,8 +169,7 @@ public: free(a); } - // Limits - static void SetMaximumDiffingTime(int Seconds); + static void DiffTimerExpired(); // Building blocks class EncodingBuffer @@ -192,4 +216,3 @@ public: #include "MemLeakFindOff.h" #endif // BACKUPSTOREFILE__H - diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp index c1d0492d..571f4351 100644 --- a/lib/backupclient/BackupStoreFileDiff.cpp +++ b/lib/backupclient/BackupStoreFileDiff.cpp @@ -11,11 +11,11 @@ #include #include -#include -#ifdef WIN32 -#include -#else -#include + +#ifdef HAVE_TIME_H + #include +#elif HAVE_SYS_TIME_H + #include #endif #include "BackupStoreFile.h" @@ -43,33 +43,37 @@ using namespace BackupStoreFileCreation; static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis); static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); -static void SearchForMatchingBlocks(IOStream &rFile, std::map &rFoundBlocks, BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); +static void SearchForMatchingBlocks(IOStream &rFile, + std::map &rFoundBlocks, BlocksAvailableEntry *pIndex, + int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], + DiffTimer *pDiffTimer); static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable); static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map &rFoundBlocks); static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map &rFoundBlocks, int64_t SizeOfInputFile); -// Avoid running on too long with diffs -static int sMaximumDiffTime = 600; // maximum time to spend diffing -static bool sDiffTimedOut = false; -static bool sSetTimerSignelHandler = false; -static void TimerSignalHandler(int signal); -static void StartDiffTimer(); +// sDiffTimerExpired flags when the diff timer has expired. When true, the +// diff routine should check the wall clock as soon as possible, to determine +// whether it's time for a keepalive to be sent, or whether the diff has been +// running for too long and should be terminated. +static bool sDiffTimerExpired = false; // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreFile::SetMaximumDiffingTime(int) -// Purpose: Sets the maximum time to spend diffing, in seconds. Time is -// process virutal time. -// Created: 19/3/04 +// Name: BackupStoreFile::DiffTimerExpired() +// Purpose: Notifies BackupStoreFile object that the diff operation +// timer has expired, which may mean that a keepalive should +// be sent, or the diff should be terminated. Called from an +// external timer, so it should not do more than set a flag. +// +// Created: 19/1/06 // // -------------------------------------------------------------------------- -void BackupStoreFile::SetMaximumDiffingTime(int Seconds) +void BackupStoreFile::DiffTimerExpired() { - sMaximumDiffTime = Seconds; - TRACE1("Set maximum diffing time to %d seconds\n", Seconds); + sDiffTimerExpired = true; } @@ -139,9 +143,12 @@ void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream) // Created: 12/1/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupStoreFile::EncodeFileDiff(const char *Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, - int Timeout, int64_t *pModificationTime, bool *pIsCompletelyDifferent) +std::auto_ptr BackupStoreFile::EncodeFileDiff +( + const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, + IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, + int64_t *pModificationTime, bool *pIsCompletelyDifferent) { // Is it a symlink? { @@ -180,9 +187,6 @@ std::auto_ptr BackupStoreFile::EncodeFileDiff(const char *Filename, in // Pointer to recipe we're going to create BackupStoreFileEncodeStream::Recipe *precipe = 0; - - // Start the timeout timer, so that the operation doesn't continue for ever - StartDiffTimer(); try { @@ -204,7 +208,8 @@ std::auto_ptr BackupStoreFile::EncodeFileDiff(const char *Filename, in // Get size of file sizeOfInputFile = file.BytesLeftToRead(); // Find all those lovely matching blocks - SearchForMatchingBlocks(file, foundBlocks, pindex, blocksInIndex, sizesToScan); + SearchForMatchingBlocks(file, foundBlocks, pindex, + blocksInIndex, sizesToScan, pDiffTimer); // Is it completely different? completelyDifferent = (foundBlocks.size() == 0); @@ -475,8 +480,20 @@ static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, i // // -------------------------------------------------------------------------- static void SearchForMatchingBlocks(IOStream &rFile, std::map &rFoundBlocks, - BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) + BlocksAvailableEntry *pIndex, int64_t NumBlocks, + int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer) { + time_t TimeMgmtEpoch = 0; + int MaximumDiffingTime = 0; + int KeepAliveTime = 0; + + if (pDiffTimer) + { + TimeMgmtEpoch = pDiffTimer->GetTimeMgmtEpoch(); + MaximumDiffingTime = pDiffTimer->GetMaximumDiffingTime(); + KeepAliveTime = pDiffTimer->GetKeepaliveTime(); + } + std::map goodnessOfFit; // Allocate the hash lookup table @@ -643,8 +660,39 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map // End this loop, so the final byte isn't used again break; } + + bool DiffTimedOut = false; + + if(sDiffTimerExpired) + { + ASSERT(TimeMgmtEpoch > 0); + ASSERT(pDiffTimer != NULL); + + time_t tTotalRunIntvl = time(NULL) - TimeMgmtEpoch; + + if(MaximumDiffingTime > 0 && + tTotalRunIntvl >= MaximumDiffingTime) + { + TRACE0("MaximumDiffingTime reached - " + "suspending file diff\n"); + DiffTimedOut = true; + } + else if(KeepAliveTime > 0) + { + TRACE0("KeepAliveTime reached - " + "initiating keep-alive\n"); + pDiffTimer->DoKeepAlive(); + } + + sDiffTimerExpired = false; + } + + int64_t NumBlocksFound = static_cast( + rFoundBlocks.size()); + int64_t MaxBlocksFound = NumBlocks * + BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE; - if(static_cast(rFoundBlocks.size()) > (NumBlocks * BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE) || sDiffTimedOut) + if(NumBlocksFound > MaxBlocksFound || DiffTimedOut) { abortSearch = true; break; @@ -1005,55 +1053,3 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA } #endif } - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static TimerSignalHandler(int) -// Purpose: Signal handler -// Created: 19/3/04 -// -// -------------------------------------------------------------------------- -void TimerSignalHandler(int signal) -{ - sDiffTimedOut = true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static StartDiffTimer() -// Purpose: Starts the diff timeout timer -// Created: 19/3/04 -// -// -------------------------------------------------------------------------- -void StartDiffTimer() -{ - // Set timer signal handler - if(!sSetTimerSignelHandler) - { - ::signal(SIGVTALRM, TimerSignalHandler); - sSetTimerSignelHandler = true; - } - - struct itimerval timeout; - // Don't want this to repeat - timeout.it_interval.tv_sec = 0; - timeout.it_interval.tv_usec = 0; - // Single timeout after the specified number of seconds - timeout.it_value.tv_sec = sMaximumDiffTime; - timeout.it_value.tv_usec = 0; - // Set timer - if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0) - { - TRACE0("WARNING: couldn't set diff timeout\n"); - } - - // Unset flag (last thing) - sDiffTimedOut = false; -} - - - -- cgit v1.2.3