diff options
| -rw-r--r-- | bin/bbackupd/BackupClientContext.cpp | 180 | ||||
| -rw-r--r-- | bin/bbackupd/BackupClientContext.h | 69 | ||||
| -rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.cpp | 23 | ||||
| -rw-r--r-- | bin/bbackupd/BackupDaemon.cpp | 10 | ||||
| -rw-r--r-- | bin/bbstored/BackupCommands.cpp | 17 | ||||
| -rw-r--r-- | bin/bbstored/backupprotocol.txt | 7 | ||||
| -rw-r--r-- | configure.ac | 3 | ||||
| -rw-r--r-- | lib/backupclient/BackupDaemonConfigVerify.cpp | 1 | ||||
| -rw-r--r-- | lib/backupclient/BackupStoreFile.h | 35 | ||||
| -rw-r--r-- | lib/backupclient/BackupStoreFileDiff.cpp | 156 | ||||
| -rw-r--r-- | test/backupdiff/testbackupdiff.cpp | 28 | ||||
| -rw-r--r-- | test/backupstore/testbackupstore.cpp | 13 | ||||
| -rw-r--r-- | test/backupstorepatch/testbackupstorepatch.cpp | 13 |
13 files changed, 442 insertions, 113 deletions
diff --git a/bin/bbackupd/BackupClientContext.cpp b/bin/bbackupd/BackupClientContext.cpp index 38d0b98c..ae4e1cad 100644 --- a/bin/bbackupd/BackupClientContext.cpp +++ b/bin/bbackupd/BackupClientContext.cpp @@ -9,8 +9,14 @@ #include "Box.h" -#ifndef WIN32 -#include <syslog.h> +#ifdef HAVE_SYSLOG_H + #include <syslog.h> +#endif +#ifdef HAVE_SIGNAL_H + #include <signal.h> +#endif +#ifdef HAVE_SYS_TIME_H + #include <sys/time.h> #endif #include "BoxPortsAndFiles.h" @@ -22,6 +28,7 @@ #include "BackupStoreException.h" #include "BackupDaemon.h" #include "autogen_BackupProtocolClient.h" +#include "BackupStoreFile.h" #include "MemLeakFindOn.h" @@ -48,7 +55,11 @@ BackupClientContext::BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLS mpNewIDMap(0), mStorageLimitExceeded(false), mpExcludeFiles(0), - mpExcludeDirs(0) + mpExcludeDirs(0), + mbIsManaged(false), + mTimeMgmtEpoch(0), + mMaximumDiffTime(600), + mKeepAliveTime(0) { } @@ -453,3 +464,166 @@ bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirec } +// maximum time to spend diffing +static int sMaximumDiffTime = 600; +// maximum time of SSL inactivity (keep-alive interval) +static int sKeepAliveTime = 0; + +void BackupClientContext::SetMaximumDiffingTime(int iSeconds) +{ + sMaximumDiffTime = iSeconds < 0 ? 0 : iSeconds; + TRACE1("Set maximum diffing time to %d seconds\n", sMaximumDiffTime); +} + +void BackupClientContext::SetKeepAliveTime(int iSeconds) +{ + sKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; + TRACE1("Set keep-alive time to %d seconds\n", sKeepAliveTime); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static TimerSigHandler(int) +// Purpose: Signal handler +// Created: 19/3/04 +// +// -------------------------------------------------------------------------- +static void TimerSigHandler(int iUnused) +{ + BackupStoreFile::DiffTimerExpired(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::ManageDiffProcess() +// Purpose: Initiates a file diff control timer +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::ManageDiffProcess() +{ + if (mbIsManaged || !mpConnection) + return; + + ASSERT(mTimeMgmtEpoch == 0); + +#ifdef PLATFORM_CYGWIN + ::signal(SIGALRM, TimerSigHandler); +#else + ::signal(SIGVTALRM, TimerSigHandler); +#endif // PLATFORM_CYGWIN + + struct itimerval timeout; + memset(&timeout, 0, sizeof(timeout)); + + // + // + // + if (sMaximumDiffTime <= 0 && sKeepAliveTime <= 0) + { + TRACE0("Diff control not requested - letting things run wild\n"); + return; + } + else if (sMaximumDiffTime > 0 && sKeepAliveTime > 0) + { + timeout.it_value.tv_sec = sKeepAliveTime < sMaximumDiffTime ? sKeepAliveTime : sMaximumDiffTime; + timeout.it_interval.tv_sec = sKeepAliveTime < sMaximumDiffTime ? sKeepAliveTime : sMaximumDiffTime; + } + else + { + timeout.it_value.tv_sec = sKeepAliveTime > 0 ? sKeepAliveTime : sMaximumDiffTime; + timeout.it_interval.tv_sec = sKeepAliveTime > 0 ? sKeepAliveTime : sMaximumDiffTime; + } + + // avoid race + mTimeMgmtEpoch = time(NULL); + +#ifdef PLATFORM_CYGWIN + if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) +#else + if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0) +#endif // PLATFORM_CYGWIN + { + mTimeMgmtEpoch = 0; + + TRACE0("WARNING: couldn't set file diff control timeout\n"); + THROW_EXCEPTION(BackupStoreException, Internal) + } + + mbIsManaged = true; + TRACE0("Initiated timer for file diff control\n"); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::UnManageDiffProcess() +// Purpose: suspends file diff control timer +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::UnManageDiffProcess() +{ + if (!mbIsManaged /* don't test for active connection, just do it */) + return; + + struct itimerval timeout; + memset(&timeout, 0, sizeof(timeout)); + +#ifdef PLATFORM_CYGWIN + if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) +#else + if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0) +#endif // PLATFORM_CYGWIN + { + TRACE0("WARNING: couldn't clear file diff control timeout\n"); + THROW_EXCEPTION(BackupStoreException, Internal) + } + + mbIsManaged = false; + mTimeMgmtEpoch = 0; + + TRACE0("Suspended timer for file diff control\n"); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::DoKeepAlive() +// Purpose: Does something inconsequential over the SSL link to keep it up +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +void BackupClientContext::DoKeepAlive() +{ + if (!mpConnection) + return; + + mpConnection->QueryGetIsAlive(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientContext::GetTimeMgmtEpoch() +// Purpose: Returns the unix time when the diff was started, or zero +// if the diff process is unmanaged. +// Created: 04/19/2005 +// +// -------------------------------------------------------------------------- +time_t BackupClientContext::GetTimeMgmtEpoch() +{ + return mTimeMgmtEpoch; +} + +int BackupClientContext::GetMaximumDiffingTime() +{ + return mMaximumDiffTime; +} + +int BackupClientContext::GetKeepaliveTime() +{ + return mKeepAliveTime; +} diff --git a/bin/bbackupd/BackupClientContext.h b/bin/bbackupd/BackupClientContext.h index 3933dbed..234868db 100644 --- a/bin/bbackupd/BackupClientContext.h +++ b/bin/bbackupd/BackupClientContext.h @@ -12,6 +12,7 @@ #include "BoxTime.h" #include "BackupClientDeleteList.h" +#include "BackupStoreFile.h" #include "ExcludeList.h" class TLSContext; @@ -31,12 +32,12 @@ class BackupStoreFilenameClear; // Created: 2003/10/08 // // -------------------------------------------------------------------------- -class BackupClientContext +class BackupClientContext : public DiffTimer { public: BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLSContext, const std::string &rHostname, int32_t AccountNumber, bool ExtendedLogging); - ~BackupClientContext(); + virtual ~BackupClientContext(); private: BackupClientContext(const BackupClientContext &); public: @@ -134,6 +135,60 @@ public: bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer = 0, box_time_t *pAttributesHashOnServer = 0, BackupStoreFilenameClear *pLeafname = 0); // not const as may connect to server + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetMaximumDiffingTime() + // Purpose: Sets the maximum time that will be spent diffing a file + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + static void SetMaximumDiffingTime(int iSeconds); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::SetKeepAliveTime() + // Purpose: Sets the time interval for repetitive keep-alive operation + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + static void SetKeepAliveTime(int iSeconds); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::ManageDiffProcess() + // Purpose: Initiates an SSL connection/session keep-alive process + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void ManageDiffProcess(); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::UnManageDiffProcess() + // Purpose: Suspends an SSL connection/session keep-alive process + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + void UnManageDiffProcess(); + + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupClientContext::DoKeepAlive() + // Purpose: Does something inconsequential over the SSL link to + // keep it up, implements DiffTimer interface + // Created: 04/19/2005 + // + // -------------------------------------------------------------------------- + virtual void DoKeepAlive(); + virtual time_t GetTimeMgmtEpoch(); + virtual int GetMaximumDiffingTime(); + virtual int GetKeepaliveTime(); + private: BackupDaemon &mrDaemon; TLSContext &mrTLSContext; @@ -149,8 +204,16 @@ private: bool mStorageLimitExceeded; ExcludeList *mpExcludeFiles; ExcludeList *mpExcludeDirs; + + bool mbIsManaged; + // unix time when diff was started + time_t mTimeMgmtEpoch; + // maximum time to spend diffing, in seconds + int mMaximumDiffTime; + // maximum time of SSL inactivity (keep-alive interval), in seconds + int mKeepAliveTime; + }; #endif // BACKUPCLIENTCONTEXT__H - diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index 1a124333..3e8ceeb2 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -1093,14 +1093,28 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn // Found an old version -- get the index std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); + // // Diff the file + // + + rParams.mrContext.ManageDiffProcess(); + bool isCompletelyDifferent = false; - std::auto_ptr<IOStream> patchStream(BackupStoreFile::EncodeFileDiff(rFilename.c_str(), + std::auto_ptr<IOStream> patchStream( + BackupStoreFile::EncodeFileDiff( + rFilename.c_str(), mObjectID, /* containing directory */ rStoreFilename, diffFromID, *blockIndexStream, - connection.GetTimeout(), 0 /* not interested in the modification time */, &isCompletelyDifferent)); + connection.GetTimeout(), + &rParams.mrContext, // DiffTimer implementation + 0 /* not interested in the modification time */, + &isCompletelyDifferent)); + rParams.mrContext.UnManageDiffProcess(); + + // // Upload the patch to the store + // std::auto_ptr<BackupProtocolClientSuccess> stored(connection.QueryStoreFile(mObjectID, ModificationTime, AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream)); @@ -1130,6 +1144,8 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn } catch(BoxException &e) { + rParams.mrContext.UnManageDiffProcess(); + if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) { // Check and see what error the protocol has -- as it might be an error... @@ -1210,6 +1226,3 @@ BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, Backu BackupClientDirectoryRecord::SyncParams::~SyncParams() { } - - - diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp index 954504a6..07a8db4d 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/bin/bbackupd/BackupDaemon.cpp @@ -435,10 +435,14 @@ void BackupDaemon::Run2() // Set up the keys for various things BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str()); - // Set maximum diffing time? + // max diffing time, keep-alive time if(conf.KeyExists("MaximumDiffingTime")) { - BackupStoreFile::SetMaximumDiffingTime(conf.GetKeyValueInt("MaximumDiffingTime")); + BackupClientContext::SetMaximumDiffingTime(conf.GetKeyValueInt("MaximumDiffingTime")); + } + if(conf.KeyExists("KeepAliveTime")) + { + BackupClientContext::SetKeepAliveTime(conf.GetKeyValueInt("KeepAliveTime")); } // Setup various timings @@ -885,7 +889,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla { #ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET bool uidOK = true; - ::syslog(LOG_ERR, "On this platform, no security check can be made on the credientials of peers connecting to the command socket. (bbackupctl)"); + ::syslog(LOG_WARNING, "On this platform, no security check can be made on the credientials of peers connecting to the command socket. (bbackupctl)"); #else // Security check -- does the process connecting to this socket have // the same UID as this process? diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index 1a021a58..35bc095d 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -859,3 +859,20 @@ std::auto_ptr<ProtocolObject> BackupProtocolServerGetAccountUsage::DoCommand(Bac )); } +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &, BackupContext &) +// Purpose: Return the amount of disc space used +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr<ProtocolObject> BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + + // + // NOOP + // + return std::auto_ptr<ProtocolObject>(new BackupProtocolServerIsAlive()); +} diff --git a/bin/bbstored/backupprotocol.txt b/bin/bbstored/backupprotocol.txt index 39cb1fb3..106ac24c 100644 --- a/bin/bbstored/backupprotocol.txt +++ b/bin/bbstored/backupprotocol.txt @@ -219,3 +219,10 @@ AccountUsage 41 Reply int64 BlocksSoftLimit int64 BlocksHardLimit int32 BlockSize + +GetIsAlive 42 Command(IsAlive) + # no data members + +IsAlive 43 Reply + # no data members + diff --git a/configure.ac b/configure.ac index 3322cf91..2ed3812a 100644 --- a/configure.ac +++ b/configure.ac @@ -75,7 +75,8 @@ AC_HEADER_DIRENT AC_HEADER_STDC AC_HEADER_SYS_WAIT AC_CHECK_HEADERS([execinfo.h netinet/in.h regex.h sys/types.h sys/xattr.h]) -AC_CHECK_HEADERS([sys/endian.h asm/byteorder.h]) +AC_CHECK_HEADERS([sys/endian.h asm/byteorder.h syslog.h signal.h sys/time.h]) +AC_CHECK_HEADERS([time.h]) ### Checks for typedefs, structures, and compiler characteristics. diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index eec2ceaf..f1d5bd16 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -84,6 +84,7 @@ static const ConfigurationVerifyKey verifyrootkeys[] = {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file {"CommandSocket", 0, 0, 0}, // not compulsory to have this + {"KeepAliveTime", 0, ConfigTest_IsInt, 0}, // optional {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h index 446e9422..27b2ef98 100644 --- a/lib/backupclient/BackupStoreFile.h +++ b/lib/backupclient/BackupStoreFile.h @@ -38,6 +38,24 @@ typedef struct // -------------------------------------------------------------------------- // // Class +// Name: DiffTimer +// Purpose: Interface for classes that can keep track of diffing time, +// and send SSL keepalive messages +// Created: 2006/01/19 +// +// -------------------------------------------------------------------------- +class DiffTimer +{ +public: + virtual void DoKeepAlive() = 0; + virtual time_t GetTimeMgmtEpoch() = 0; + virtual int GetMaximumDiffingTime() = 0; + virtual int GetKeepaliveTime() = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class // Name: BackupStoreFile // Purpose: Class to hold together utils for maniplating files. // Created: 2003/08/28 @@ -95,9 +113,16 @@ public: // Main interface static std::auto_ptr<IOStream> EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0); - static std::auto_ptr<IOStream> EncodeFileDiff(const char *Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, - int Timeout, int64_t *pModificationTime = 0, bool *pIsCompletelyDifferent = 0); + static std::auto_ptr<IOStream> EncodeFileDiff + ( + const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, + int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, + int Timeout, + DiffTimer *pDiffTimer, + int64_t *pModificationTime = 0, + bool *pIsCompletelyDifferent = 0 + ); static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); @@ -144,8 +169,7 @@ public: free(a); } - // Limits - static void SetMaximumDiffingTime(int Seconds); + static void DiffTimerExpired(); // Building blocks class EncodingBuffer @@ -192,4 +216,3 @@ public: #include "MemLeakFindOff.h" #endif // BACKUPSTOREFILE__H - diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp index c1d0492d..571f4351 100644 --- a/lib/backupclient/BackupStoreFileDiff.cpp +++ b/lib/backupclient/BackupStoreFileDiff.cpp @@ -11,11 +11,11 @@ #include <new> #include <map> -#include <signal.h> -#ifdef WIN32 -#include <time.h> -#else -#include <sys/time.h> + +#ifdef HAVE_TIME_H + #include <time.h> +#elif HAVE_SYS_TIME_H + #include <sys/time.h> #endif #include "BackupStoreFile.h" @@ -43,33 +43,37 @@ using namespace BackupStoreFileCreation; static void LoadIndex(IOStream &rBlockIndex, int64_t ThisID, BlocksAvailableEntry **ppIndex, int64_t &rNumBlocksOut, int Timeout, bool &rCanDiffFromThis); static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); -static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]); +static void SearchForMatchingBlocks(IOStream &rFile, + std::map<int64_t, int64_t> &rFoundBlocks, BlocksAvailableEntry *pIndex, + int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], + DiffTimer *pDiffTimer); static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t BlockSize, BlocksAvailableEntry **pHashTable); static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChecksum &fastSum, uint8_t *pBeginnings, uint8_t *pEndings, int Offset, int32_t BlockSize, int64_t FileBlockNumber, BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks); static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile); -// Avoid running on too long with diffs -static int sMaximumDiffTime = 600; // maximum time to spend diffing -static bool sDiffTimedOut = false; -static bool sSetTimerSignelHandler = false; -static void TimerSignalHandler(int signal); -static void StartDiffTimer(); +// sDiffTimerExpired flags when the diff timer has expired. When true, the +// diff routine should check the wall clock as soon as possible, to determine +// whether it's time for a keepalive to be sent, or whether the diff has been +// running for too long and should be terminated. +static bool sDiffTimerExpired = false; // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreFile::SetMaximumDiffingTime(int) -// Purpose: Sets the maximum time to spend diffing, in seconds. Time is -// process virutal time. -// Created: 19/3/04 +// Name: BackupStoreFile::DiffTimerExpired() +// Purpose: Notifies BackupStoreFile object that the diff operation +// timer has expired, which may mean that a keepalive should +// be sent, or the diff should be terminated. Called from an +// external timer, so it should not do more than set a flag. +// +// Created: 19/1/06 // // -------------------------------------------------------------------------- -void BackupStoreFile::SetMaximumDiffingTime(int Seconds) +void BackupStoreFile::DiffTimerExpired() { - sMaximumDiffTime = Seconds; - TRACE1("Set maximum diffing time to %d seconds\n", Seconds); + sDiffTimerExpired = true; } @@ -139,9 +143,12 @@ void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream) // Created: 12/1/04 // // -------------------------------------------------------------------------- -std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff(const char *Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, IOStream &rDiffFromBlockIndex, - int Timeout, int64_t *pModificationTime, bool *pIsCompletelyDifferent) +std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff +( + const char *Filename, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, + IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, + int64_t *pModificationTime, bool *pIsCompletelyDifferent) { // Is it a symlink? { @@ -180,9 +187,6 @@ std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff(const char *Filename, in // Pointer to recipe we're going to create BackupStoreFileEncodeStream::Recipe *precipe = 0; - - // Start the timeout timer, so that the operation doesn't continue for ever - StartDiffTimer(); try { @@ -204,7 +208,8 @@ std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff(const char *Filename, in // Get size of file sizeOfInputFile = file.BytesLeftToRead(); // Find all those lovely matching blocks - SearchForMatchingBlocks(file, foundBlocks, pindex, blocksInIndex, sizesToScan); + SearchForMatchingBlocks(file, foundBlocks, pindex, + blocksInIndex, sizesToScan, pDiffTimer); // Is it completely different? completelyDifferent = (foundBlocks.size() == 0); @@ -475,8 +480,20 @@ static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, i // // -------------------------------------------------------------------------- static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> &rFoundBlocks, - BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES]) + BlocksAvailableEntry *pIndex, int64_t NumBlocks, + int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer) { + time_t TimeMgmtEpoch = 0; + int MaximumDiffingTime = 0; + int KeepAliveTime = 0; + + if (pDiffTimer) + { + TimeMgmtEpoch = pDiffTimer->GetTimeMgmtEpoch(); + MaximumDiffingTime = pDiffTimer->GetMaximumDiffingTime(); + KeepAliveTime = pDiffTimer->GetKeepaliveTime(); + } + std::map<int64_t, int32_t> goodnessOfFit; // Allocate the hash lookup table @@ -643,8 +660,39 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> // End this loop, so the final byte isn't used again break; } + + bool DiffTimedOut = false; + + if(sDiffTimerExpired) + { + ASSERT(TimeMgmtEpoch > 0); + ASSERT(pDiffTimer != NULL); + + time_t tTotalRunIntvl = time(NULL) - TimeMgmtEpoch; + + if(MaximumDiffingTime > 0 && + tTotalRunIntvl >= MaximumDiffingTime) + { + TRACE0("MaximumDiffingTime reached - " + "suspending file diff\n"); + DiffTimedOut = true; + } + else if(KeepAliveTime > 0) + { + TRACE0("KeepAliveTime reached - " + "initiating keep-alive\n"); + pDiffTimer->DoKeepAlive(); + } + + sDiffTimerExpired = false; + } + + int64_t NumBlocksFound = static_cast<int64_t>( + rFoundBlocks.size()); + int64_t MaxBlocksFound = NumBlocks * + BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE; - if(static_cast<int64_t>(rFoundBlocks.size()) > (NumBlocks * BACKUP_FILE_DIFF_MAX_BLOCK_FIND_MULTIPLE) || sDiffTimedOut) + if(NumBlocksFound > MaxBlocksFound || DiffTimedOut) { abortSearch = true; break; @@ -1005,55 +1053,3 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA } #endif } - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static TimerSignalHandler(int) -// Purpose: Signal handler -// Created: 19/3/04 -// -// -------------------------------------------------------------------------- -void TimerSignalHandler(int signal) -{ - sDiffTimedOut = true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: static StartDiffTimer() -// Purpose: Starts the diff timeout timer -// Created: 19/3/04 -// -// -------------------------------------------------------------------------- -void StartDiffTimer() -{ - // Set timer signal handler - if(!sSetTimerSignelHandler) - { - ::signal(SIGVTALRM, TimerSignalHandler); - sSetTimerSignelHandler = true; - } - - struct itimerval timeout; - // Don't want this to repeat - timeout.it_interval.tv_sec = 0; - timeout.it_interval.tv_usec = 0; - // Single timeout after the specified number of seconds - timeout.it_value.tv_sec = sMaximumDiffTime; - timeout.it_value.tv_usec = 0; - // Set timer - if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0) - { - TRACE0("WARNING: couldn't set diff timeout\n"); - } - - // Unset flag (last thing) - sDiffTimedOut = false; -} - - - diff --git a/test/backupdiff/testbackupdiff.cpp b/test/backupdiff/testbackupdiff.cpp index 4408b69f..584dd70c 100644 --- a/test/backupdiff/testbackupdiff.cpp +++ b/test/backupdiff/testbackupdiff.cpp @@ -169,9 +169,17 @@ void test_diff(int from, int to, int new_blocks_expected, int old_blocks_expecte { BackupStoreFilenameClear f1name("filename"); FileStream out(to_diff, O_WRONLY | O_CREAT | O_EXCL); - std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff(to_orig, 1 /* dir ID */, f1name, - 1000 + from /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite, - 0, &completelyDifferent)); + std::auto_ptr<IOStream> encoded( + BackupStoreFile::EncodeFileDiff( + to_orig, + 1 /* dir ID */, + f1name, + 1000 + from /* object ID of the file diffing from */, + blockindex, + IOStream::TimeOutInfinite, + NULL, // DiffTimer interface + 0, + &completelyDifferent)); encoded->CopyStreamTo(out); } TEST_THAT(completelyDifferent == expect_completely_different); @@ -443,9 +451,17 @@ int test(int argc, const char *argv[]) BackupStoreFilenameClear f1name("filename"); FileStream out("testfiles/f2.symlink.diff", O_WRONLY | O_CREAT | O_EXCL); - std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFileDiff("testfiles/f2.symlink", 1 /* dir ID */, f1name, - 1001 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite, - 0, &completelyDifferent)); + std::auto_ptr<IOStream> encoded( + BackupStoreFile::EncodeFileDiff( + "testfiles/f2.symlink", + 1 /* dir ID */, + f1name, + 1001 /* object ID of the file diffing from */, + blockindex, + IOStream::TimeOutInfinite, + NULL, // DiffTimer interface + 0, + &completelyDifferent)); encoded->CopyStreamTo(out); } TEST_THAT(completelyDifferent == true); diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp index 1d297f16..f89d2ff7 100644 --- a/test/backupstore/testbackupstore.cpp +++ b/test/backupstore/testbackupstore.cpp @@ -1072,9 +1072,16 @@ int test_server(const char *hostname) // Do the patching bool isCompletelyDifferent = false; int64_t modtime; - std::auto_ptr<IOStream> patchstream(BackupStoreFile::EncodeFileDiff(TEST_FILE_FOR_PATCHING ".mod", BackupProtocolClientListDirectory::RootDirectory, - uploads[UPLOAD_PATCH_EN].name, uploads[UPLOAD_PATCH_EN].allocated_objid, *blockIndexStream, - IOStream::TimeOutInfinite, &modtime, &isCompletelyDifferent)); + std::auto_ptr<IOStream> patchstream( + BackupStoreFile::EncodeFileDiff( + TEST_FILE_FOR_PATCHING ".mod", + BackupProtocolClientListDirectory::RootDirectory, + uploads[UPLOAD_PATCH_EN].name, + uploads[UPLOAD_PATCH_EN].allocated_objid, + *blockIndexStream, + IOStream::TimeOutInfinite, + NULL, // pointer to DiffTimer impl + &modtime, &isCompletelyDifferent)); TEST_THAT(isCompletelyDifferent == false); // Sent this to a file, so we can check the size, rather than uploading it directly { diff --git a/test/backupstorepatch/testbackupstorepatch.cpp b/test/backupstorepatch/testbackupstorepatch.cpp index a4a1ece3..5d77c3fd 100644 --- a/test/backupstorepatch/testbackupstorepatch.cpp +++ b/test/backupstorepatch/testbackupstorepatch.cpp @@ -376,10 +376,17 @@ int test(int argc, const char *argv[]) char filename[64]; ::sprintf(filename, "testfiles/%d.test", f); bool isCompletelyDifferent = false; - std::auto_ptr<IOStream> patchStream(BackupStoreFile::EncodeFileDiff(filename, + std::auto_ptr<IOStream> patchStream( + BackupStoreFile::EncodeFileDiff( + filename, BackupProtocolClientListDirectory::RootDirectory, /* containing directory */ - storeFilename, diffFromID, *blockIndexStream, - protocol.GetTimeout(), 0 /* not interested in the modification time */, &isCompletelyDifferent)); + storeFilename, + diffFromID, + *blockIndexStream, + protocol.GetTimeout(), + NULL, // DiffTimer impl + 0 /* not interested in the modification time */, + &isCompletelyDifferent)); // Upload the patch to the store std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile( |