From c7662795f519d2b6797e4b1ac7fa4a22afa26310 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 27 Jul 2006 23:18:35 +0000 Subject: * merge - This is my current patch queue. I think that all of these are safe to apply. This is just under half of the pending changes in chris/general (the easy half). --- bin/bbstored/BBStoreDHousekeeping.cpp | 174 ++++++++++++++++++++------------- bin/bbstored/BackupCommands.cpp | 33 ++++++- bin/bbstored/BackupContext.cpp | 20 +++- bin/bbstored/BackupStoreDaemon.cpp | 19 +++- bin/bbstored/BackupStoreDaemon.h | 13 +++ bin/bbstored/HousekeepStoreAccount.cpp | 7 ++ 6 files changed, 190 insertions(+), 76 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index d3656630..86b2468b 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -10,7 +10,10 @@ #include "Box.h" #include + +#ifdef HAVE_SYSLOG_H #include +#endif #include "BackupStoreDaemon.h" #include "BackupStoreAccountDatabase.h" @@ -29,95 +32,128 @@ // Created: 11/12/03 // // -------------------------------------------------------------------------- +void BackupStoreDaemon::HousekeepingInit() +{ + + mLastHousekeepingRun = 0; +} + +#ifndef WIN32 void BackupStoreDaemon::HousekeepingProcess() { + HousekeepingInit(); + // Get the time between housekeeping runs const Configuration &rconfig(GetConfiguration()); int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); - - int64_t lastHousekeepingRun = 0; while(!StopRun()) { - // Time now + RunHousekeepingIfNeeded(); + + // Calculate how long should wait before doing the next housekeeping run int64_t timeNow = GetCurrentBoxTime(); - // Do housekeeping if the time interval has elapsed since the last check - if((timeNow - lastHousekeepingRun) >= housekeepingInterval) - { - // Store the time - lastHousekeepingRun = timeNow; - ::syslog(LOG_INFO, "Starting housekeeping"); + time_t secondsToGo = BoxTimeToSeconds((mLastHousekeepingRun + housekeepingInterval) - timeNow); + if(secondsToGo < 1) secondsToGo = 1; + if(secondsToGo > 60) secondsToGo = 60; + int32_t millisecondsToGo = ((int)secondsToGo) * 1000; + + // Check to see if there's any message pending + CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); + } +} +#endif - // Get the list of accounts - std::vector accounts; - if(mpAccountDatabase) - { - mpAccountDatabase->GetAllAccountIDs(accounts); - } +void BackupStoreDaemon::RunHousekeepingIfNeeded() +{ + // Get the time between housekeeping runs + const Configuration &rconfig(GetConfiguration()); + int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); + + // Time now + int64_t timeNow = GetCurrentBoxTime(); + // Do housekeeping if the time interval has elapsed since the last check + if((timeNow - mLastHousekeepingRun) < housekeepingInterval) + { + return; + } + + // Store the time + mLastHousekeepingRun = timeNow; + ::syslog(LOG_INFO, "Starting housekeeping"); + + // Get the list of accounts + std::vector accounts; + if(mpAccountDatabase) + { + mpAccountDatabase->GetAllAccountIDs(accounts); + } - SetProcessTitle("housekeeping, active"); + SetProcessTitle("housekeeping, active"); - // Check them all - for(std::vector::const_iterator i = accounts.begin(); i != accounts.end(); ++i) + // Check them all + for(std::vector::const_iterator i = accounts.begin(); i != accounts.end(); ++i) + { + try + { + if(mpAccounts) { - try - { - if(mpAccounts) - { - // Get the account root - std::string rootDir; - int discSet = 0; - mpAccounts->GetAccountRoot(*i, rootDir, discSet); - - // Do housekeeping on this account - HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this); - housekeeping.DoHousekeeping(); - } - } - catch(BoxException &e) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", - *i, e.what(), e.GetType(), e.GetSubType()); - } - catch(std::exception &e) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", - *i, e.what()); - } - catch(...) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", - *i); - } + // Get the account root + std::string rootDir; + int discSet = 0; + mpAccounts->GetAccountRoot(*i, rootDir, discSet); - // Check to see if there's any message pending - CheckForInterProcessMsg(0 /* no account */); - - // Stop early? - if(StopRun()) - { - break; - } + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this); + housekeeping.DoHousekeeping(); } - - ::syslog(LOG_INFO, "Finished housekeeping"); } - - // Placed here for accuracy, if StopRun() is true, for example. - SetProcessTitle("housekeeping, idle"); - - // Calculate how long should wait before doing the next housekeeping run - timeNow = GetCurrentBoxTime(); - time_t secondsToGo = BoxTimeToSeconds((lastHousekeepingRun + housekeepingInterval) - timeNow); - if(secondsToGo < 1) secondsToGo = 1; - if(secondsToGo > 60) secondsToGo = 60; - int32_t millisecondsToGo = ((int)secondsToGo) * 1000; + catch(BoxException &e) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", + *i, e.what(), e.GetType(), e.GetSubType()); + } + catch(std::exception &e) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", + *i, e.what()); + } + catch(...) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", + *i); + } +#ifndef WIN32 // Check to see if there's any message pending - CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); + CheckForInterProcessMsg(0 /* no account */); +#endif + + // Stop early? + if(StopRun()) + { + break; + } } + + ::syslog(LOG_INFO, "Finished housekeeping"); + + // Placed here for accuracy, if StopRun() is true, for example. + SetProcessTitle("housekeeping, idle"); } +#ifdef WIN32 +void BackupStoreDaemon::OnIdle() +{ + if (!mHousekeepingInited) + { + HousekeepingInit(); + mHousekeepingInited = true; + } + + RunHousekeepingIfNeeded(); +} +#endif // -------------------------------------------------------------------------- // @@ -128,6 +164,7 @@ void BackupStoreDaemon::HousekeepingProcess() // Created: 11/12/03 // // -------------------------------------------------------------------------- +#ifndef WIN32 bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitTime) { // First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate. @@ -171,5 +208,6 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT return false; } +#endif diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index 35bc095d..845903b2 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -9,7 +9,12 @@ #include "Box.h" +#ifdef HAVE_SYSLOG_H #include +#endif + +#include +#include #include "autogen_BackupProtocolServer.h" #include "BackupConstants.h" @@ -327,8 +332,15 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto std::auto_ptr diff2(rContext.OpenObject(patchID)); // Choose a temporary filename for the result of the combination - std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp", - p + 16 /* rotate which disc it's on */)); +#ifdef WIN32 + std::ostringstream fs(rContext.GetStoreRoot()); + fs << ".recombinetemp."; + fs << p; + std::string tempFn(fs.str()); + tempFn = RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), tempFn, p + 16); +#else + std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp", p + 16 /* rotate which disc it's on */)); +#endif // Open the temporary file std::auto_ptr combined; @@ -336,14 +348,23 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto { { // Write nastily to allow this to work with gcc 2.x +#ifdef WIN32 + combined.reset(new FileStream( + tempFn.c_str(), + O_RDWR | O_CREAT | O_EXCL | + O_BINARY | O_TRUNC)); +#else std::auto_ptr t(new FileStream(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL)); combined = t; +#endif } +#ifndef WIN32 // Unlink immediately as it's a temporary file if(::unlink(tempFn.c_str()) != 0) { THROW_EXCEPTION(CommonException, OSFileError); } +#endif } catch(...) { @@ -359,6 +380,9 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto combined->Seek(0, IOStream::SeekType_Absolute); // Then shuffle round for the next go +#ifdef WIN32 + if (from.get()) from->Close(); +#endif from = combined; } @@ -396,8 +420,9 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto stream = t; } - // Object will be deleted when the stream is deleted, so can release the object auto_ptr here to - // avoid premature deletiong + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid + // premature deletion object.release(); } diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp index c796c13a..cd17812c 100644 --- a/bin/bbstored/BackupContext.cpp +++ b/bin/bbstored/BackupContext.cpp @@ -132,6 +132,7 @@ bool BackupContext::AttemptToGetWriteLock() // Request the lock bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); +#ifndef WIN32 if(!gotLock) { // The housekeeping process might have the thing open -- ask it to stop @@ -150,6 +151,7 @@ bool BackupContext::AttemptToGetWriteLock() } while(!gotLock && tries > 0); } +#endif if(gotLock) { @@ -453,13 +455,21 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod try { // Open it twice +#ifdef WIN32 + FileStream diff(tempFn.c_str(), + O_RDWR | O_CREAT | O_BINARY); + FileStream diff2(tempFn.c_str(), + O_RDWR | O_BINARY); +#else FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); FileStream diff2(tempFn.c_str(), O_RDONLY); - // Unlink it immediately, so it definately goes away + + // Unlink it immediately, so it definitely goes away if(::unlink(tempFn.c_str()) != 0) { THROW_EXCEPTION(CommonException, OSFileError); } +#endif // Stream the incoming diff to this temporary file if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) @@ -508,6 +518,14 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod ::unlink(tempFn.c_str()); throw; } + +#ifdef WIN32 + // we can't delete the file while it's open, above + if(::unlink(tempFn.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } +#endif } // Get the blocks used diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 2752893a..24a81ceb 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -11,9 +11,12 @@ #include #include -#include #include +#ifdef HAVE_SYSLOG_H +#include +#endif + #include "BackupContext.h" #include "BackupStoreDaemon.h" #include "BackupStoreConfigVerify.h" @@ -39,7 +42,11 @@ BackupStoreDaemon::BackupStoreDaemon() mExtendedLogging(false), mHaveForkedHousekeeping(false), mIsHousekeepingProcess(false), +#ifdef WIN32 + mHousekeepingInited(false) +#else mInterProcessComms(mInterProcessCommsSocket) +#endif { } @@ -156,6 +163,7 @@ void BackupStoreDaemon::Run() const Configuration &config(GetConfiguration()); mExtendedLogging = config.GetKeyValueBool("ExtendedLogging"); +#ifndef WIN32 // Fork off housekeeping daemon -- must only do this the first time Run() is called if(!mHaveForkedHousekeeping) { @@ -219,9 +227,11 @@ void BackupStoreDaemon::Run() } else { +#endif // !WIN32 // In server process -- use the base class to do the magic ServerTLS::Run(); +#ifndef WIN32 // Why did it stop? Tell the housekeeping process to do the same if(IsReloadConfigWanted()) { @@ -232,6 +242,7 @@ void BackupStoreDaemon::Run() mInterProcessCommsSocket.Write("t\n", 2); } } +#endif } @@ -297,6 +308,8 @@ void BackupStoreDaemon::LogConnectionStats(const char *commonName, // Log the amount of data transferred ::syslog(LOG_INFO, "Connection statistics for %s: " "IN=%lld OUT=%lld TOTAL=%lld\n", commonName, - s.GetBytesRead(), s.GetBytesWritten(), - s.GetBytesRead() + s.GetBytesWritten()); + (long long)s.GetBytesRead(), + (long long)s.GetBytesWritten(), + (long long)s.GetBytesRead() + + (long long)s.GetBytesWritten()); } diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 2fbe486d..857a9356 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -38,11 +38,13 @@ private: BackupStoreDaemon(const BackupStoreDaemon &rToCopy); public: +#ifndef WIN32 // For BackupContext to comminicate with housekeeping process void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) { mInterProcessCommsSocket.Write(Msg, MsgLen); } +#endif protected: @@ -57,9 +59,11 @@ protected: const ConfigurationVerify *GetConfigVerify() const; +#ifndef WIN32 // Housekeeping functions void HousekeepingProcess(); bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0); +#endif void LogConnectionStats(const char *commonName, const SocketStreamTLS &s); @@ -70,8 +74,17 @@ private: bool mHaveForkedHousekeeping; bool mIsHousekeepingProcess; +#ifdef WIN32 + virtual void OnIdle(); + bool mHousekeepingInited; +#else SocketStream mInterProcessCommsSocket; IOStreamGetLine mInterProcessComms; +#endif + + void HousekeepingInit(); + void RunHousekeepingIfNeeded(); + int64_t mLastHousekeepingRun; }; diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp index 4aa1999e..91945306 100644 --- a/bin/bbstored/HousekeepStoreAccount.cpp +++ b/bin/bbstored/HousekeepStoreAccount.cpp @@ -225,6 +225,7 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // -------------------------------------------------------------------------- bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { +#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -235,6 +236,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) return false; } } +#endif // Get the filename std::string objectFilename; @@ -251,6 +253,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Read the directory in BackupStoreDirectory dir; dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + dirStream->Close(); // Is it empty? if(dir.GetNumberOfEntries() == 0) @@ -485,6 +488,7 @@ bool HousekeepStoreAccount::DeleteFiles() // (there is likely to be more in the set than should be actually deleted). for(std::set::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i) { +#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -495,6 +499,7 @@ bool HousekeepStoreAccount::DeleteFiles() return true; } } +#endif // Load up the directory it's in // Get the filename @@ -729,6 +734,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() // Go through list for(std::vector::const_iterator i(mEmptyDirectories.begin()); i != mEmptyDirectories.end(); ++i) { +#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -739,6 +745,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() return true; } } +#endif // Do not delete the root directory if(*i == BACKUPSTORE_ROOT_DIRECTORY_ID) -- cgit v1.2.3 From 6932579c38836aaec62ede8cd77bf66bb2e82d11 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 14:02:30 +0000 Subject: * bin/bbstored/BackupCommands.cpp - Revert to trunk --- bin/bbstored/BackupCommands.cpp | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index 845903b2..35bc095d 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -9,12 +9,7 @@ #include "Box.h" -#ifdef HAVE_SYSLOG_H #include -#endif - -#include -#include #include "autogen_BackupProtocolServer.h" #include "BackupConstants.h" @@ -332,15 +327,8 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto std::auto_ptr diff2(rContext.OpenObject(patchID)); // Choose a temporary filename for the result of the combination -#ifdef WIN32 - std::ostringstream fs(rContext.GetStoreRoot()); - fs << ".recombinetemp."; - fs << p; - std::string tempFn(fs.str()); - tempFn = RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), tempFn, p + 16); -#else - std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp", p + 16 /* rotate which disc it's on */)); -#endif + std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp", + p + 16 /* rotate which disc it's on */)); // Open the temporary file std::auto_ptr combined; @@ -348,23 +336,14 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto { { // Write nastily to allow this to work with gcc 2.x -#ifdef WIN32 - combined.reset(new FileStream( - tempFn.c_str(), - O_RDWR | O_CREAT | O_EXCL | - O_BINARY | O_TRUNC)); -#else std::auto_ptr t(new FileStream(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL)); combined = t; -#endif } -#ifndef WIN32 // Unlink immediately as it's a temporary file if(::unlink(tempFn.c_str()) != 0) { THROW_EXCEPTION(CommonException, OSFileError); } -#endif } catch(...) { @@ -380,9 +359,6 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto combined->Seek(0, IOStream::SeekType_Absolute); // Then shuffle round for the next go -#ifdef WIN32 - if (from.get()) from->Close(); -#endif from = combined; } @@ -420,9 +396,8 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto stream = t; } - // Object will be deleted when the stream is deleted, - // so can release the object auto_ptr here to avoid - // premature deletion + // Object will be deleted when the stream is deleted, so can release the object auto_ptr here to + // avoid premature deletiong object.release(); } -- cgit v1.2.3 From 824756aadab385140ae5c1ccdf382c387f646ca6 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 14:18:40 +0000 Subject: * bin/bbstored/BackupCommands.cpp - Can't unlink open files on Win32. This is not the correct fix, but it does work around the problem. - Only include syslog.h if we have it --- bin/bbstored/BackupCommands.cpp | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index 35bc095d..845903b2 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -9,7 +9,12 @@ #include "Box.h" +#ifdef HAVE_SYSLOG_H #include +#endif + +#include +#include #include "autogen_BackupProtocolServer.h" #include "BackupConstants.h" @@ -327,8 +332,15 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto std::auto_ptr diff2(rContext.OpenObject(patchID)); // Choose a temporary filename for the result of the combination - std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp", - p + 16 /* rotate which disc it's on */)); +#ifdef WIN32 + std::ostringstream fs(rContext.GetStoreRoot()); + fs << ".recombinetemp."; + fs << p; + std::string tempFn(fs.str()); + tempFn = RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), tempFn, p + 16); +#else + std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp", p + 16 /* rotate which disc it's on */)); +#endif // Open the temporary file std::auto_ptr combined; @@ -336,14 +348,23 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto { { // Write nastily to allow this to work with gcc 2.x +#ifdef WIN32 + combined.reset(new FileStream( + tempFn.c_str(), + O_RDWR | O_CREAT | O_EXCL | + O_BINARY | O_TRUNC)); +#else std::auto_ptr t(new FileStream(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL)); combined = t; +#endif } +#ifndef WIN32 // Unlink immediately as it's a temporary file if(::unlink(tempFn.c_str()) != 0) { THROW_EXCEPTION(CommonException, OSFileError); } +#endif } catch(...) { @@ -359,6 +380,9 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto combined->Seek(0, IOStream::SeekType_Absolute); // Then shuffle round for the next go +#ifdef WIN32 + if (from.get()) from->Close(); +#endif from = combined; } @@ -396,8 +420,9 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto stream = t; } - // Object will be deleted when the stream is deleted, so can release the object auto_ptr here to - // avoid premature deletiong + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid + // premature deletion object.release(); } -- cgit v1.2.3 From 88a078d6c0a97dc5c8212dc75f6c1109b8007b64 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 18:20:08 +0000 Subject: * bin/bbstored/BackupContext.cpp - Revert to trunk --- bin/bbstored/BackupContext.cpp | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp index cd17812c..c796c13a 100644 --- a/bin/bbstored/BackupContext.cpp +++ b/bin/bbstored/BackupContext.cpp @@ -132,7 +132,6 @@ bool BackupContext::AttemptToGetWriteLock() // Request the lock bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); -#ifndef WIN32 if(!gotLock) { // The housekeeping process might have the thing open -- ask it to stop @@ -151,7 +150,6 @@ bool BackupContext::AttemptToGetWriteLock() } while(!gotLock && tries > 0); } -#endif if(gotLock) { @@ -455,21 +453,13 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod try { // Open it twice -#ifdef WIN32 - FileStream diff(tempFn.c_str(), - O_RDWR | O_CREAT | O_BINARY); - FileStream diff2(tempFn.c_str(), - O_RDWR | O_BINARY); -#else FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); FileStream diff2(tempFn.c_str(), O_RDONLY); - - // Unlink it immediately, so it definitely goes away + // Unlink it immediately, so it definately goes away if(::unlink(tempFn.c_str()) != 0) { THROW_EXCEPTION(CommonException, OSFileError); } -#endif // Stream the incoming diff to this temporary file if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) @@ -518,14 +508,6 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod ::unlink(tempFn.c_str()); throw; } - -#ifdef WIN32 - // we can't delete the file while it's open, above - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } -#endif } // Get the blocks used -- cgit v1.2.3 From d66c1cf54727d91ed8a1ef937c07ff32abb879d4 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 18:24:04 +0000 Subject: * bin/bbstored/BackupContext.cpp - Removed locking on Win32, there is no housekeeping process to lock against - Open files with O_BINARY on Win32 - Fixed a cosmetic spelling mistake in a comment - Unlink file later on Windows, since we can't do it while it's open --- bin/bbstored/BackupContext.cpp | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp index c796c13a..a3f614a5 100644 --- a/bin/bbstored/BackupContext.cpp +++ b/bin/bbstored/BackupContext.cpp @@ -125,6 +125,7 @@ void BackupContext::ReceivedFinishCommand() // -------------------------------------------------------------------------- bool BackupContext::AttemptToGetWriteLock() { +#ifndef WIN32 // Make the filename of the write lock file std::string writeLockFile; StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); @@ -150,7 +151,7 @@ bool BackupContext::AttemptToGetWriteLock() } while(!gotLock && tries > 0); } - + if(gotLock) { // Got the lock, mark as not read only @@ -158,6 +159,10 @@ bool BackupContext::AttemptToGetWriteLock() } return gotLock; +#else // WIN32 + // no housekeeping process, we do have the lock + return true; +#endif // !WIN32 } @@ -453,13 +458,21 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod try { // Open it twice +#ifdef WIN32 + FileStream diff(tempFn.c_str(), + O_RDWR | O_CREAT | O_BINARY); + FileStream diff2(tempFn.c_str(), + O_RDWR | O_BINARY); +#else FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); FileStream diff2(tempFn.c_str(), O_RDONLY); - // Unlink it immediately, so it definately goes away + + // Unlink it immediately, so it definitely goes away if(::unlink(tempFn.c_str()) != 0) { THROW_EXCEPTION(CommonException, OSFileError); } +#endif // Stream the incoming diff to this temporary file if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) @@ -508,6 +521,14 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod ::unlink(tempFn.c_str()); throw; } + +#ifdef WIN32 + // we can't delete the file while it's open, above + if(::unlink(tempFn.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } +#endif } // Get the blocks used -- cgit v1.2.3 From 0d0ad416458a2bf4e04dfa35f549063b381c4e50 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 18:26:11 +0000 Subject: * bin/bbstored/BackupStoreDaemon.cpp - Revert to trunk --- bin/bbstored/BackupStoreDaemon.cpp | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 24a81ceb..2752893a 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -11,11 +11,8 @@ #include #include -#include - -#ifdef HAVE_SYSLOG_H #include -#endif +#include #include "BackupContext.h" #include "BackupStoreDaemon.h" @@ -42,11 +39,7 @@ BackupStoreDaemon::BackupStoreDaemon() mExtendedLogging(false), mHaveForkedHousekeeping(false), mIsHousekeepingProcess(false), -#ifdef WIN32 - mHousekeepingInited(false) -#else mInterProcessComms(mInterProcessCommsSocket) -#endif { } @@ -163,7 +156,6 @@ void BackupStoreDaemon::Run() const Configuration &config(GetConfiguration()); mExtendedLogging = config.GetKeyValueBool("ExtendedLogging"); -#ifndef WIN32 // Fork off housekeeping daemon -- must only do this the first time Run() is called if(!mHaveForkedHousekeeping) { @@ -227,11 +219,9 @@ void BackupStoreDaemon::Run() } else { -#endif // !WIN32 // In server process -- use the base class to do the magic ServerTLS::Run(); -#ifndef WIN32 // Why did it stop? Tell the housekeeping process to do the same if(IsReloadConfigWanted()) { @@ -242,7 +232,6 @@ void BackupStoreDaemon::Run() mInterProcessCommsSocket.Write("t\n", 2); } } -#endif } @@ -308,8 +297,6 @@ void BackupStoreDaemon::LogConnectionStats(const char *commonName, // Log the amount of data transferred ::syslog(LOG_INFO, "Connection statistics for %s: " "IN=%lld OUT=%lld TOTAL=%lld\n", commonName, - (long long)s.GetBytesRead(), - (long long)s.GetBytesWritten(), - (long long)s.GetBytesRead() + - (long long)s.GetBytesWritten()); + s.GetBytesRead(), s.GetBytesWritten(), + s.GetBytesRead() + s.GetBytesWritten()); } -- cgit v1.2.3 From f9fbe34b3d409749264bc69497d7e51bf744fbf3 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 18:26:45 +0000 Subject: * bin/bbstored/BackupStoreDaemon.cpp - Only include if we have it --- bin/bbstored/BackupStoreDaemon.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 2752893a..fc09b624 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -11,9 +11,12 @@ #include #include -#include #include +#ifdef HAVE_SYSLOG_H + #include +#endif + #include "BackupContext.h" #include "BackupStoreDaemon.h" #include "BackupStoreConfigVerify.h" -- cgit v1.2.3 From 55c923c1ce31cb02aec672b6f78098cfdecafa40 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 18:28:29 +0000 Subject: * bin/bbstored/BackupStoreDaemon.h - Revert to trunk --- bin/bbstored/BackupStoreDaemon.h | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 857a9356..2fbe486d 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -38,13 +38,11 @@ private: BackupStoreDaemon(const BackupStoreDaemon &rToCopy); public: -#ifndef WIN32 // For BackupContext to comminicate with housekeeping process void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) { mInterProcessCommsSocket.Write(Msg, MsgLen); } -#endif protected: @@ -59,11 +57,9 @@ protected: const ConfigurationVerify *GetConfigVerify() const; -#ifndef WIN32 // Housekeeping functions void HousekeepingProcess(); bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0); -#endif void LogConnectionStats(const char *commonName, const SocketStreamTLS &s); @@ -74,17 +70,8 @@ private: bool mHaveForkedHousekeeping; bool mIsHousekeepingProcess; -#ifdef WIN32 - virtual void OnIdle(); - bool mHousekeepingInited; -#else SocketStream mInterProcessCommsSocket; IOStreamGetLine mInterProcessComms; -#endif - - void HousekeepingInit(); - void RunHousekeepingIfNeeded(); - int64_t mLastHousekeepingRun; }; -- cgit v1.2.3 From 27bb0b1740173c7c6b7b313678abf3760f94402b Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 18:31:54 +0000 Subject: * bin/bbstored/BackupStoreDaemon.cpp - Cast off_t to long long, in case they differ in size (e.g. Win32) --- bin/bbstored/BackupStoreDaemon.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index fc09b624..ce7263da 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -300,6 +300,8 @@ void BackupStoreDaemon::LogConnectionStats(const char *commonName, // Log the amount of data transferred ::syslog(LOG_INFO, "Connection statistics for %s: " "IN=%lld OUT=%lld TOTAL=%lld\n", commonName, - s.GetBytesRead(), s.GetBytesWritten(), - s.GetBytesRead() + s.GetBytesWritten()); + (long long)s.GetBytesRead(), + (long long)s.GetBytesWritten(), + (long long)s.GetBytesRead() + + (long long)s.GetBytesWritten()); } -- cgit v1.2.3 From 7286b52c4b2f703d8862088b1197bb848bba36da Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 18:37:38 +0000 Subject: * bin/bbstored/BBStoreDHousekeeping.cpp - Revert to trunk --- bin/bbstored/BBStoreDHousekeeping.cpp | 174 +++++++++++++--------------------- 1 file changed, 68 insertions(+), 106 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index 86b2468b..d3656630 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -10,10 +10,7 @@ #include "Box.h" #include - -#ifdef HAVE_SYSLOG_H #include -#endif #include "BackupStoreDaemon.h" #include "BackupStoreAccountDatabase.h" @@ -32,128 +29,95 @@ // Created: 11/12/03 // // -------------------------------------------------------------------------- -void BackupStoreDaemon::HousekeepingInit() -{ - - mLastHousekeepingRun = 0; -} - -#ifndef WIN32 void BackupStoreDaemon::HousekeepingProcess() { - HousekeepingInit(); - // Get the time between housekeeping runs const Configuration &rconfig(GetConfiguration()); int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); + + int64_t lastHousekeepingRun = 0; while(!StopRun()) { - RunHousekeepingIfNeeded(); - - // Calculate how long should wait before doing the next housekeeping run + // Time now int64_t timeNow = GetCurrentBoxTime(); - time_t secondsToGo = BoxTimeToSeconds((mLastHousekeepingRun + housekeepingInterval) - timeNow); - if(secondsToGo < 1) secondsToGo = 1; - if(secondsToGo > 60) secondsToGo = 60; - int32_t millisecondsToGo = ((int)secondsToGo) * 1000; - - // Check to see if there's any message pending - CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); - } -} -#endif - -void BackupStoreDaemon::RunHousekeepingIfNeeded() -{ - // Get the time between housekeeping runs - const Configuration &rconfig(GetConfiguration()); - int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); - - // Time now - int64_t timeNow = GetCurrentBoxTime(); - // Do housekeeping if the time interval has elapsed since the last check - if((timeNow - mLastHousekeepingRun) < housekeepingInterval) - { - return; - } - - // Store the time - mLastHousekeepingRun = timeNow; - ::syslog(LOG_INFO, "Starting housekeeping"); + // Do housekeeping if the time interval has elapsed since the last check + if((timeNow - lastHousekeepingRun) >= housekeepingInterval) + { + // Store the time + lastHousekeepingRun = timeNow; + ::syslog(LOG_INFO, "Starting housekeeping"); - // Get the list of accounts - std::vector accounts; - if(mpAccountDatabase) - { - mpAccountDatabase->GetAllAccountIDs(accounts); - } + // Get the list of accounts + std::vector accounts; + if(mpAccountDatabase) + { + mpAccountDatabase->GetAllAccountIDs(accounts); + } - SetProcessTitle("housekeeping, active"); + SetProcessTitle("housekeeping, active"); - // Check them all - for(std::vector::const_iterator i = accounts.begin(); i != accounts.end(); ++i) - { - try - { - if(mpAccounts) + // Check them all + for(std::vector::const_iterator i = accounts.begin(); i != accounts.end(); ++i) { - // Get the account root - std::string rootDir; - int discSet = 0; - mpAccounts->GetAccountRoot(*i, rootDir, discSet); + try + { + if(mpAccounts) + { + // Get the account root + std::string rootDir; + int discSet = 0; + mpAccounts->GetAccountRoot(*i, rootDir, discSet); + + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this); + housekeeping.DoHousekeeping(); + } + } + catch(BoxException &e) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", + *i, e.what(), e.GetType(), e.GetSubType()); + } + catch(std::exception &e) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", + *i, e.what()); + } + catch(...) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", + *i); + } - // Do housekeeping on this account - HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this); - housekeeping.DoHousekeeping(); + // Check to see if there's any message pending + CheckForInterProcessMsg(0 /* no account */); + + // Stop early? + if(StopRun()) + { + break; + } } + + ::syslog(LOG_INFO, "Finished housekeeping"); } - catch(BoxException &e) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", - *i, e.what(), e.GetType(), e.GetSubType()); - } - catch(std::exception &e) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", - *i, e.what()); - } - catch(...) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", - *i); - } + + // Placed here for accuracy, if StopRun() is true, for example. + SetProcessTitle("housekeeping, idle"); + + // Calculate how long should wait before doing the next housekeeping run + timeNow = GetCurrentBoxTime(); + time_t secondsToGo = BoxTimeToSeconds((lastHousekeepingRun + housekeepingInterval) - timeNow); + if(secondsToGo < 1) secondsToGo = 1; + if(secondsToGo > 60) secondsToGo = 60; + int32_t millisecondsToGo = ((int)secondsToGo) * 1000; -#ifndef WIN32 // Check to see if there's any message pending - CheckForInterProcessMsg(0 /* no account */); -#endif - - // Stop early? - if(StopRun()) - { - break; - } + CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); } - - ::syslog(LOG_INFO, "Finished housekeeping"); - - // Placed here for accuracy, if StopRun() is true, for example. - SetProcessTitle("housekeeping, idle"); } -#ifdef WIN32 -void BackupStoreDaemon::OnIdle() -{ - if (!mHousekeepingInited) - { - HousekeepingInit(); - mHousekeepingInited = true; - } - - RunHousekeepingIfNeeded(); -} -#endif // -------------------------------------------------------------------------- // @@ -164,7 +128,6 @@ void BackupStoreDaemon::OnIdle() // Created: 11/12/03 // // -------------------------------------------------------------------------- -#ifndef WIN32 bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitTime) { // First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate. @@ -208,6 +171,5 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT return false; } -#endif -- cgit v1.2.3 From 5989931d7c88696a1e4858c9ea4a93a8312efc53 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 18:52:01 +0000 Subject: * bin/bbstored/BackupStoreDaemon.h - Removed SendMessageToHousekeepingProcess() on Win32, no longer needed --- bin/bbstored/BackupStoreDaemon.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 2fbe486d..3123b21f 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -38,11 +38,13 @@ private: BackupStoreDaemon(const BackupStoreDaemon &rToCopy); public: - // For BackupContext to comminicate with housekeeping process +#ifndef WIN32 + // For BackupContext to communicate with housekeeping process void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) { mInterProcessCommsSocket.Write(Msg, MsgLen); } +#endif protected: -- cgit v1.2.3 From 82c4503253a2fb311c1fda5c6b5f545200e89ad7 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 18:54:31 +0000 Subject: * bin/bbstored/BBStoreDHousekeeping.cpp - Only include syslog.h if we have one --- bin/bbstored/BBStoreDHousekeeping.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index d3656630..be6d71f5 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -10,7 +10,10 @@ #include "Box.h" #include -#include + +#ifdef HAVE_SYSLOG_H + #include +#endif #include "BackupStoreDaemon.h" #include "BackupStoreAccountDatabase.h" -- cgit v1.2.3 From f2fedcebe908b2fb77a2743575098230ca67236a Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 19:01:05 +0000 Subject: * bin/bbstored/BackupStoreDaemon.h * bin/bbstored/BBStoreDHousekeeping.cpp - Split housekeeping process into separate initialisation, process loop and run methods (we don't want the process loop on Win32) --- bin/bbstored/BBStoreDHousekeeping.cpp | 158 ++++++++++++++++++++-------------- bin/bbstored/BackupStoreDaemon.h | 4 + 2 files changed, 96 insertions(+), 66 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index be6d71f5..8c725b0f 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -32,95 +32,121 @@ // Created: 11/12/03 // // -------------------------------------------------------------------------- +void BackupStoreDaemon::HousekeepingInit() +{ + + mLastHousekeepingRun = 0; +} + void BackupStoreDaemon::HousekeepingProcess() { + HousekeepingInit(); + // Get the time between housekeeping runs const Configuration &rconfig(GetConfiguration()); int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); - - int64_t lastHousekeepingRun = 0; while(!StopRun()) { - // Time now + RunHousekeepingIfNeeded(); + + // Calculate how long should wait before doing the next housekeeping run int64_t timeNow = GetCurrentBoxTime(); - // Do housekeeping if the time interval has elapsed since the last check - if((timeNow - lastHousekeepingRun) >= housekeepingInterval) - { - // Store the time - lastHousekeepingRun = timeNow; - ::syslog(LOG_INFO, "Starting housekeeping"); + time_t secondsToGo = BoxTimeToSeconds( + (mLastHousekeepingRun + housekeepingInterval) - + timeNow); + if(secondsToGo < 1) secondsToGo = 1; + if(secondsToGo > 60) secondsToGo = 60; + int32_t millisecondsToGo = ((int)secondsToGo) * 1000; + + // Check to see if there's any message pending + CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); + } +} - // Get the list of accounts - std::vector accounts; - if(mpAccountDatabase) - { - mpAccountDatabase->GetAllAccountIDs(accounts); - } +void BackupStoreDaemon::RunHousekeepingIfNeeded() +{ + // Get the time between housekeeping runs + const Configuration &rconfig(GetConfiguration()); + int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); + + // Time now + int64_t timeNow = GetCurrentBoxTime(); + // Do housekeeping if the time interval has elapsed since the last check + if((timeNow - mLastHousekeepingRun) < housekeepingInterval) + { + return; + } + + // Store the time + mLastHousekeepingRun = timeNow; + ::syslog(LOG_INFO, "Starting housekeeping"); + + // Get the list of accounts + std::vector accounts; + if(mpAccountDatabase) + { + mpAccountDatabase->GetAllAccountIDs(accounts); + } - SetProcessTitle("housekeeping, active"); + SetProcessTitle("housekeeping, active"); - // Check them all - for(std::vector::const_iterator i = accounts.begin(); i != accounts.end(); ++i) + // Check them all + for(std::vector::const_iterator i = accounts.begin(); i != accounts.end(); ++i) + { + try + { + if(mpAccounts) { - try - { - if(mpAccounts) - { - // Get the account root - std::string rootDir; - int discSet = 0; - mpAccounts->GetAccountRoot(*i, rootDir, discSet); - - // Do housekeeping on this account - HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this); - housekeeping.DoHousekeeping(); - } - } - catch(BoxException &e) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", - *i, e.what(), e.GetType(), e.GetSubType()); - } - catch(std::exception &e) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", - *i, e.what()); - } - catch(...) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", - *i); - } + // Get the account root + std::string rootDir; + int discSet = 0; + mpAccounts->GetAccountRoot(*i, rootDir, discSet); - // Check to see if there's any message pending - CheckForInterProcessMsg(0 /* no account */); - - // Stop early? - if(StopRun()) - { - break; - } + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this); + housekeeping.DoHousekeeping(); } - - ::syslog(LOG_INFO, "Finished housekeeping"); } - - // Placed here for accuracy, if StopRun() is true, for example. - SetProcessTitle("housekeeping, idle"); - - // Calculate how long should wait before doing the next housekeeping run - timeNow = GetCurrentBoxTime(); - time_t secondsToGo = BoxTimeToSeconds((lastHousekeepingRun + housekeepingInterval) - timeNow); + catch(BoxException &e) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", + *i, e.what(), e.GetType(), e.GetSubType()); + } + catch(std::exception &e) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", + *i, e.what()); + } + catch(...) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", + *i); + } + + int64_t timeNow = GetCurrentBoxTime(); + time_t secondsToGo = BoxTimeToSeconds( + (mLastHousekeepingRun + housekeepingInterval) - + timeNow); if(secondsToGo < 1) secondsToGo = 1; if(secondsToGo > 60) secondsToGo = 60; int32_t millisecondsToGo = ((int)secondsToGo) * 1000; - + // Check to see if there's any message pending CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); + + // Stop early? + if(StopRun()) + { + break; + } } -} + + ::syslog(LOG_INFO, "Finished housekeeping"); + // Placed here for accuracy, if StopRun() is true, for example. + SetProcessTitle("housekeeping, idle"); +} // -------------------------------------------------------------------------- // diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 3123b21f..47f29327 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -74,6 +74,10 @@ private: SocketStream mInterProcessCommsSocket; IOStreamGetLine mInterProcessComms; + + void HousekeepingInit(); + void RunHousekeepingIfNeeded(); + int64_t mLastHousekeepingRun; }; -- cgit v1.2.3 From 2c64f7e2f6c0dbf7106b63af2c1cd39caa2078f9 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 19:20:35 +0000 Subject: * bin/bbstored/HousekeepStoreAccount.cpp - Revert to trunk --- bin/bbstored/HousekeepStoreAccount.cpp | 7 ------- 1 file changed, 7 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp index 91945306..4aa1999e 100644 --- a/bin/bbstored/HousekeepStoreAccount.cpp +++ b/bin/bbstored/HousekeepStoreAccount.cpp @@ -225,7 +225,6 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // -------------------------------------------------------------------------- bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { -#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -236,7 +235,6 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) return false; } } -#endif // Get the filename std::string objectFilename; @@ -253,7 +251,6 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Read the directory in BackupStoreDirectory dir; dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); - dirStream->Close(); // Is it empty? if(dir.GetNumberOfEntries() == 0) @@ -488,7 +485,6 @@ bool HousekeepStoreAccount::DeleteFiles() // (there is likely to be more in the set than should be actually deleted). for(std::set::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i) { -#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -499,7 +495,6 @@ bool HousekeepStoreAccount::DeleteFiles() return true; } } -#endif // Load up the directory it's in // Get the filename @@ -734,7 +729,6 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() // Go through list for(std::vector::const_iterator i(mEmptyDirectories.begin()); i != mEmptyDirectories.end(); ++i) { -#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -745,7 +739,6 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() return true; } } -#endif // Do not delete the root directory if(*i == BACKUPSTORE_ROOT_DIRECTORY_ID) -- cgit v1.2.3 From c283b5715077a8e9c273e53816b3ad473888610a Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 30 Aug 2006 19:29:23 +0000 Subject: * bin/bbstored/HousekeepStoreAccount.cpp - Disable checks for inter-process messages on Win32 (there is only one process) - Close directory immediately after we finish reading it --- bin/bbstored/HousekeepStoreAccount.cpp | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'bin/bbstored') diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp index 4aa1999e..91945306 100644 --- a/bin/bbstored/HousekeepStoreAccount.cpp +++ b/bin/bbstored/HousekeepStoreAccount.cpp @@ -225,6 +225,7 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // -------------------------------------------------------------------------- bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { +#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -235,6 +236,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) return false; } } +#endif // Get the filename std::string objectFilename; @@ -251,6 +253,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Read the directory in BackupStoreDirectory dir; dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + dirStream->Close(); // Is it empty? if(dir.GetNumberOfEntries() == 0) @@ -485,6 +488,7 @@ bool HousekeepStoreAccount::DeleteFiles() // (there is likely to be more in the set than should be actually deleted). for(std::set::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i) { +#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -495,6 +499,7 @@ bool HousekeepStoreAccount::DeleteFiles() return true; } } +#endif // Load up the directory it's in // Get the filename @@ -729,6 +734,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() // Go through list for(std::vector::const_iterator i(mEmptyDirectories.begin()); i != mEmptyDirectories.end(); ++i) { +#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -739,6 +745,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() return true; } } +#endif // Do not delete the root directory if(*i == BACKUPSTORE_ROOT_DIRECTORY_ID) -- cgit v1.2.3 From 94212acf332d56413ae0dc2a6c9c9fccb02f6bee Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 31 Aug 2006 19:59:02 +0000 Subject: * bin/bbstored/BackupCommands.cpp - Use the same code for file names and file closing on other platforms that's needed on Win32 --- bin/bbstored/BackupCommands.cpp | 6 ------ 1 file changed, 6 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index 845903b2..60a660d1 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -332,15 +332,11 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto std::auto_ptr diff2(rContext.OpenObject(patchID)); // Choose a temporary filename for the result of the combination -#ifdef WIN32 std::ostringstream fs(rContext.GetStoreRoot()); fs << ".recombinetemp."; fs << p; std::string tempFn(fs.str()); tempFn = RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), tempFn, p + 16); -#else - std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp", p + 16 /* rotate which disc it's on */)); -#endif // Open the temporary file std::auto_ptr combined; @@ -380,9 +376,7 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto combined->Seek(0, IOStream::SeekType_Absolute); // Then shuffle round for the next go -#ifdef WIN32 if (from.get()) from->Close(); -#endif from = combined; } -- cgit v1.2.3 From b97df587f4ebf1cd445d72d9f129a24bb8d29b67 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 31 Aug 2006 20:08:11 +0000 Subject: * bin/bbstored/BackupContext.cpp - Delete the temporary file on Win32 just like on other platforms (note that this reduces the guarantees that the file will be deleted, especially if an exception is thrown, refs #819) --- bin/bbstored/BackupContext.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp index a3f614a5..d78ed9b1 100644 --- a/bin/bbstored/BackupContext.cpp +++ b/bin/bbstored/BackupContext.cpp @@ -125,7 +125,6 @@ void BackupContext::ReceivedFinishCommand() // -------------------------------------------------------------------------- bool BackupContext::AttemptToGetWriteLock() { -#ifndef WIN32 // Make the filename of the write lock file std::string writeLockFile; StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); @@ -159,10 +158,6 @@ bool BackupContext::AttemptToGetWriteLock() } return gotLock; -#else // WIN32 - // no housekeeping process, we do have the lock - return true; -#endif // !WIN32 } @@ -466,12 +461,6 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod #else FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); FileStream diff2(tempFn.c_str(), O_RDONLY); - - // Unlink it immediately, so it definitely goes away - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } #endif // Stream the incoming diff to this temporary file @@ -514,6 +503,14 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod spaceAdjustFromDiff = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed; // Everything cleans up here... + diff.Close(); + diff2.Close(); + + // Unlink the temporary file + if(::unlink(tempFn.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } } catch(...) { -- cgit v1.2.3 From b4f24408e408ca7f9f8c64955477f5c2aa47a459 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 31 Aug 2006 20:11:19 +0000 Subject: * bin/bbstored/BackupStoreDaemon.h - Reinstate SendMessageToHousekeepingProcess() on Win32, but make it do nothing (refs #3) --- bin/bbstored/BackupStoreDaemon.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 47f29327..0ce6f21f 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -38,13 +38,13 @@ private: BackupStoreDaemon(const BackupStoreDaemon &rToCopy); public: -#ifndef WIN32 // For BackupContext to communicate with housekeeping process void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) { +#ifndef WIN32 mInterProcessCommsSocket.Write(Msg, MsgLen); - } #endif + } protected: -- cgit v1.2.3 From 9635631135e1f0df36a0b038f047384dd9d8c8c0 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Fri, 13 Oct 2006 23:03:23 +0000 Subject: * Added support for Win32 temporary files * Added InvisibleTempFileStream class and unit tests for it * Use InvisibleTempFileStream instead of FileStream for temporary files (refs #3) --- bin/bbstored/BackupCommands.cpp | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index 60a660d1..cf8025e4 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -29,6 +29,7 @@ #include "BackupStoreInfo.h" #include "RaidFileController.h" #include "FileStream.h" +#include "InvisibleTempFileStream.h" #include "MemLeakFindOn.h" @@ -344,23 +345,14 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto { { // Write nastily to allow this to work with gcc 2.x -#ifdef WIN32 - combined.reset(new FileStream( - tempFn.c_str(), - O_RDWR | O_CREAT | O_EXCL | - O_BINARY | O_TRUNC)); -#else - std::auto_ptr t(new FileStream(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL)); + std::auto_ptr t( + new InvisibleTempFileStream( + tempFn.c_str(), + O_RDWR | O_CREAT | + O_EXCL | O_BINARY | + O_TRUNC)); combined = t; -#endif - } -#ifndef WIN32 - // Unlink immediately as it's a temporary file - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); } -#endif } catch(...) { -- cgit v1.2.3 From e3273ee1e61dc94baa733aeba533af886717142a Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 15 Oct 2006 19:49:11 +0000 Subject: Revert patch [862] (refs #3) --- bin/bbstored/BackupContext.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp index d78ed9b1..a3f614a5 100644 --- a/bin/bbstored/BackupContext.cpp +++ b/bin/bbstored/BackupContext.cpp @@ -125,6 +125,7 @@ void BackupContext::ReceivedFinishCommand() // -------------------------------------------------------------------------- bool BackupContext::AttemptToGetWriteLock() { +#ifndef WIN32 // Make the filename of the write lock file std::string writeLockFile; StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); @@ -158,6 +159,10 @@ bool BackupContext::AttemptToGetWriteLock() } return gotLock; +#else // WIN32 + // no housekeeping process, we do have the lock + return true; +#endif // !WIN32 } @@ -461,6 +466,12 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod #else FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); FileStream diff2(tempFn.c_str(), O_RDONLY); + + // Unlink it immediately, so it definitely goes away + if(::unlink(tempFn.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } #endif // Stream the incoming diff to this temporary file @@ -503,14 +514,6 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod spaceAdjustFromDiff = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed; // Everything cleans up here... - diff.Close(); - diff2.Close(); - - // Unlink the temporary file - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } } catch(...) { -- cgit v1.2.3 From 0b4fae0d39ee4ef1df329cfbc852ef16463c26a0 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 15 Oct 2006 19:59:30 +0000 Subject: Revert patch [825] (mostly). Re-enable locking code on Win32 (although it's never used). Use InvisibleTempFileStream to ensure that the temporary file is always deleted. (refs #3) --- bin/bbstored/BackupContext.cpp | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp index a3f614a5..fa1d88a6 100644 --- a/bin/bbstored/BackupContext.cpp +++ b/bin/bbstored/BackupContext.cpp @@ -24,6 +24,7 @@ #include "BackupStoreDaemon.h" #include "RaidFileController.h" #include "FileStream.h" +#include "InvisibleTempFileStream.h" #include "MemLeakFindOn.h" @@ -125,7 +126,6 @@ void BackupContext::ReceivedFinishCommand() // -------------------------------------------------------------------------- bool BackupContext::AttemptToGetWriteLock() { -#ifndef WIN32 // Make the filename of the write lock file std::string writeLockFile; StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); @@ -151,7 +151,7 @@ bool BackupContext::AttemptToGetWriteLock() } while(!gotLock && tries > 0); } - + if(gotLock) { // Got the lock, mark as not read only @@ -159,10 +159,6 @@ bool BackupContext::AttemptToGetWriteLock() } return gotLock; -#else // WIN32 - // no housekeeping process, we do have the lock - return true; -#endif // !WIN32 } @@ -459,9 +455,9 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod { // Open it twice #ifdef WIN32 - FileStream diff(tempFn.c_str(), + InvisibleTempFileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_BINARY); - FileStream diff2(tempFn.c_str(), + InvisibleTempFileStream diff2(tempFn.c_str(), O_RDWR | O_BINARY); #else FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); @@ -521,14 +517,6 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod ::unlink(tempFn.c_str()); throw; } - -#ifdef WIN32 - // we can't delete the file while it's open, above - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } -#endif } // Get the blocks used -- cgit v1.2.3 From a8fe12e18fcdb50d9f6cf93d9a642e22f25c0184 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 18 Oct 2006 19:48:50 +0000 Subject: Reinstate #ifdefs on Win32. (refs #3) --- bin/bbstored/BackupStoreDaemon.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index ce7263da..f4968a46 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -42,6 +42,7 @@ BackupStoreDaemon::BackupStoreDaemon() mExtendedLogging(false), mHaveForkedHousekeeping(false), mIsHousekeepingProcess(false), + mHousekeepingInited(false), mInterProcessComms(mInterProcessCommsSocket) { } @@ -159,6 +160,9 @@ void BackupStoreDaemon::Run() const Configuration &config(GetConfiguration()); mExtendedLogging = config.GetKeyValueBool("ExtendedLogging"); +#ifdef WIN32 + // Housekeeping runs synchronously on Win32 +#else // Fork off housekeeping daemon -- must only do this the first time Run() is called if(!mHaveForkedHousekeeping) { @@ -214,6 +218,7 @@ void BackupStoreDaemon::Run() THROW_EXCEPTION(ServerException, SocketCloseError) } } +#endif // WIN32 if(mIsHousekeepingProcess) { -- cgit v1.2.3 From db5c857339b5079d82901249af93bba2b960ee72 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 18 Oct 2006 19:50:07 +0000 Subject: Catch exceptions from BackupStoreDaemon::Run and log them without killing the server process, on platforms where forking is disabled (Win32). (refs #3) --- bin/bbstored/BackupStoreDaemon.cpp | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index f4968a46..196cacfb 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -228,13 +228,38 @@ void BackupStoreDaemon::Run() else { // In server process -- use the base class to do the magic - ServerTLS::Run(); - + try + { + ServerTLS::Run(); + } + catch(BoxException &e) + { + ::syslog(LOG_ERR, "%s: disconnecting due to " + "exception %s (%d/%d)", DaemonName(), + e.what(), e.GetType(), e.GetSubType()); + } + catch(std::exception &e) + { + ::syslog(LOG_ERR, "%s: disconnecting due to " + "exception %s", DaemonName(), e.what()); + } + catch(...) + { + ::syslog(LOG_ERR, "%s: disconnecting due to " + "unknown exception", DaemonName()); + } + + if (!mInterProcessCommsSocket.IsOpened()) + { + return; + } + // Why did it stop? Tell the housekeeping process to do the same if(IsReloadConfigWanted()) { mInterProcessCommsSocket.Write("h\n", 2); } + if(IsTerminateWanted()) { mInterProcessCommsSocket.Write("t\n", 2); -- cgit v1.2.3 From ee4dc408e4ce8364322ea19a4fabbb5e8edf4d46 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 6 Nov 2006 20:40:46 +0000 Subject: Compile fix --- bin/bbstored/BackupStoreDaemon.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 196cacfb..e09bd17d 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -42,7 +42,6 @@ BackupStoreDaemon::BackupStoreDaemon() mExtendedLogging(false), mHaveForkedHousekeeping(false), mIsHousekeepingProcess(false), - mHousekeepingInited(false), mInterProcessComms(mInterProcessCommsSocket) { } -- cgit v1.2.3 From 7e22c206a755fa481da252e8286251fea4d5b1ce Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 13 Nov 2006 19:52:05 +0000 Subject: Revert [1096] as it causes infinite loops if the listening socket can't be opened (refs #3) --- bin/bbstored/BackupStoreDaemon.cpp | 65 +++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 29 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index e09bd17d..ca28ed57 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -227,38 +227,13 @@ void BackupStoreDaemon::Run() else { // In server process -- use the base class to do the magic - try - { - ServerTLS::Run(); - } - catch(BoxException &e) - { - ::syslog(LOG_ERR, "%s: disconnecting due to " - "exception %s (%d/%d)", DaemonName(), - e.what(), e.GetType(), e.GetSubType()); - } - catch(std::exception &e) - { - ::syslog(LOG_ERR, "%s: disconnecting due to " - "exception %s", DaemonName(), e.what()); - } - catch(...) - { - ::syslog(LOG_ERR, "%s: disconnecting due to " - "unknown exception", DaemonName()); - } - - if (!mInterProcessCommsSocket.IsOpened()) - { - return; - } - + ServerTLS::Run(); + // Why did it stop? Tell the housekeeping process to do the same if(IsReloadConfigWanted()) { mInterProcessCommsSocket.Write("h\n", 2); } - if(IsTerminateWanted()) { mInterProcessCommsSocket.Write("t\n", 2); @@ -266,16 +241,48 @@ void BackupStoreDaemon::Run() } } - // -------------------------------------------------------------------------- // // Function // Name: BackupStoreDaemon::Connection(SocketStreamTLS &) -// Purpose: Handles a connection +// Purpose: Handles a connection, by catching exceptions and +// delegating to Connection2 // Created: 2003/08/20 // // -------------------------------------------------------------------------- void BackupStoreDaemon::Connection(SocketStreamTLS &rStream) +{ + try + { + Connection2(rStream); + } + catch(BoxException &e) + { + ::syslog(LOG_ERR, "%s: disconnecting due to " + "exception %s (%d/%d)", DaemonName(), + e.what(), e.GetType(), e.GetSubType()); + } + catch(std::exception &e) + { + ::syslog(LOG_ERR, "%s: disconnecting due to " + "exception %s", DaemonName(), e.what()); + } + catch(...) + { + ::syslog(LOG_ERR, "%s: disconnecting due to " + "unknown exception", DaemonName()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::Connection2(SocketStreamTLS &) +// Purpose: Handles a connection from bbackupd +// Created: 2006/11/12 +// +// -------------------------------------------------------------------------- +void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) { // Get the common name from the certificate std::string clientCommonName(rStream.GetPeerCommonName()); -- cgit v1.2.3 From a3583d8e0883e0232a3380e4cf1aabed6e3275eb Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 13 Nov 2006 20:01:39 +0000 Subject: Properly revert [1096] (refs #3) --- bin/bbstored/BackupStoreDaemon.cpp | 36 ++---------------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index ca28ed57..91d7cd12 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -241,48 +241,16 @@ void BackupStoreDaemon::Run() } } + // -------------------------------------------------------------------------- // // Function // Name: BackupStoreDaemon::Connection(SocketStreamTLS &) -// Purpose: Handles a connection, by catching exceptions and -// delegating to Connection2 +// Purpose: Handles a connection // Created: 2003/08/20 // // -------------------------------------------------------------------------- void BackupStoreDaemon::Connection(SocketStreamTLS &rStream) -{ - try - { - Connection2(rStream); - } - catch(BoxException &e) - { - ::syslog(LOG_ERR, "%s: disconnecting due to " - "exception %s (%d/%d)", DaemonName(), - e.what(), e.GetType(), e.GetSubType()); - } - catch(std::exception &e) - { - ::syslog(LOG_ERR, "%s: disconnecting due to " - "exception %s", DaemonName(), e.what()); - } - catch(...) - { - ::syslog(LOG_ERR, "%s: disconnecting due to " - "unknown exception", DaemonName()); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreDaemon::Connection2(SocketStreamTLS &) -// Purpose: Handles a connection from bbackupd -// Created: 2006/11/12 -// -// -------------------------------------------------------------------------- -void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) { // Get the common name from the certificate std::string clientCommonName(rStream.GetPeerCommonName()); -- cgit v1.2.3 From 2b2acef11f34df1420447b4fcfa7ca5c401481a9 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 13 Nov 2006 20:04:01 +0000 Subject: Don't try to write to the interprocess socket if it's not open (refs #3) --- bin/bbstored/BackupStoreDaemon.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 91d7cd12..74885c57 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -228,12 +228,18 @@ void BackupStoreDaemon::Run() { // In server process -- use the base class to do the magic ServerTLS::Run(); - + + if (!mInterProcessCommsSocket.IsOpened()) + { + return; + } + // Why did it stop? Tell the housekeeping process to do the same if(IsReloadConfigWanted()) { mInterProcessCommsSocket.Write("h\n", 2); } + if(IsTerminateWanted()) { mInterProcessCommsSocket.Write("t\n", 2); -- cgit v1.2.3 From c05cc844114f1f2289432a9ec220e6c527adbf66 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 26 Nov 2006 19:47:49 +0000 Subject: Catch any exceptions while handling a connection and report to user rather than terminating. Useful for non-forking servers like bbstored on Windows. (refs #3) --- bin/bbstored/BackupStoreDaemon.cpp | 36 ++++++++++++++++++++++++++++++++++-- bin/bbstored/BackupStoreDaemon.h | 3 ++- 2 files changed, 36 insertions(+), 3 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 74885c57..06d529c3 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -247,16 +247,48 @@ void BackupStoreDaemon::Run() } } - // -------------------------------------------------------------------------- // // Function // Name: BackupStoreDaemon::Connection(SocketStreamTLS &) -// Purpose: Handles a connection +// Purpose: Handles a connection, by catching exceptions and +// delegating to Connection2 // Created: 2003/08/20 // // -------------------------------------------------------------------------- void BackupStoreDaemon::Connection(SocketStreamTLS &rStream) +{ + try + { + Connection2(rStream); + } + catch(BoxException &e) + { + ::syslog(LOG_ERR, "%s: disconnecting due to " + "exception %s (%d/%d)", DaemonName(), + e.what(), e.GetType(), e.GetSubType()); + } + catch(std::exception &e) + { + ::syslog(LOG_ERR, "%s: disconnecting due to " + "exception %s", DaemonName(), e.what()); + } + catch(...) + { + ::syslog(LOG_ERR, "%s: disconnecting due to " + "unknown exception", DaemonName()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::Connection2(SocketStreamTLS &) +// Purpose: Handles a connection from bbackupd +// Created: 2006/11/12 +// +// -------------------------------------------------------------------------- +void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) { // Get the common name from the certificate std::string clientCommonName(rStream.GetPeerCommonName()); diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 0ce6f21f..eea47284 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -52,7 +52,8 @@ protected: virtual void Run(); - void Connection(SocketStreamTLS &rStream); + virtual void Connection(SocketStreamTLS &rStream); + void Connection2(SocketStreamTLS &rStream); virtual const char *DaemonName() const; virtual const char *DaemonBanner() const; -- cgit v1.2.3 From 3dc6cc0f31914aecc0900583dff6c80b00987611 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 17 Jan 2007 20:42:44 +0000 Subject: Use BufferedStream to speed up housekeeping by about ten times. (refs #3) --- bin/bbstored/HousekeepStoreAccount.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp index 91945306..8d2d9e4c 100644 --- a/bin/bbstored/HousekeepStoreAccount.cpp +++ b/bin/bbstored/HousekeepStoreAccount.cpp @@ -23,6 +23,7 @@ #include "NamedLock.h" #include "autogen_BackupStoreException.h" #include "BackupStoreFile.h" +#include "BufferedStream.h" #include "MemLeakFindOn.h" @@ -252,7 +253,8 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Read the directory in BackupStoreDirectory dir; - dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + BufferedStream buf(*dirStream); + dir.ReadFromStream(buf, IOStream::TimeOutInfinite); dirStream->Close(); // Is it empty? -- cgit v1.2.3 From 7f228e13623761c744f98d9cfde8144439085e59 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 17 Jan 2007 22:23:04 +0000 Subject: Buffer directory reads (refs #3) --- bin/bbstored/BackupContext.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp index fa1d88a6..16388099 100644 --- a/bin/bbstored/BackupContext.cpp +++ b/bin/bbstored/BackupContext.cpp @@ -25,6 +25,7 @@ #include "RaidFileController.h" #include "FileStream.h" #include "InvisibleTempFileStream.h" +#include "BufferedStream.h" #include "MemLeakFindOn.h" @@ -306,7 +307,8 @@ BackupStoreDirectory &BackupContext::GetDirectoryInternal(int64_t ObjectID) std::auto_ptr dir(new BackupStoreDirectory); // Read it from the stream, then set it's revision ID - dir->ReadFromStream(*objectFile, IOStream::TimeOutInfinite); + BufferedStream buf(*objectFile); + dir->ReadFromStream(buf, IOStream::TimeOutInfinite); dir->SetRevisionID(revID); // Make sure the size of the directory is available for writing the dir back -- cgit v1.2.3 From 782e1cfe59bc1d819c549a2d7a3459448c1a088d Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 17 Jan 2007 22:23:36 +0000 Subject: Buffer store file integrity checks (refs #3) --- bin/bbstored/BackupCommands.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index cf8025e4..fd19713d 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -30,6 +30,7 @@ #include "RaidFileController.h" #include "FileStream.h" #include "InvisibleTempFileStream.h" +#include "BufferedStream.h" #include "MemLeakFindOn.h" @@ -388,9 +389,10 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto // Open the object std::auto_ptr object(rContext.OpenObject(mObjectID)); + BufferedStream buf(*object); // Verify it - if(!BackupStoreFile::VerifyEncodedFileFormat(*object)) + if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) { return std::auto_ptr(new BackupProtocolServerError( BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_FileDoesNotVerify)); -- cgit v1.2.3 From 9cc598dc355819a99b1d3a9a15e44bd4468a33b9 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 8 Mar 2007 22:59:11 +0000 Subject: Run housekeeping in idle time on Win32 (refs #3) --- bin/bbstored/BBStoreDHousekeeping.cpp | 22 +++++++++++++++++++++- bin/bbstored/BackupStoreDaemon.cpp | 1 + bin/bbstored/BackupStoreDaemon.h | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index 8c725b0f..a4b26a93 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -50,7 +50,8 @@ void BackupStoreDaemon::HousekeepingProcess() { RunHousekeepingIfNeeded(); - // Calculate how long should wait before doing the next housekeeping run + // Calculate how long should wait before doing the next + // housekeeping run int64_t timeNow = GetCurrentBoxTime(); time_t secondsToGo = BoxTimeToSeconds( (mLastHousekeepingRun + housekeepingInterval) - @@ -72,6 +73,7 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() // Time now int64_t timeNow = GetCurrentBoxTime(); + // Do housekeeping if the time interval has elapsed since the last check if((timeNow - mLastHousekeepingRun) < housekeepingInterval) { @@ -148,6 +150,19 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() SetProcessTitle("housekeeping, idle"); } +void BackupStoreDaemon::OnIdle() +{ + #ifdef WIN32 + if (!mHousekeepingInited) + { + HousekeepingInit(); + mHousekeepingInited = true; + } + + RunHousekeepingIfNeeded(); + #endif +} + // -------------------------------------------------------------------------- // // Function @@ -159,6 +174,11 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() // -------------------------------------------------------------------------- bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitTime) { + if(!mInterProcessCommsSocket.IsOpened()) + { + return false; + } + // First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate. if(mInterProcessComms.IsEOF()) { diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 06d529c3..335135ce 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -42,6 +42,7 @@ BackupStoreDaemon::BackupStoreDaemon() mExtendedLogging(false), mHaveForkedHousekeeping(false), mIsHousekeepingProcess(false), + mHousekeepingInited(false), mInterProcessComms(mInterProcessCommsSocket) { } diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index eea47284..eb665440 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -72,10 +72,12 @@ private: bool mExtendedLogging; bool mHaveForkedHousekeeping; bool mIsHousekeepingProcess; + bool mHousekeepingInited; SocketStream mInterProcessCommsSocket; IOStreamGetLine mInterProcessComms; + virtual void OnIdle(); void HousekeepingInit(); void RunHousekeepingIfNeeded(); int64_t mLastHousekeepingRun; -- cgit v1.2.3 From 86a9d5654378efc3dfedb627d28af5324f0edc52 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 24 Mar 2007 23:00:41 +0000 Subject: Remove newlines from syslog() messages. (refs #3, merges [1447]) --- bin/bbstored/BackupStoreDaemon.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 335135ce..c9601575 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -295,7 +295,7 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) std::string clientCommonName(rStream.GetPeerCommonName()); // Log the name - ::syslog(LOG_INFO, "Certificate CN: %s\n", clientCommonName.c_str()); + ::syslog(LOG_INFO, "Certificate CN: %s", clientCommonName.c_str()); // Check it int32_t id; @@ -342,7 +342,7 @@ void BackupStoreDaemon::LogConnectionStats(const char *commonName, { // Log the amount of data transferred ::syslog(LOG_INFO, "Connection statistics for %s: " - "IN=%lld OUT=%lld TOTAL=%lld\n", commonName, + "IN=%lld OUT=%lld TOTAL=%lld", commonName, (long long)s.GetBytesRead(), (long long)s.GetBytesWritten(), (long long)s.GetBytesRead() + -- cgit v1.2.3 From 718b0a1122e0067187d991b52927c1eeef68ec50 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 24 Mar 2007 23:41:48 +0000 Subject: Initialise logging framework and set our program name to "Box Backup (bbstored)". (refs #3, merges [1462]) --- bin/bbstored/bbstored.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'bin/bbstored') diff --git a/bin/bbstored/bbstored.cpp b/bin/bbstored/bbstored.cpp index 3eaf2639..c0ea1199 100644 --- a/bin/bbstored/bbstored.cpp +++ b/bin/bbstored/bbstored.cpp @@ -10,6 +10,7 @@ #include "Box.h" #include "BackupStoreDaemon.h" #include "MainHelper.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -17,6 +18,10 @@ int main(int argc, const char *argv[]) { MAINHELPER_START + Logging::SetProgramName("Box Backup (bbstored)"); + Logging::ToConsole(true); + Logging::ToSyslog (true); + BackupStoreDaemon daemon; return daemon.Main(BOX_FILE_BBSTORED_DEFAULT_CONFIG, argc, argv); -- cgit v1.2.3 From 498e58eb188d98c6ec78e57cdc5f0c211f1f4dcf Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 26 Jul 2007 22:11:03 +0000 Subject: Make Configuration take a std::string filename instead of a char array, in C++ style. Add a function to get default config file paths at runtime, dependent on the location of the executable being run. Pass the config file name directly to Daemon::Main, instead of faking argv. No default raid file path at compile time on Windows, depends on executable location when run. Determine RaidFile path at runtime if not supplied in config file on Windows. Don't define default locations for config files at compile time on Windows, provide macros to determine them at runtime instead. Make FileHandleGuard take a std::string instead of a char array, C++ style. Determine config file location at runtime instead of hard-coding on Windows. Thanks to Paul MacKenzie, Per Thomsen, Pete Jalajas, Stuart Sanders, Dave Bamford and Gary for pushing me to do this. (fixes #12) Determine config file path at runtime. Call Daemon::Main with config file name instead of building fake argv. (refs #3, merges [1684] [1685] [1686] [1687] [1688] [1689] [1690] [1691] [1692]) --- bin/bbstored/BackupStoreDaemon.cpp | 18 +++++++++++++++++- bin/bbstored/bbstored.cpp | 9 ++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index c9601575..049c0ae4 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -132,7 +132,23 @@ void BackupStoreDaemon::SetupInInitialProcess() // Initialise the raid files controller RaidFileController &rcontroller = RaidFileController::GetController(); - rcontroller.Initialise(config.GetKeyValue("RaidFileConf").c_str()); + + std::string raidFileConfig; + + #ifdef WIN32 + if (!config.KeyExists("RaidFileConf")) + { + raidFileConfig = BOX_GET_DEFAULT_RAIDFILE_CONFIG_FILE; + } + else + { + raidFileConfig = config.GetKeyValue("RaidFileConf"); + } + #else + raidFileConfig = config.GetKeyValue("RaidFileConf"); + #endif + + rcontroller.Initialise(raidFileConfig); // Load the account database std::auto_ptr pdb(BackupStoreAccountDatabase::Read(config.GetKeyValue("AccountDatabase").c_str())); diff --git a/bin/bbstored/bbstored.cpp b/bin/bbstored/bbstored.cpp index c0ea1199..54858dd4 100644 --- a/bin/bbstored/bbstored.cpp +++ b/bin/bbstored/bbstored.cpp @@ -23,7 +23,14 @@ int main(int argc, const char *argv[]) Logging::ToSyslog (true); BackupStoreDaemon daemon; - return daemon.Main(BOX_FILE_BBSTORED_DEFAULT_CONFIG, argc, argv); + + #ifdef WIN32 + return daemon.Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE, + argc, argv); + #else + return daemon.Main(BOX_FILE_BBSTORED_DEFAULT_CONFIG, + argc, argv); + #endif MAINHELPER_END } -- cgit v1.2.3 From 0b2fd98dd63c533e00c0d61ce022f37ee75857cc Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 31 Jul 2007 23:18:05 +0000 Subject: Replace almost all calls to syslog() with logging framework. (refs #3) --- bin/bbstored/BBStoreDHousekeeping.cpp | 33 +++++++++-------- bin/bbstored/BackupCommands.cpp | 65 ++++++++++++++++++++++++---------- bin/bbstored/BackupStoreDaemon.cpp | 28 +++++++-------- bin/bbstored/HousekeepStoreAccount.cpp | 59 ++++++++++++++++++++++-------- 4 files changed, 124 insertions(+), 61 deletions(-) (limited to 'bin/bbstored') diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index a4b26a93..16a1432a 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -11,10 +11,6 @@ #include -#ifdef HAVE_SYSLOG_H - #include -#endif - #include "BackupStoreDaemon.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreAccounts.h" @@ -82,7 +78,7 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() // Store the time mLastHousekeepingRun = timeNow; - ::syslog(LOG_INFO, "Starting housekeeping"); + BOX_INFO("Starting housekeeping"); // Get the list of accounts std::vector accounts; @@ -112,18 +108,25 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() } catch(BoxException &e) { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", - *i, e.what(), e.GetType(), e.GetSubType()); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(*i) << " threw exception, " + "aborting run for this account: " << + e.what() << " (" << + e.GetType() << "/" << e.GetSubType() << ")"); } catch(std::exception &e) { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", - *i, e.what()); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(*i) << " threw exception, " + "aborting run for this account: " << + e.what()); } catch(...) { - ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", - *i); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(*i) << " threw exception, " + "aborting run for this account: " + "unknown exception"); } int64_t timeNow = GetCurrentBoxTime(); @@ -144,7 +147,7 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() } } - ::syslog(LOG_INFO, "Finished housekeeping"); + BOX_INFO("Finished housekeeping"); // Placed here for accuracy, if StopRun() is true, for example. SetProcessTitle("housekeeping, idle"); @@ -190,7 +193,7 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT std::string line; if(mInterProcessComms.GetLine(line, false /* no pre-processing */, MaximumWaitTime)) { - TRACE1("housekeeping received command '%s' over interprocess comms\n", line.c_str()); + TRACE1("Housekeeping received command '%s' over interprocess comms\n", line.c_str()); int account = 0; @@ -212,7 +215,9 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT if(account == AccountNum) { // Yes! -- need to stop now so when it retries to get the lock, it will succeed - ::syslog(LOG_INFO, "Housekeeping giving way to connection for account 0x%08x", AccountNum); + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(AccountNum) << + "giving way to client connection"); return true; } } diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index fd19713d..bca52c04 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -9,10 +9,6 @@ #include "Box.h" -#ifdef HAVE_SYSLOG_H -#include -#endif - #include #include @@ -88,11 +84,26 @@ std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtoco // Check given client ID against the ID in the certificate certificate // and that the client actually has an account on this machine - if(mClientID != rContext.GetClientID() || !rContext.GetClientHasAccount()) + if(mClientID != rContext.GetClientID()) { - ::syslog(LOG_INFO, "Failed login: Client ID presented was %08X", mClientID); - return std::auto_ptr(new BackupProtocolServerError( - BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_BadLogin)); + BOX_WARNING("Failed login from client ID " << + BOX_FORMAT_ACCOUNT(mClientID) << + ": wrong certificate for this account"); + return std::auto_ptr( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_BadLogin)); + } + + if(!rContext.GetClientHasAccount()) + { + BOX_WARNING("Failed login from client ID " << + BOX_FORMAT_ACCOUNT(mClientID) << + ": no such account on this server"); + return std::auto_ptr( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_BadLogin)); } // If we need to write, check that nothing else has got a write lock @@ -101,9 +112,12 @@ std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtoco // See if the context will get the lock if(!rContext.AttemptToGetWriteLock()) { - ::syslog(LOG_INFO, "Failed to get write lock (for Client ID %08X)", mClientID); - return std::auto_ptr(new BackupProtocolServerError( - BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_CannotLockStoreForWriting)); + BOX_WARNING("Failed to get write lock for Client ID " << + BOX_FORMAT_ACCOUNT(mClientID)); + return std::auto_ptr( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_CannotLockStoreForWriting)); } // Debug: check we got the lock @@ -120,7 +134,11 @@ std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtoco rContext.SetPhase(BackupContext::Phase_Commands); // Log login - ::syslog(LOG_INFO, "Login: Client ID %08X, %s", mClientID, ((mFlags & Flags_ReadOnly) != Flags_ReadOnly)?"Read/Write":"Read-only"); + BOX_NOTICE("Login from Client ID " << + BOX_FORMAT_ACCOUNT(mClientID) << + " " << + (((mFlags & Flags_ReadOnly) != Flags_ReadOnly) + ?"Read/Write":"Read-only")); // Get the usage info for reporting to the client int64_t blocksUsed = 0, blocksSoftLimit = 0, blocksHardLimit = 0; @@ -140,7 +158,8 @@ std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtoco // -------------------------------------------------------------------------- std::auto_ptr BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) { - ::syslog(LOG_INFO, "Session finished"); + BOX_NOTICE("Session finished for Client ID " << + BOX_FORMAT_ACCOUNT(rContext.GetClientID())); // Let the context know about it rContext.ReceivedFinishCommand(); @@ -311,13 +330,23 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto en = rdir.FindEntryByID(id); if(en == 0) { - ::syslog(LOG_ERR, "Object %llx in dir %llx for account %x references object %llx which does not exist in dir", - mObjectID, mInDirectory, rContext.GetClientID(), id); - return std::auto_ptr(new BackupProtocolServerError( - BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_PatchConsistencyError)); + BOX_ERROR("Object " << + BOX_FORMAT_OBJECTID(mObjectID) << + " in dir " << + BOX_FORMAT_OBJECTID(mInDirectory) << + " for account " << + BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << + " references object " << + BOX_FORMAT_OBJECTID(id) << + " which does not exist in dir"); + return std::auto_ptr( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_PatchConsistencyError)); } id = en->GetDependsNewer(); - } while(en != 0 && id != 0); + } + while(en != 0 && id != 0); // OK! The last entry in the chain is the full file, the others are patches back from it. // Open the last one, which is the current from file diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 049c0ae4..c5d5fe40 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -208,7 +208,7 @@ void BackupStoreDaemon::Run() // Change the log name ::openlog("bbstored/hk", LOG_PID, LOG_LOCAL6); // Log that housekeeping started - ::syslog(LOG_INFO, "Housekeeping process started"); + BOX_INFO("Housekeeping process started"); // Ignore term and hup // Parent will handle these and alert the child via the socket, don't want to randomly die ::signal(SIGHUP, SIG_IGN); @@ -281,19 +281,19 @@ void BackupStoreDaemon::Connection(SocketStreamTLS &rStream) } catch(BoxException &e) { - ::syslog(LOG_ERR, "%s: disconnecting due to " - "exception %s (%d/%d)", DaemonName(), - e.what(), e.GetType(), e.GetSubType()); + BOX_ERROR("Error in child process, terminating connection: " << + e.what() << " (" << e.GetType() << "/" << + e.GetSubType() << ")"); } catch(std::exception &e) { - ::syslog(LOG_ERR, "%s: disconnecting due to " - "exception %s", DaemonName(), e.what()); + BOX_ERROR("Error in child process, terminating connection: " << + e.what()); } catch(...) { - ::syslog(LOG_ERR, "%s: disconnecting due to " - "unknown exception", DaemonName()); + BOX_ERROR("Error in child process, terminating connection: " << + "unknown exception"); } } @@ -311,7 +311,7 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) std::string clientCommonName(rStream.GetPeerCommonName()); // Log the name - ::syslog(LOG_INFO, "Certificate CN: %s", clientCommonName.c_str()); + BOX_INFO("Client certificate CN: " << clientCommonName); // Check it int32_t id; @@ -357,10 +357,8 @@ void BackupStoreDaemon::LogConnectionStats(const char *commonName, const SocketStreamTLS &s) { // Log the amount of data transferred - ::syslog(LOG_INFO, "Connection statistics for %s: " - "IN=%lld OUT=%lld TOTAL=%lld", commonName, - (long long)s.GetBytesRead(), - (long long)s.GetBytesWritten(), - (long long)s.GetBytesRead() + - (long long)s.GetBytesWritten()); + BOX_INFO("Connection statistics for " << commonName << ":" + " IN=" << s.GetBytesRead() << + " OUT=" << s.GetBytesWritten() << + " TOTAL=" << (s.GetBytesRead() + s.GetBytesWritten())); } diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp index 8d2d9e4c..9f4239e7 100644 --- a/bin/bbstored/HousekeepStoreAccount.cpp +++ b/bin/bbstored/HousekeepStoreAccount.cpp @@ -9,9 +9,10 @@ #include "Box.h" -#include #include +#include + #include "HousekeepStoreAccount.h" #include "BackupStoreDaemon.h" #include "StoreStructure.h" @@ -137,11 +138,18 @@ void HousekeepStoreAccount::DoHousekeeping() || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles || usedDirectories != mBlocksInDirectories) { // Log this - ::syslog(LOG_ERR, "On housekeeping, sizes in store do not match calculated sizes, correcting"); - ::syslog(LOG_ERR, "different (store,calc): acc 0x%08x, used (%lld,%lld), old (%lld,%lld), deleted (%lld,%lld), dirs (%lld,%lld)", - mAccountID, - (used + mBlocksUsedDelta), mBlocksUsed, (usedOld + mBlocksInOldFilesDelta), mBlocksInOldFiles, - (usedDeleted + mBlocksInDeletedFilesDelta), mBlocksInDeletedFiles, usedDirectories, mBlocksInDirectories); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " found " + "and fixed wrong block counts: " + "used (" << + (used + mBlocksUsedDelta) << "," << + mBlocksUsed << "), old (" << + (usedOld + mBlocksInOldFilesDelta) << "," << + mBlocksInOldFiles << "), deleted (" << + (usedDeleted + mBlocksInDeletedFilesDelta) << + "," << mBlocksInDeletedFiles << "), dirs (" << + usedDirectories << "," << mBlocksInDirectories + << ")"); } // If the current values don't match, store them @@ -173,17 +181,33 @@ void HousekeepStoreAccount::DoHousekeeping() // Log deletion if anything was deleted if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) { - ::syslog(LOG_INFO, "Account 0x%08x, removed %lld blocks (%lld files, %lld dirs)%s", mAccountID, 0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta), - mFilesDeleted, mEmptyDirectoriesDeleted, - deleteInterrupted?" was interrupted":""); + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " " + "removed " << + (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << + " blocks (" << mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs)" << + (deleteInterrupted?" and was interrupted":"")); } // Make sure the delta's won't cause problems if the counts are really wrong, and // it wasn't fixed because the store was updated during the scan. - if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) mBlocksUsedDelta = (0 - info->GetBlocksUsed()); - if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles())) mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); - if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles())) mBlocksInDeletedFilesDelta =(0 - info->GetBlocksInDeletedFiles()); - if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories())) mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories()); + if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) + { + mBlocksUsedDelta = (0 - info->GetBlocksUsed()); + } + if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles())) + { + mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); + } + if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles())) + { + mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); + } + if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories())) + { + mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories()); + } // Update the usage counts in the store info->ChangeBlocksUsed(mBlocksUsedDelta); @@ -554,7 +578,14 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID); if(pentry == 0) { - ::syslog(LOG_ERR, "acc 0x%08x, object %lld not found in dir %lld, logic error/corruption? Run bbstoreaccounts check fix", mAccountID, ObjectID, InDirectory); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " " + "found error: object " << + BOX_FORMAT_OBJECTID(ObjectID) << " " + "not found in dir " << + BOX_FORMAT_OBJECTID(InDirectory) << ", " + "indicates logic error/corruption? Run " + "bbstoreaccounts check fix"); return; } -- cgit v1.2.3