diff options
Diffstat (limited to 'bin/bbackupd/BackupDaemon.cpp')
| -rw-r--r-- | bin/bbackupd/BackupDaemon.cpp | 1699 |
1 files changed, 749 insertions, 950 deletions
diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp index e762bbdc..3615b848 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/bin/bbackupd/BackupDaemon.cpp @@ -10,6 +10,7 @@ #include "Box.h" #include <stdio.h> +#include <stdlib.h> #include <string.h> #ifdef HAVE_UNISTD_H @@ -47,36 +48,34 @@ #include "BoxPortsAndFiles.h" #include "SSLLib.h" -#include "TLSContext.h" -#include "BackupDaemon.h" -#include "BackupDaemonConfigVerify.h" +#include "autogen_BackupProtocolClient.h" +#include "autogen_ClientException.h" +#include "autogen_ConversionException.h" +#include "Archive.h" #include "BackupClientContext.h" +#include "BackupClientCryptoKeys.h" #include "BackupClientDirectoryRecord.h" -#include "BackupStoreDirectory.h" #include "BackupClientFileAttributes.h" -#include "BackupStoreFilenameClear.h" #include "BackupClientInodeToIDMap.h" -#include "autogen_BackupProtocolClient.h" -#include "autogen_ConversionException.h" -#include "BackupClientCryptoKeys.h" -#include "BannerText.h" +#include "BackupClientMakeExcludeList.h" +#include "BackupDaemon.h" +#include "BackupDaemonConfigVerify.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "Random.h" +#include "BackupStoreFilenameClear.h" +#include "BannerText.h" +#include "Conversion.h" #include "ExcludeList.h" -#include "BackupClientMakeExcludeList.h" -#include "IOStreamGetLine.h" -#include "Utils.h" #include "FileStream.h" -#include "BackupStoreException.h" -#include "BackupStoreConstants.h" -#include "LocalProcessStream.h" #include "IOStreamGetLine.h" -#include "Conversion.h" -#include "Archive.h" -#include "Timer.h" +#include "LocalProcessStream.h" #include "Logging.h" -#include "autogen_ClientException.h" +#include "Random.h" +#include "Timer.h" +#include "Utils.h" #ifdef WIN32 #include "Win32ServiceFunctions.h" @@ -93,25 +92,6 @@ static const time_t MAX_SLEEP_TIME = 1024; // This prevents repetative cycles of load on the server #define SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY 6 -#ifdef WIN32 -// -------------------------------------------------------------------------- -// -// Function -// Name: HelperThread() -// Purpose: Background thread function, called by Windows, -// calls the BackupDaemon's RunHelperThread method -// to listen for and act on control communications -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -unsigned int WINAPI HelperThread(LPVOID lpParam) -{ - ((BackupDaemon *)lpParam)->RunHelperThread(); - - return 0; -} -#endif - // -------------------------------------------------------------------------- // // Function @@ -122,9 +102,23 @@ unsigned int WINAPI HelperThread(LPVOID lpParam) // -------------------------------------------------------------------------- BackupDaemon::BackupDaemon() : mState(BackupDaemon::State_Initialising), - mpCommandSocketInfo(0), + mDeleteRedundantLocationsAfter(0), + mLastNotifiedEvent(SysadminNotifier::MAX), mDeleteUnusedRootDirEntriesAfter(0), - mLogAllFileAccess(false) + mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown), + mStorageLimitExceeded(false), + mReadErrorsOnFilesystemObjects(false), + mLastSyncTime(0), + mNextSyncTime(0), + mCurrentSyncStartTime(0), + mUpdateStoreInterval(0), + mDeleteStoreObjectInfoFile(false), + mDoSyncForcedByPreviousSyncError(false), + mLogAllFileAccess(false), + mpProgressNotifier(this), + mpLocationResolver(this), + mpRunStatusProvider(this), + mpSysadminNotifier(this) #ifdef WIN32 , mInstallService(false), mRemoveService(false), @@ -134,40 +128,6 @@ BackupDaemon::BackupDaemon() { // Only ever one instance of a daemon SSLLib::Initialise(); - - // Initialise notification sent status - for(int l = 0; l < NotifyEvent__MAX; ++l) - { - mNotificationsSent[l] = false; - } - - #ifdef WIN32 - // Create the event object to signal from main thread to - // worker when new messages are queued to be sent to the - // command socket. - - mhMessageToSendEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - if(mhMessageToSendEvent == INVALID_HANDLE_VALUE) - { - BOX_ERROR("Failed to create event object: error " << - GetLastError()); - exit(1); - } - - // Create the event object to signal from worker to main thread - // when a command has been received on the command socket. - - mhCommandReceivedEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - if(mhCommandReceivedEvent == INVALID_HANDLE_VALUE) - { - BOX_ERROR("Failed to create event object: error " << - GetLastError()); - exit(1); - } - - // Create the critical section to protect the message queue - InitializeCriticalSection(&mMessageQueueLock); - #endif } // -------------------------------------------------------------------------- @@ -182,12 +142,6 @@ BackupDaemon::~BackupDaemon() { DeleteAllLocations(); DeleteAllIDMaps(); - - if(mpCommandSocketInfo != 0) - { - delete mpCommandSocketInfo; - mpCommandSocketInfo = 0; - } } // -------------------------------------------------------------------------- @@ -262,12 +216,12 @@ void BackupDaemon::SetupInInitialProcess() if(GetConfiguration().KeyExists("CommandSocket")) { BOX_WARNING( - "==============================================================================\n" - "SECURITY WARNING: This platform cannot check the credentials of connections to\n" - "the command socket. This is a potential DoS security problem.\n" - "Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n" - "is not used.\n" - "==============================================================================\n" + "==============================================================================\n" + "SECURITY WARNING: This platform cannot check the credentials of connections to\n" + "the command socket. This is a potential DoS security problem.\n" + "Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n" + "is not used.\n" + "==============================================================================\n" ); } } @@ -294,7 +248,7 @@ void BackupDaemon::DeleteAllLocations() // Clear the contents of the map, so it is empty mLocations.clear(); - // And delete everything from the assoicated mount vector + // And delete everything from the associated mount vector mIDMapMounts.clear(); } @@ -322,6 +276,7 @@ int BackupDaemon::ProcessOption(signed int option) case 'S': { mServiceName = optarg; + Logging::SetProgramName(mServiceName); return 0; } @@ -356,8 +311,6 @@ int BackupDaemon::Main(const std::string &rConfigFileName) return RemoveService(mServiceName); } - Logging::SetProgramName("Box Backup (" + mServiceName + ")"); - int returnCode; if (mRunAsService) @@ -377,220 +330,6 @@ int BackupDaemon::Main(const std::string &rConfigFileName) return returnCode; } - -void BackupDaemon::RunHelperThread(void) -{ - const Configuration &conf(GetConfiguration()); - mpCommandSocketInfo = new CommandSocketInfo; - WinNamedPipeStream& rSocket(mpCommandSocketInfo->mListeningSocket); - - // loop until the parent process exits, or we decide - // to kill the thread ourselves - while (!IsTerminateWanted()) - { - try - { - std::string socket = conf.GetKeyValue("CommandSocket"); - rSocket.Accept(socket); - } - catch (BoxException &e) - { - BOX_ERROR("Failed to open command socket: " << - e.what()); - SetTerminateWanted(); - break; // this is fatal to listening thread - } - catch(std::exception &e) - { - BOX_ERROR("Failed to open command socket: " << - e.what()); - SetTerminateWanted(); - break; // this is fatal to listening thread - } - catch(...) - { - BOX_ERROR("Failed to open command socket: " - "unknown error"); - SetTerminateWanted(); - break; // this is fatal to listening thread - } - - try - { - // Errors here do not kill the thread, - // only the current connection. - - // This next section comes from Ben's original function - // Log - BOX_INFO("Connection from command socket"); - - // Send a header line summarising the configuration - // and current state - char summary[256]; - size_t summarySize = sprintf(summary, - "bbackupd: %d %d %d %d\nstate %d\n", - conf.GetKeyValueBool("AutomaticBackup"), - conf.GetKeyValueInt("UpdateStoreInterval"), - conf.GetKeyValueInt("MinimumFileAge"), - conf.GetKeyValueInt("MaxUploadWait"), - mState); - - rSocket.Write(summary, summarySize); - rSocket.Write("ping\n", 5); - - // old queued messages are not useful - EnterCriticalSection(&mMessageQueueLock); - mMessageList.clear(); - ResetEvent(mhMessageToSendEvent); - LeaveCriticalSection(&mMessageQueueLock); - - IOStreamGetLine readLine(rSocket); - std::string command; - - while (rSocket.IsConnected() && !IsTerminateWanted()) - { - HANDLE handles[2]; - handles[0] = mhMessageToSendEvent; - handles[1] = rSocket.GetReadableEvent(); - - BOX_TRACE("Received command '" << command - << "' over command socket"); - - DWORD result = WaitForMultipleObjects( - sizeof(handles)/sizeof(*handles), - handles, FALSE, 1000); - - if(result == 0) - { - ResetEvent(mhMessageToSendEvent); - - EnterCriticalSection(&mMessageQueueLock); - try - { - while (mMessageList.size() > 0) - { - std::string message = *(mMessageList.begin()); - mMessageList.erase(mMessageList.begin()); - printf("Sending '%s' to waiting client... ", message.c_str()); - message += "\n"; - rSocket.Write(message.c_str(), - message.length()); - - printf("done.\n"); - } - } - catch (...) - { - LeaveCriticalSection(&mMessageQueueLock); - throw; - } - LeaveCriticalSection(&mMessageQueueLock); - continue; - } - else if(result == WAIT_TIMEOUT) - { - continue; - } - else if(result != 1) - { - BOX_ERROR("WaitForMultipleObjects returned invalid result " << result); - continue; - } - - if(!readLine.GetLine(command)) - { - BOX_ERROR("Failed to read line"); - continue; - } - - BOX_INFO("Received command " << command << - " from client"); - - bool sendOK = false; - bool sendResponse = true; - bool disconnect = false; - - // Command to process! - if(command == "quit" || command == "") - { - // Close the socket. - disconnect = true; - sendResponse = false; - } - else if(command == "sync") - { - // Sync now! - this->mDoSyncFlagOut = true; - this->mSyncIsForcedOut = false; - sendOK = true; - SetEvent(mhCommandReceivedEvent); - } - else if(command == "force-sync") - { - // Sync now (forced -- overrides any SyncAllowScript) - this->mDoSyncFlagOut = true; - this->mSyncIsForcedOut = true; - sendOK = true; - SetEvent(mhCommandReceivedEvent); - } - else if(command == "reload") - { - // Reload the configuration - SetReloadConfigWanted(); - sendOK = true; - SetEvent(mhCommandReceivedEvent); - } - else if(command == "terminate") - { - // Terminate the daemon cleanly - SetTerminateWanted(); - sendOK = true; - SetEvent(mhCommandReceivedEvent); - } - else - { - BOX_ERROR("Received unknown command " - "'" << command << "' " - "from client"); - sendResponse = true; - sendOK = false; - } - - // Send a response back? - if(sendResponse) - { - const char* response = sendOK ? "ok\n" : "error\n"; - rSocket.Write( - response, strlen(response)); - } - - if(disconnect) - { - break; - } - } - - rSocket.Close(); - } - catch(BoxException &e) - { - BOX_ERROR("Communication error with " - "control client: " << e.what()); - } - catch(std::exception &e) - { - BOX_ERROR("Internal error in command socket " - "thread: " << e.what()); - } - catch(...) - { - BOX_ERROR("Communication error with control client"); - } - } - - CloseHandle(mhCommandReceivedEvent); - CloseHandle(mhMessageToSendEvent); -} #endif // -------------------------------------------------------------------------- @@ -606,36 +345,29 @@ void BackupDaemon::Run() // initialise global timer mechanism Timers::Init(); - #ifdef WIN32 - // Create a thread to handle the named pipe - HANDLE hThread; - unsigned int dwThreadId; - - hThread = (HANDLE) _beginthreadex( - NULL, // default security attributes - 0, // use default stack size - HelperThread, // thread function - this, // argument to thread function - 0, // use default creation flags - &dwThreadId); // returns the thread identifier - #else + #ifndef WIN32 // Ignore SIGPIPE so that if a command connection is broken, // the daemon doesn't terminate. ::signal(SIGPIPE, SIG_IGN); + #endif - // Create a command socket? - const Configuration &conf(GetConfiguration()); - if(conf.KeyExists("CommandSocket")) - { - // Yes, create a local UNIX socket - mpCommandSocketInfo = new CommandSocketInfo; - const char *socketName = - conf.GetKeyValue("CommandSocket").c_str(); + // Create a command socket? + const Configuration &conf(GetConfiguration()); + if(conf.KeyExists("CommandSocket")) + { + // Yes, create a local UNIX socket + mapCommandSocketInfo.reset(new CommandSocketInfo); + const char *socketName = + conf.GetKeyValue("CommandSocket").c_str(); + #ifdef WIN32 + mapCommandSocketInfo->mListeningSocket.Listen( + socketName); + #else ::unlink(socketName); - mpCommandSocketInfo->mListeningSocket.Listen( + mapCommandSocketInfo->mListeningSocket.Listen( Socket::TypeUNIX, socketName); - } - #endif // !WIN32 + #endif + } // Handle things nicely on exceptions try @@ -644,16 +376,11 @@ void BackupDaemon::Run() } catch(...) { - #ifdef WIN32 - // Don't delete the socket, as the helper thread - // is probably still using it. Let Windows clean - // up after us. - #else - if(mpCommandSocketInfo != 0) + if(mapCommandSocketInfo.get()) { try { - delete mpCommandSocketInfo; + mapCommandSocketInfo.reset(); } catch(std::exception &e) { @@ -666,91 +393,63 @@ void BackupDaemon::Run() BOX_WARNING("Error closing command socket " "after exception, ignored."); } - mpCommandSocketInfo = 0; } - #endif // WIN32 Timers::Cleanup(); throw; } - #ifndef WIN32 - // Clean up - if(mpCommandSocketInfo != 0) - { - delete mpCommandSocketInfo; - mpCommandSocketInfo = 0; - } - #endif - + // Clean up + mapCommandSocketInfo.reset(); Timers::Cleanup(); } -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::Run2() -// Purpose: Run function for daemon (second stage) -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupDaemon::Run2() +void BackupDaemon::InitCrypto() { // Read in the certificates creating a TLS context - TLSContext tlsContext; const Configuration &conf(GetConfiguration()); std::string certFile(conf.GetKeyValue("CertificateFile")); std::string keyFile(conf.GetKeyValue("PrivateKeyFile")); std::string caFile(conf.GetKeyValue("TrustedCAsFile")); - tlsContext.Initialise(false /* as client */, certFile.c_str(), keyFile.c_str(), caFile.c_str()); + mTlsContext.Initialise(false /* as client */, certFile.c_str(), + keyFile.c_str(), caFile.c_str()); // Set up the keys for various things BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str()); +} - // Setup various timings - int maximumDiffingTime = 600; - int keepAliveTime = 60; +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Run2() +// Purpose: Run function for daemon (second stage) +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Run2() +{ + InitCrypto(); - // max diffing time, keep-alive time - if(conf.KeyExists("MaximumDiffingTime")) - { - maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); - } - if(conf.KeyExists("KeepAliveTime")) - { - keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); - } + const Configuration &conf(GetConfiguration()); // How often to connect to the store (approximate) - box_time_t updateStoreInterval = SecondsToBoxTime(conf.GetKeyValueInt("UpdateStoreInterval")); + mUpdateStoreInterval = SecondsToBoxTime( + conf.GetKeyValueInt("UpdateStoreInterval")); // But are we connecting automatically? bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup"); - // The minimum age a file needs to be before it will be considered for uploading - box_time_t minimumFileAge = SecondsToBoxTime(conf.GetKeyValueInt("MinimumFileAge")); - - // The maximum time we'll wait to upload a file, regardless of how often it's modified - box_time_t maxUploadWait = SecondsToBoxTime(conf.GetKeyValueInt("MaxUploadWait")); - // Adjust by subtracting the minimum file age, so is relative to sync period end in comparisons - maxUploadWait = (maxUploadWait > minimumFileAge)?(maxUploadWait - minimumFileAge):(0); - // When the next sync should take place -- which is ASAP - box_time_t nextSyncTime = 0; + mNextSyncTime = 0; // When the last sync started (only updated if the store was not full when the sync ended) - box_time_t lastSyncTime = 0; + mLastSyncTime = 0; // -------------------------------------------------------------------------------------------- - // And what's the current client store marker? - int64_t clientStoreMarker = - BackupClientContext::ClientStoreMarker_NotKnown; - // haven't contacted the store yet - - bool deleteStoreObjectInfoFile = DeserializeStoreObjectInfo( - clientStoreMarker, lastSyncTime, nextSyncTime); + mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo( + mLastSyncTime, mNextSyncTime); // -------------------------------------------------------------------------------------------- @@ -758,91 +457,98 @@ void BackupDaemon::Run2() // Set state SetState(State_Idle); + mDoSyncForcedByPreviousSyncError = false; + // Loop around doing backups do { // Flags used below bool storageLimitExceeded = false; bool doSync = false; - bool doSyncForcedByCommand = false; + bool mDoSyncForcedByCommand = false; // Is a delay necessary? + box_time_t currentTime; + + do { - box_time_t currentTime; - do + // Check whether we should be stopping, + // and don't run a sync if so. + if(StopRun()) break; + + currentTime = GetCurrentBoxTime(); + + // Pause a while, but no more than + // MAX_SLEEP_TIME seconds (use the conditional + // because times are unsigned) + box_time_t requiredDelay = + (mNextSyncTime < currentTime) + ? (0) + : (mNextSyncTime - currentTime); + + // If there isn't automatic backup happening, + // set a long delay. And limit delays at the + // same time. + if(!automaticBackup && !mDoSyncForcedByPreviousSyncError) { - // Check whether we should be stopping, - // and don't run a sync if so. - if(StopRun()) break; - - currentTime = GetCurrentBoxTime(); - - // Pause a while, but no more than - // MAX_SLEEP_TIME seconds (use the conditional - // because times are unsigned) - box_time_t requiredDelay = - (nextSyncTime < currentTime) - ? (0) - : (nextSyncTime - currentTime); - - // If there isn't automatic backup happening, - // set a long delay. And limit delays at the - // same time. - if(!automaticBackup || requiredDelay > - SecondsToBoxTime(MAX_SLEEP_TIME)) + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + else if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) + { + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + + // Only delay if necessary + if(requiredDelay > 0) + { + // Sleep somehow. There are choices + // on how this should be done, + // depending on the state of the + // control connection + if(mapCommandSocketInfo.get() != 0) { - requiredDelay = SecondsToBoxTime( - MAX_SLEEP_TIME); + // A command socket exists, + // so sleep by waiting on it + WaitOnCommandSocket(requiredDelay, + doSync, mDoSyncForcedByCommand); } - - // Only delay if necessary - if(requiredDelay > 0) + else { - // Sleep somehow. There are choices - // on how this should be done, - // depending on the state of the - // control connection - if(mpCommandSocketInfo != 0) - { - // A command socket exists, - // so sleep by waiting on it - WaitOnCommandSocket( - requiredDelay, doSync, - doSyncForcedByCommand); - } - else - { - // No command socket or - // connection, just do a - // normal sleep - time_t sleepSeconds = - BoxTimeToSeconds( - requiredDelay); - ::sleep((sleepSeconds <= 0) - ? 1 - : sleepSeconds); - } + // No command socket or + // connection, just do a + // normal sleep + time_t sleepSeconds = + BoxTimeToSeconds(requiredDelay); + ::sleep((sleepSeconds <= 0) + ? 1 : sleepSeconds); } - - } while((!automaticBackup || (currentTime < nextSyncTime)) && !doSync && !StopRun()); + } + + if ((automaticBackup || mDoSyncForcedByPreviousSyncError) + && currentTime >= mNextSyncTime) + { + doSync = true; + } } + while(!doSync && !StopRun()); // Time of sync start, and if it's time for another sync // (and we're doing automatic syncs), set the flag - box_time_t currentSyncStartTime = GetCurrentBoxTime(); - if(automaticBackup && currentSyncStartTime >= nextSyncTime) + mCurrentSyncStartTime = GetCurrentBoxTime(); + if((automaticBackup || mDoSyncForcedByPreviousSyncError) && + mCurrentSyncStartTime >= mNextSyncTime) { doSync = true; } // Use a script to see if sync is allowed now? - if(!doSyncForcedByCommand && doSync && !StopRun()) + if(!mDoSyncForcedByCommand && doSync && !StopRun()) { int d = UseScriptToSeeIfSyncAllowed(); if(d > 0) { // Script has asked for a delay - nextSyncTime = GetCurrentBoxTime() + + mNextSyncTime = GetCurrentBoxTime() + SecondsToBoxTime(d); doSync = false; } @@ -852,383 +558,448 @@ void BackupDaemon::Run2() // to be stopping) if(doSync && !StopRun()) { - // Touch a file to record times in filesystem - TouchFileInWorkingDir("last_sync_start"); + RunSyncNowWithExceptionHandling(); + } - // Tell anything connected to the command socket - SendSyncStartOrFinish(true /* start */); - - // Reset statistics on uploads - BackupStoreFile::ResetStats(); - - // Calculate the sync period of files to examine - box_time_t syncPeriodStart = lastSyncTime; - box_time_t syncPeriodEnd = currentSyncStartTime - - minimumFileAge; + // Set state + SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); - if(syncPeriodStart >= syncPeriodEnd && - syncPeriodStart - syncPeriodEnd < minimumFileAge) - { - // This can happen if we receive a force-sync - // command less than minimumFileAge after - // the last sync. Deal with it by moving back - // syncPeriodStart, which should not do any - // damage. - syncPeriodStart = syncPeriodEnd - - SecondsToBoxTime(1); - } + } while(!StopRun()); + + // Make sure we have a clean start next time round (if restart) + DeleteAllLocations(); + DeleteAllIDMaps(); +} - if(syncPeriodStart >= syncPeriodEnd) - { - BOX_ERROR("Invalid (negative) sync period: " - "perhaps your clock is going " - "backwards (" << syncPeriodStart << - " to " << syncPeriodEnd << ")"); - THROW_EXCEPTION(ClientException, - ClockWentBackwards); - } +void BackupDaemon::RunSyncNowWithExceptionHandling() +{ + OnBackupStart(); - // Check logic - ASSERT(syncPeriodEnd > syncPeriodStart); - // Paranoid check on sync times - if(syncPeriodStart >= syncPeriodEnd) continue; - - // Adjust syncPeriodEnd to emulate snapshot - // behaviour properly - box_time_t syncPeriodEndExtended = syncPeriodEnd; - // Using zero min file age? - if(minimumFileAge == 0) - { - // Add a year on to the end of the end time, - // to make sure we sync files which are - // modified after the scan run started. - // Of course, they may be eligible to be - // synced again the next time round, - // but this should be OK, because the changes - // only upload should upload no data. - syncPeriodEndExtended += SecondsToBoxTime( - (time_t)(356*24*3600)); - } + // Do sync + bool errorOccurred = false; + int errorCode = 0, errorSubCode = 0; + const char* errorString = "unknown"; - // Delete the serialised store object file, - // so that we don't try to reload it after a - // partially completed backup - if(deleteStoreObjectInfoFile && - !DeleteStoreObjectInfo()) - { - BOX_ERROR("Failed to delete the " - "StoreObjectInfoFile, backup cannot " - "continue safely."); - THROW_EXCEPTION(ClientException, - FailedToDeleteStoreObjectInfoFile); - } + try + { + RunSyncNow(); + } + catch(BoxException &e) + { + errorOccurred = true; + errorString = e.what(); + errorCode = e.GetType(); + errorSubCode = e.GetSubType(); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error during backup run: " << e.what()); + errorOccurred = true; + errorString = e.what(); + } + catch(...) + { + // TODO: better handling of exceptions here... + // need to be very careful + errorOccurred = true; + } - // In case the backup throws an exception, - // we should not try to delete the store info - // object file again. - deleteStoreObjectInfoFile = false; - - // Do sync - bool errorOccurred = false; - int errorCode = 0, errorSubCode = 0; - const char* errorString = "unknown"; + // do not retry immediately without a good reason + mDoSyncForcedByPreviousSyncError = false; + + if(errorOccurred) + { + // Is it a berkely db failure? + bool isBerkelyDbFailure = false; - try - { - // Set state and log start - SetState(State_Connected); - BOX_NOTICE("Beginning scan of local files"); + if (errorCode == BackupStoreException::ExceptionType + && errorSubCode == BackupStoreException::BerkelyDBFailure) + { + isBerkelyDbFailure = true; + } - std::string extendedLogFile; - if (conf.KeyExists("ExtendedLogFile")) - { - extendedLogFile = conf.GetKeyValue( - "ExtendedLogFile"); - } - - if (conf.KeyExists("LogAllFileAccess")) - { - mLogAllFileAccess = - conf.GetKeyValueBool( - "LogAllFileAccess"); - } - - // Then create a client context object (don't - // just connect, as this may be unnecessary) - BackupClientContext clientContext - ( - *this, - tlsContext, - conf.GetKeyValue("StoreHostname"), - conf.GetKeyValueInt("AccountNumber"), - conf.GetKeyValueBool("ExtendedLogging"), - conf.KeyExists("ExtendedLogFile"), - extendedLogFile - ); - - // Set up the sync parameters - BackupClientDirectoryRecord::SyncParams params( - *this, *this, clientContext); - params.mSyncPeriodStart = syncPeriodStart; - params.mSyncPeriodEnd = syncPeriodEndExtended; - // use potentially extended end time - params.mMaxUploadWait = maxUploadWait; - params.mFileTrackingSizeThreshold = - conf.GetKeyValueInt( - "FileTrackingSizeThreshold"); - params.mDiffingUploadSizeThreshold = - conf.GetKeyValueInt( - "DiffingUploadSizeThreshold"); - params.mMaxFileTimeInFuture = - SecondsToBoxTime( - conf.GetKeyValueInt( - "MaxFileTimeInFuture")); - mDeleteRedundantLocationsAfter = - conf.GetKeyValueInt( - "DeleteRedundantLocationsAfter"); - - clientContext.SetMaximumDiffingTime(maximumDiffingTime); - clientContext.SetKeepAliveTime(keepAliveTime); - - // Set store marker - clientContext.SetClientStoreMarker(clientStoreMarker); - - // Set up the locations, if necessary -- - // need to do it here so we have a - // (potential) connection to use - if(mLocations.empty()) - { - const Configuration &locations( - conf.GetSubConfiguration( - "BackupLocations")); - - // Make sure all the directory records - // are set up - SetupLocations(clientContext, locations); - } - - // Get some ID maps going - SetupIDMapsForSync(); - - // Delete any unused directories? - DeleteUnusedRootDirEntries(clientContext); - - // Notify administrator - NotifySysadmin(NotifyEvent_BackupStart); - - // Go through the records, syncing them - for(std::vector<Location *>::const_iterator - i(mLocations.begin()); - i != mLocations.end(); ++i) - { - // Set current and new ID map pointers - // in the context - clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], mNewIDMaps[(*i)->mIDMapIndex]); - - // Set exclude lists (context doesn't - // take ownership) - clientContext.SetExcludeLists( - (*i)->mpExcludeFiles, - (*i)->mpExcludeDirs); - - // Sync the directory - (*i)->mpDirectoryRecord->SyncDirectory( - params, - BackupProtocolClientListDirectory::RootDirectory, - (*i)->mPath); + if(isBerkelyDbFailure) + { + // Delete corrupt files + DeleteCorruptBerkelyDbFiles(); + } - // Unset exclude lists (just in case) - clientContext.SetExcludeLists(0, 0); - } - - // Errors reading any files? - if(params.mReadErrorsOnFilesystemObjects) - { - // Notify administrator - NotifySysadmin(NotifyEvent_ReadError); - } - else - { - // Unset the read error flag, so the // error is reported again if it - // happens again - mNotificationsSent[NotifyEvent_ReadError] = false; - } - - // Perform any deletions required -- these are - // delayed until the end to allow renaming to - // happen neatly. - clientContext.PerformDeletions(); + // Clear state data + // Go back to beginning of time + mLastSyncTime = 0; + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything + DeleteAllLocations(); + DeleteAllIDMaps(); - // Close any open connection - clientContext.CloseAnyOpenConnection(); - - // Get the new store marker - clientStoreMarker = clientContext.GetClientStoreMarker(); - - // Check the storage limit - if(clientContext.StorageLimitExceeded()) - { - // Tell the sysadmin about this - NotifySysadmin(NotifyEvent_StoreFull); - } - else - { - // The start time of the next run is - // the end time of this run. - // This is only done if the storage - // limit wasn't exceeded (as things - // won't have been done properly if - // it was) - lastSyncTime = syncPeriodEnd; - - // unflag the storage full notify flag - // so that next time the store is full, - // an alert will be sent - mNotificationsSent[NotifyEvent_StoreFull] = false; - } - - // Calculate when the next sync run should be - nextSyncTime = currentSyncStartTime + - updateStoreInterval + - Random::RandomInt(updateStoreInterval >> + // Handle restart? + if(StopRun()) + { + BOX_NOTICE("Exception (" << errorCode + << "/" << errorSubCode + << ") due to signal"); + OnBackupFinish(); + return; + } + + NotifySysadmin(SysadminNotifier::BackupError); + + // If the Berkely db files get corrupted, + // delete them and try again immediately. + if(isBerkelyDbFailure) + { + BOX_ERROR("Berkely db inode map files corrupted, " + "deleting and restarting scan. Renamed files " + "and directories will not be tracked until " + "after this scan."); + ::sleep(1); + } + else + { + // Not restart/terminate, pause and retry + // Notify administrator + SetState(State_Error); + BOX_ERROR("Exception caught (" << errorString << + " " << errorCode << "/" << errorSubCode << + "), reset state and waiting to retry..."); + ::sleep(10); + mNextSyncTime = mCurrentSyncStartTime + + SecondsToBoxTime(100) + + Random::RandomInt(mUpdateStoreInterval >> SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); - - // Commit the ID Maps - CommitIDMapsAfterSync(); + } + } + // Notify system administrator about the final state of the backup + else if(mReadErrorsOnFilesystemObjects) + { + NotifySysadmin(SysadminNotifier::ReadError); + } + else if(mStorageLimitExceeded) + { + NotifySysadmin(SysadminNotifier::StoreFull); + } + else + { + NotifySysadmin(SysadminNotifier::BackupOK); + } + + // If we were retrying after an error, and this backup succeeded, + // then now would be a good time to stop :-) + mDoSyncForcedByPreviousSyncError = errorOccurred; - // Log - BOX_NOTICE("Finished scan of local files"); + OnBackupFinish(); +} - // Notify administrator - NotifySysadmin(NotifyEvent_BackupFinish); +void BackupDaemon::RunSyncNow() +{ + // Delete the serialised store object file, + // so that we don't try to reload it after a + // partially completed backup + if(mDeleteStoreObjectInfoFile && + !DeleteStoreObjectInfo()) + { + BOX_ERROR("Failed to delete the StoreObjectInfoFile, " + "backup cannot continue safely."); + THROW_EXCEPTION(ClientException, + FailedToDeleteStoreObjectInfoFile); + } - // -------------------------------------------------------------------------------------------- + // In case the backup throws an exception, + // we should not try to delete the store info + // object file again. + mDeleteStoreObjectInfoFile = false; - // We had a successful backup, save the store - // info. If we save successfully, we must - // delete the file next time we start a backup + const Configuration &conf(GetConfiguration()); - deleteStoreObjectInfoFile = - SerializeStoreObjectInfo( - clientStoreMarker, - lastSyncTime, nextSyncTime); + std::auto_ptr<FileLogger> fileLogger; - // -------------------------------------------------------------------------------------------- - } - catch(BoxException &e) - { - errorOccurred = true; - errorString = e.what(); - errorCode = e.GetType(); - errorSubCode = e.GetSubType(); - } - catch(std::exception &e) - { - BOX_ERROR("Internal error during " - "backup run: " << e.what()); - errorOccurred = true; - errorString = e.what(); - } - catch(...) - { - // TODO: better handling of exceptions here... - // need to be very careful - errorOccurred = true; - } - - if(errorOccurred) - { - // Is it a berkely db failure? - bool isBerkelyDbFailure = false; + if (conf.KeyExists("LogFile")) + { + Log::Level level = Log::INFO; + if (conf.KeyExists("LogFileLevel")) + { + level = Logging::GetNamedLevel( + conf.GetKeyValue("LogFileLevel")); + } + fileLogger.reset(new FileLogger(conf.GetKeyValue("LogFile"), + level)); + } - if (errorCode == BackupStoreException::ExceptionType - && errorSubCode == BackupStoreException::BerkelyDBFailure) - { - isBerkelyDbFailure = true; - } + std::string extendedLogFile; + if (conf.KeyExists("ExtendedLogFile")) + { + extendedLogFile = conf.GetKeyValue("ExtendedLogFile"); + } + + if (conf.KeyExists("LogAllFileAccess")) + { + mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess"); + } + + // Then create a client context object (don't + // just connect, as this may be unnecessary) + BackupClientContext clientContext + ( + *mpLocationResolver, + mTlsContext, + conf.GetKeyValue("StoreHostname"), + conf.GetKeyValueInt("StorePort"), + conf.GetKeyValueInt("AccountNumber"), + conf.GetKeyValueBool("ExtendedLogging"), + conf.KeyExists("ExtendedLogFile"), + extendedLogFile, *mpProgressNotifier + ); + + // The minimum age a file needs to be before it will be + // considered for uploading + box_time_t minimumFileAge = SecondsToBoxTime( + conf.GetKeyValueInt("MinimumFileAge")); - if(isBerkelyDbFailure) - { - // Delete corrupt files - DeleteCorruptBerkelyDbFiles(); - } + // The maximum time we'll wait to upload a file, regardless + // of how often it's modified + box_time_t maxUploadWait = SecondsToBoxTime( + conf.GetKeyValueInt("MaxUploadWait")); + // Adjust by subtracting the minimum file age, so is relative + // to sync period end in comparisons + if (maxUploadWait > minimumFileAge) + { + maxUploadWait -= minimumFileAge; + } + else + { + maxUploadWait = 0; + } - // Clear state data - syncPeriodStart = 0; - // go back to beginning of time - clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything - DeleteAllLocations(); - DeleteAllIDMaps(); + // Calculate the sync period of files to examine + box_time_t syncPeriodStart = mLastSyncTime; + box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge; - // Handle restart? - if(StopRun()) - { - BOX_NOTICE("Exception (" << errorCode - << "/" << errorSubCode - << ") due to signal"); - return; - } + if(syncPeriodStart >= syncPeriodEnd && + syncPeriodStart - syncPeriodEnd < minimumFileAge) + { + // This can happen if we receive a force-sync command less + // than minimumFileAge after the last sync. Deal with it by + // moving back syncPeriodStart, which should not do any + // damage. + syncPeriodStart = syncPeriodEnd - + SecondsToBoxTime(1); + } - // If the Berkely db files get corrupted, delete them and try again immediately - if(isBerkelyDbFailure) - { - BOX_ERROR("Berkely db inode map files corrupted, deleting and restarting scan. Renamed files and directories will not be tracked until after this scan."); - ::sleep(1); - } - else - { - // Not restart/terminate, pause and retry - // Notify administrator - NotifySysadmin(NotifyEvent_BackupError); - SetState(State_Error); - BOX_ERROR("Exception caught (" - << errorString - << " " << errorCode - << "/" << errorSubCode - << "), reset state and " - "waiting to retry..."); - ::sleep(10); - nextSyncTime = currentSyncStartTime + - SecondsToBoxTime(90) + - Random::RandomInt( - updateStoreInterval >> - SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); - } - } + if(syncPeriodStart >= syncPeriodEnd) + { + BOX_ERROR("Invalid (negative) sync period: " + "perhaps your clock is going " + "backwards (" << syncPeriodStart << + " to " << syncPeriodEnd << ")"); + THROW_EXCEPTION(ClientException, + ClockWentBackwards); + } - // Log the stats - BOX_NOTICE("File statistics: total file size uploaded " - << BackupStoreFile::msStats.mBytesInEncodedFiles - << ", bytes already on server " - << BackupStoreFile::msStats.mBytesAlreadyOnServer - << ", encoded size " - << BackupStoreFile::msStats.mTotalFileStreamSize); - BackupStoreFile::ResetStats(); + // Check logic + ASSERT(syncPeriodEnd > syncPeriodStart); + // Paranoid check on sync times + if(syncPeriodStart >= syncPeriodEnd) return; + + // Adjust syncPeriodEnd to emulate snapshot + // behaviour properly + box_time_t syncPeriodEndExtended = syncPeriodEnd; + + // Using zero min file age? + if(minimumFileAge == 0) + { + // Add a year on to the end of the end time, + // to make sure we sync files which are + // modified after the scan run started. + // Of course, they may be eligible to be + // synced again the next time round, + // but this should be OK, because the changes + // only upload should upload no data. + syncPeriodEndExtended += SecondsToBoxTime( + (time_t)(356*24*3600)); + } + + // Set up the sync parameters + BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider, + *mpSysadminNotifier, *mpProgressNotifier, clientContext); + params.mSyncPeriodStart = syncPeriodStart; + params.mSyncPeriodEnd = syncPeriodEndExtended; + // use potentially extended end time + params.mMaxUploadWait = maxUploadWait; + params.mFileTrackingSizeThreshold = + conf.GetKeyValueInt("FileTrackingSizeThreshold"); + params.mDiffingUploadSizeThreshold = + conf.GetKeyValueInt("DiffingUploadSizeThreshold"); + params.mMaxFileTimeInFuture = + SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); + mDeleteRedundantLocationsAfter = + conf.GetKeyValueInt("DeleteRedundantLocationsAfter"); + mStorageLimitExceeded = false; + mReadErrorsOnFilesystemObjects = false; - // Tell anything connected to the command socket - SendSyncStartOrFinish(false /* finish */); + // Setup various timings + int maximumDiffingTime = 600; + int keepAliveTime = 60; - // Touch a file to record times in filesystem - TouchFileInWorkingDir("last_sync_finish"); - } + // max diffing time, keep-alive time + if(conf.KeyExists("MaximumDiffingTime")) + { + maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); + } + if(conf.KeyExists("KeepAliveTime")) + { + keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); + } + + clientContext.SetMaximumDiffingTime(maximumDiffingTime); + clientContext.SetKeepAliveTime(keepAliveTime); + + // Set store marker + clientContext.SetClientStoreMarker(mClientStoreMarker); + + // Set up the locations, if necessary -- + // need to do it here so we have a + // (potential) connection to use + if(mLocations.empty()) + { + const Configuration &locations( + conf.GetSubConfiguration( + "BackupLocations")); - // Set state - SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); + // Make sure all the directory records + // are set up + SetupLocations(clientContext, locations); + } + + mpProgressNotifier->NotifyIDMapsSetup(clientContext); + + // Get some ID maps going + SetupIDMapsForSync(); - } while(!StopRun()); + // Delete any unused directories? + DeleteUnusedRootDirEntries(clientContext); + + // Go through the records, syncing them + for(std::vector<Location *>::const_iterator + i(mLocations.begin()); + i != mLocations.end(); ++i) + { + // Set current and new ID map pointers + // in the context + clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], + mNewIDMaps[(*i)->mIDMapIndex]); - // Make sure we have a clean start next time round (if restart) - DeleteAllLocations(); - DeleteAllIDMaps(); + // Set exclude lists (context doesn't + // take ownership) + clientContext.SetExcludeLists( + (*i)->mpExcludeFiles, + (*i)->mpExcludeDirs); + + // Sync the directory + (*i)->mpDirectoryRecord->SyncDirectory( + params, + BackupProtocolClientListDirectory::RootDirectory, + (*i)->mPath, std::string("/") + (*i)->mName); + + // Unset exclude lists (just in case) + clientContext.SetExcludeLists(0, 0); + } + + // Perform any deletions required -- these are + // delayed until the end to allow renaming to + // happen neatly. + clientContext.PerformDeletions(); + + // Close any open connection + clientContext.CloseAnyOpenConnection(); + + // Get the new store marker + mClientStoreMarker = clientContext.GetClientStoreMarker(); + mStorageLimitExceeded = clientContext.StorageLimitExceeded(); + mReadErrorsOnFilesystemObjects = + params.mReadErrorsOnFilesystemObjects; + + if(!mStorageLimitExceeded) + { + // The start time of the next run is the end time of this + // run. This is only done if the storage limit wasn't + // exceeded (as things won't have been done properly if + // it was) + mLastSyncTime = syncPeriodEnd; + } + + // Commit the ID Maps + CommitIDMapsAfterSync(); + + // Calculate when the next sync run should be + mNextSyncTime = mCurrentSyncStartTime + + mUpdateStoreInterval + + Random::RandomInt(mUpdateStoreInterval >> + SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); + + // -------------------------------------------------------------------------------------------- + + // We had a successful backup, save the store + // info. If we save successfully, we must + // delete the file next time we start a backup + + mDeleteStoreObjectInfoFile = + SerializeStoreObjectInfo(mLastSyncTime, + mNextSyncTime); + + // -------------------------------------------------------------------------------------------- } +void BackupDaemon::OnBackupStart() +{ + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_start"); + + // Reset statistics on uploads + BackupStoreFile::ResetStats(); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(true /* start */); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupStart); + + // Set state and log start + SetState(State_Connected); + BOX_NOTICE("Beginning scan of local files"); +} + +void BackupDaemon::OnBackupFinish() +{ + // Log + BOX_NOTICE("Finished scan of local files"); + + // Log the stats + BOX_NOTICE("File statistics: total file size uploaded " + << BackupStoreFile::msStats.mBytesInEncodedFiles + << ", bytes already on server " + << BackupStoreFile::msStats.mBytesAlreadyOnServer + << ", encoded size " + << BackupStoreFile::msStats.mTotalFileStreamSize); + + // Reset statistics again + BackupStoreFile::ResetStats(); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupFinish); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(false /* finish */); + + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_finish"); +} // -------------------------------------------------------------------------- // // Function // Name: BackupDaemon::UseScriptToSeeIfSyncAllowed() -// Purpose: Private. Use a script to see if the sync should be allowed (if configured) -// Returns -1 if it's allowed, time in seconds to wait otherwise. +// Purpose: Private. Use a script to see if the sync should be +// allowed now (if configured). Returns -1 if it's +// allowed, time in seconds to wait otherwise. // Created: 21/6/04 // // -------------------------------------------------------------------------- @@ -1250,7 +1021,8 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() pid_t pid = 0; try { - std::auto_ptr<IOStream> pscript(LocalProcessStream(conf.GetKeyValue("SyncAllowScript").c_str(), pid)); + std::auto_ptr<IOStream> pscript(LocalProcessStream( + conf.GetKeyValue("SyncAllowScript").c_str(), pid)); // Read in the result IOStreamGetLine getLine(*pscript); @@ -1323,33 +1095,13 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() // -------------------------------------------------------------------------- void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) { -#ifdef WIN32 - DWORD requiredDelayMs = BoxTimeToMilliSeconds(RequiredDelay); - - DWORD result = WaitForSingleObject(mhCommandReceivedEvent, - (DWORD)requiredDelayMs); - - if(result == WAIT_OBJECT_0) - { - DoSyncFlagOut = this->mDoSyncFlagOut; - SyncIsForcedOut = this->mSyncIsForcedOut; - ResetEvent(mhCommandReceivedEvent); - } - else if(result == WAIT_TIMEOUT) + ASSERT(mapCommandSocketInfo.get()); + if(!mapCommandSocketInfo.get()) { - DoSyncFlagOut = false; - SyncIsForcedOut = false; - } - else - { - BOX_ERROR("Unexpected result from WaitForSingleObject: " - "error " << GetLastError()); + // failure case isn't too bad + ::sleep(1); + return; } - - return; -#else // ! WIN32 - ASSERT(mpCommandSocketInfo != 0); - if(mpCommandSocketInfo == 0) {::sleep(1); return;} // failure case isn't too bad BOX_TRACE("Wait on command socket, delay = " << RequiredDelay); @@ -1362,12 +1114,12 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla if(timeout == INFTIM) timeout = 100000; // Wait for socket connection, or handle a command? - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { // No connection, listen for a new one - mpCommandSocketInfo->mpConnectedSocket.reset(mpCommandSocketInfo->mListeningSocket.Accept(timeout).release()); + mapCommandSocketInfo->mpConnectedSocket.reset(mapCommandSocketInfo->mListeningSocket.Accept(timeout).release()); - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { // If a connection didn't arrive, there was a timeout, which means we've // waited long enough and it's time to go. @@ -1386,7 +1138,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla { uid_t remoteEUID = 0xffff; gid_t remoteEGID = 0xffff; - if(mpCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID)) + if(mapCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID)) { // Credentials are available -- check UID if(remoteEUID == ::getuid()) @@ -1403,7 +1155,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla { // Dump the connection BOX_ERROR("Incoming command connection from peer had different user ID than this process, or security check could not be completed."); - mpCommandSocketInfo->mpConnectedSocket.reset(); + mapCommandSocketInfo->mpConnectedSocket.reset(); return; } else @@ -1420,7 +1172,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla conf.GetKeyValueInt("MinimumFileAge"), conf.GetKeyValueInt("MaxUploadWait"), mState); - mpCommandSocketInfo->mpConnectedSocket->Write(summary, summarySize); + mapCommandSocketInfo->mpConnectedSocket->Write(summary, summarySize); // Set the timeout to something very small, so we don't wait too long on waiting // for any incoming data @@ -1430,22 +1182,22 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla } // So there must be a connection now. - ASSERT(mpCommandSocketInfo->mpConnectedSocket.get() != 0); + ASSERT(mapCommandSocketInfo->mpConnectedSocket.get() != 0); // Is there a getline object ready? - if(mpCommandSocketInfo->mpGetLine == 0) + if(mapCommandSocketInfo->mpGetLine == 0) { // Create a new one - mpCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mpCommandSocketInfo->mpConnectedSocket.get())); + mapCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mapCommandSocketInfo->mpConnectedSocket.get())); } // Ping the remote side, to provide errors which will mean the socket gets closed - mpCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5); + mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5); // Wait for a command or something on the socket std::string command; - while(mpCommandSocketInfo->mpGetLine != 0 && !mpCommandSocketInfo->mpGetLine->IsEOF() - && mpCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) + while(mapCommandSocketInfo->mpGetLine != 0 && !mapCommandSocketInfo->mpGetLine->IsEOF() + && mapCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) { BOX_TRACE("Receiving command '" << command << "' over command socket"); @@ -1490,7 +1242,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla // Send a response back? if(sendResponse) { - mpCommandSocketInfo->mpConnectedSocket->Write(sendOK?"ok\n":"error\n", sendOK?3:6); + mapCommandSocketInfo->mpConnectedSocket->Write(sendOK?"ok\n":"error\n", sendOK?3:6); } // Set timeout to something very small, so this just checks for data which is waiting @@ -1498,18 +1250,39 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla } // Close on EOF? - if(mpCommandSocketInfo->mpGetLine != 0 && mpCommandSocketInfo->mpGetLine->IsEOF()) + if(mapCommandSocketInfo->mpGetLine != 0 && mapCommandSocketInfo->mpGetLine->IsEOF()) { CloseCommandConnection(); } } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write to command socket: " << ce.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } catch(std::exception &e) { - BOX_ERROR("Internal error in command socket thread: " - << e.what()); - // If an error occurs, and there is a connection active, just close that - // connection and continue. Otherwise, let the error propagate. - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + BOX_ERROR("Failed to write to command socket: " << + e.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { throw; // thread will die } @@ -1521,9 +1294,13 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla } catch(...) { - // If an error occurs, and there is a connection active, just close that - // connection and continue. Otherwise, let the error propagate. - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + BOX_ERROR("Failed to write to command socket: unknown error"); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { throw; // thread will die } @@ -1533,7 +1310,6 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla CloseCommandConnection(); } } -#endif // WIN32 } @@ -1547,17 +1323,16 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla // -------------------------------------------------------------------------- void BackupDaemon::CloseCommandConnection() { -#ifndef WIN32 try { BOX_TRACE("Closing command connection"); - if(mpCommandSocketInfo->mpGetLine) + if(mapCommandSocketInfo->mpGetLine) { - delete mpCommandSocketInfo->mpGetLine; - mpCommandSocketInfo->mpGetLine = 0; + delete mapCommandSocketInfo->mpGetLine; + mapCommandSocketInfo->mpGetLine = 0; } - mpCommandSocketInfo->mpConnectedSocket.reset(); + mapCommandSocketInfo->mpConnectedSocket.reset(); } catch(std::exception &e) { @@ -1568,7 +1343,6 @@ void BackupDaemon::CloseCommandConnection() { // Ignore any errors } -#endif } @@ -1586,27 +1360,15 @@ void BackupDaemon::SendSyncStartOrFinish(bool SendStart) // The bbackupctl program can't rely on a state change, because it // may never change if the server doesn't need to be contacted. - if(mpCommandSocketInfo != NULL && -#ifdef WIN32 - mpCommandSocketInfo->mListeningSocket.IsConnected() -#else - mpCommandSocketInfo->mpConnectedSocket.get() != 0 -#endif - ) + if(mapCommandSocketInfo.get() && + mapCommandSocketInfo->mpConnectedSocket.get() != 0) { std::string message = SendStart ? "start-sync" : "finish-sync"; try { -#ifdef WIN32 - EnterCriticalSection(&mMessageQueueLock); - mMessageList.push_back(message); - SetEvent(mhMessageToSendEvent); - LeaveCriticalSection(&mMessageQueueLock); -#else message += "\n"; - mpCommandSocketInfo->mpConnectedSocket->Write( + mapCommandSocketInfo->mpConnectedSocket->Write( message.c_str(), message.size()); -#endif } catch(std::exception &e) { @@ -1668,14 +1430,20 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // Just a check to make sure it's right. DeleteAllLocations(); - // Going to need a copy of the root directory. Get a connection, and fetch it. + // Going to need a copy of the root directory. Get a connection, + // and fetch it. BackupProtocolClient &connection(rClientContext.GetConnection()); - // Ask server for a list of everything in the root directory, which is a directory itself - std::auto_ptr<BackupProtocolClientSuccess> dirreply(connection.QueryListDirectory( + // Ask server for a list of everything in the root directory, + // which is a directory itself + std::auto_ptr<BackupProtocolClientSuccess> dirreply( + connection.QueryListDirectory( BackupProtocolClientListDirectory::RootDirectory, - BackupProtocolClientListDirectory::Flags_Dir, // only directories - BackupProtocolClientListDirectory::Flags_Deleted | BackupProtocolClientListDirectory::Flags_OldVersion, // exclude old/deleted stuff + // only directories + BackupProtocolClientListDirectory::Flags_Dir, + // exclude old/deleted stuff + BackupProtocolClientListDirectory::Flags_Deleted | + BackupProtocolClientListDirectory::Flags_OldVersion, false /* no attributes */)); // Retrieve the directory from the stream following @@ -1755,33 +1523,41 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con #endif // HAVE_STRUCT_MNTENT_MNT_DIR // Check sorting and that things are as we expect ASSERT(mountPoints.size() > 0); -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD { std::set<std::string, mntLenCompare>::reverse_iterator i(mountPoints.rbegin()); ASSERT(*i == "/"); } -#endif // n NDEBUG +#endif // n BOX_RELEASE_BUILD #endif // n HAVE_STRUCT_STATFS_F_MNTONNAME || n HAVE_STRUCT_STATVFS_F_MNTONNAME #endif // HAVE_MOUNTS // Then... go through each of the entries in the configuration, // making sure there's a directory created for it. - for(std::list<std::pair<std::string, Configuration> >::const_iterator i = rLocationsConf.mSubConfigurations.begin(); - i != rLocationsConf.mSubConfigurations.end(); ++i) - { - BOX_TRACE("new location: " << i->first); + std::vector<std::string> locNames = + rLocationsConf.GetSubConfigurationNames(); + + for(std::vector<std::string>::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + const Configuration& rConfig( + rLocationsConf.GetSubConfiguration(*pLocName)); + BOX_TRACE("new location: " << *pLocName); + // Create a record for it std::auto_ptr<Location> apLoc(new Location); try { // Setup names in the location record - apLoc->mName = i->first; - apLoc->mPath = i->second.GetKeyValue("Path"); + apLoc->mName = *pLocName; + apLoc->mPath = rConfig.GetKeyValue("Path"); // Read the exclude lists from the Configuration - apLoc->mpExcludeFiles = BackupClientMakeExcludeList_Files(i->second); - apLoc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(i->second); + apLoc->mpExcludeFiles = BackupClientMakeExcludeList_Files(rConfig); + apLoc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(rConfig); // Does this exist on the server? // Remove from dir object early, so that if we fail @@ -1814,10 +1590,9 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con if(::statfs(apLoc->mPath.c_str(), &s) != 0) #endif // HAVE_STRUCT_STATVFS_F_MNTONNAME { - BOX_WARNING("Failed to stat location " + BOX_LOG_SYS_WARNING("Failed to stat location " "path '" << apLoc->mPath << - "' (" << strerror(errno) << - "), skipping location '" << + "', skipping location '" << apLoc->mName << "'"); continue; } @@ -1929,7 +1704,8 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // Create and store the directory object for the root of this location ASSERT(oid != 0); - BackupClientDirectoryRecord *precord = new BackupClientDirectoryRecord(oid, i->first); + BackupClientDirectoryRecord *precord = + new BackupClientDirectoryRecord(oid, *pLocName); apLoc->mpDirectoryRecord.reset(precord); // Push it back on the vector of locations @@ -2007,8 +1783,8 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // -------------------------------------------------------------------------- void BackupDaemon::SetupIDMapsForSync() { - // Need to do different things depending on whether it's an in memory implementation, - // or whether it's all stored on disc. + // Need to do different things depending on whether it's an + // in memory implementation, or whether it's all stored on disc. #ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION @@ -2016,8 +1792,8 @@ void BackupDaemon::SetupIDMapsForSync() DeleteIDMapVector(mNewIDMaps); FillIDMapVector(mNewIDMaps, true /* new maps */); - // Then make sure that the current maps have objects, even if they are empty - // (for the very first run) + // Then make sure that the current maps have objects, + // even if they are empty (for the very first run) if(mCurrentIDMaps.empty()) { FillIDMapVector(mCurrentIDMaps, false /* current maps */); @@ -2191,9 +1967,8 @@ void BackupDaemon::CommitIDMapsAfterSync() #endif if(::rename(newmap.c_str(), target.c_str()) != 0) { - BOX_ERROR("failed to rename ID map: " << newmap - << " to " << target << ": " - << strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to rename ID map: " << + newmap << " to " << target); THROW_EXCEPTION(CommonException, OSFileError) } } @@ -2281,20 +2056,14 @@ void BackupDaemon::SetState(int State) sprintf(newState, "state %d", State); std::string message = newState; -#ifdef WIN32 - EnterCriticalSection(&mMessageQueueLock); - mMessageList.push_back(newState); - SetEvent(mhMessageToSendEvent); - LeaveCriticalSection(&mMessageQueueLock); -#else message += "\n"; - if(mpCommandSocketInfo == 0) + if(!mapCommandSocketInfo.get()) { return; } - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { return; } @@ -2302,22 +2071,27 @@ void BackupDaemon::SetState(int State) // Something connected to the command socket, tell it about the new state try { - mpCommandSocketInfo->mpConnectedSocket->Write(message.c_str(), + mapCommandSocketInfo->mpConnectedSocket->Write(message.c_str(), message.length()); } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write state to command socket: " << + ce.what()); + CloseCommandConnection(); + } catch(std::exception &e) { - BOX_ERROR("Internal error while writing state " - "to command socket: " << e.what()); + BOX_ERROR("Failed to write state to command socket: " << + e.what()); CloseCommandConnection(); } catch(...) { - BOX_ERROR("Internal error while writing state " - "to command socket: unknown error"); + BOX_ERROR("Failed to write state to command socket: " + "unknown error"); CloseCommandConnection(); } -#endif } @@ -2351,7 +2125,7 @@ void BackupDaemon::TouchFileInWorkingDir(const char *Filename) // Created: 25/2/04 // // -------------------------------------------------------------------------- -void BackupDaemon::NotifySysadmin(int Event) +void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event) { static const char *sEventNames[] = { @@ -2360,31 +2134,47 @@ void BackupDaemon::NotifySysadmin(int Event) "backup-error", "backup-start", "backup-finish", + "backup-ok", 0 }; - BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); - BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); - BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); - ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == NotifyEvent__MAX + 1); + // BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); + // BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); + // BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); + ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == SysadminNotifier::MAX + 1); - BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << - sEventNames[Event]); - - if(Event < 0 || Event >= NotifyEvent__MAX) + if(Event < 0 || Event >= SysadminNotifier::MAX) { + BOX_ERROR("BackupDaemon::NotifySysadmin() called for " + "invalid event code " << Event); THROW_EXCEPTION(BackupStoreException, BadNotifySysadminEventCode); } - // Don't send lots of repeated messages - if(mNotificationsSent[Event] && - Event != NotifyEvent_BackupStart && - Event != NotifyEvent_BackupFinish) + BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << + sEventNames[Event]); + + if(!GetConfiguration().KeyExists("NotifyAlways") || + !GetConfiguration().GetKeyValueBool("NotifyAlways")) { - BOX_WARNING("Suppressing duplicate notification about " << - sEventNames[Event]); - return; + // Don't send lots of repeated messages + // Note: backup-start and backup-finish will always be + // logged, because mLastNotifiedEvent is never set to + // these values and therefore they are never "duplicates". + if(mLastNotifiedEvent == Event) + { + if(Event == SysadminNotifier::BackupOK) + { + BOX_INFO("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + else + { + BOX_WARNING("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + return; + } } // Is there a notification script? @@ -2392,10 +2182,10 @@ void BackupDaemon::NotifySysadmin(int Event) if(!conf.KeyExists("NotifyScript")) { // Log, and then return - if(Event != NotifyEvent_BackupStart && - Event != NotifyEvent_BackupFinish) + if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) { - BOX_ERROR("Not notifying administrator about event " + BOX_INFO("Not notifying administrator about event " << sEventNames[Event] << " -- set NotifyScript " "to do this in future"); } @@ -2407,20 +2197,22 @@ void BackupDaemon::NotifySysadmin(int Event) sEventNames[Event]); // Log what we're about to do - BOX_NOTICE("About to notify administrator about event " + BOX_INFO("About to notify administrator about event " << sEventNames[Event] << ", running script '" << script << "'"); // Then do it - if(::system(script.c_str()) != 0) + int returnCode = ::system(script.c_str()); + if(returnCode != 0) { - BOX_ERROR("Notify script returned an error code. ('" - << script << "')"); + BOX_WARNING("Notify script returned error code: " << + returnCode << " ('" << script << "')"); + } + else if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) + { + mLastNotifiedEvent = Event; } - - // Flag that this is done so the administrator isn't constantly - // bombarded with lots of errors - mNotificationsSent[Event] = true; } @@ -2461,13 +2253,13 @@ void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) // Entries to delete, and it's the right time to do so... BOX_NOTICE("Deleting unused locations from store root..."); BackupProtocolClient &connection(rContext.GetConnection()); - for(std::vector<std::pair<int64_t,std::string> >::iterator i(mUnusedRootDirEntries.begin()); i != mUnusedRootDirEntries.end(); ++i) + for(std::vector<std::pair<int64_t,std::string> >::iterator + i(mUnusedRootDirEntries.begin()); + i != mUnusedRootDirEntries.end(); ++i) { connection.QueryDeleteDirectory(i->first); - - // Log this - BOX_NOTICE("Deleted " << i->second << " (ID " << i->first - << ") from store root"); + rContext.GetProgressNotifier().NotifyFileDeleted( + i->first, i->second); } // Reset state @@ -2738,9 +2530,12 @@ BackupDaemon::CommandSocketInfo::~CommandSocketInfo() // -------------------------------------------------------------------------- // // Function -// Name: BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) -// Purpose: Serializes remote directory and file information into a stream of bytes, using an Archive abstraction. -// +// Name: BackupDaemon::SerializeStoreObjectInfo( +// box_time_t theLastSyncTime, +// box_time_t theNextSyncTime) +// Purpose: Serializes remote directory and file information +// into a stream of bytes, using an Archive +// abstraction. // Created: 2005/04/11 // // -------------------------------------------------------------------------- @@ -2749,7 +2544,8 @@ static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; static const int STOREOBJECTINFO_VERSION = 2; -bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const +bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime, + box_time_t theNextSyncTime) const { if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) { @@ -2778,7 +2574,7 @@ bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); anArchive.Write(STOREOBJECTINFO_VERSION); anArchive.Write(GetLoadedConfigModifiedTime()); - anArchive.Write(aClientStoreMarker); + anArchive.Write(mClientStoreMarker); anArchive.Write(theLastSyncTime); anArchive.Write(theNextSyncTime); @@ -2830,15 +2626,13 @@ bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time } catch(std::exception &e) { - BOX_ERROR("Internal error writing store object " - "info file (" << StoreObjectInfoFile << "): " - << e.what()); + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": " << e.what()); } catch(...) { - BOX_ERROR("Internal error writing store object " - "info file (" << StoreObjectInfoFile << "): " - "unknown error"); + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": unknown error"); } return created; @@ -2847,13 +2641,17 @@ bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time // -------------------------------------------------------------------------- // // Function -// Name: BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime) -// Purpose: Deserializes remote directory and file information from a stream of bytes, using an Archive abstraction. -// +// Name: BackupDaemon::DeserializeStoreObjectInfo( +// box_time_t & theLastSyncTime, +// box_time_t & theNextSyncTime) +// Purpose: Deserializes remote directory and file information +// from a stream of bytes, using an Archive +// abstraction. // Created: 2005/04/11 // // -------------------------------------------------------------------------- -bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime) +bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, + box_time_t & theNextSyncTime) { // // @@ -2945,7 +2743,7 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ // // this is it, go at it // - anArchive.Read(aClientStoreMarker); + anArchive.Read(mClientStoreMarker); anArchive.Read(theLastSyncTime); anArchive.Read(theNextSyncTime); @@ -3023,7 +2821,7 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ DeleteAllLocations(); - aClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; theLastSyncTime = 0; theNextSyncTime = 0; @@ -3057,9 +2855,10 @@ bool BackupDaemon::DeleteStoreObjectInfo() const // Check to see if the file exists if(!FileExists(storeObjectInfoFile.c_str())) { - // File doesn't exist -- so can't be deleted. But something isn't quite right, so log a message - BOX_WARNING("Store object info file did not exist when it " - "was supposed to. (" << storeObjectInfoFile << ")"); + // File doesn't exist -- so can't be deleted. But something + // isn't quite right, so log a message + BOX_WARNING("StoreObjectInfoFile did not exist when it " + "was supposed to: " << storeObjectInfoFile); // Return true to stop things going around in a loop return true; @@ -3068,8 +2867,8 @@ bool BackupDaemon::DeleteStoreObjectInfo() const // Actually delete it if(::unlink(storeObjectInfoFile.c_str()) != 0) { - BOX_ERROR("Failed to delete the old store object info file: " - << storeObjectInfoFile << ": "<< strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to delete the old " + "StoreObjectInfoFile: " << storeObjectInfoFile); return false; } |