From c7662795f519d2b6797e4b1ac7fa4a22afa26310 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Thu, 27 Jul 2006 23:18:35 +0000 Subject: * merge - This is my current patch queue. I think that all of these are safe to apply. This is just under half of the pending changes in chris/general (the easy half). --- bin/bbackupctl/bbackupctl.cpp | 104 ++++++++-- bin/bbackupd/BackupClientContext.cpp | 2 +- bin/bbackupd/BackupClientDirectoryRecord.cpp | 98 +++++++--- bin/bbackupd/BackupDaemon.cpp | 282 +++++++++++++++++++-------- bin/bbackupd/BackupDaemon.h | 14 +- bin/bbackupd/Win32BackupService.cpp | 47 +++-- bin/bbackupd/Win32BackupService.h | 2 +- bin/bbackupd/Win32ServiceFunctions.cpp | 184 ++++++++++++----- bin/bbackupd/Win32ServiceFunctions.h | 6 +- bin/bbackupd/bbackupd.cpp | 56 +++--- bin/bbackupquery/BackupQueries.cpp | 168 ++++++++++++---- bin/bbackupquery/BackupQueries.h | 2 + bin/bbackupquery/documentation.txt | 1 + bin/bbstored/BBStoreDHousekeeping.cpp | 174 ++++++++++------- bin/bbstored/BackupCommands.cpp | 33 +++- bin/bbstored/BackupContext.cpp | 20 +- bin/bbstored/BackupStoreDaemon.cpp | 19 +- bin/bbstored/BackupStoreDaemon.h | 13 ++ bin/bbstored/HousekeepStoreAccount.cpp | 7 + 19 files changed, 895 insertions(+), 337 deletions(-) (limited to 'bin') diff --git a/bin/bbackupctl/bbackupctl.cpp b/bin/bbackupctl/bbackupctl.cpp index dfadc37d..37d5c377 100644 --- a/bin/bbackupctl/bbackupctl.cpp +++ b/bin/bbackupctl/bbackupctl.cpp @@ -32,12 +32,14 @@ void PrintUsageAndExit() { printf("Usage: bbackupctl [-q] [-c config_file] \n" "Commands are:\n" - " sync -- start a syncronisation run now\n" - " force-sync -- force the start of a syncronisation run, " + " sync -- start a synchronisation (backup) run now\n" + " force-sync -- force the start of a synchronisation run, " "even if SyncAllowScript says no\n" " reload -- reload daemon configuration\n" " terminate -- terminate daemon now\n" " wait-for-sync -- wait until the next sync starts, then exit\n" + " wait-for-end -- wait until the next sync finishes, then exit\n" + " sync-and-wait -- start sync, wait until it finishes, then exit\n" ); exit(1); } @@ -107,7 +109,10 @@ int main(int argc, const char *argv[]) // Check there's a socket defined in the config file if(!conf.KeyExists("CommandSocket")) { - printf("Daemon isn't using a control socket, could not execute command.\nAdd a CommandSocket declaration to the bbackupd.conf file.\n"); + printf("Daemon isn't using a control socket, " + "could not execute command.\n" + "Add a CommandSocket declaration to the " + "bbackupd.conf file.\n"); return 1; } @@ -188,27 +193,74 @@ int main(int argc, const char *argv[]) // Print summary? if(!quiet) { - printf("Daemon configuration summary:\n" \ - " AutomaticBackup = %s\n" \ - " UpdateStoreInterval = %d seconds\n" \ - " MinimumFileAge = %d seconds\n" \ - " MaxUploadWait = %d seconds\n", - autoBackup?"true":"false", updateStoreInterval, minimumFileAge, maxUploadWait); + printf("Daemon configuration summary:\n" + " AutomaticBackup = %s\n" + " UpdateStoreInterval = %d seconds\n" + " MinimumFileAge = %d seconds\n" + " MaxUploadWait = %d seconds\n", + autoBackup?"true":"false", updateStoreInterval, + minimumFileAge, maxUploadWait); + } + + std::string stateLine; + if(!getLine.GetLine(stateLine) || getLine.IsEOF()) + { +#if defined WIN32 && ! defined NDEBUG + syslog(LOG_ERR, "Failed to receive state line from daemon"); +#else + printf("Failed to receive state line from daemon\n"); +#endif + return 1; + } + + // Decode it + int currentState; + if(::sscanf(stateLine.c_str(), "state %d", ¤tState) != 1) + { + printf("State line didn't decode\n"); + return 1; } // Is the command the "wait for sync to start" command? bool areWaitingForSync = false; - if(::strcmp(argv[0], "wait-for-sync") == 0) + bool areWaitingForSyncEnd = false; + + if(::strcmp(argv[0], "wait-for-sync") == 0 || + ::strcmp(argv[0], "wait-for-end") == 0) { - // Check that it's not in non-automatic mode, because then it'll never start + // Check that it's not in non-automatic mode, + // because then it'll never start if(!autoBackup) { - printf("ERROR: Daemon is not in automatic mode -- sync will never start!\n"); + printf("ERROR: Daemon is not in automatic mode -- " + "sync will never start!\n"); return 1; } - // Yes... set the flag so we know what we're waiting for a sync to start - areWaitingForSync = true; + // Yes... set the flag so we know that + // we're waiting for a sync to start/end + if(::strcmp(argv[0], "wait-for-sync") == 0) + { + areWaitingForSync = true; + } + else if (::strcmp(argv[0], "wait-for-end") == 0) + { + areWaitingForSyncEnd = true; + } + } + else if (::strcmp(argv[0], "sync-and-wait") == 0) + { + // send a sync command + std::string cmd("force-sync\n"); + connection.Write(cmd.c_str(), cmd.size()); + connection.WriteAllBuffered(); + areWaitingForSyncEnd = true; + + if (currentState != 0) + { + printf("Waiting for current sync/error state " + "to finish...\n"); + } } else { @@ -220,6 +272,8 @@ int main(int argc, const char *argv[]) // Read the response std::string line; + bool syncIsRunning = false; + while(!getLine.IsEOF() && getLine.GetLine(line)) { if(areWaitingForSync) @@ -234,6 +288,28 @@ int main(int argc, const char *argv[]) break; } } + else if(areWaitingForSyncEnd) + { + if(line == "start-sync") + { + if (!quiet) printf("Sync started...\n"); + syncIsRunning = true; + } + else if(line == "finish-sync" && syncIsRunning) + { + if (!quiet) printf("Sync finished.\n"); + // Send a quit command to finish nicely + connection.Write("quit\n", 5); + + // And we're done + break; + } + else if(line == "finish-sync") + { + if (!quiet) printf("Previous sync finished.\n"); + } + // daemon must still be busy + } else { // Is this an OK or error line? diff --git a/bin/bbackupd/BackupClientContext.cpp b/bin/bbackupd/BackupClientContext.cpp index 25852b19..4b84fecb 100644 --- a/bin/bbackupd/BackupClientContext.cpp +++ b/bin/bbackupd/BackupClientContext.cpp @@ -176,7 +176,7 @@ BackupProtocolClient &BackupClientContext::GetConnection() // no -- flag so only things like deletions happen mStorageLimitExceeded = true; // Log - ::syslog(LOG_INFO, "Exceeded storage limits on server -- not uploading changes to files"); + ::syslog(LOG_WARNING, "Exceeded storage limits on server -- not uploading changes to files"); } } catch(...) diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index 5a566d84..37bd3fea 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -645,34 +645,75 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // Need to update? // // Condition for upload: - // modifiction time within sync period + // modification time within sync period // if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store - if( - ( - // Check the file modified within the acceptable time period we're checking - // If the file isn't on the server, the acceptable time starts at zero. - // Check pDirOnStore and en, because if we didn't download a directory listing, - // pDirOnStore will be zero, but we know it's on the server. - ( ((pDirOnStore != 0 && en == 0) || (modTime >= rParams.mSyncPeriodStart)) && modTime < rParams.mSyncPeriodEnd) - - // However, just in case things are continually modified, we check the first seen time. - // The two compares of syncPeriodEnd and pendingFirstSeenTime are because the values are unsigned. - || (pendingFirstSeenTime != 0 && - (rParams.mSyncPeriodEnd > pendingFirstSeenTime) - && ((rParams.mSyncPeriodEnd - pendingFirstSeenTime) > rParams.mMaxUploadWait)) - - // Then make sure that if files are added with a time less than the sync period start - // (which can easily happen on file server), it gets uploaded. The directory contents checksum - // will pick up the fact it has been added, so the store listing will be available when this happens. - || ((modTime <= rParams.mSyncPeriodStart) && (en != 0) && (en->GetModificationTime() != modTime)) - - // And just to catch really badly off clocks in the future for file server clients, - // just upload the file if it's madly in the future. - || (modTime > rParams.mUploadAfterThisTimeInTheFuture) - ) - // But even then, only upload it if the mod time locally is different to that on the server. - && (en == 0 || en->GetModificationTime() != modTime)) + + bool doUpload = false; + + // Only upload a file if the mod time locally is + // different to that on the server. + + if (en == 0 || en->GetModificationTime() != modTime) + { + // Check the file modified within the acceptable time period we're checking + // If the file isn't on the server, the acceptable time starts at zero. + // Check pDirOnStore and en, because if we didn't download a directory listing, + // pDirOnStore will be zero, but we know it's on the server. + if (modTime < rParams.mSyncPeriodEnd) + { + if (pDirOnStore != 0 && en == 0) + { + doUpload = true; + } + else if (modTime >= rParams.mSyncPeriodStart) + { + doUpload = true; + } + } + + // However, just in case things are continually + // modified, we check the first seen time. + // The two compares of syncPeriodEnd and + // pendingFirstSeenTime are because the values + // are unsigned. + + if (!doUpload && + pendingFirstSeenTime != 0 && + rParams.mSyncPeriodEnd > pendingFirstSeenTime && + (rParams.mSyncPeriodEnd - pendingFirstSeenTime) + > rParams.mMaxUploadWait) + { + doUpload = true; + } + + // Then make sure that if files are added with a + // time less than the sync period start + // (which can easily happen on file server), it + // gets uploaded. The directory contents checksum + // will pick up the fact it has been added, so the + // store listing will be available when this happens. + + if (!doUpload && + modTime <= rParams.mSyncPeriodStart && + en != 0 && + en->GetModificationTime() != modTime) + { + doUpload = true; + } + + // And just to catch really badly off clocks in + // the future for file server clients, + // just upload the file if it's madly in the future. + + if (!doUpload && modTime > + rParams.mUploadAfterThisTimeInTheFuture) + { + doUpload = true; + } + } + + if (doUpload) { // Make sure we're connected -- must connect here so we know whether // the storage limit has been exceeded, and hence whether or not @@ -1022,7 +1063,10 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP BackupClientDirectoryRecord *rec = e->second; mSubDirectories.erase(e); delete rec; - TRACE2("Deleted directory record for %s/%s\n", rLocalPath.c_str(), dirname.GetClearFilename().c_str()); + TRACE2("Deleted directory record for " + "%s" DIRECTORY_SEPARATOR "%s\n", + rLocalPath.c_str(), + dirname.GetClearFilename().c_str()); } } } diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp index ebf5a1ea..e2d7c5e7 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/bin/bbackupd/BackupDaemon.cpp @@ -125,6 +125,29 @@ BackupDaemon::BackupDaemon() } #ifdef WIN32 + // Create the event object to signal from main thread to worker + // when new messages are queued to be sent to the command socket. + mhMessageToSendEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if (mhMessageToSendEvent == INVALID_HANDLE_VALUE) + { + syslog(LOG_ERR, "Failed to create event object: error %d", + GetLastError); + exit(1); + } + + // Create the event object to signal from worker to main thread + // when a command has been received on the command socket. + mhCommandReceivedEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if (mhCommandReceivedEvent == INVALID_HANDLE_VALUE) + { + syslog(LOG_ERR, "Failed to create event object: error %d", + GetLastError); + exit(1); + } + + // Create the critical section to protect the message queue + InitializeCriticalSection(&mMessageQueueLock); + // Create a thread to handle the named pipe HANDLE hThread; unsigned int dwThreadId; @@ -263,15 +286,34 @@ void BackupDaemon::DeleteAllLocations() void BackupDaemon::RunHelperThread(void) { mpCommandSocketInfo = new CommandSocketInfo; - this->mReceivedCommandConn = false; + WinNamedPipeStream& rSocket(mpCommandSocketInfo->mListeningSocket); // loop until the parent process exits - while (TRUE) + while (!IsTerminateWanted()) { try { - mpCommandSocketInfo->mListeningSocket.Accept( - BOX_NAMED_PIPE_NAME); + rSocket.Accept(BOX_NAMED_PIPE_NAME); + } + catch (BoxException &e) + { + ::syslog(LOG_ERR, "Failed to open command socket: %s", + e.what()); + SetTerminateWanted(); + break; // this is fatal + } + catch (...) + { + ::syslog(LOG_ERR, "Failed to open command socket: " + "unknown error"); + SetTerminateWanted(); + break; // this is fatal + } + + try + { + // Errors here do not kill the thread, + // only the current connection. // This next section comes from Ben's original function // Log @@ -289,17 +331,72 @@ void BackupDaemon::RunHelperThread(void) conf.GetKeyValueInt("MaxUploadWait"), mState); - mpCommandSocketInfo->mListeningSocket.Write(summary, summarySize); - mpCommandSocketInfo->mListeningSocket.Write("ping\n", 5); + rSocket.Write(summary, summarySize); + rSocket.Write("ping\n", 5); + + // old queued messages are not useful + EnterCriticalSection(&mMessageQueueLock); + mMessageList.clear(); + ResetEvent(mhMessageToSendEvent); + LeaveCriticalSection(&mMessageQueueLock); - IOStreamGetLine readLine(mpCommandSocketInfo->mListeningSocket); + IOStreamGetLine readLine(rSocket); std::string command; - while (mpCommandSocketInfo->mListeningSocket.IsConnected() && - readLine.GetLine(command) ) + while (rSocket.IsConnected() && !IsTerminateWanted()) { - TRACE1("Receiving command '%s' over " - "command socket\n", command.c_str()); + HANDLE handles[2]; + handles[0] = mhMessageToSendEvent; + handles[1] = rSocket.GetReadableEvent(); + + DWORD result = WaitForMultipleObjects( + sizeof(handles)/sizeof(*handles), + handles, FALSE, 1000); + + if (result == 0) + { + ResetEvent(mhMessageToSendEvent); + + EnterCriticalSection(&mMessageQueueLock); + try + { + while (mMessageList.size() > 0) + { + std::string message = *(mMessageList.begin()); + mMessageList.erase(mMessageList.begin()); + printf("Sending '%s' to waiting client... ", message.c_str()); + message += "\n"; + rSocket.Write(message.c_str(), + message.length()); + + printf("done.\n"); + } + } + catch (...) + { + LeaveCriticalSection(&mMessageQueueLock); + throw; + } + LeaveCriticalSection(&mMessageQueueLock); + continue; + } + else if (result == WAIT_TIMEOUT) + { + continue; + } + else if (result != 1) + { + ::syslog(LOG_ERR, "WaitForMultipleObjects returned invalid result %d", result); + continue; + } + + if (!readLine.GetLine(command)) + { + ::syslog(LOG_ERR, "Failed to read line"); + continue; + } + + printf("Received command '%s' from client\n", command.c_str()); bool sendOK = false; bool sendResponse = true; @@ -338,12 +435,18 @@ void BackupDaemon::RunHelperThread(void) SetTerminateWanted(); sendOK = true; } + else + { + ::syslog(LOG_ERR, "Received unknown command '%s' from client", command.c_str()); + sendResponse = true; + sendOK = false; + } // Send a response back? if (sendResponse) { const char* response = sendOK ? "ok\n" : "error\n"; - mpCommandSocketInfo->mListeningSocket.Write( + rSocket.Write( response, strlen(response)); } @@ -352,10 +455,10 @@ void BackupDaemon::RunHelperThread(void) break; } - this->mReceivedCommandConn = true; + // this->mReceivedCommandConn = true; } - mpCommandSocketInfo->mListeningSocket.Close(); + rSocket.Close(); } catch (BoxException &e) { @@ -504,8 +607,8 @@ void BackupDaemon::Run2() BackupClientContext::ClientStoreMarker_NotKnown; // haven't contacted the store yet - bool deserialised = DeserializeStoreObjectInfo(clientStoreMarker, - lastSyncTime, nextSyncTime); + bool deleteStoreObjectInfoFile = DeserializeStoreObjectInfo( + clientStoreMarker, lastSyncTime, nextSyncTime); // -------------------------------------------------------------------------------------------- @@ -611,7 +714,8 @@ void BackupDaemon::Run2() // Delete the serialised store object file, // so that we don't try to reload it after a // partially completed backup - if(deserialised && !DeleteStoreObjectInfo()) + if(deleteStoreObjectInfoFile && + !DeleteStoreObjectInfo()) { ::syslog(LOG_ERR, "Failed to delete the " "StoreObjectInfoFile, backup cannot " @@ -621,6 +725,11 @@ void BackupDaemon::Run2() ::sleep(60); continue; } + + // In case the backup throws an exception, + // we should not try to delete the store info + // object file again. + deleteStoreObjectInfoFile = false; // Do sync bool errorOccurred = false; @@ -729,8 +838,14 @@ void BackupDaemon::Run2() // -------------------------------------------------------------------------------------------- - // We had a successful backup, save the store info - SerializeStoreObjectInfo(clientStoreMarker, lastSyncTime, nextSyncTime); + // We had a successful backup, save the store + // info. If we save successfully, we must + // delete the file next time we start a backup + + deleteStoreObjectInfoFile = + SerializeStoreObjectInfo( + clientStoreMarker, + lastSyncTime, nextSyncTime); // -------------------------------------------------------------------------------------------- } @@ -902,25 +1017,27 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) { #ifdef WIN32 - // Really could use some interprocess protection, mutex etc - // any side effect should be too bad???? :) - DWORD timeout = (DWORD)BoxTimeToMilliSeconds(RequiredDelay); + DWORD requiredDelayMs = BoxTimeToMilliSeconds(RequiredDelay); - while ( this->mReceivedCommandConn == false ) - { - Sleep(1); + DWORD result = WaitForSingleObject(mhCommandReceivedEvent, + (DWORD)requiredDelayMs); - if ( timeout == 0 ) - { - DoSyncFlagOut = false; - SyncIsForcedOut = false; - return; - } - timeout--; + if (result == WAIT_OBJECT_0) + { + DoSyncFlagOut = this->mDoSyncFlagOut; + SyncIsForcedOut = this->mSyncIsForcedOut; + ResetEvent(mhCommandReceivedEvent); + } + else if (result == WAIT_TIMEOUT) + { + DoSyncFlagOut = false; + SyncIsForcedOut = false; + } + else + { + ::syslog(LOG_ERR, "Unexpected result from " + "WaitForSingleObject: error %d", GetLastError()); } - this->mReceivedCommandConn = false; - DoSyncFlagOut = this->mDoSyncFlagOut; - SyncIsForcedOut = this->mSyncIsForcedOut; return; #else // ! WIN32 @@ -953,7 +1070,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla { #ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET bool uidOK = true; - ::syslog(LOG_WARNING, "On this platform, no security check can be made on the credientials of peers connecting to the command socket. (bbackupctl)"); + ::syslog(LOG_WARNING, "On this platform, no security check can be made on the credentials of peers connecting to the command socket. (bbackupctl)"); #else // Security check -- does the process connecting to this socket have // the same UID as this process? @@ -1023,8 +1140,13 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla while(mpCommandSocketInfo->mpGetLine != 0 && !mpCommandSocketInfo->mpGetLine->IsEOF() && mpCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) { - TRACE1("Receiving command '%s' over command socket\n", command.c_str()); - + TRACE1("Receiving command '%s' over command socket\n", + command.c_str()); + + #ifdef WIN32 + SetEvent(mhCommandReceivedEvent); + #endif + bool sendOK = false; bool sendResponse = true; @@ -1137,12 +1259,8 @@ void BackupDaemon::CloseCommandConnection() // -------------------------------------------------------------------------- void BackupDaemon::SendSyncStartOrFinish(bool SendStart) { - // The bbackupctl program can't rely on a state change, because it may never - // change if the server doesn't need to be contacted. - -#ifdef __MINGW32__ -#warning race condition: what happens if socket is closed? -#endif + // The bbackupctl program can't rely on a state change, because it + // may never change if the server doesn't need to be contacted. if (mpCommandSocketInfo != NULL && #ifdef WIN32 @@ -1152,15 +1270,18 @@ void BackupDaemon::SendSyncStartOrFinish(bool SendStart) #endif ) { - const char* message = SendStart ? "start-sync\n" : "finish-sync\n"; + std::string message = SendStart ? "start-sync" : "finish-sync"; try { #ifdef WIN32 - mpCommandSocketInfo->mListeningSocket.Write(message, - (int)strlen(message)); + EnterCriticalSection(&mMessageQueueLock); + mMessageList.push_back(message); + SetEvent(mhMessageToSendEvent); + LeaveCriticalSection(&mMessageQueueLock); #else - mpCommandSocketInfo->mpConnectedSocket->Write(message, - strlen(message)); + message += "\n"; + mpCommandSocketInfo->mpConnectedSocket->Write( + message.c_str(), message.size()); #endif } catch(...) @@ -1756,39 +1877,36 @@ void BackupDaemon::SetState(int State) // command socket if there's an error char newState[64]; - char newStateSize = sprintf(newState, "state %d\n", State); + sprintf(newState, "state %d", State); + std::string message = newState; #ifdef WIN32 - #ifndef _MSC_VER - #warning FIX ME: race condition - #endif + EnterCriticalSection(&mMessageQueueLock); + mMessageList.push_back(newState); + SetEvent(mhMessageToSendEvent); + LeaveCriticalSection(&mMessageQueueLock); +#else + message += "\n"; - // what happens if the socket is closed by the other thread before - // we can write to it? Null pointer deref at best. - if (mpCommandSocketInfo && - mpCommandSocketInfo->mListeningSocket.IsConnected()) + if(mpCommandSocketInfo == 0) { - try - { - mpCommandSocketInfo->mListeningSocket.Write(newState, newStateSize); - } - catch(...) - { - CloseCommandConnection(); - } + return; } -#else - if(mpCommandSocketInfo != 0 && mpCommandSocketInfo->mpConnectedSocket.get() != 0) + + if (mpCommandSocketInfo->mpConnectedSocket.get() == 0) { - // Something connected to the command socket, tell it about the new state - try - { - mpCommandSocketInfo->mpConnectedSocket->Write(newState, newStateSize); - } - catch(...) - { - CloseCommandConnection(); - } + return; + } + + // Something connected to the command socket, tell it about the new state + try + { + mpCommandSocketInfo->mpConnectedSocket->Write(message.c_str(), + message.length()); + } + catch(...) + { + CloseCommandConnection(); } #endif } @@ -2173,11 +2291,11 @@ static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; static const int STOREOBJECTINFO_VERSION = 1; -void BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const +bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const { if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) { - return; + return false; } std::string StoreObjectInfoFile = @@ -2185,13 +2303,17 @@ void BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time if (StoreObjectInfoFile.size() <= 0) { - return; + return false; } + bool created = false; + try { FileStream aFile(StoreObjectInfoFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC); + created = true; + Archive anArchive(aFile, 0); anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE); @@ -2236,6 +2358,8 @@ void BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time "not accessible or could not be created", StoreObjectInfoFile.c_str()); } + + return created; } // -------------------------------------------------------------------------- diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h index 3bd15fad..94a142f5 100644 --- a/bin/bbackupd/BackupDaemon.h +++ b/bin/bbackupd/BackupDaemon.h @@ -46,9 +46,12 @@ public: ~BackupDaemon(); private: - // methods below do partial (specialized) serialization of client state only - void SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const; - bool DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime); + // methods below do partial (specialized) serialization of + // client state only + bool SerializeStoreObjectInfo(int64_t aClientStoreMarker, + box_time_t theLastSyncTime, box_time_t theNextSyncTime) const; + bool DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, + box_time_t & theLastSyncTime, box_time_t & theNextSyncTime); bool DeleteStoreObjectInfo() const; BackupDaemon(const BackupDaemon &); public: @@ -182,7 +185,10 @@ private: void RunHelperThread(void); private: - bool mDoSyncFlagOut, mSyncIsForcedOut, mReceivedCommandConn; + bool mDoSyncFlagOut, mSyncIsForcedOut; + HANDLE mhMessageToSendEvent, mhCommandReceivedEvent; + CRITICAL_SECTION mMessageQueueLock; + std::vector mMessageList; #endif }; diff --git a/bin/bbackupd/Win32BackupService.cpp b/bin/bbackupd/Win32BackupService.cpp index aa3bf55c..7cbf4828 100644 --- a/bin/bbackupd/Win32BackupService.cpp +++ b/bin/bbackupd/Win32BackupService.cpp @@ -12,40 +12,51 @@ #include "Win32BackupService.h" -Win32BackupService gDaemonService; +Win32BackupService* gpDaemonService = NULL; extern HANDLE gStopServiceEvent; unsigned int WINAPI RunService(LPVOID lpParameter) { - DWORD retVal = gDaemonService.WinService(); - SetEvent( gStopServiceEvent ); + DWORD retVal = gpDaemonService->WinService((const char*) lpParameter); + SetEvent(gStopServiceEvent); return retVal; } void TerminateService(void) { - gDaemonService.SetTerminateWanted(); + gpDaemonService->SetTerminateWanted(); } -DWORD Win32BackupService::WinService(void) +DWORD Win32BackupService::WinService(const char* pConfigFileName) { - int argc = 2; - //first off get the path name for the default - char buf[MAX_PATH]; - - GetModuleFileName(NULL, buf, sizeof(buf)); - std::string buffer(buf); - std::string conf( "-c"); - std::string cfile(buffer.substr(0,(buffer.find("bbackupd.exe"))) - + "bbackupd.conf"); + char exepath[MAX_PATH]; + GetModuleFileName(NULL, exepath, sizeof(exepath)); - const char *argv[] = {conf.c_str(), cfile.c_str()}; + std::string configfile; + + if (pConfigFileName != NULL) + { + configfile = pConfigFileName; + } + else + { + // make the default config file name, + // based on the program path + configfile = exepath; + configfile = configfile.substr(0, + configfile.rfind(DIRECTORY_SEPARATOR_ASCHAR)); + configfile += DIRECTORY_SEPARATOR "bbackupd.conf"; + } + + const char *argv[] = {exepath, "-c", configfile.c_str()}; + int argc = sizeof(argv) / sizeof(*argv); + DWORD ret; MAINHELPER_START - - return this->Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); - + ret = this->Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); MAINHELPER_END + + return ret; } #endif // WIN32 diff --git a/bin/bbackupd/Win32BackupService.h b/bin/bbackupd/Win32BackupService.h index 38cebacc..e7f077f2 100644 --- a/bin/bbackupd/Win32BackupService.h +++ b/bin/bbackupd/Win32BackupService.h @@ -12,7 +12,7 @@ class BackupDaemon; class Win32BackupService : public BackupDaemon { public: - DWORD WinService(void); + DWORD WinService(const char* pConfigFileName); }; #endif // WIN32 diff --git a/bin/bbackupd/Win32ServiceFunctions.cpp b/bin/bbackupd/Win32ServiceFunctions.cpp index b74c7e09..1b50a7fa 100644 --- a/bin/bbackupd/Win32ServiceFunctions.cpp +++ b/bin/bbackupd/Win32ServiceFunctions.cpp @@ -93,28 +93,30 @@ void WINAPI ServiceControlHandler( DWORD controlCode ) // It also returns on any error because the // service cannot start if there is an eror. +static char* spConfigFileName; + VOID ServiceMain(DWORD argc, LPTSTR *argv) { - // initialise service status - gServiceStatus.dwServiceType = SERVICE_WIN32; - gServiceStatus.dwCurrentState = SERVICE_STOPPED; - gServiceStatus.dwControlsAccepted = 0; - gServiceStatus.dwWin32ExitCode = NO_ERROR; - gServiceStatus.dwServiceSpecificExitCode = NO_ERROR; - gServiceStatus.dwCheckPoint = 0; - gServiceStatus.dwWaitHint = 0; - - gServiceStatusHandle = RegisterServiceCtrlHandler(gServiceName, - ServiceControlHandler); - - if (gServiceStatusHandle) - { - // service is starting - gServiceStatus.dwCurrentState = SERVICE_START_PENDING; - SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - - // do initialisation here - gStopServiceEvent = CreateEvent( 0, TRUE, FALSE, 0 ); + // initialise service status + gServiceStatus.dwServiceType = SERVICE_WIN32; + gServiceStatus.dwCurrentState = SERVICE_STOPPED; + gServiceStatus.dwControlsAccepted = 0; + gServiceStatus.dwWin32ExitCode = NO_ERROR; + gServiceStatus.dwServiceSpecificExitCode = NO_ERROR; + gServiceStatus.dwCheckPoint = 0; + gServiceStatus.dwWaitHint = 0; + + gServiceStatusHandle = RegisterServiceCtrlHandler(gServiceName, + ServiceControlHandler); + + if (gServiceStatusHandle) + { + // service is starting + gServiceStatus.dwCurrentState = SERVICE_START_PENDING; + SetServiceStatus(gServiceStatusHandle, &gServiceStatus); + + // do initialisation here + gStopServiceEvent = CreateEvent(0, TRUE, FALSE, 0); if (!gStopServiceEvent) { gServiceStatus.dwControlsAccepted &= @@ -129,7 +131,7 @@ VOID ServiceMain(DWORD argc, LPTSTR *argv) NULL, 0, RunService, - 0, + spConfigFileName, CREATE_SUSPENDED, NULL); @@ -138,7 +140,7 @@ VOID ServiceMain(DWORD argc, LPTSTR *argv) // we are now running so tell the SCM gServiceStatus.dwControlsAccepted |= - (SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); + (SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); gServiceStatus.dwCurrentState = SERVICE_RUNNING; SetServiceStatus(gServiceStatusHandle, &gServiceStatus); @@ -156,11 +158,13 @@ VOID ServiceMain(DWORD argc, LPTSTR *argv) ~(SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN); gServiceStatus.dwCurrentState = SERVICE_STOPPED; SetServiceStatus(gServiceStatusHandle, &gServiceStatus); - } + } } -void OurService(void) +void OurService(char* pConfigFileName) { + spConfigFileName = pConfigFileName; + SERVICE_TABLE_ENTRY serviceTable[] = { { SERVICE_NAME, (LPSERVICE_MAIN_FUNCTION) ServiceMain }, @@ -179,28 +183,52 @@ void OurService(void) } } -void InstallService(void) +int InstallService(const char* pConfigFileName) { - SC_HANDLE newService, scm; + if (pConfigFileName != NULL) + { + struct stat st; - scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); + if (emu_stat(pConfigFileName, &st) != 0) + { + syslog(LOG_ERR, "Failed to open configuration file: " + "%s: %s", pConfigFileName, strerror(errno)); + return 1; + } + + if (! st.st_mode & S_IFREG) + { + + syslog(LOG_ERR, "Failed to open configuration file: " + "%s: not a file", pConfigFileName); + return 1; + } + } + + SC_HANDLE scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); if (!scm) { syslog(LOG_ERR, "Failed to open service control manager: " "error %d", GetLastError()); - return; + return 1; } char cmd[MAX_PATH]; GetModuleFileName(NULL, cmd, sizeof(cmd)-1); cmd[sizeof(cmd)-1] = 0; - char cmd_args[MAX_PATH]; - _snprintf(cmd_args, sizeof(cmd_args)-1, "%s --service", cmd); - cmd_args[sizeof(cmd_args)-1] = 0; + std::string cmdWithArgs(cmd); + cmdWithArgs += " --service"; - newService = CreateService( + if (pConfigFileName != NULL) + { + cmdWithArgs += " \""; + cmdWithArgs += pConfigFileName; + cmdWithArgs += "\""; + } + + SC_HANDLE newService = CreateService( scm, SERVICE_NAME, "Box Backup", @@ -208,14 +236,36 @@ void InstallService(void) SERVICE_WIN32_OWN_PROCESS, SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, - cmd_args, + cmdWithArgs.c_str(), 0,0,0,0,0); + DWORD err = GetLastError(); + CloseServiceHandle(scm); + if (!newService) { - ::syslog(LOG_ERR, "Failed to create Box Backup service: " - "error %d", GetLastError()); - return; + if (err == ERROR_SERVICE_EXISTS) + { + ::syslog(LOG_ERR, "Failed to create Box Backup " + "service: it already exists"); + } + else if (err == ERROR_SERVICE_MARKED_FOR_DELETE) + { + ::syslog(LOG_ERR, "Failed to create Box Backup " + "service: it is waiting to be deleted"); + } + else if (err == ERROR_DUPLICATE_SERVICE_NAME) + { + ::syslog(LOG_ERR, "Failed to create Box Backup " + "service: a service with this name " + "already exists"); + } + else + { + ::syslog(LOG_ERR, "Failed to create Box Backup " + "service: error %d", err); + } + return 1; } ::syslog(LOG_INFO, "Created Box Backup service"); @@ -231,45 +281,75 @@ void InstallService(void) } CloseServiceHandle(newService); - CloseServiceHandle(scm); + + return 0; } -void RemoveService(void) +int RemoveService(void) { - SC_HANDLE service, scm; - SERVICE_STATUS status; - - scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); + SC_HANDLE scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); if (!scm) { syslog(LOG_ERR, "Failed to open service control manager: " "error %d", GetLastError()); - return; + return 1; } - service = OpenService(scm, SERVICE_NAME, SERVICE_ALL_ACCESS|DELETE); - ControlService(service, SERVICE_CONTROL_STOP, &status); + SC_HANDLE service = OpenService(scm, SERVICE_NAME, + SERVICE_ALL_ACCESS|DELETE); + DWORD err = GetLastError(); + CloseServiceHandle(scm); if (!service) { - syslog(LOG_ERR, "Failed to open Box Backup service: " - "error %d", GetLastError()); - return; + if (err == ERROR_SERVICE_DOES_NOT_EXIST || + err == ERROR_IO_PENDING) + // hello microsoft? anyone home? + { + syslog(LOG_ERR, "Failed to open Box Backup service: " + "not installed or not found"); + } + else + { + syslog(LOG_ERR, "Failed to open Box Backup service: " + "error %d", err); + } + return 1; } - if (DeleteService(service)) + SERVICE_STATUS status; + if (!ControlService(service, SERVICE_CONTROL_STOP, &status)) + { + err = GetLastError(); + if (err != ERROR_SERVICE_NOT_ACTIVE) + { + syslog(LOG_WARNING, "Failed to stop Box Backup " + "service: error %d", err); + } + } + + BOOL deleted = DeleteService(service); + err = GetLastError(); + CloseServiceHandle(service); + + if (deleted) { syslog(LOG_INFO, "Box Backup service deleted"); + return 0; + } + else if (err == ERROR_SERVICE_MARKED_FOR_DELETE) + { + syslog(LOG_ERR, "Failed to remove Box Backup service: " + "it is already being deleted"); } else { syslog(LOG_ERR, "Failed to remove Box Backup service: " - "error %d", GetLastError()); + "error %d", err); } - CloseServiceHandle(service); - CloseServiceHandle(scm); + return 1; } #endif // WIN32 diff --git a/bin/bbackupd/Win32ServiceFunctions.h b/bin/bbackupd/Win32ServiceFunctions.h index 72f65479..3fe77187 100644 --- a/bin/bbackupd/Win32ServiceFunctions.h +++ b/bin/bbackupd/Win32ServiceFunctions.h @@ -12,8 +12,8 @@ #ifndef WIN32SERVICEFUNCTIONS_H #define WIN32SERVICEFUNCTIONS_H -void RemoveService(void); -void InstallService(void); -void OurService(void); +int RemoveService(void); +int InstallService(const char* pConfigFilePath); +void OurService(char* pConfigFileName); #endif diff --git a/bin/bbackupd/bbackupd.cpp b/bin/bbackupd/bbackupd.cpp index 089b2d09..e00d3628 100644 --- a/bin/bbackupd/bbackupd.cpp +++ b/bin/bbackupd/bbackupd.cpp @@ -19,7 +19,7 @@ #include "Win32ServiceFunctions.h" #include "Win32BackupService.h" - extern Win32BackupService gDaemonService; + extern Win32BackupService* gpDaemonService; #endif int main(int argc, const char *argv[]) @@ -28,7 +28,7 @@ int main(int argc, const char *argv[]) #ifdef WIN32 - ::openlog("Box Backup (bbackupd)", 0, 0); + ::openlog("Box Backup (bbackupd)", LOG_PID, LOG_LOCAL6); if(argc == 2 && (::strcmp(argv[1], "--help") == 0 || @@ -40,32 +40,25 @@ int main(int argc, const char *argv[]) } if(argc == 2 && ::strcmp(argv[1], "-r") == 0) { - RemoveService(); - return 0; + return RemoveService(); } - if(argc == 2 && ::strcmp(argv[1], "-i") == 0) + if((argc == 2 || argc == 3) && ::strcmp(argv[1], "-i") == 0) { - InstallService(); - return 0; + const char* config = NULL; + if (argc == 3) + { + config = argv[2]; + } + return InstallService(config); } bool runAsWin32Service = false; - if (argc == 2 && ::strcmp(argv[1], "--service") == 0) + if (argc >= 2 && ::strcmp(argv[1], "--service") == 0) { runAsWin32Service = true; } - - // Under win32 we must initialise the Winsock library - // before using sockets - - WSADATA info; - if (WSAStartup(0x0101, &info) == SOCKET_ERROR) - { - // box backup will not run without sockets - ::syslog(LOG_ERR, "Failed to initialise Windows Sockets"); - THROW_EXCEPTION(BackupStoreException, Internal) - } + gpDaemonService = new Win32BackupService(); EnableBackupRights(); @@ -73,20 +66,33 @@ int main(int argc, const char *argv[]) if (runAsWin32Service) { - syslog(LOG_INFO,"Starting Box Backup Service"); - OurService(); + syslog(LOG_INFO, "Box Backup service starting"); + + char* config = NULL; + if (argc >= 3) + { + config = strdup(argv[2]); + } + + OurService(config); + + if (config) + { + free(config); + } + + syslog(LOG_INFO, "Box Backup service shut down"); } else { - ExitCode = gDaemonService.Main( + ExitCode = gpDaemonService->Main( BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); } - // Clean up our sockets - WSACleanup(); - ::closelog(); + delete gpDaemonService; + return ExitCode; #else // !WIN32 diff --git a/bin/bbackupquery/BackupQueries.cpp b/bin/bbackupquery/BackupQueries.cpp index b7207720..05e34b7e 100644 --- a/bin/bbackupquery/BackupQueries.cpp +++ b/bin/bbackupquery/BackupQueries.cpp @@ -70,7 +70,11 @@ BackupQueries::BackupQueries(BackupProtocolClient &rConnection, const Configurat mWarnedAboutOwnerAttributes(false), mReturnCode(0) // default return code { + #ifdef WIN32 + mRunningAsRoot = TRUE; + #else mRunningAsRoot = (::geteuid() == 0); + #endif } // -------------------------------------------------------------------------- @@ -85,6 +89,12 @@ BackupQueries::~BackupQueries() { } +typedef struct cmd_info +{ + const char* name; + const char* opts; +} cmd_info_t; + // -------------------------------------------------------------------------- // // Function @@ -162,8 +172,24 @@ void BackupQueries::DoCommand(const char *Command) } // Data about commands - static const char *commandNames[] = {"quit", "exit", "list", "pwd", "cd", "lcd", "sh", "getobject", "get", "compare", "restore", "help", "usage", "undelete", 0}; - static const char *validOptions[] = {"", "", "rodIFtsh", "", "od", "", "", "", "i", "alcqE", "dri", "", "", "", 0}; + static cmd_info_t commands[] = + { + { "quit", "" }, + { "exit", "" }, + { "list", "rodIFtTsh", }, + { "pwd", "" }, + { "cd", "od" }, + { "lcd", "" }, + { "sh", "" }, + { "getobject", "" }, + { "get", "i" }, + { "compare", "alcqAE" }, + { "restore", "dri" }, + { "help", "" }, + { "usage", "" }, + { "undelete", "" }, + { NULL, NULL } + }; #define COMMAND_Quit 0 #define COMMAND_Exit 1 #define COMMAND_List 2 @@ -183,11 +209,11 @@ void BackupQueries::DoCommand(const char *Command) // Work out which command it is... int cmd = 0; - while(commandNames[cmd] != 0 && ::strcmp(cmdElements[0].c_str(), commandNames[cmd]) != 0) + while(commands[cmd].name != 0 && ::strcmp(cmdElements[0].c_str(), commands[cmd].name) != 0) { cmd++; } - if(commandNames[cmd] == 0) + if(commands[cmd].name == 0) { // Check for aliases int a; @@ -222,9 +248,10 @@ void BackupQueries::DoCommand(const char *Command) while(*c != 0) { // Valid option? - if(::strchr(validOptions[cmd], *c) == NULL) + if(::strchr(commands[cmd].opts, *c) == NULL) { - printf("Invalid option '%c' for command %s\n", *c, commandNames[cmd]); + printf("Invalid option '%c' for command %s\n", + *c, commands[cmd].name); return; } opts[(int)*c] = true; @@ -319,8 +346,9 @@ void BackupQueries::CommandList(const std::vector &args, const bool #define LIST_OPTION_ALLOWOLD 'o' #define LIST_OPTION_ALLOWDELETED 'd' #define LIST_OPTION_NOOBJECTID 'I' - #define LIST_OPTION_NOFLAGS 'F' - #define LIST_OPTION_TIMES 't' + #define LIST_OPTION_NOFLAGS 'F' + #define LIST_OPTION_TIMES_LOCAL 't' + #define LIST_OPTION_TIMES_UTC 'T' #define LIST_OPTION_SIZEINBLOCKS 's' #define LIST_OPTION_DISPLAY_HASH 'h' @@ -362,7 +390,7 @@ void BackupQueries::CommandList(const std::vector &args, const bool // -------------------------------------------------------------------------- // // Function -// Name: BackupQueries::CommandList2(int64_t, const std::string &, const bool *) +// Name: BackupQueries::List(int64_t, const std::string &, const bool *, bool) // Purpose: Do the actual listing of directories and files // Created: 2003/10/10 // @@ -437,9 +465,9 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool } } - if(opts[LIST_OPTION_TIMES]) + if(opts[LIST_OPTION_TIMES_LOCAL]) { - // Show times... + // Show local times... std::string time = BoxTimeToISO8601String( en->GetModificationTime()); printf("%s ", time.c_str()); @@ -844,13 +872,44 @@ void BackupQueries::CommandGet(const std::vector &args, const bool } // Find object ID somehow - int64_t id; + int64_t fileId; + int64_t dirId = GetCurrentDirectoryID(); std::string localName; + // BLOCK { +#ifdef WIN32 + std::string fileName; + if(!ConvertConsoleToUtf8(args[0].c_str(), fileName)) + return; +#else + std::string fileName(args[0]); +#endif + + if(!opts['i']) + { + // does this remote filename include a path? + std::string::size_type index = fileName.rfind('/'); + if(index != std::string::npos) + { + std::string dirName(fileName.substr(0, index)); + fileName = fileName.substr(index + 1); + + dirId = FindDirectoryObjectID(dirName); + if(dirId == 0) + { + printf("Directory '%s' not found\n", + dirName.c_str()); + return; + } + } + } + + BackupStoreFilenameClear fn(fileName); + // Need to look it up in the current directory mrConnection.QueryListDirectory( - GetCurrentDirectoryID(), + dirId, BackupProtocolClientListDirectory::Flags_File, // just files (opts['i'])?(BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING):(BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted), // only current versions false /* don't want attributes */); @@ -863,17 +922,23 @@ void BackupQueries::CommandGet(const std::vector &args, const bool if(opts['i']) { // Specified as ID. - id = ::strtoll(args[0].c_str(), 0, 16); - if(id == std::numeric_limits::min() || id == std::numeric_limits::max() || id == 0) + fileId = ::strtoll(args[0].c_str(), 0, 16); + if(fileId == std::numeric_limits::min() || + fileId == std::numeric_limits::max() || + fileId == 0) { printf("Not a valid object ID (specified in hex)\n"); return; } // Check that the item is actually in the directory - if(dir.FindEntryByID(id) == 0) + if(dir.FindEntryByID(fileId) == 0) { - printf("ID '%08llx' not found in current directory on store.\n(You can only download objects by ID from the current directory.)\n", id); + printf("ID '%08llx' not found in current " + "directory on store.\n" + "(You can only download objects by ID " + "from the current directory.)\n", + fileId); return; } @@ -884,26 +949,22 @@ void BackupQueries::CommandGet(const std::vector &args, const bool { // Specified by name, find the object in the directory to get the ID BackupStoreDirectory::Iterator i(dir); -#ifdef WIN32 - std::string fileName; - if(!ConvertConsoleToUtf8(args[0].c_str(), fileName)) - return; - BackupStoreFilenameClear fn(fileName); -#else - BackupStoreFilenameClear fn(args[0]); -#endif BackupStoreDirectory::Entry *en = i.FindMatchingClearName(fn); if(en == 0) { - printf("Filename '%s' not found in current directory on store.\n(Subdirectories in path not searched.)\n", args[0].c_str()); + printf("Filename '%s' not found in current " + "directory on store.\n" + "(Subdirectories in path not " + "searched.)\n", args[0].c_str()); return; } - id = en->GetObjectID(); + fileId = en->GetObjectID(); - // Local name is the last argument, which is either the looked up filename, or - // a filename specified by the user. + // Local name is the last argument, which is either + // the looked up filename, or a filename specified + // by the user. localName = args[args.size() - 1]; } } @@ -920,7 +981,7 @@ void BackupQueries::CommandGet(const std::vector &args, const bool try { // Request object - mrConnection.QueryGetFile(GetCurrentDirectoryID(), id); + mrConnection.QueryGetFile(dirId, fileId); // Stream containing encoded file std::auto_ptr objectStream(mrConnection.ReceiveStream()); @@ -929,7 +990,7 @@ void BackupQueries::CommandGet(const std::vector &args, const bool BackupStoreFile::DecodeFile(*objectStream, localName.c_str(), mrConnection.GetTimeout()); // Done. - printf("Object ID %08llx fetched sucessfully.\n", id); + printf("Object ID %08llx fetched sucessfully.\n", fileId); } catch(...) { @@ -950,8 +1011,10 @@ void BackupQueries::CommandGet(const std::vector &args, const bool BackupQueries::CompareParams::CompareParams() : mQuickCompare(false), mIgnoreExcludes(false), + mIgnoreAttributes(false), mDifferences(0), mDifferencesExplainedByModTime(0), + mUncheckedFiles(0), mExcludedDirs(0), mExcludedFiles(0), mpExcludeFiles(0), @@ -1012,6 +1075,7 @@ void BackupQueries::CommandCompare(const std::vector &args, const b BackupQueries::CompareParams params; params.mQuickCompare = opts['q']; params.mIgnoreExcludes = opts['E']; + params.mIgnoreAttributes = opts['A']; // Try and work out the time before which all files should be on the server { @@ -1074,13 +1138,29 @@ void BackupQueries::CommandCompare(const std::vector &args, const b return; } - printf("\n[ %d (of %d) differences probably due to file modifications after the last upload ]\nDifferences: %d (%d dirs excluded, %d files excluded)\n", - params.mDifferencesExplainedByModTime, params.mDifferences, params.mDifferences, params.mExcludedDirs, params.mExcludedFiles); + printf("\n[ %d (of %d) differences probably due to file " + "modifications after the last upload ]\n" + "Differences: %d (%d dirs excluded, %d files excluded, " + "%d files not checked)\n", + params.mDifferencesExplainedByModTime, params.mDifferences, + params.mDifferences, params.mExcludedDirs, + params.mExcludedFiles, params.mUncheckedFiles); // Set return code? if(opts['c']) { - SetReturnCode((params.mDifferences == 0)?COMPARE_RETURN_SAME:COMPARE_RETURN_DIFFERENT); + if (params.mUncheckedFiles != 0) + { + SetReturnCode(COMPARE_RETURN_ERROR); + } + else if (params.mDifferences != 0) + { + SetReturnCode(COMPARE_RETURN_DIFFERENT); + } + else + { + SetReturnCode(COMPARE_RETURN_SAME); + } } } @@ -1214,11 +1294,13 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s "(compared to server directory '%s')\n", localDirDisplay.c_str(), storeDirDisplay.c_str()); + rParams.mDifferences ++; } else { printf("ERROR: stat on local dir '%s'\n", localDirDisplay.c_str()); + rParams.mUncheckedFiles ++; } return; } @@ -1268,6 +1350,7 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s { printf("ERROR: opendir on local dir '%s'\n", localDirDisplay.c_str()); + rParams.mUncheckedFiles ++; return; } try @@ -1455,11 +1538,12 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s } // Compare attributes - BackupClientFileAttributes localAttr; box_time_t fileModTime = 0; + BackupClientFileAttributes localAttr; localAttr.ReadAttributes(localPath.c_str(), false /* don't zero mod times */, &fileModTime); modifiedAfterLastSync = (fileModTime > rParams.mLatestFileUploadTime); - if(!localAttr.Compare(fileOnServerStream->GetAttributes(), + if(!rParams.mIgnoreAttributes && + !localAttr.Compare(fileOnServerStream->GetAttributes(), true /* ignore attr mod time */, fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) { @@ -1554,10 +1638,12 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s e.GetType(), e.GetSubType(), storePathDisplay.c_str()); + rParams.mUncheckedFiles ++; } catch(...) - { + { printf("ERROR: (unknown) during file fetch and comparison for '%s'\n", storePathDisplay.c_str()); + rParams.mUncheckedFiles ++; } // Remove from set so that we know it's been compared @@ -1793,6 +1879,14 @@ void BackupQueries::CommandRestore(const std::vector &args, const b printf("The target directory exists. You cannot restore over an existing directory.\n"); break; + #ifdef WIN32 + case Restore_TargetPathNotFound: + printf("The target directory path does not exist.\n" + "To restore to a directory whose parent " + "does not exist, create the parent first.\n"); + break; + #endif + default: printf("ERROR: Unknown restore result.\n"); break; diff --git a/bin/bbackupquery/BackupQueries.h b/bin/bbackupquery/BackupQueries.h index e84de6ab..42fd54fe 100644 --- a/bin/bbackupquery/BackupQueries.h +++ b/bin/bbackupquery/BackupQueries.h @@ -68,8 +68,10 @@ private: void DeleteExcludeLists(); bool mQuickCompare; bool mIgnoreExcludes; + bool mIgnoreAttributes; int mDifferences; int mDifferencesExplainedByModTime; + int mUncheckedFiles; int mExcludedDirs; int mExcludedFiles; const ExcludeList *mpExcludeFiles; diff --git a/bin/bbackupquery/documentation.txt b/bin/bbackupquery/documentation.txt index 429caabe..0fb0bacb 100644 --- a/bin/bbackupquery/documentation.txt +++ b/bin/bbackupquery/documentation.txt @@ -104,6 +104,7 @@ compare -c -- set return code -q -- quick compare. Only checks file contents against checksums, doesn't do a full download + -A -- ignore attribute differences -E -- ignore exclusion settings Comparing with the root directory is an error, use -a option instead. diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index d3656630..86b2468b 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -10,7 +10,10 @@ #include "Box.h" #include + +#ifdef HAVE_SYSLOG_H #include +#endif #include "BackupStoreDaemon.h" #include "BackupStoreAccountDatabase.h" @@ -29,95 +32,128 @@ // Created: 11/12/03 // // -------------------------------------------------------------------------- +void BackupStoreDaemon::HousekeepingInit() +{ + + mLastHousekeepingRun = 0; +} + +#ifndef WIN32 void BackupStoreDaemon::HousekeepingProcess() { + HousekeepingInit(); + // Get the time between housekeeping runs const Configuration &rconfig(GetConfiguration()); int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); - - int64_t lastHousekeepingRun = 0; while(!StopRun()) { - // Time now + RunHousekeepingIfNeeded(); + + // Calculate how long should wait before doing the next housekeeping run int64_t timeNow = GetCurrentBoxTime(); - // Do housekeeping if the time interval has elapsed since the last check - if((timeNow - lastHousekeepingRun) >= housekeepingInterval) - { - // Store the time - lastHousekeepingRun = timeNow; - ::syslog(LOG_INFO, "Starting housekeeping"); + time_t secondsToGo = BoxTimeToSeconds((mLastHousekeepingRun + housekeepingInterval) - timeNow); + if(secondsToGo < 1) secondsToGo = 1; + if(secondsToGo > 60) secondsToGo = 60; + int32_t millisecondsToGo = ((int)secondsToGo) * 1000; + + // Check to see if there's any message pending + CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); + } +} +#endif - // Get the list of accounts - std::vector accounts; - if(mpAccountDatabase) - { - mpAccountDatabase->GetAllAccountIDs(accounts); - } +void BackupStoreDaemon::RunHousekeepingIfNeeded() +{ + // Get the time between housekeeping runs + const Configuration &rconfig(GetConfiguration()); + int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping")); + + // Time now + int64_t timeNow = GetCurrentBoxTime(); + // Do housekeeping if the time interval has elapsed since the last check + if((timeNow - mLastHousekeepingRun) < housekeepingInterval) + { + return; + } + + // Store the time + mLastHousekeepingRun = timeNow; + ::syslog(LOG_INFO, "Starting housekeeping"); + + // Get the list of accounts + std::vector accounts; + if(mpAccountDatabase) + { + mpAccountDatabase->GetAllAccountIDs(accounts); + } - SetProcessTitle("housekeeping, active"); + SetProcessTitle("housekeeping, active"); - // Check them all - for(std::vector::const_iterator i = accounts.begin(); i != accounts.end(); ++i) + // Check them all + for(std::vector::const_iterator i = accounts.begin(); i != accounts.end(); ++i) + { + try + { + if(mpAccounts) { - try - { - if(mpAccounts) - { - // Get the account root - std::string rootDir; - int discSet = 0; - mpAccounts->GetAccountRoot(*i, rootDir, discSet); - - // Do housekeeping on this account - HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this); - housekeeping.DoHousekeeping(); - } - } - catch(BoxException &e) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", - *i, e.what(), e.GetType(), e.GetSubType()); - } - catch(std::exception &e) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", - *i, e.what()); - } - catch(...) - { - ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", - *i); - } + // Get the account root + std::string rootDir; + int discSet = 0; + mpAccounts->GetAccountRoot(*i, rootDir, discSet); - // Check to see if there's any message pending - CheckForInterProcessMsg(0 /* no account */); - - // Stop early? - if(StopRun()) - { - break; - } + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this); + housekeeping.DoHousekeeping(); } - - ::syslog(LOG_INFO, "Finished housekeeping"); } - - // Placed here for accuracy, if StopRun() is true, for example. - SetProcessTitle("housekeeping, idle"); - - // Calculate how long should wait before doing the next housekeeping run - timeNow = GetCurrentBoxTime(); - time_t secondsToGo = BoxTimeToSeconds((lastHousekeepingRun + housekeepingInterval) - timeNow); - if(secondsToGo < 1) secondsToGo = 1; - if(secondsToGo > 60) secondsToGo = 60; - int32_t millisecondsToGo = ((int)secondsToGo) * 1000; + catch(BoxException &e) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", + *i, e.what(), e.GetType(), e.GetSubType()); + } + catch(std::exception &e) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", + *i, e.what()); + } + catch(...) + { + ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", + *i); + } +#ifndef WIN32 // Check to see if there's any message pending - CheckForInterProcessMsg(0 /* no account */, millisecondsToGo); + CheckForInterProcessMsg(0 /* no account */); +#endif + + // Stop early? + if(StopRun()) + { + break; + } } + + ::syslog(LOG_INFO, "Finished housekeeping"); + + // Placed here for accuracy, if StopRun() is true, for example. + SetProcessTitle("housekeeping, idle"); } +#ifdef WIN32 +void BackupStoreDaemon::OnIdle() +{ + if (!mHousekeepingInited) + { + HousekeepingInit(); + mHousekeepingInited = true; + } + + RunHousekeepingIfNeeded(); +} +#endif // -------------------------------------------------------------------------- // @@ -128,6 +164,7 @@ void BackupStoreDaemon::HousekeepingProcess() // Created: 11/12/03 // // -------------------------------------------------------------------------- +#ifndef WIN32 bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitTime) { // First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate. @@ -171,5 +208,6 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT return false; } +#endif diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index 35bc095d..845903b2 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -9,7 +9,12 @@ #include "Box.h" +#ifdef HAVE_SYSLOG_H #include +#endif + +#include +#include #include "autogen_BackupProtocolServer.h" #include "BackupConstants.h" @@ -327,8 +332,15 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto std::auto_ptr diff2(rContext.OpenObject(patchID)); // Choose a temporary filename for the result of the combination - std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp", - p + 16 /* rotate which disc it's on */)); +#ifdef WIN32 + std::ostringstream fs(rContext.GetStoreRoot()); + fs << ".recombinetemp."; + fs << p; + std::string tempFn(fs.str()); + tempFn = RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), tempFn, p + 16); +#else + std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(rContext.GetStoreDiscSet(), rContext.GetStoreRoot() + ".recombinetemp", p + 16 /* rotate which disc it's on */)); +#endif // Open the temporary file std::auto_ptr combined; @@ -336,14 +348,23 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto { { // Write nastily to allow this to work with gcc 2.x +#ifdef WIN32 + combined.reset(new FileStream( + tempFn.c_str(), + O_RDWR | O_CREAT | O_EXCL | + O_BINARY | O_TRUNC)); +#else std::auto_ptr t(new FileStream(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL)); combined = t; +#endif } +#ifndef WIN32 // Unlink immediately as it's a temporary file if(::unlink(tempFn.c_str()) != 0) { THROW_EXCEPTION(CommonException, OSFileError); } +#endif } catch(...) { @@ -359,6 +380,9 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto combined->Seek(0, IOStream::SeekType_Absolute); // Then shuffle round for the next go +#ifdef WIN32 + if (from.get()) from->Close(); +#endif from = combined; } @@ -396,8 +420,9 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto stream = t; } - // Object will be deleted when the stream is deleted, so can release the object auto_ptr here to - // avoid premature deletiong + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid + // premature deletion object.release(); } diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp index c796c13a..cd17812c 100644 --- a/bin/bbstored/BackupContext.cpp +++ b/bin/bbstored/BackupContext.cpp @@ -132,6 +132,7 @@ bool BackupContext::AttemptToGetWriteLock() // Request the lock bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); +#ifndef WIN32 if(!gotLock) { // The housekeeping process might have the thing open -- ask it to stop @@ -150,6 +151,7 @@ bool BackupContext::AttemptToGetWriteLock() } while(!gotLock && tries > 0); } +#endif if(gotLock) { @@ -453,13 +455,21 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod try { // Open it twice +#ifdef WIN32 + FileStream diff(tempFn.c_str(), + O_RDWR | O_CREAT | O_BINARY); + FileStream diff2(tempFn.c_str(), + O_RDWR | O_BINARY); +#else FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); FileStream diff2(tempFn.c_str(), O_RDONLY); - // Unlink it immediately, so it definately goes away + + // Unlink it immediately, so it definitely goes away if(::unlink(tempFn.c_str()) != 0) { THROW_EXCEPTION(CommonException, OSFileError); } +#endif // Stream the incoming diff to this temporary file if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) @@ -508,6 +518,14 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod ::unlink(tempFn.c_str()); throw; } + +#ifdef WIN32 + // we can't delete the file while it's open, above + if(::unlink(tempFn.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } +#endif } // Get the blocks used diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 2752893a..24a81ceb 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -11,9 +11,12 @@ #include #include -#include #include +#ifdef HAVE_SYSLOG_H +#include +#endif + #include "BackupContext.h" #include "BackupStoreDaemon.h" #include "BackupStoreConfigVerify.h" @@ -39,7 +42,11 @@ BackupStoreDaemon::BackupStoreDaemon() mExtendedLogging(false), mHaveForkedHousekeeping(false), mIsHousekeepingProcess(false), +#ifdef WIN32 + mHousekeepingInited(false) +#else mInterProcessComms(mInterProcessCommsSocket) +#endif { } @@ -156,6 +163,7 @@ void BackupStoreDaemon::Run() const Configuration &config(GetConfiguration()); mExtendedLogging = config.GetKeyValueBool("ExtendedLogging"); +#ifndef WIN32 // Fork off housekeeping daemon -- must only do this the first time Run() is called if(!mHaveForkedHousekeeping) { @@ -219,9 +227,11 @@ void BackupStoreDaemon::Run() } else { +#endif // !WIN32 // In server process -- use the base class to do the magic ServerTLS::Run(); +#ifndef WIN32 // Why did it stop? Tell the housekeeping process to do the same if(IsReloadConfigWanted()) { @@ -232,6 +242,7 @@ void BackupStoreDaemon::Run() mInterProcessCommsSocket.Write("t\n", 2); } } +#endif } @@ -297,6 +308,8 @@ void BackupStoreDaemon::LogConnectionStats(const char *commonName, // Log the amount of data transferred ::syslog(LOG_INFO, "Connection statistics for %s: " "IN=%lld OUT=%lld TOTAL=%lld\n", commonName, - s.GetBytesRead(), s.GetBytesWritten(), - s.GetBytesRead() + s.GetBytesWritten()); + (long long)s.GetBytesRead(), + (long long)s.GetBytesWritten(), + (long long)s.GetBytesRead() + + (long long)s.GetBytesWritten()); } diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 2fbe486d..857a9356 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -38,11 +38,13 @@ private: BackupStoreDaemon(const BackupStoreDaemon &rToCopy); public: +#ifndef WIN32 // For BackupContext to comminicate with housekeeping process void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) { mInterProcessCommsSocket.Write(Msg, MsgLen); } +#endif protected: @@ -57,9 +59,11 @@ protected: const ConfigurationVerify *GetConfigVerify() const; +#ifndef WIN32 // Housekeeping functions void HousekeepingProcess(); bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0); +#endif void LogConnectionStats(const char *commonName, const SocketStreamTLS &s); @@ -70,8 +74,17 @@ private: bool mHaveForkedHousekeeping; bool mIsHousekeepingProcess; +#ifdef WIN32 + virtual void OnIdle(); + bool mHousekeepingInited; +#else SocketStream mInterProcessCommsSocket; IOStreamGetLine mInterProcessComms; +#endif + + void HousekeepingInit(); + void RunHousekeepingIfNeeded(); + int64_t mLastHousekeepingRun; }; diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp index 4aa1999e..91945306 100644 --- a/bin/bbstored/HousekeepStoreAccount.cpp +++ b/bin/bbstored/HousekeepStoreAccount.cpp @@ -225,6 +225,7 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // -------------------------------------------------------------------------- bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { +#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -235,6 +236,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) return false; } } +#endif // Get the filename std::string objectFilename; @@ -251,6 +253,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Read the directory in BackupStoreDirectory dir; dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + dirStream->Close(); // Is it empty? if(dir.GetNumberOfEntries() == 0) @@ -485,6 +488,7 @@ bool HousekeepStoreAccount::DeleteFiles() // (there is likely to be more in the set than should be actually deleted). for(std::set::iterator i(mPotentialDeletions.begin()); i != mPotentialDeletions.end(); ++i) { +#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -495,6 +499,7 @@ bool HousekeepStoreAccount::DeleteFiles() return true; } } +#endif // Load up the directory it's in // Get the filename @@ -729,6 +734,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() // Go through list for(std::vector::const_iterator i(mEmptyDirectories.begin()); i != mEmptyDirectories.end(); ++i) { +#ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; @@ -739,6 +745,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() return true; } } +#endif // Do not delete the root directory if(*i == BACKUPSTORE_ROOT_DIRECTORY_ID) -- cgit v1.2.3