diff options
160 files changed, 12057 insertions, 2712 deletions
diff --git a/LICENSE.txt b/LICENSE.txt index fc3f9f3e..d08aa509 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,18 +1,24 @@ - -Copyright (c) 2003, 2005 - Ben Summers. All rights reserved. +Box Backup, http://www.fluffy.co.uk/boxbackup + +Copyright (c) 2003-2007 Ben Summers and contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + 3. All use of this software and associated advertising materials must display the following acknowledgement: - This product includes software developed by Ben Summers. + + This product includes software developed by Ben Summers and + contributors. + 4. The names of the Authors may not be used to endorse or promote products derived from this software without specific prior written permission. @@ -32,6 +38,3 @@ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -
\ No newline at end of file diff --git a/bin/bbackupctl/bbackupctl.cpp b/bin/bbackupctl/bbackupctl.cpp index 30b5def8..edbc252f 100644 --- a/bin/bbackupctl/bbackupctl.cpp +++ b/bin/bbackupctl/bbackupctl.cpp @@ -56,17 +56,23 @@ int main(int argc, const char *argv[]) { int returnCode = 0; -#if defined WIN32 && ! defined NDEBUG - ::openlog("Box Backup (bbackupctl)", 0, 0); -#endif - MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbackupctl.memleaks", "bbackupctl") MAINHELPER_START +#if defined WIN32 && ! defined NDEBUG + ::openlog("Box Backup (bbackupctl)", 0, 0); +#endif + // Filename for configuration file? - const char *configFilename = BOX_FILE_BBACKUPD_DEFAULT_CONFIG; + std::string configFilename; + + #ifdef WIN32 + configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; + #else + configFilename = BOX_FILE_BBACKUPD_DEFAULT_CONFIG; + #endif // Quiet? bool quiet = false; @@ -103,12 +109,16 @@ int main(int argc, const char *argv[]) } // Read in the configuration file - if(!quiet) printf("Using configuration file %s\n", configFilename); + if(!quiet) BOX_NOTICE("Using configuration file " << configFilename); + std::string errs; - std::auto_ptr<Configuration> config(Configuration::LoadAndVerify(configFilename, &BackupDaemonConfigVerify, errs)); + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + (configFilename, &BackupDaemonConfigVerify, errs)); + if(config.get() == 0 || !errs.empty()) { - printf("Invalid configuration file:\n%s", errs.c_str()); + BOX_ERROR("Invalid configuration file: " << errs); return 1; } // Easier coding @@ -117,10 +127,10 @@ int main(int argc, const char *argv[]) // Check there's a socket defined in the config file if(!conf.KeyExists("CommandSocket")) { - printf("Daemon isn't using a control socket, " + BOX_ERROR("Daemon isn't using a control socket, " "could not execute command.\n" "Add a CommandSocket declaration to the " - "bbackupd.conf file.\n"); + "bbackupd.conf file."); return 1; } @@ -142,18 +152,14 @@ int main(int argc, const char *argv[]) } catch(...) { - printf("Failed to connect to daemon control socket.\n" + BOX_ERROR("Failed to connect to daemon control socket.\n" "Possible causes:\n" " * Daemon not running\n" " * Daemon busy syncing with store server\n" " * Another bbackupctl process is communicating with the daemon\n" - " * Daemon is waiting to recover from an error\n" + " * Daemon is waiting to recover from an error" ); -#if defined WIN32 && ! defined NDEBUG - syslog(LOG_ERR,"Failed to connect to the command socket"); -#endif - return 1; } @@ -164,29 +170,16 @@ int main(int argc, const char *argv[]) std::string configSummary; if(!getLine.GetLine(configSummary)) { -#if defined WIN32 && ! defined NDEBUG - syslog(LOG_ERR, "Failed to receive configuration summary " + BOX_ERROR("Failed to receive configuration summary " "from daemon"); -#else - printf("Failed to receive configuration summary from daemon\n"); -#endif - return 1; } // Was the connection rejected by the server? if(getLine.IsEOF()) { -#if defined WIN32 && ! defined NDEBUG - syslog(LOG_ERR, "Server rejected the connection. " - "Are you running bbackupctl as the same user " - "as the daemon?"); -#else - printf("Server rejected the connection. " - "Are you running bbackupctl as the same user " - "as the daemon?\n"); -#endif - + BOX_ERROR("Server rejected the connection. Are you running " + "bbackupctl as the same user as the daemon?"); return 1; } @@ -195,29 +188,25 @@ int main(int argc, const char *argv[]) if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", &autoBackup, &updateStoreInterval, &minimumFileAge, &maxUploadWait) != 4) { - printf("Config summary didn't decode\n"); + BOX_ERROR("Config summary didn't decode."); return 1; } // Print summary? if(!quiet) { - printf("Daemon configuration summary:\n" - " AutomaticBackup = %s\n" - " UpdateStoreInterval = %d seconds\n" - " MinimumFileAge = %d seconds\n" - " MaxUploadWait = %d seconds\n", - autoBackup?"true":"false", updateStoreInterval, - minimumFileAge, maxUploadWait); + BOX_INFO("Daemon configuration summary:\n" + " AutomaticBackup = " << + (autoBackup?"true":"false") << "\n" + " UpdateStoreInterval = " << updateStoreInterval << + " seconds\n" + " MinimumFileAge = " << minimumFileAge << " seconds\n" + " MaxUploadWait = " << maxUploadWait << " seconds\n"); } std::string stateLine; if(!getLine.GetLine(stateLine) || getLine.IsEOF()) { -#if defined WIN32 && ! defined NDEBUG - syslog(LOG_ERR, "Failed to receive state line from daemon"); -#else - printf("Failed to receive state line from daemon\n"); -#endif + BOX_ERROR("Failed to receive state line from daemon"); return 1; } @@ -225,7 +214,7 @@ int main(int argc, const char *argv[]) int currentState; if(::sscanf(stateLine.c_str(), "state %d", ¤tState) != 1) { - printf("State line didn't decode\n"); + BOX_ERROR("Received invalid state line from daemon"); return 1; } @@ -255,8 +244,8 @@ int main(int argc, const char *argv[]) if(!autoBackup) { - printf("ERROR: Daemon is not in automatic mode -- " - "sync will never start!\n"); + BOX_ERROR("Daemon is not in automatic mode, " + "sync will never start!"); return 1; } @@ -272,8 +261,8 @@ int main(int argc, const char *argv[]) if (currentState != 0) { - printf("Waiting for current sync/error state " - "to finish...\n"); + BOX_INFO("Waiting for current sync/error state " + "to finish..."); } } break; @@ -316,14 +305,14 @@ int main(int argc, const char *argv[]) { if(line == "start-sync") { - if (!quiet) printf("Sync started...\n"); + if (!quiet) BOX_INFO("Sync started..."); syncIsRunning = true; } else if(line == "finish-sync") { if (syncIsRunning) { - if (!quiet) printf("Sync finished.\n"); + if (!quiet) BOX_INFO("Sync finished.\n"); // Send a quit command to finish nicely connection.Write("quit\n", 5); @@ -332,7 +321,7 @@ int main(int argc, const char *argv[]) } else { - if (!quiet) printf("Previous sync finished.\n"); + if (!quiet) BOX_INFO("Previous sync finished."); } // daemon must still be busy } @@ -346,13 +335,13 @@ int main(int argc, const char *argv[]) { if(!quiet) { - printf("Succeeded.\n"); + BOX_INFO("Succeeded.\n"); } finished = true; } else if(line == "error") { - printf("ERROR. (Check command spelling)\n"); + BOX_ERROR("Check command spelling"); returnCode = 1; finished = true; } diff --git a/bin/bbackupd/BackupClientContext.cpp b/bin/bbackupd/BackupClientContext.cpp index 4b84fecb..4b4efd90 100644 --- a/bin/bbackupd/BackupClientContext.cpp +++ b/bin/bbackupd/BackupClientContext.cpp @@ -9,12 +9,10 @@ #include "Box.h" -#ifdef HAVE_SYSLOG_H - #include <syslog.h> -#endif #ifdef HAVE_SIGNAL_H #include <signal.h> #endif + #ifdef HAVE_SYS_TIME_H #include <sys/time.h> #endif @@ -29,19 +27,28 @@ #include "BackupDaemon.h" #include "autogen_BackupProtocolClient.h" #include "BackupStoreFile.h" +#include "Logging.h" #include "MemLeakFindOn.h" // -------------------------------------------------------------------------- // // Function -// Name: BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool) +// Name: BackupClientContext::BackupClientContext(BackupDaemon &, TLSContext &, const std::string &, int32_t, bool, bool, std::string) // Purpose: Constructor // Created: 2003/10/08 // // -------------------------------------------------------------------------- -BackupClientContext::BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLSContext, const std::string &rHostname, - int32_t AccountNumber, bool ExtendedLogging) +BackupClientContext::BackupClientContext +( + BackupDaemon &rDaemon, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile +) : mrDaemon(rDaemon), mrTLSContext(rTLSContext), mHostname(rHostname), @@ -49,6 +56,9 @@ BackupClientContext::BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLS mpSocket(0), mpConnection(0), mExtendedLogging(ExtendedLogging), + mExtendedLogToFile(ExtendedLogToFile), + mExtendedLogFile(ExtendedLogFile), + mpExtendedLogFileHandle(NULL), mClientStoreMarker(ClientStoreMarker_NotKnown), mpDeleteList(0), mpCurrentIDMap(0), @@ -56,8 +66,8 @@ BackupClientContext::BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLS mStorageLimitExceeded(false), mpExcludeFiles(0), mpExcludeDirs(0), - mbIsManaged(false), - mTimeMgmtEpoch(0) + mKeepAliveTimer(0), + mbIsManaged(false) { } @@ -115,7 +125,8 @@ BackupProtocolClient &BackupClientContext::GetConnection() } // Log intention - ::syslog(LOG_INFO, "Opening connection to server %s...", mHostname.c_str()); + BOX_INFO("Opening connection to server '" << + mHostname << "'..."); // Connect! mpSocket->Open(mrTLSContext, Socket::TypeINET, mHostname.c_str(), BOX_PORT_BBSTORED); @@ -126,6 +137,24 @@ BackupProtocolClient &BackupClientContext::GetConnection() // Set logging option mpConnection->SetLogToSysLog(mExtendedLogging); + if (mExtendedLogToFile) + { + ASSERT(mpExtendedLogFileHandle == NULL); + + mpExtendedLogFileHandle = fopen( + mExtendedLogFile.c_str(), "a+"); + + if (!mpExtendedLogFileHandle) + { + BOX_ERROR("Failed to open extended log " + "file: " << strerror(errno)); + } + else + { + mpConnection->SetLogToFile(mpExtendedLogFileHandle); + } + } + // Handshake mpConnection->Handshake(); @@ -164,19 +193,16 @@ BackupProtocolClient &BackupClientContext::GetConnection() } // Log success - ::syslog(LOG_INFO, "Connection made, login successful"); + BOX_INFO("Connection made, login successful"); // Check to see if there is any space available on the server - int64_t softLimit = loginConf->GetBlocksSoftLimit(); - int64_t hardLimit = loginConf->GetBlocksHardLimit(); - // Threshold for uploading new stuff - int64_t stopUploadThreshold = softLimit + ((hardLimit - softLimit) / 3); - if(loginConf->GetBlocksUsed() > stopUploadThreshold) + if(loginConf->GetBlocksUsed() >= loginConf->GetBlocksHardLimit()) { // no -- flag so only things like deletions happen mStorageLimitExceeded = true; // Log - ::syslog(LOG_WARNING, "Exceeded storage limits on server -- not uploading changes to files"); + BOX_WARNING("Exceeded storage hard-limit on server, " + "not uploading changes to files"); } } catch(...) @@ -256,6 +282,12 @@ void BackupClientContext::CloseAnyOpenConnection() delete mpDeleteList; mpDeleteList = 0; } + + if (mpExtendedLogFileHandle != NULL) + { + fclose(mpExtendedLogFileHandle); + mpExtendedLogFileHandle = NULL; + } } @@ -303,8 +335,8 @@ BackupClientDeleteList &BackupClientContext::GetDeleteList() // -------------------------------------------------------------------------- // // Function -// Name: -// Purpose: +// Name: BackupClientContext::PerformDeletions() +// Purpose: Perform any pending file deletions. // Created: 10/11/03 // // -------------------------------------------------------------------------- @@ -461,35 +493,18 @@ bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirec return true; } - -// maximum time to spend diffing -static int sMaximumDiffTime = 600; -// maximum time of SSL inactivity (keep-alive interval) -static int sKeepAliveTime = 0; - void BackupClientContext::SetMaximumDiffingTime(int iSeconds) { - sMaximumDiffTime = iSeconds < 0 ? 0 : iSeconds; - TRACE1("Set maximum diffing time to %d seconds\n", sMaximumDiffTime); + mMaximumDiffingTime = iSeconds < 0 ? 0 : iSeconds; + BOX_TRACE("Set maximum diffing time to " << mMaximumDiffingTime << + " seconds"); } void BackupClientContext::SetKeepAliveTime(int iSeconds) { - sKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; - TRACE1("Set keep-alive time to %d seconds\n", sKeepAliveTime); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: static TimerSigHandler(int) -// Purpose: Signal handler -// Created: 19/3/04 -// -// -------------------------------------------------------------------------- -static void TimerSigHandler(int iUnused) -{ - BackupStoreFile::DiffTimerExpired(); + mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; + BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); + mKeepAliveTimer = Timer(mKeepAliveTime); } // -------------------------------------------------------------------------- @@ -502,59 +517,8 @@ static void TimerSigHandler(int iUnused) // -------------------------------------------------------------------------- void BackupClientContext::ManageDiffProcess() { - if (mbIsManaged || !mpConnection) - return; - - ASSERT(mTimeMgmtEpoch == 0); - -#ifdef PLATFORM_CYGWIN - ::signal(SIGALRM, TimerSigHandler); -#elif defined WIN32 - // no support for SIGVTALRM - SetTimerHandler(TimerSigHandler); -#else - ::signal(SIGVTALRM, TimerSigHandler); -#endif // PLATFORM_CYGWIN - - struct itimerval timeout; - memset(&timeout, 0, sizeof(timeout)); - - // - // - // - if (sMaximumDiffTime <= 0 && sKeepAliveTime <= 0) - { - TRACE0("Diff control not requested - letting things run wild\n"); - return; - } - else if (sMaximumDiffTime > 0 && sKeepAliveTime > 0) - { - timeout.it_value.tv_sec = sKeepAliveTime < sMaximumDiffTime ? sKeepAliveTime : sMaximumDiffTime; - timeout.it_interval.tv_sec = sKeepAliveTime < sMaximumDiffTime ? sKeepAliveTime : sMaximumDiffTime; - } - else - { - timeout.it_value.tv_sec = sKeepAliveTime > 0 ? sKeepAliveTime : sMaximumDiffTime; - timeout.it_interval.tv_sec = sKeepAliveTime > 0 ? sKeepAliveTime : sMaximumDiffTime; - } - - // avoid race - mTimeMgmtEpoch = time(NULL); - -#ifdef PLATFORM_CYGWIN - if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) -#else - if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0) -#endif // PLATFORM_CYGWIN - { - mTimeMgmtEpoch = 0; - - TRACE0("WARNING: couldn't set file diff control timeout\n"); - THROW_EXCEPTION(BackupStoreException, Internal) - } - + ASSERT(!mbIsManaged); mbIsManaged = true; - TRACE0("Initiated timer for file diff control\n"); } // -------------------------------------------------------------------------- @@ -567,33 +531,16 @@ void BackupClientContext::ManageDiffProcess() // -------------------------------------------------------------------------- void BackupClientContext::UnManageDiffProcess() { - if (!mbIsManaged /* don't test for active connection, just do it */) - return; - - struct itimerval timeout; - memset(&timeout, 0, sizeof(timeout)); - -#ifdef PLATFORM_CYGWIN - if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) -#else - if(::setitimer(ITIMER_VIRTUAL, &timeout, NULL) != 0) -#endif // PLATFORM_CYGWIN - { - TRACE0("WARNING: couldn't clear file diff control timeout\n"); - THROW_EXCEPTION(BackupStoreException, Internal) - } - + // ASSERT(mbIsManaged); mbIsManaged = false; - mTimeMgmtEpoch = 0; - - TRACE0("Suspended timer for file diff control\n"); } // -------------------------------------------------------------------------- // // Function // Name: BackupClientContext::DoKeepAlive() -// Purpose: Does something inconsequential over the SSL link to keep it up +// Purpose: Check whether it's time to send a KeepAlive +// message over the SSL link, and if so, send it. // Created: 04/19/2005 // // -------------------------------------------------------------------------- @@ -601,33 +548,26 @@ void BackupClientContext::DoKeepAlive() { if (!mpConnection) { - ::syslog(LOG_ERR, "DoKeepAlive() called with no connection!"); + return; + } + + if (mKeepAliveTime == 0) + { return; } + if (!mKeepAliveTimer.HasExpired()) + { + return; + } + + BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); mpConnection->QueryGetIsAlive(); -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupClientContext::GetTimeMgmtEpoch() -// Purpose: Returns the unix time when the diff was started, or zero -// if the diff process is unmanaged. -// Created: 04/19/2005 -// -// -------------------------------------------------------------------------- -time_t BackupClientContext::GetTimeMgmtEpoch() -{ - return mTimeMgmtEpoch; + + mKeepAliveTimer = Timer(mKeepAliveTime); } int BackupClientContext::GetMaximumDiffingTime() { - return sMaximumDiffTime; -} - -int BackupClientContext::GetKeepaliveTime() -{ - return sKeepAliveTime; + return mMaximumDiffingTime; } diff --git a/bin/bbackupd/BackupClientContext.h b/bin/bbackupd/BackupClientContext.h index a0cf6e1f..152d8556 100644 --- a/bin/bbackupd/BackupClientContext.h +++ b/bin/bbackupd/BackupClientContext.h @@ -14,6 +14,7 @@ #include "BackupClientDeleteList.h" #include "BackupStoreFile.h" #include "ExcludeList.h" +#include "Timer.h" class TLSContext; class BackupProtocolClient; @@ -35,8 +36,16 @@ class BackupStoreFilenameClear; class BackupClientContext : public DiffTimer { public: - BackupClientContext(BackupDaemon &rDaemon, TLSContext &rTLSContext, const std::string &rHostname, - int32_t AccountNumber, bool ExtendedLogging); + BackupClientContext + ( + BackupDaemon &rDaemon, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile + ); virtual ~BackupClientContext(); private: BackupClientContext(const BackupClientContext &); @@ -143,7 +152,7 @@ public: // Created: 04/19/2005 // // -------------------------------------------------------------------------- - static void SetMaximumDiffingTime(int iSeconds); + void SetMaximumDiffingTime(int iSeconds); // -------------------------------------------------------------------------- // @@ -153,7 +162,7 @@ public: // Created: 04/19/2005 // // -------------------------------------------------------------------------- - static void SetKeepAliveTime(int iSeconds); + void SetKeepAliveTime(int iSeconds); // -------------------------------------------------------------------------- // @@ -175,19 +184,18 @@ public: // -------------------------------------------------------------------------- void UnManageDiffProcess(); - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------- // // Function // Name: BackupClientContext::DoKeepAlive() - // Purpose: Does something inconsequential over the SSL link to - // keep it up, implements DiffTimer interface + // Purpose: Check whether it's time to send a KeepAlive + // message over the SSL link, and if so, send it. // Created: 04/19/2005 // - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------- virtual void DoKeepAlive(); - virtual time_t GetTimeMgmtEpoch(); virtual int GetMaximumDiffingTime(); - virtual int GetKeepaliveTime(); + virtual bool IsManaged() { return mbIsManaged; } private: BackupDaemon &mrDaemon; @@ -197,6 +205,9 @@ private: SocketStreamTLS *mpSocket; BackupProtocolClient *mpConnection; bool mExtendedLogging; + bool mExtendedLogToFile; + std::string mExtendedLogFile; + FILE* mpExtendedLogFileHandle; int64_t mClientStoreMarker; BackupClientDeleteList *mpDeleteList; const BackupClientInodeToIDMap *mpCurrentIDMap; @@ -204,11 +215,10 @@ private: bool mStorageLimitExceeded; ExcludeList *mpExcludeFiles; ExcludeList *mpExcludeDirs; - + Timer mKeepAliveTimer; bool mbIsManaged; - // unix time when diff was started - time_t mTimeMgmtEpoch; + int mKeepAliveTime; + int mMaximumDiffingTime; }; - #endif // BACKUPCLIENTCONTEXT__H diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index 1db6cbe5..e5a57bc1 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -29,6 +29,9 @@ #include "BackupDaemon.h" #include "BackupStoreException.h" #include "Archive.h" +#include "PathUtils.h" +#include "Logging.h" +#include "ReadLoggingStream.h" #include "MemLeakFindOn.h" @@ -97,33 +100,6 @@ void BackupClientDirectoryRecord::DeleteSubDirectories() // -------------------------------------------------------------------------- // // Function -// Name: MakeFullPath(const std::string& rDir, const std::string& rFile) -// Purpose: Combine directory and file name -// Created: 2006/08/10 -// -// -------------------------------------------------------------------------- -static std::string MakeFullPath(const std::string& rDir, - const std::string& rFile) -{ - std::string result; - - if (rDir.size() > 0 && - rDir[rDir.size()-1] == DIRECTORY_SEPARATOR_ASCHAR) - { - result = rDir + rFile; - } - else - { - result = rDir + DIRECTORY_SEPARATOR + rFile; - } - - return result; -} - - -// -------------------------------------------------------------------------- -// -// Function // Name: BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &, int64_t, const std::string &, bool) // Purpose: Syncronise, recusively, a local directory with the server. // Created: 2003/10/08 @@ -164,8 +140,8 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn { // The directory has probably been deleted, so just ignore this error. // In a future scan, this deletion will be noticed, deleted from server, and this object deleted. - TRACE1("Stat failed for '%s' (directory)\n", - rLocalPath.c_str()); + rParams.GetProgressNotifier().NotifyDirStatFailed( + this, rLocalPath, strerror(errno)); return; } // Store inode number in map so directories are tracked in case they're renamed @@ -193,15 +169,48 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn std::vector<std::string> dirs; std::vector<std::string> files; bool downloadDirectoryRecordBecauseOfFutureFiles = false; + + struct stat dir_st; + if(::lstat(rLocalPath.c_str(), &dir_st) != 0) + { + // Report the error (logs and + // eventual email to administrator) + rParams.GetProgressNotifier().NotifyFileStatFailed(this, + rLocalPath, strerror(errno)); + + // FIXME move to NotifyFileStatFailed() + SetErrorWhenReadingFilesystemObject(rParams, + rLocalPath.c_str()); + + // This shouldn't happen, so we'd better not continue + THROW_EXCEPTION(CommonException, OSFileError) + } + // BLOCK { // read the contents... DIR *dirHandle = 0; try { + rParams.GetProgressNotifier().NotifyScanDirectory( + this, rLocalPath); + dirHandle = ::opendir(rLocalPath.c_str()); if(dirHandle == 0) { + // Report the error (logs and + // eventual email to administrator) + if (errno == EACCES) + { + rParams.GetProgressNotifier().NotifyDirListFailed( + this, rLocalPath, "Access denied"); + } + else + { + rParams.GetProgressNotifier().NotifyDirListFailed(this, + rLocalPath, strerror(errno)); + } + // Report the error (logs and eventual email to administrator) SetErrorWhenReadingFilesystemObject(rParams, rLocalPath.c_str()); // Ignore this directory for now. @@ -223,6 +232,8 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn std::string filename; while((en = ::readdir(dirHandle)) != 0) { + rParams.mrContext.DoKeepAlive(); + // Don't need to use LinuxWorkaround_FinishDirentStruct(en, rLocalPath.c_str()); // on Linux, as a stat is performed to get all this info @@ -252,6 +263,10 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn { // Report the error (logs and // eventual email to administrator) + rParams.GetProgressNotifier().NotifyFileStatFailed(this, + filename, strerror(errno)); + + // FIXME move to NotifyFileStatFailed() SetErrorWhenReadingFilesystemObject( rParams, filename.c_str()); @@ -259,6 +274,14 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn continue; } + if(st.st_dev != dir_st.st_dev) + { + rParams.GetProgressNotifier() + .NotifyMountPointSkipped(this, + filename); + continue; + } + int type = st.st_mode & S_IFMT; #endif @@ -269,6 +292,11 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Exclude it? if(rParams.mrContext.ExcludeFile(filename)) { + rParams.GetProgressNotifier() + .NotifyFileExcluded( + this, + filename); + // Next item! continue; } @@ -283,6 +311,11 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Exclude it? if(rParams.mrContext.ExcludeDir(filename)) { + rParams.GetProgressNotifier() + .NotifyDirExcluded( + this, + filename); + // Next item! continue; } @@ -292,13 +325,22 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn } else { - #ifdef WIN32 - ::syslog(LOG_ERR, "Unknown file type: " - "%d (%s)", type, - filename.c_str()); - #endif - SetErrorWhenReadingFilesystemObject( - rParams, filename.c_str()); + if(rParams.mrContext.ExcludeFile(filename)) + { + rParams.GetProgressNotifier() + .NotifyFileExcluded( + this, + filename); + } + else + { + rParams.GetProgressNotifier() + .NotifyUnsupportedFileType( + this, filename); + SetErrorWhenReadingFilesystemObject( + rParams, filename.c_str()); + } + continue; } @@ -310,6 +352,11 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // but now we need the information. if(::lstat(filename.c_str(), &st) != 0) { + rParams.GetProgressNotifier() + .NotifyFileStatFailed(this, + filename, + strerror(errno)); + // Report the error (logs and // eventual email to administrator) SetErrorWhenReadingFilesystemObject( @@ -318,6 +365,14 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Ignore this entry for now. continue; } + + if(st.st_dev != dir_st.st_dev) + { + rParams.GetProgressNotifier() + .NotifyMountPointSkipped(this, + filename); + continue; + } #endif checksum_info.mModificationTime = FileModificationTime(st); @@ -335,8 +390,8 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Log that this has happened if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) { - ::syslog(LOG_ERR, "Some files have modification times excessively in the future. Check clock syncronisation.\n"); - ::syslog(LOG_ERR, "Example file (only one shown) : %s\n", filename.c_str()); + rParams.GetProgressNotifier().NotifyFileModifiedInFuture( + this, filename); rParams.mHaveLoggedWarningAboutFutureFileTimes = true; } } @@ -574,6 +629,9 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP for(std::vector<std::string>::const_iterator f = rFiles.begin(); f != rFiles.end(); ++f) { + // Send keep-alive message if needed + rParams.mrContext.DoKeepAlive(); + // Filename of this file std::string filename(MakeFullPath(rLocalPath, *f)); @@ -589,7 +647,16 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP struct stat st; if(::lstat(filename.c_str(), &st) != 0) { - THROW_EXCEPTION(CommonException, OSFileError) + rParams.GetProgressNotifier().NotifyFileStatFailed(this, + filename, strerror(errno)); + + // Report the error (logs and + // eventual email to administrator) + SetErrorWhenReadingFilesystemObject(rParams, + filename.c_str()); + + // Ignore this entry for now. + continue; } // Extract required data @@ -728,10 +795,14 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP if (pDirOnStore != 0 && en == 0) { doUpload = true; + BOX_TRACE(filename << ": will upload " + "(not on server)"); } else if (modTime >= rParams.mSyncPeriodStart) { doUpload = true; + BOX_TRACE(filename << ": will upload " + "(modified since last sync)"); } } @@ -748,6 +819,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP > rParams.mMaxUploadWait) { doUpload = true; + BOX_TRACE(filename << ": will upload " + "(continually modified)"); } // Then make sure that if files are added with a @@ -763,6 +836,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP en->GetModificationTime() != modTime) { doUpload = true; + BOX_TRACE(filename << ": will upload " + "(mod time changed)"); } // And just to catch really badly off clocks in @@ -773,9 +848,20 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP rParams.mUploadAfterThisTimeInTheFuture) { doUpload = true; + BOX_TRACE(filename << ": will upload " + "(mod time in the future)"); } } + if (!doUpload) + { + BOX_TRACE(filename << ": will not upload " + "(no reason to upload, mod time is " + << modTime << " versus sync period " + << rParams.mSyncPeriodStart << " to " + << rParams.mSyncPeriodEnd << ")"); + } + if (doUpload) { // Make sure we're connected -- must connect here so we know whether @@ -801,6 +887,9 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP { // Connection errors should just be passed on to the main handler, retries // would probably just cause more problems. + rParams.GetProgressNotifier() + .NotifyFileUploadException( + this, filename, e); throw; } catch(BoxException &e) @@ -809,8 +898,9 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP allUpdatedSuccessfully = false; // Log it. SetErrorWhenReadingFilesystemObject(rParams, filename.c_str()); - // Log error. - ::syslog(LOG_ERR, "Error code when uploading was (%d/%d), %s", e.GetType(), e.GetSubType(), e.what()); + rParams.GetProgressNotifier() + .NotifyFileUploadException( + this, filename, e); } // Update structures if the file was uploaded successfully. @@ -823,6 +913,11 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP } } } + else + { + rParams.GetProgressNotifier().NotifyFileSkippedServerFull(this, + filename); + } } else if(en != 0 && en->GetAttributesHash() != attributesHash) { @@ -904,6 +999,9 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP } } } + + rParams.GetProgressNotifier().NotifyFileSynchronised(this, + filename, fileSize); } // Erase contents of files to save space when recursing @@ -921,6 +1019,9 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP for(std::vector<std::string>::const_iterator d = rDirs.begin(); d != rDirs.end(); ++d) { + // Send keep-alive message if needed + rParams.mrContext.DoKeepAlive(); + // Get the local filename std::string dirname(MakeFullPath(rLocalPath, *d)); @@ -1217,7 +1318,11 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn if(diffFromID != 0) { - // Found an old version -- get the index + // Found an old version + rParams.GetProgressNotifier().NotifyFileUploadingPatch(this, + rFilename); + + // Get the index std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); // @@ -1253,9 +1358,13 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn if(doNormalUpload) { // below threshold or nothing to diff from, so upload whole + rParams.GetProgressNotifier().NotifyFileUploading(this, + rFilename); // Prepare to upload, getting a stream which will encode the file as we go along - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(rFilename.c_str(), mObjectID, rStoreFilename)); + std::auto_ptr<IOStream> upload( + BackupStoreFile::EncodeFile(rFilename.c_str(), + mObjectID, rStoreFilename)); // Send to store std::auto_ptr<BackupProtocolClientSuccess> stored( @@ -1275,14 +1384,20 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) { - // Check and see what error the protocol has -- as it might be an error... + // Check and see what error the protocol has, + // this is more useful to users than the exception. int type, subtype; - if(connection.GetLastError(type, subtype) - && type == BackupProtocolClientError::ErrorType - && subtype == BackupProtocolClientError::Err_StorageLimitExceeded) + if(connection.GetLastError(type, subtype)) { - // The hard limit was exceeded on the server, notify! - rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull); + if(type == BackupProtocolClientError::ErrorType + && subtype == BackupProtocolClientError::Err_StorageLimitExceeded) + { + // The hard limit was exceeded on the server, notify! + rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull); + } + rParams.GetProgressNotifier() + .NotifyFileUploadServerError( + this, rFilename, type, subtype); } } @@ -1290,6 +1405,8 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn throw; } + rParams.GetProgressNotifier().NotifyFileUploaded(this, rFilename, FileSize); + // Return the new object ID of this file return objID; } @@ -1309,8 +1426,11 @@ void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClie // Zero hash, so it gets synced properly next time round. ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); - // Log the error - ::syslog(LOG_ERR, "Backup object failed, error when reading %s", Filename); + // Log the error - already done by caller + /* + rParams.GetProgressNotifier().NotifyFileReadFailed(this, + Filename, strerror(errno)); + */ // Mark that an error occured in the parameters object rParams.mReadErrorsOnFilesystemObjects = true; @@ -1326,8 +1446,10 @@ void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClie // Created: 8/3/04 // // -------------------------------------------------------------------------- -BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, BackupClientContext &rContext) - : mSyncPeriodStart(0), +BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, + ProgressNotifier &rProgressNotifier, BackupClientContext &rContext) + : mrProgressNotifier(rProgressNotifier), + mSyncPeriodStart(0), mSyncPeriodEnd(0), mMaxUploadWait(0), mMaxFileTimeInFuture(99999999999999999LL), diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h index b7b84984..9e4dda7a 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.h +++ b/bin/bbackupd/BackupClientDirectoryRecord.h @@ -25,6 +25,82 @@ class BackupDaemon; // -------------------------------------------------------------------------- // // Class +// Name: ProgressNotifier +// Purpose: Provides methods for the backup library to inform the user +// interface about its progress with the backup +// Created: 2005/11/20 +// +// -------------------------------------------------------------------------- +class BackupClientDirectoryRecord; + +class ProgressNotifier +{ + public: + virtual ~ProgressNotifier() { } + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyMountPointSkipped( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyUnsupportedFileType( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) = 0; + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) = 0; + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) = 0; + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class // Name: BackupClientDirectoryRecord // Purpose: Implementation of record about directory for backup client // Created: 2003/10/08 @@ -59,14 +135,18 @@ public: class SyncParams { public: - SyncParams(BackupDaemon &rDaemon, BackupClientContext &rContext); + SyncParams( + BackupDaemon &rDaemon, + ProgressNotifier &rProgressNotifier, + BackupClientContext &rContext); ~SyncParams(); private: // No copying SyncParams(const SyncParams&); SyncParams &operator=(const SyncParams&); + ProgressNotifier &mrProgressNotifier; + public: - // Data members are public, as accessors are not justified here box_time_t mSyncPeriodStart; box_time_t mSyncPeriodEnd; @@ -81,6 +161,11 @@ public: // Member variables modified by syncing process box_time_t mUploadAfterThisTimeInTheFuture; bool mHaveLoggedWarningAboutFutureFileTimes; + + ProgressNotifier& GetProgressNotifier() const + { + return mrProgressNotifier; + } }; void SyncDirectory(SyncParams &rParams, int64_t ContainingDirectoryID, const std::string &rLocalPath, diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp index 7d508743..cfeb320e 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/bin/bbackupd/BackupDaemon.cpp @@ -18,9 +18,6 @@ #ifdef HAVE_SIGNAL_H #include <signal.h> #endif -#ifdef HAVE_SYSLOG_H - #include <syslog.h> -#endif #ifdef HAVE_SYS_PARAM_H #include <sys/param.h> #endif @@ -75,6 +72,9 @@ #include "IOStreamGetLine.h" #include "Conversion.h" #include "Archive.h" +#include "Timer.h" +#include "Logging.h" +#include "autogen_ClientException.h" #include "MemLeakFindOn.h" @@ -114,18 +114,42 @@ unsigned int WINAPI HelperThread(LPVOID lpParam) BackupDaemon::BackupDaemon() : mState(BackupDaemon::State_Initialising), mpCommandSocketInfo(0), - mDeleteUnusedRootDirEntriesAfter(0) + mDeleteUnusedRootDirEntriesAfter(0), + mLogAllFileAccess(false) { // Only ever one instance of a daemon SSLLib::Initialise(); - // Initialise notifcation sent status - for(int l = 0; l <= NotifyEvent__MAX; ++l) + // Initialise notification sent status + for(int l = 0; l < NotifyEvent__MAX; ++l) { mNotificationsSent[l] = false; } #ifdef WIN32 + // Create the event object to signal from main thread to worker + // when new messages are queued to be sent to the command socket. + mhMessageToSendEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if(mhMessageToSendEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to create event object: error " << + GetLastError()); + exit(1); + } + + // Create the event object to signal from worker to main thread + // when a command has been received on the command socket. + mhCommandReceivedEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if(mhCommandReceivedEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to create event object: error " << + GetLastError()); + exit(1); + } + + // Create the critical section to protect the message queue + InitializeCriticalSection(&mMessageQueueLock); + // Create a thread to handle the named pipe HANDLE hThread; unsigned int dwThreadId; @@ -223,7 +247,7 @@ void BackupDaemon::SetupInInitialProcess() // Print a warning on this platform if the CommandSocket is used. if(GetConfiguration().KeyExists("CommandSocket")) { - printf( + BOX_WARNING( "==============================================================================\n" "SECURITY WARNING: This platform cannot check the credentials of connections to\n" "the command socket. This is a potential DoS security problem.\n" @@ -276,29 +300,21 @@ void BackupDaemon::RunHelperThread(void) } catch (BoxException &e) { - ::syslog(LOG_ERR, "Failed to open command socket: %s", + BOX_ERROR("Failed to open command socket: " << e.what()); SetTerminateWanted(); break; // this is fatal to listening thread } - catch (...) - { - ::syslog(LOG_ERR, "Failed to open command socket: " - "unknown error"); - SetTerminateWanted(); - break; // this is fatal to listening thread - } - } catch(std::exception &e) { - ::syslog(LOG_ERR, "Failed to open command socket: " - "%s", e.what()); + BOX_ERROR("Failed to open command socket: " << + e.what()); SetTerminateWanted(); break; // this is fatal to listening thread } catch(...) { - ::syslog(LOG_ERR, "Failed to open command socket: " + BOX_ERROR("Failed to open command socket: " "unknown error"); SetTerminateWanted(); break; // this is fatal to listening thread @@ -311,7 +327,7 @@ void BackupDaemon::RunHelperThread(void) // This next section comes from Ben's original function // Log - ::syslog(LOG_INFO, "Connection from command socket"); + BOX_INFO("Connection from command socket"); // Send a header line summarising the configuration // and current state @@ -328,15 +344,73 @@ void BackupDaemon::RunHelperThread(void) rSocket.Write(summary, summarySize); rSocket.Write("ping\n", 5); + // old queued messages are not useful + EnterCriticalSection(&mMessageQueueLock); + mMessageList.clear(); + ResetEvent(mhMessageToSendEvent); + LeaveCriticalSection(&mMessageQueueLock); + IOStreamGetLine readLine(rSocket); std::string command; - while (rSocket.IsConnected() && - readLine.GetLine(command) && - !IsTerminateWanted()) + while (rSocket.IsConnected() && !IsTerminateWanted()) { - TRACE1("Received command '%s' over " - "command socket\n", command.c_str()); + HANDLE handles[2]; + handles[0] = mhMessageToSendEvent; + handles[1] = rSocket.GetReadableEvent(); + + BOX_TRACE("Received command '" << command + << "' over command socket"); + + DWORD result = WaitForMultipleObjects( + sizeof(handles)/sizeof(*handles), + handles, FALSE, 1000); + + if(result == 0) + { + ResetEvent(mhMessageToSendEvent); + + EnterCriticalSection(&mMessageQueueLock); + try + { + while (mMessageList.size() > 0) + { + std::string message = *(mMessageList.begin()); + mMessageList.erase(mMessageList.begin()); + printf("Sending '%s' to waiting client... ", message.c_str()); + message += "\n"; + rSocket.Write(message.c_str(), + message.length()); + + printf("done.\n"); + } + } + catch (...) + { + LeaveCriticalSection(&mMessageQueueLock); + throw; + } + LeaveCriticalSection(&mMessageQueueLock); + continue; + } + else if(result == WAIT_TIMEOUT) + { + continue; + } + else if(result != 1) + { + BOX_ERROR("WaitForMultipleObjects returned invalid result " << result); + continue; + } + + if(!readLine.GetLine(command)) + { + BOX_ERROR("Failed to read line"); + continue; + } + + BOX_INFO("Received command " << command << + " from client"); bool sendOK = false; bool sendResponse = true; @@ -355,6 +429,7 @@ void BackupDaemon::RunHelperThread(void) this->mDoSyncFlagOut = true; this->mSyncIsForcedOut = false; sendOK = true; + SetEvent(mhCommandReceivedEvent); } else if(command == "force-sync") { @@ -362,22 +437,27 @@ void BackupDaemon::RunHelperThread(void) this->mDoSyncFlagOut = true; this->mSyncIsForcedOut = true; sendOK = true; + SetEvent(mhCommandReceivedEvent); } else if(command == "reload") { // Reload the configuration SetReloadConfigWanted(); sendOK = true; + SetEvent(mhCommandReceivedEvent); } else if(command == "terminate") { // Terminate the daemon cleanly SetTerminateWanted(); sendOK = true; + SetEvent(mhCommandReceivedEvent); } else { - ::syslog(LOG_ERR, "Received unknown command '%s' from client", command.c_str()); + BOX_ERROR("Received unknown command " + "'" << command << "' " + "from client"); sendResponse = true; sendOK = false; } @@ -394,27 +474,28 @@ void BackupDaemon::RunHelperThread(void) { break; } - - this->mReceivedCommandConn = true; } rSocket.Close(); } catch(BoxException &e) { - ::syslog(LOG_ERR, "Communication error with " - "control client: %s", e.what()); + BOX_ERROR("Communication error with " + "control client: " << e.what()); } catch(std::exception &e) { - ::syslog(LOG_ERR, "Internal error in command socket " - "thread: %s", e.what()); + BOX_ERROR("Internal error in command socket " + "thread: " << e.what()); } catch(...) { - ::syslog(LOG_ERR, "Communication error with control client"); + BOX_ERROR("Communication error with control client"); } } + + CloseHandle(mhCommandReceivedEvent); + CloseHandle(mhMessageToSendEvent); } #endif @@ -428,21 +509,19 @@ void BackupDaemon::RunHelperThread(void) // -------------------------------------------------------------------------- void BackupDaemon::Run() { + // initialise global timer mechanism + Timers::Init(); + #ifdef WIN32 - // init our own timer for file diff timeouts - InitTimer(); - try { Run2(); } catch(...) { - FiniTimer(); + Timers::Cleanup(); throw; } - - FiniTimer(); #else // ! WIN32 // Ignore SIGPIPE (so that if a command connection is broken, the daemon doesn't terminate) ::signal(SIGPIPE, SIG_IGN); @@ -473,19 +552,20 @@ void BackupDaemon::Run() } catch(std::exception &e) { - ::syslog(LOG_ERR, "Internal error while " + BOX_WARNING("Internal error while " "closing command socket after " - "another exception: %s", e.what()); + "another exception: " << e.what()); } catch(...) { - ::syslog(LOG_WARNING, - "Error closing command socket " + BOX_WARNING("Error closing command socket " "after exception, ignored."); } mpCommandSocketInfo = 0; } + Timers::Cleanup(); + throw; } @@ -496,6 +576,8 @@ void BackupDaemon::Run() mpCommandSocketInfo = 0; } #endif + + Timers::Cleanup(); } // -------------------------------------------------------------------------- @@ -519,18 +601,20 @@ void BackupDaemon::Run2() // Set up the keys for various things BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str()); + // Setup various timings + int maximumDiffingTime = 600; + int keepAliveTime = 60; + // max diffing time, keep-alive time if(conf.KeyExists("MaximumDiffingTime")) { - BackupClientContext::SetMaximumDiffingTime(conf.GetKeyValueInt("MaximumDiffingTime")); + maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); } if(conf.KeyExists("KeepAliveTime")) { - BackupClientContext::SetKeepAliveTime(conf.GetKeyValueInt("KeepAliveTime")); + keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); } - // Setup various timings - // How often to connect to the store (approximate) box_time_t updateStoreInterval = SecondsToBoxTime(conf.GetKeyValueInt("UpdateStoreInterval")); @@ -580,38 +664,64 @@ void BackupDaemon::Run2() box_time_t currentTime; do { - // Need to check the stop run thing here too, so this loop isn't run if we should be stopping + // Check whether we should be stopping, + // and don't run a sync if so. if(StopRun()) break; currentTime = GetCurrentBoxTime(); - // Pause a while, but no more than MAX_SLEEP_TIME seconds (use the conditional because times are unsigned) - box_time_t requiredDelay = (nextSyncTime < currentTime)?(0):(nextSyncTime - currentTime); - // If there isn't automatic backup happening, set a long delay. And limit delays at the same time. - if(!automaticBackup || requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) - requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + // Pause a while, but no more than + // MAX_SLEEP_TIME seconds (use the conditional + // because times are unsigned) + box_time_t requiredDelay = + (nextSyncTime < currentTime) + ? (0) + : (nextSyncTime - currentTime); + + // If there isn't automatic backup happening, + // set a long delay. And limit delays at the + // same time. + if(!automaticBackup || requiredDelay > + SecondsToBoxTime(MAX_SLEEP_TIME)) + { + requiredDelay = SecondsToBoxTime( + MAX_SLEEP_TIME); + } - // Only do the delay if there is a delay required + // Only delay if necessary if(requiredDelay > 0) { - // Sleep somehow. There are choices on how this should be done, depending on the state of the control connection + // Sleep somehow. There are choices + // on how this should be done, + // depending on the state of the + // control connection if(mpCommandSocketInfo != 0) { - // A command socket exists, so sleep by handling connections with it - WaitOnCommandSocket(requiredDelay, doSync, doSyncForcedByCommand); + // A command socket exists, + // so sleep by waiting on it + WaitOnCommandSocket( + requiredDelay, doSync, + doSyncForcedByCommand); } else { - // No command socket or connection, just do a normal sleep - time_t sleepSeconds = BoxTimeToSeconds(requiredDelay); - ::sleep((sleepSeconds <= 0)?1:sleepSeconds); + // No command socket or + // connection, just do a + // normal sleep + time_t sleepSeconds = + BoxTimeToSeconds( + requiredDelay); + ::sleep((sleepSeconds <= 0) + ? 1 + : sleepSeconds); } } } while((!automaticBackup || (currentTime < nextSyncTime)) && !doSync && !StopRun()); } - // Time of sync start, and if it's time for another sync (and we're doing automatic syncs), set the flag + // Time of sync start, and if it's time for another sync + // (and we're doing automatic syncs), set the flag box_time_t currentSyncStartTime = GetCurrentBoxTime(); if(automaticBackup && currentSyncStartTime >= nextSyncTime) { @@ -625,12 +735,14 @@ void BackupDaemon::Run2() if(d > 0) { // Script has asked for a delay - nextSyncTime = GetCurrentBoxTime() + SecondsToBoxTime(d); + nextSyncTime = GetCurrentBoxTime() + + SecondsToBoxTime(d); doSync = false; } } - // Ready to sync? (but only if we're not supposed to be stopping) + // Ready to sync? (but only if we're not supposed + // to be stopping) if(doSync && !StopRun()) { // Touch a file to record times in filesystem @@ -644,22 +756,51 @@ void BackupDaemon::Run2() // Calculate the sync period of files to examine box_time_t syncPeriodStart = lastSyncTime; - box_time_t syncPeriodEnd = currentSyncStartTime - minimumFileAge; + box_time_t syncPeriodEnd = currentSyncStartTime - + minimumFileAge; + + if(syncPeriodStart >= syncPeriodEnd && + syncPeriodStart - syncPeriodEnd < minimumFileAge) + { + // This can happen if we receive a force-sync + // command less than minimumFileAge after + // the last sync. Deal with it by moving back + // syncPeriodStart, which should not do any + // damage. + syncPeriodStart = syncPeriodEnd - + SecondsToBoxTime(1); + } + + if(syncPeriodStart >= syncPeriodEnd) + { + BOX_ERROR("Invalid (negative) sync period: " + "perhaps your clock is going " + "backwards (" << syncPeriodStart << + " to " << syncPeriodEnd << ")"); + THROW_EXCEPTION(ClientException, + ClockWentBackwards); + } + // Check logic ASSERT(syncPeriodEnd > syncPeriodStart); // Paranoid check on sync times if(syncPeriodStart >= syncPeriodEnd) continue; - // Adjust syncPeriodEnd to emulate snapshot behaviour properly + // Adjust syncPeriodEnd to emulate snapshot + // behaviour properly box_time_t syncPeriodEndExtended = syncPeriodEnd; // Using zero min file age? if(minimumFileAge == 0) { - // Add a year on to the end of the end time, to make sure we sync - // files which are modified after the scan run started. - // Of course, they may be eligable to be synced again the next time round, - // but this should be OK, because the changes only upload should upload no data. - syncPeriodEndExtended += SecondsToBoxTime((time_t)(356*24*3600)); + // Add a year on to the end of the end time, + // to make sure we sync files which are + // modified after the scan run started. + // Of course, they may be eligible to be + // synced again the next time round, + // but this should be OK, because the changes + // only upload should upload no data. + syncPeriodEndExtended += SecondsToBoxTime( + (time_t)(356*24*3600)); } // Delete the serialised store object file, @@ -668,13 +809,11 @@ void BackupDaemon::Run2() if(deleteStoreObjectInfoFile && !DeleteStoreObjectInfo()) { - ::syslog(LOG_ERR, "Failed to delete the " + BOX_ERROR("Failed to delete the " "StoreObjectInfoFile, backup cannot " "continue safely."); - // prevent runaway process where the logs fill up -- without this - // the log message will be emitted in a tight loop. - ::sleep(60); - continue; + THROW_EXCEPTION(ClientException, + FailedToDeleteStoreObjectInfoFile); } // In case the backup throws an exception, @@ -691,30 +830,70 @@ void BackupDaemon::Run2() { // Set state and log start SetState(State_Connected); - ::syslog(LOG_INFO, "Beginning scan of local files"); + BOX_NOTICE("Beginning scan of local files"); - // Then create a client context object (don't just connect, as this may be unnecessary) - BackupClientContext clientContext(*this, tlsContext, conf.GetKeyValue("StoreHostname"), - conf.GetKeyValueInt("AccountNumber"), conf.GetKeyValueBool("ExtendedLogging")); + std::string extendedLogFile; + if (conf.KeyExists("ExtendedLogFile")) + { + extendedLogFile = conf.GetKeyValue( + "ExtendedLogFile"); + } + + if (conf.KeyExists("LogAllFileAccess")) + { + mLogAllFileAccess = + conf.GetKeyValueBool( + "LogAllFileAccess"); + } + + // Then create a client context object (don't + // just connect, as this may be unnecessary) + BackupClientContext clientContext + ( + *this, + tlsContext, + conf.GetKeyValue("StoreHostname"), + conf.GetKeyValueInt("AccountNumber"), + conf.GetKeyValueBool("ExtendedLogging"), + conf.KeyExists("ExtendedLogFile"), + extendedLogFile + ); // Set up the sync parameters - BackupClientDirectoryRecord::SyncParams params(*this, clientContext); + BackupClientDirectoryRecord::SyncParams params( + *this, *this, clientContext); params.mSyncPeriodStart = syncPeriodStart; - params.mSyncPeriodEnd = syncPeriodEndExtended; // use potentially extended end time + params.mSyncPeriodEnd = syncPeriodEndExtended; + // use potentially extended end time params.mMaxUploadWait = maxUploadWait; - params.mFileTrackingSizeThreshold = conf.GetKeyValueInt("FileTrackingSizeThreshold"); - params.mDiffingUploadSizeThreshold = conf.GetKeyValueInt("DiffingUploadSizeThreshold"); - params.mMaxFileTimeInFuture = SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); + params.mFileTrackingSizeThreshold = + conf.GetKeyValueInt( + "FileTrackingSizeThreshold"); + params.mDiffingUploadSizeThreshold = + conf.GetKeyValueInt( + "DiffingUploadSizeThreshold"); + params.mMaxFileTimeInFuture = + SecondsToBoxTime( + conf.GetKeyValueInt( + "MaxFileTimeInFuture")); + + clientContext.SetMaximumDiffingTime(maximumDiffingTime); + clientContext.SetKeepAliveTime(keepAliveTime); // Set store marker clientContext.SetClientStoreMarker(clientStoreMarker); - // Set up the locations, if necessary -- need to do it here so we have a (potential) connection to use + // Set up the locations, if necessary -- + // need to do it here so we have a + // (potential) connection to use if(mLocations.empty()) { - const Configuration &locations(conf.GetSubConfiguration("BackupLocations")); + const Configuration &locations( + conf.GetSubConfiguration( + "BackupLocations")); - // Make sure all the directory records are set up + // Make sure all the directory records + // are set up SetupLocations(clientContext, locations); } @@ -724,17 +903,29 @@ void BackupDaemon::Run2() // Delete any unused directories? DeleteUnusedRootDirEntries(clientContext); + // Notify administrator + NotifySysadmin(NotifyEvent_BackupStart); + // Go through the records, syncing them - for(std::vector<Location *>::const_iterator i(mLocations.begin()); i != mLocations.end(); ++i) + for(std::vector<Location *>::const_iterator + i(mLocations.begin()); + i != mLocations.end(); ++i) { - // Set current and new ID map pointers in the context + // Set current and new ID map pointers + // in the context clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], mNewIDMaps[(*i)->mIDMapIndex]); - // Set exclude lists (context doesn't take ownership) - clientContext.SetExcludeLists((*i)->mpExcludeFiles, (*i)->mpExcludeDirs); + // Set exclude lists (context doesn't + // take ownership) + clientContext.SetExcludeLists( + (*i)->mpExcludeFiles, + (*i)->mpExcludeDirs); // Sync the directory - (*i)->mpDirectoryRecord->SyncDirectory(params, BackupProtocolClientListDirectory::RootDirectory, (*i)->mPath); + (*i)->mpDirectoryRecord->SyncDirectory( + params, + BackupProtocolClientListDirectory::RootDirectory, + (*i)->mPath); // Unset exclude lists (just in case) clientContext.SetExcludeLists(0, 0); @@ -748,13 +939,14 @@ void BackupDaemon::Run2() } else { - // Unset the read error flag, so the error is - // reported again in the future + // Unset the read error flag, so the // error is reported again if it + // happens again mNotificationsSent[NotifyEvent_ReadError] = false; } - // Perform any deletions required -- these are delayed until the end - // to allow renaming to happen neatly. + // Perform any deletions required -- these are + // delayed until the end to allow renaming to + // happen neatly. clientContext.PerformDeletions(); // Close any open connection @@ -771,21 +963,34 @@ void BackupDaemon::Run2() } else { - // The start time of the next run is the end time of this run - // This is only done if the storage limit wasn't exceeded (as things won't have been done properly if it was) + // The start time of the next run is + // the end time of this run. + // This is only done if the storage + // limit wasn't exceeded (as things + // won't have been done properly if + // it was) lastSyncTime = syncPeriodEnd; - // unflag the storage full notify flag so that next time the store is full, and alert will be sent + + // unflag the storage full notify flag + // so that next time the store is full, + // an alert will be sent mNotificationsSent[NotifyEvent_StoreFull] = false; } // Calculate when the next sync run should be - nextSyncTime = currentSyncStartTime + updateStoreInterval + Random::RandomInt(updateStoreInterval >> SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); + nextSyncTime = currentSyncStartTime + + updateStoreInterval + + Random::RandomInt(updateStoreInterval >> + SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); // Commit the ID Maps CommitIDMapsAfterSync(); // Log - ::syslog(LOG_INFO, "Finished scan of local files"); + BOX_NOTICE("Finished scan of local files"); + + // Notify administrator + NotifySysadmin(NotifyEvent_BackupFinish); // -------------------------------------------------------------------------------------------- @@ -809,21 +1014,29 @@ void BackupDaemon::Run2() } catch(std::exception &e) { - ::syslog(LOG_ERR, "Internal error during " - "backup run: %s", e.what()); + BOX_ERROR("Internal error during " + "backup run: " << e.what()); errorOccurred = true; + errorString = e.what(); } catch(...) { - // TODO: better handling of exceptions here... need to be very careful + // TODO: better handling of exceptions here... + // need to be very careful errorOccurred = true; } if(errorOccurred) { // Is it a berkely db failure? - bool isBerkelyDbFailure = (errorCode == BackupStoreException::ExceptionType - && errorSubCode == BackupStoreException::BerkelyDBFailure); + bool isBerkelyDbFailure = false; + + if (errorCode == BackupStoreException::ExceptionType + && errorSubCode == BackupStoreException::BerkelyDBFailure) + { + isBerkelyDbFailure = true; + } + if(isBerkelyDbFailure) { // Delete corrupt files @@ -831,7 +1044,8 @@ void BackupDaemon::Run2() } // Clear state data - syncPeriodStart = 0; // go back to beginning of time + syncPeriodStart = 0; + // go back to beginning of time clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything DeleteAllLocations(); DeleteAllIDMaps(); @@ -839,26 +1053,30 @@ void BackupDaemon::Run2() // Handle restart? if(StopRun()) { - ::syslog(LOG_INFO, "Exception (%d/%d) due to signal", errorCode, errorSubCode); + BOX_NOTICE("Exception (" << errorCode + << "/" << errorSubCode + << ") due to signal"); return; } // If the Berkely db files get corrupted, delete them and try again immediately if(isBerkelyDbFailure) { - ::syslog(LOG_ERR, "Berkely db inode map files corrupted, deleting and restarting scan. Renamed files and directories will not be tracked until after this scan.\n"); + BOX_ERROR("Berkely db inode map files corrupted, deleting and restarting scan. Renamed files and directories will not be tracked until after this scan."); ::sleep(1); } else { // Not restart/terminate, pause and retry + // Notify administrator + NotifySysadmin(NotifyEvent_BackupError); SetState(State_Error); - ::syslog(LOG_ERR, - "Exception caught (%s %d/%d), " - "reset state and waiting " - "to retry...", - errorString, errorCode, - errorSubCode); + BOX_ERROR("Exception caught (" + << errorString + << " " << errorCode + << "/" << errorSubCode + << "), reset state and " + "waiting to retry..."); ::sleep(10); nextSyncTime = currentSyncStartTime + SecondsToBoxTime(90) + @@ -869,9 +1087,12 @@ void BackupDaemon::Run2() } // Log the stats - ::syslog(LOG_INFO, "File statistics: total file size uploaded %lld, bytes already on server %lld, encoded size %lld", - BackupStoreFile::msStats.mBytesInEncodedFiles, BackupStoreFile::msStats.mBytesAlreadyOnServer, - BackupStoreFile::msStats.mTotalFileStreamSize); + BOX_NOTICE("File statistics: total file size uploaded " + << BackupStoreFile::msStats.mBytesInEncodedFiles + << ", bytes already on server " + << BackupStoreFile::msStats.mBytesAlreadyOnServer + << ", encoded size " + << BackupStoreFile::msStats.mTotalFileStreamSize); BackupStoreFile::ResetStats(); // Tell anything connected to the command socket @@ -926,7 +1147,7 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() std::string line; if(getLine.GetLine(line, true, 30000)) // 30 seconds should be enough { - // Got a string, intepret + // Got a string, interpret if(line == "now") { // Script says do it now. Obey. @@ -941,44 +1162,39 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() } catch(ConversionException &e) { - ::syslog(LOG_ERR, "Invalid output " - "from SyncAllowScript '%s': " - "'%s'", - conf.GetKeyValue("SyncAllowScript").c_str(), - line.c_str()); + BOX_ERROR("Invalid output " + "from SyncAllowScript '" + << conf.GetKeyValue("SyncAllowScript") + << "': '" << line << "'"); throw; } - ::syslog(LOG_INFO, "Delaying sync by %d seconds (SyncAllowScript '%s')", waitInSeconds, conf.GetKeyValue("SyncAllowScript").c_str()); + BOX_NOTICE("Delaying sync by " << waitInSeconds + << " seconds (SyncAllowScript '" + << conf.GetKeyValue("SyncAllowScript") + << "')"); } } - // Wait and then cleanup child process - int status = 0; - ::waitpid(pid, &status, 0); } catch(std::exception &e) { - ::syslog(LOG_ERR, "Internal error running SyncAllowScript: " - "%s", e.what()); - // Clean up - if(pid != 0) - { - int status = 0; - ::waitpid(pid, &status, 0); - } + BOX_ERROR("Internal error running SyncAllowScript: " + << e.what()); } catch(...) { // Ignore any exceptions // Log that something bad happened - ::syslog(LOG_ERR, "Error running SyncAllowScript '%s'", conf.GetKeyValue("SyncAllowScript").c_str()); - // Clean up though - if(pid != 0) - { - int status = 0; - ::waitpid(pid, &status, 0); - } + BOX_ERROR("Error running SyncAllowScript '" + << conf.GetKeyValue("SyncAllowScript") << "'"); + } + + // Wait and then cleanup child process, if any + if(pid != 0) + { + int status = 0; + ::waitpid(pid, &status, 0); } return waitInSeconds; @@ -998,32 +1214,34 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) { #ifdef WIN32 - // Really could use some interprocess protection, mutex etc - // any side effect should be too bad???? :) - DWORD timeout = (DWORD)BoxTimeToMilliSeconds(RequiredDelay); + DWORD requiredDelayMs = BoxTimeToMilliSeconds(RequiredDelay); - while ( this->mReceivedCommandConn == false ) - { - Sleep(1); + DWORD result = WaitForSingleObject(mhCommandReceivedEvent, + (DWORD)requiredDelayMs); - if( timeout == 0 ) - { - DoSyncFlagOut = false; - SyncIsForcedOut = false; - return; - } - timeout--; + if(result == WAIT_OBJECT_0) + { + DoSyncFlagOut = this->mDoSyncFlagOut; + SyncIsForcedOut = this->mSyncIsForcedOut; + ResetEvent(mhCommandReceivedEvent); + } + else if(result == WAIT_TIMEOUT) + { + DoSyncFlagOut = false; + SyncIsForcedOut = false; + } + else + { + BOX_ERROR("Unexpected result from WaitForSingleObject: " + "error " << GetLastError()); } - this->mReceivedCommandConn = false; - DoSyncFlagOut = this->mDoSyncFlagOut; - SyncIsForcedOut = this->mSyncIsForcedOut; return; #else // ! WIN32 ASSERT(mpCommandSocketInfo != 0); if(mpCommandSocketInfo == 0) {::sleep(1); return;} // failure case isn't too bad - TRACE1("Wait on command socket, delay = %lld\n", RequiredDelay); + BOX_TRACE("Wait on command socket, delay = " << RequiredDelay); try { @@ -1049,7 +1267,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla { #ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET bool uidOK = true; - ::syslog(LOG_WARNING, "On this platform, no security check can be made on the credentials of peers connecting to the command socket. (bbackupctl)"); + BOX_WARNING("On this platform, no security check can be made on the credentials of peers connecting to the command socket. (bbackupctl)"); #else // Security check -- does the process connecting to this socket have // the same UID as this process? @@ -1074,14 +1292,14 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla if(!uidOK) { // Dump the connection - ::syslog(LOG_ERR, "Incoming command connection from peer had different user ID than this process, or security check could not be completed."); + BOX_ERROR("Incoming command connection from peer had different user ID than this process, or security check could not be completed."); mpCommandSocketInfo->mpConnectedSocket.reset(); return; } else { // Log - ::syslog(LOG_INFO, "Connection from command socket"); + BOX_INFO("Connection from command socket"); // Send a header line summarising the configuration and current state const Configuration &conf(GetConfiguration()); @@ -1119,7 +1337,8 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla while(mpCommandSocketInfo->mpGetLine != 0 && !mpCommandSocketInfo->mpGetLine->IsEOF() && mpCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) { - TRACE1("Receiving command '%s' over command socket\n", command.c_str()); + BOX_TRACE("Receiving command '" << command + << "' over command socket"); bool sendOK = false; bool sendResponse = true; @@ -1176,9 +1395,19 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla } catch(std::exception &e) { - ::syslog(LOG_ERR, "Internal error in command socket thread: " - "%s", e.what()); - throw; // thread will die + BOX_ERROR("Internal error in command socket thread: " + << e.what()); + // If an error occurs, and there is a connection active, just close that + // connection and continue. Otherwise, let the error propagate. + if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } } catch(...) { @@ -1211,7 +1440,7 @@ void BackupDaemon::CloseCommandConnection() #ifndef WIN32 try { - TRACE0("Closing command connection\n"); + BOX_TRACE("Closing command connection"); if(mpCommandSocketInfo->mpGetLine) { @@ -1222,8 +1451,8 @@ void BackupDaemon::CloseCommandConnection() } catch(std::exception &e) { - ::syslog(LOG_ERR, "Internal error while closing command " - "socket: %s", e.what()); + BOX_ERROR("Internal error while closing command " + "socket: " << e.what()); } catch(...) { @@ -1247,10 +1476,6 @@ void BackupDaemon::SendSyncStartOrFinish(bool SendStart) // The bbackupctl program can't rely on a state change, because it // may never change if the server doesn't need to be contacted. -#ifdef __MINGW32__ -#warning race condition: what happens if socket is closed? -#endif - if(mpCommandSocketInfo != NULL && #ifdef WIN32 mpCommandSocketInfo->mListeningSocket.IsConnected() @@ -1259,21 +1484,24 @@ void BackupDaemon::SendSyncStartOrFinish(bool SendStart) #endif ) { - const char* message = SendStart ? "start-sync\n" : "finish-sync\n"; + std::string message = SendStart ? "start-sync" : "finish-sync"; try { #ifdef WIN32 - mpCommandSocketInfo->mListeningSocket.Write(message, - (int)strlen(message)); + EnterCriticalSection(&mMessageQueueLock); + mMessageList.push_back(message); + SetEvent(mhMessageToSendEvent); + LeaveCriticalSection(&mMessageQueueLock); #else - mpCommandSocketInfo->mpConnectedSocket->Write(message, - strlen(message)); + message += "\n"; + mpCommandSocketInfo->mpConnectedSocket->Write( + message.c_str(), message.size()); #endif } catch(std::exception &e) { - ::syslog(LOG_ERR, "Internal error while sending to " - "command socket client: %s", e.what()); + BOX_ERROR("Internal error while sending to " + "command socket client: " << e.what()); CloseCommandConnection(); } catch(...) @@ -1376,7 +1604,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con struct mntent *entry = 0; while((entry = ::getmntent(mountPointsFile)) != 0) { - TRACE1("Found mount point at %s\n", entry->mnt_dir); + BOX_TRACE("Found mount point at " << entry->mnt_dir); mountPoints.insert(std::string(entry->mnt_dir)); } @@ -1398,12 +1626,11 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con try { - // Read all the entries, and put them in the set struct mnttab entry; while(getmntent(mountPointsFile, &entry) == 0) { - TRACE1("Found mount point at %s\n", entry.mnt_mountp); + BOX_TRACE("Found mount point at " << entry.mnt_mountp); mountPoints.insert(std::string(entry.mnt_mountp)); } @@ -1432,7 +1659,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con for(std::list<std::pair<std::string, Configuration> >::const_iterator i = rLocationsConf.mSubConfigurations.begin(); i != rLocationsConf.mSubConfigurations.end(); ++i) { -TRACE0("new location\n"); + BOX_TRACE("new location"); // Create a record for it Location *ploc = new Location; try @@ -1444,7 +1671,23 @@ TRACE0("new location\n"); // Read the exclude lists from the Configuration ploc->mpExcludeFiles = BackupClientMakeExcludeList_Files(i->second); ploc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(i->second); - + // Does this exist on the server? + // Remove from dir object early, so that if we fail + // to stat the local directory, we still don't + // consider to remote one for deletion. + BackupStoreDirectory::Iterator iter(dir); + BackupStoreFilenameClear dirname(ploc->mName); // generate the filename + BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname); + int64_t oid = 0; + if(en != 0) + { + oid = en->GetObjectID(); + + // Delete the entry from the directory, so we get a list of + // unused root directories at the end of this. + dir.DeleteEntry(oid); + } + // Do a fsstat on the pathname to find out which mount it's on { @@ -1459,7 +1702,11 @@ TRACE0("new location\n"); if(::statfs(ploc->mPath.c_str(), &s) != 0) #endif // HAVE_STRUCT_STATVFS_F_MNTONNAME { - THROW_EXCEPTION(CommonException, OSFileError) + BOX_WARNING("Failed to stat location: " + << ploc->mPath + << ": " << strerror(errno)); + THROW_EXCEPTION(CommonException, + OSFileError) } // Where the filesystem is mounted @@ -1470,19 +1717,22 @@ TRACE0("new location\n"); // Warn in logs if the directory isn't absolute if(ploc->mPath[0] != '/') { - ::syslog(LOG_ERR, "Location path '%s' isn't absolute", ploc->mPath.c_str()); + BOX_WARNING("Location path '" + << ploc->mPath + << "' is not absolute"); } // Go through the mount points found, and find a suitable one std::string mountName("/"); { std::set<std::string, mntLenCompare>::const_iterator i(mountPoints.begin()); - TRACE1("%d potential mount points\n", mountPoints.size()); + BOX_TRACE(mountPoints.size() + << " potential mount points"); for(; i != mountPoints.end(); ++i) { // Compare first n characters with the filename // If it matches, the file belongs in that mount point // (sorting order ensures this) - TRACE1("checking against mount point %s\n", i->c_str()); + BOX_TRACE("checking against mount point " << *i); if(::strncmp(i->c_str(), ploc->mPath.c_str(), i->size()) == 0) { // Match @@ -1490,7 +1740,9 @@ TRACE0("new location\n"); break; } } - TRACE2("mount point chosen for %s is %s\n", ploc->mPath.c_str(), mountName.c_str()); + BOX_TRACE("mount point chosen for " + << ploc->mPath << " is " + << mountName); } #endif @@ -1517,35 +1769,46 @@ TRACE0("new location\n"); } // Does this exist on the server? - BackupStoreDirectory::Iterator iter(dir); - BackupStoreFilenameClear dirname(ploc->mName); // generate the filename - BackupStoreDirectory::Entry *en = iter.FindMatchingClearName(dirname); - int64_t oid = 0; - if(en != 0) - { - oid = en->GetObjectID(); - - // Delete the entry from the directory, so we get a list of - // unused root directories at the end of this. - dir.DeleteEntry(oid); - } - else + if(en == 0) { // Doesn't exist, so it has to be created on the server. Let's go! // First, get the directory's attributes and modification time box_time_t attrModTime = 0; BackupClientFileAttributes attr; - attr.ReadAttributes(ploc->mPath.c_str(), true /* directories have zero mod times */, - 0 /* not interested in mod time */, &attrModTime /* get the attribute modification time */); + try + { + attr.ReadAttributes(ploc->mPath.c_str(), + true /* directories have zero mod times */, + 0 /* not interested in mod time */, + &attrModTime /* get the attribute modification time */); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to get attributes " + "for path '" << ploc->mPath + << "', skipping."); + continue; + } // Execute create directory command - MemBlockStream attrStream(attr); - std::auto_ptr<BackupProtocolClientSuccess> dirCreate(connection.QueryCreateDirectory( - BackupProtocolClientListDirectory::RootDirectory, - attrModTime, dirname, attrStream)); - - // Object ID for later creation - oid = dirCreate->GetObjectID(); + try + { + MemBlockStream attrStream(attr); + std::auto_ptr<BackupProtocolClientSuccess> dirCreate(connection.QueryCreateDirectory( + BackupProtocolClientListDirectory::RootDirectory, + attrModTime, dirname, attrStream)); + + // Object ID for later creation + oid = dirCreate->GetObjectID(); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to create remote " + "directory '/" << ploc->mName << + "', skipping location."); + continue; + } + } // Create and store the directory object for the root of this location @@ -1556,10 +1819,26 @@ TRACE0("new location\n"); // Push it back on the vector of locations mLocations.push_back(ploc); } - catch(...) + catch (std::exception &e) { delete ploc; ploc = 0; + BOX_ERROR("Failed to configure location '" + << ploc->mName << "' path '" + << ploc->mPath << "': " << e.what() << + ": please check for previous errors"); + throw; + } + catch(...) + { + BOX_ERROR("Failed to configure location '" + << ploc->mName << "' path '" + << ploc->mPath << "': please check for " + "previous errors"); + + delete ploc; + ploc = NULL; + throw; } } @@ -1567,8 +1846,27 @@ TRACE0("new location\n"); // Any entries in the root directory which need deleting? if(dir.GetNumberOfEntries() > 0) { - ::syslog(LOG_INFO, "%d redundant locations in root directory found, will delete from store after %d seconds.", - dir.GetNumberOfEntries(), BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER); + box_time_t now = GetCurrentBoxTime(); + + // This should reset the timer if the list of unused + // locations changes, but it will not if the number of + // unused locations does not change, but the locations + // do change, e.g. one mysteriously appears and another + // mysteriously appears. (FIXME) + if (dir.GetNumberOfEntries() != mUnusedRootDirEntries.size() || + mDeleteUnusedRootDirEntriesAfter == 0) + { + mDeleteUnusedRootDirEntriesAfter = now + + SecondsToBoxTime( + BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER); + } + + int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter + - now); + + BOX_NOTICE(dir.GetNumberOfEntries() << " redundant locations " + "in root directory found, will delete from store " + "after " << secs << " seconds."); // Store directories in list of things to delete mUnusedRootDirEntries.clear(); @@ -1579,14 +1877,13 @@ TRACE0("new location\n"); // Add name to list BackupStoreFilenameClear clear(en->GetName()); const std::string &name(clear.GetClearFilename()); - mUnusedRootDirEntries.push_back(std::pair<int64_t,std::string>(en->GetObjectID(), name)); + mUnusedRootDirEntries.push_back( + std::pair<int64_t,std::string> + (en->GetObjectID(), name)); // Log this - ::syslog(LOG_INFO, "Unused location in root: %s", name.c_str()); + BOX_INFO("Unused location in root: " << name); } ASSERT(mUnusedRootDirEntries.size() > 0); - // Time to delete them - mDeleteUnusedRootDirEntriesAfter = - GetCurrentBoxTime() + SecondsToBoxTime(BACKUP_DELETE_UNUSED_ROOT_ENTRIES_AFTER); } } @@ -1698,14 +1995,14 @@ void BackupDaemon::DeleteCorruptBerkelyDbFiles() MakeMapBaseName(l, filename); // Delete the file - TRACE1("Deleting %s\n", filename.c_str()); + BOX_TRACE("Deleting " << filename); ::unlink(filename.c_str()); // Add a suffix for the new map filename += ".n"; // Delete that too - TRACE1("Deleting %s\n", filename.c_str()); + BOX_TRACE("Deleting " << filename); ::unlink(filename.c_str()); } } @@ -1785,6 +2082,9 @@ void BackupDaemon::CommitIDMapsAfterSync() #endif if(::rename(newmap.c_str(), target.c_str()) != 0) { + BOX_ERROR("failed to rename ID map: " << newmap + << " to " << target << ": " + << strerror(errno)); THROW_EXCEPTION(CommonException, OSFileError) } } @@ -1869,51 +2169,44 @@ void BackupDaemon::SetState(int State) // command socket if there's an error char newState[64]; - char newStateSize = sprintf(newState, "state %d\n", State); + sprintf(newState, "state %d", State); + std::string message = newState; #ifdef WIN32 - #ifndef _MSC_VER - #warning FIX ME: race condition - #endif + EnterCriticalSection(&mMessageQueueLock); + mMessageList.push_back(newState); + SetEvent(mhMessageToSendEvent); + LeaveCriticalSection(&mMessageQueueLock); +#else + message += "\n"; - // what happens if the socket is closed by the other thread before - // we can write to it? Null pointer deref at best. - if(mpCommandSocketInfo && - mpCommandSocketInfo->mListeningSocket.IsConnected()) + if(mpCommandSocketInfo == 0) { - try - { - mpCommandSocketInfo->mListeningSocket.Write(newState, newStateSize); - } - catch(std::exception &e) - { - ::syslog(LOG_ERR, "Internal error while writing state " - "to command socket: %s", e.what()); - CloseCommandConnection(); - } - catch(...) - { - CloseCommandConnection(); - } + return; } -#else - if(mpCommandSocketInfo != 0 && mpCommandSocketInfo->mpConnectedSocket.get() != 0) + + if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) { - // Something connected to the command socket, tell it about the new state - try - { - mpCommandSocketInfo->mpConnectedSocket->Write(newState, newStateSize); - } - catch(std::exception &e) - { - ::syslog(LOG_ERR, "Internal error while writing state " - "to command socket: %s", e.what()); - CloseCommandConnection(); - } - catch(...) - { - CloseCommandConnection(); - } + return; + } + + // Something connected to the command socket, tell it about the new state + try + { + mpCommandSocketInfo->mpConnectedSocket->Write(message.c_str(), + message.length()); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error while writing state " + "to command socket: " << e.what()); + CloseCommandConnection(); + } + catch(...) + { + BOX_ERROR("Internal error while writing state " + "to command socket: unknown error"); + CloseCommandConnection(); } #endif } @@ -1944,49 +2237,78 @@ void BackupDaemon::TouchFileInWorkingDir(const char *Filename) // // Function // Name: BackupDaemon::NotifySysadmin(int) -// Purpose: Run the script to tell the sysadmin about events which need attention. +// Purpose: Run the script to tell the sysadmin about events +// which need attention. // Created: 25/2/04 // // -------------------------------------------------------------------------- void BackupDaemon::NotifySysadmin(int Event) { - static const char *sEventNames[] = {"store-full", "read-error", 0}; + static const char *sEventNames[] = + { + "store-full", + "read-error", + "backup-error", + "backup-start", + "backup-finish", + 0 + }; + + BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); + BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); + BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); + ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == NotifyEvent__MAX + 1); - TRACE1("BackupDaemon::NotifySysadmin() called, event = %d\n", Event); + BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << + sEventNames[Event]); - if(Event < 0 || Event > NotifyEvent__MAX) + if(Event < 0 || Event >= NotifyEvent__MAX) { - THROW_EXCEPTION(BackupStoreException, BadNotifySysadminEventCode); + THROW_EXCEPTION(BackupStoreException, + BadNotifySysadminEventCode); } // Don't send lots of repeated messages - if(mNotificationsSent[Event]) + if(mNotificationsSent[Event] && + Event != NotifyEvent_BackupStart && + Event != NotifyEvent_BackupFinish) { + BOX_WARNING("Suppressing duplicate notification about " << + sEventNames[Event]); return; } // Is there a notifation script? const Configuration &conf(GetConfiguration()); - if(!conf.KeyExists("NotifyScript")) + if(!conf.KeyExists("NotifyScript") && + Event != NotifyEvent_BackupStart && + Event != NotifyEvent_BackupFinish) { // Log, and then return - ::syslog(LOG_ERR, "Not notifying administrator about event %s -- set NotifyScript to do this in future", sEventNames[Event]); + BOX_ERROR("Not notifying administrator about event " + << sEventNames[Event] << " -- set NotifyScript " + "to do this in future"); return; } // Script to run - std::string script(conf.GetKeyValue("NotifyScript") + ' ' + sEventNames[Event]); + std::string script(conf.GetKeyValue("NotifyScript") + ' ' + + sEventNames[Event]); // Log what we're about to do - ::syslog(LOG_INFO, "About to notify administrator about event %s, running script '%s'", sEventNames[Event], script.c_str()); + BOX_NOTICE("About to notify administrator about event " + << sEventNames[Event] << ", running script '" + << script << "'"); // Then do it if(::system(script.c_str()) != 0) { - ::syslog(LOG_ERR, "Notify script returned an error code. ('%s')", script.c_str()); + BOX_ERROR("Notify script returned an error code. ('" + << script << "')"); } - // Flag that this is done so the administrator isn't constantly bombarded with lots of errors + // Flag that this is done so the administrator isn't constantly + // bombarded with lots of errors mNotificationsSent[Event] = true; } @@ -2001,28 +2323,40 @@ void BackupDaemon::NotifySysadmin(int Event) // -------------------------------------------------------------------------- void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) { - if(mUnusedRootDirEntries.empty() || mDeleteUnusedRootDirEntriesAfter == 0) + if(mUnusedRootDirEntries.empty()) { - // Nothing to do. + BOX_INFO("Not deleting unused entries - none in list"); return; } + if(mDeleteUnusedRootDirEntriesAfter == 0) + { + BOX_INFO("Not deleting unused entries - " + "zero delete time (bad)"); + return; + } + // Check time - if(GetCurrentBoxTime() < mDeleteUnusedRootDirEntriesAfter) + box_time_t now = GetCurrentBoxTime(); + if(now < mDeleteUnusedRootDirEntriesAfter) { - // Too early to delete files + int secs = BoxTimeToSeconds(mDeleteUnusedRootDirEntriesAfter + - now); + BOX_INFO("Not deleting unused entries - too early (" + << secs << " seconds remaining)"); return; } // Entries to delete, and it's the right time to do so... - ::syslog(LOG_INFO, "Deleting unused locations from store root..."); + BOX_NOTICE("Deleting unused locations from store root..."); BackupProtocolClient &connection(rContext.GetConnection()); for(std::vector<std::pair<int64_t,std::string> >::iterator i(mUnusedRootDirEntries.begin()); i != mUnusedRootDirEntries.end(); ++i) { connection.QueryDeleteDirectory(i->first); // Log this - ::syslog(LOG_INFO, "Deleted %s (ID %08llx) from store root", i->second.c_str(), i->first); + BOX_NOTICE("Deleted " << i->second << " (ID " << i->first + << ") from store root"); } // Reset state @@ -2138,7 +2472,7 @@ void BackupDaemon::Location::Deserialize(Archive &rArchive) else { // there is something going on here - THROW_EXCEPTION(CommonException, Internal) + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); } // @@ -2163,7 +2497,7 @@ void BackupDaemon::Location::Deserialize(Archive &rArchive) else { // there is something going on here - THROW_EXCEPTION(CommonException, Internal) + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); } // @@ -2188,7 +2522,7 @@ void BackupDaemon::Location::Deserialize(Archive &rArchive) else { // there is something going on here - THROW_EXCEPTION(CommonException, Internal) + THROW_EXCEPTION(ClientException, CorruptStoreObjectInfoFile); } } @@ -2302,7 +2636,7 @@ BackupDaemon::CommandSocketInfo::~CommandSocketInfo() static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; -static const int STOREOBJECTINFO_VERSION = 1; +static const int STOREOBJECTINFO_VERSION = 2; bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const { @@ -2361,15 +2695,39 @@ bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time // // // + iCount = mUnusedRootDirEntries.size(); + anArchive.Write(iCount); + + for(int v = 0; v < iCount; v++) + { + anArchive.Write(mUnusedRootDirEntries[v].first); + anArchive.Write(mUnusedRootDirEntries[v].second); + } + + if (iCount > 0) + { + anArchive.Write(mDeleteUnusedRootDirEntriesAfter); + } + + // + // + // aFile.Close(); - ::syslog(LOG_INFO, "Saved store object info file '%s'", - StoreObjectInfoFile.c_str()); + BOX_INFO("Saved store object info file version " << + STOREOBJECTINFO_VERSION << " (" << + StoreObjectInfoFile << ")"); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error writing store object " + "info file (" << StoreObjectInfoFile << "): " + << e.what()); } catch(...) { - ::syslog(LOG_WARNING, "Requested store object info file '%s' " - "not accessible or could not be created", - StoreObjectInfoFile.c_str()); + BOX_ERROR("Internal error writing store object " + "info file (" << StoreObjectInfoFile << "): " + "unknown error"); } return created; @@ -2420,10 +2778,10 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ if(iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) { - ::syslog(LOG_WARNING, "Store object info file '%s' " + BOX_WARNING("Store object info file " "is not a valid or compatible serialised " - "archive. Will re-cache from store.", - StoreObjectInfoFile.c_str()); + "archive. Will re-cache from store. " + "(" << StoreObjectInfoFile << ")"); return false; } @@ -2435,10 +2793,10 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ if(strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) { - ::syslog(LOG_WARNING, "Store object info file '%s' " + BOX_WARNING("Store object info file " "is not a valid or compatible serialised " - "archive. Will re-cache from store.", - StoreObjectInfoFile.c_str()); + "archive. Will re-cache from store. " + "(" << StoreObjectInfoFile << ")"); return false; } @@ -2451,11 +2809,10 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ if(iVersion != STOREOBJECTINFO_VERSION) { - ::syslog(LOG_WARNING, "Store object info file '%s' " - "version %d unsupported. " - "Will re-cache from store.", - StoreObjectInfoFile.c_str(), - iVersion); + BOX_WARNING("Store object info file " + "version " << iVersion << " unsupported. " + "Will re-cache from store. " + "(" << StoreObjectInfoFile << ")"); return false; } @@ -2468,9 +2825,9 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ if(lastKnownConfigModTime != GetLoadedConfigModifiedTime()) { - ::syslog(LOG_WARNING, "Store object info file '%s' " - "out of date. Will re-cache from store", - StoreObjectInfoFile.c_str()); + BOX_WARNING("Store object info file " + "out of date. Will re-cache from store. " + "(" << StoreObjectInfoFile << ")"); return false; } @@ -2516,22 +2873,41 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ // // // - aFile.Close(); - ::syslog(LOG_INFO, "Loaded store object info file '%s', " - "version [%d]", StoreObjectInfoFile.c_str(), - iVersion); + iCount = 0; + anArchive.Read(iCount); + for(int v = 0; v < iCount; v++) + { + int64_t anId; + anArchive.Read(anId); + + std::string aName; + anArchive.Read(aName); + + mUnusedRootDirEntries.push_back(std::pair<int64_t, std::string>(anId, aName)); + } + + if (iCount > 0) + anArchive.Read(mDeleteUnusedRootDirEntriesAfter); + + // + // + // + aFile.Close(); + BOX_INFO("Loaded store object info file version " << iVersion + << " (" << StoreObjectInfoFile << ")"); + return true; } catch(std::exception &e) { - ::syslog(LOG_ERR, "Internal error reading store object " - "info file: %s", e.what()); + BOX_ERROR("Internal error reading store object info file: " + << StoreObjectInfoFile << ": " << e.what()); } catch(...) { - ::syslog(LOG_ERR, "Internal error reading store object " - "info file: unknown error"); + BOX_ERROR("Internal error reading store object info file: " + << StoreObjectInfoFile << ": unknown error"); } DeleteAllLocations(); @@ -2540,11 +2916,10 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ theLastSyncTime = 0; theNextSyncTime = 0; - ::syslog(LOG_WARNING, "Requested store object info file '%s' " - "does not exist, not accessible, or inconsistent. " - "Will re-cache from store.", - StoreObjectInfoFile.c_str()); - + BOX_WARNING("Store object info file is missing, not accessible, " + "or inconsistent. Will re-cache from store. " + "(" << StoreObjectInfoFile << ")"); + return false; } @@ -2572,9 +2947,9 @@ bool BackupDaemon::DeleteStoreObjectInfo() const if(!FileExists(storeObjectInfoFile.c_str())) { // File doesn't exist -- so can't be deleted. But something isn't quite right, so log a message - ::syslog(LOG_ERR, "Expected to be able to delete " - "store object info file '%s', but the file did not exist.", - storeObjectInfoFile.c_str()); + BOX_WARNING("Store object info file did not exist when it " + "was supposed to. (" << storeObjectInfoFile << ")"); + // Return true to stop things going around in a loop return true; } @@ -2582,9 +2957,8 @@ bool BackupDaemon::DeleteStoreObjectInfo() const // Actually delete it if(::unlink(storeObjectInfoFile.c_str()) != 0) { - ::syslog(LOG_ERR, "Failed to delete the old " - "store object info file '%s': %s", - storeObjectInfoFile.c_str(), strerror(errno)); + BOX_ERROR("Failed to delete the old store object info file: " + << storeObjectInfoFile << ": "<< strerror(errno)); return false; } diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h index 347c205b..ab64bbfa 100644 --- a/bin/bbackupd/BackupDaemon.h +++ b/bin/bbackupd/BackupDaemon.h @@ -16,9 +16,13 @@ #include "BoxTime.h" #include "Daemon.h" +#include "BackupClientDirectoryRecord.h" #include "Socket.h" #include "SocketListen.h" #include "SocketStream.h" +#include "Logging.h" +#include "autogen_BackupProtocolClient.h" + #ifdef WIN32 #include "WinNamedPipeStream.h" #endif @@ -39,7 +43,7 @@ class Archive; // Created: 2003/10/08 // // -------------------------------------------------------------------------- -class BackupDaemon : public Daemon +class BackupDaemon : public Daemon, ProgressNotifier { public: BackupDaemon(); @@ -79,8 +83,11 @@ public: enum { NotifyEvent_StoreFull = 0, - NotifyEvent_ReadError = 1, - NotifyEvent__MAX = 1 + NotifyEvent_ReadError, + NotifyEvent_BackupError, + NotifyEvent_BackupStart, + NotifyEvent_BackupFinish, + NotifyEvent__MAX // When adding notifications, remember to add strings to NotifySysadmin() }; void NotifySysadmin(int Event); @@ -174,18 +181,244 @@ private: CommandSocketInfo *mpCommandSocketInfo; // Stop notifications being repeated. - bool mNotificationsSent[NotifyEvent__MAX + 1]; + bool mNotificationsSent[NotifyEvent__MAX]; // Unused entries in the root directory wait a while before being deleted box_time_t mDeleteUnusedRootDirEntriesAfter; // time to delete them std::vector<std::pair<int64_t,std::string> > mUnusedRootDirEntries; +public: + bool StopRun() { return this->Daemon::StopRun(); } + +private: + bool mLogAllFileAccess; + + /* ProgressNotifier implementation */ +public: + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Scanning directory: " << rLocalPath); + } + } + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to access directory: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to access file: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Failed to list directory: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyMountPointSkipped( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + #ifdef WIN32 + BOX_WARNING("Ignored directory: " << rLocalPath << + ": is an NTFS junction/reparse point; create " + "a new location if you want to back it up"); + #else + BOX_WARNING("Ignored directory: " << rLocalPath << + ": is a mount point; create a new location " + "if you want to back it up"); + #endif + } + virtual void NotifyFileExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Skipping excluded file: " << rLocalPath); + } + } + virtual void NotifyDirExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Skipping excluded directory: " << rLocalPath); + } + } + virtual void NotifyUnsupportedFileType( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Ignoring file of unknown type: " << rLocalPath); + } + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { + BOX_WARNING("Error reading file: " << rLocalPath + << ": " << rErrorMsg); + } + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Some files have modification times excessively " + "in the future. Check clock synchronisation. " + "Example file (only one shown): " << rLocalPath); + } + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + BOX_WARNING("Skipped file: server is full: " << rLocalPath); + } + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) + { + if (rException.GetType() == CommonException::ExceptionType && + rException.GetSubType() == CommonException::AccessDenied) + { + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": Access denied"); + } + else + { + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": caught exception: " << rException.what() + << " (" << rException.GetType() + << "/" << rException.GetSubType() << ")"); + } + } + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) + { + std::ostringstream msgs; + if (type != BackupProtocolClientError::ErrorType) + { + msgs << "unknown error type " << type; + } + else + { + switch(subtype) + { + case BackupProtocolClientError::Err_WrongVersion: + msgs << "WrongVersion"; + break; + case BackupProtocolClientError::Err_NotInRightProtocolPhase: + msgs << "NotInRightProtocolPhase"; + break; + case BackupProtocolClientError::Err_BadLogin: + msgs << "BadLogin"; + break; + case BackupProtocolClientError::Err_CannotLockStoreForWriting: + msgs << "CannotLockStoreForWriting"; + break; + case BackupProtocolClientError::Err_SessionReadOnly: + msgs << "SessionReadOnly"; + break; + case BackupProtocolClientError::Err_FileDoesNotVerify: + msgs << "FileDoesNotVerify"; + break; + case BackupProtocolClientError::Err_DoesNotExist: + msgs << "DoesNotExist"; + break; + case BackupProtocolClientError::Err_DirectoryAlreadyExists: + msgs << "DirectoryAlreadyExists"; + break; + case BackupProtocolClientError::Err_CannotDeleteRoot: + msgs << "CannotDeleteRoot"; + break; + case BackupProtocolClientError::Err_TargetNameExists: + msgs << "TargetNameExists"; + break; + case BackupProtocolClientError::Err_StorageLimitExceeded: + msgs << "StorageLimitExceeded"; + break; + case BackupProtocolClientError::Err_DiffFromFileDoesNotExist: + msgs << "DiffFromFileDoesNotExist"; + break; + case BackupProtocolClientError::Err_DoesNotExistInDirectory: + msgs << "DoesNotExistInDirectory"; + break; + case BackupProtocolClientError::Err_PatchConsistencyError: + msgs << "PatchConsistencyError"; + break; + default: + msgs << "unknown error subtype " << subtype; + } + } + + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": server error: " << msgs.str()); + } + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Uploading complete file: " << rLocalPath); + } + } + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { + if (mLogAllFileAccess) + { + BOX_INFO("Uploading patch to file: " << rLocalPath); + } + } + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) + { + if (mLogAllFileAccess) + { + BOX_INFO("Uploaded file: " << rLocalPath); + } + } + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) + { + if (mLogAllFileAccess) + { + BOX_INFO("Synchronised file: " << rLocalPath); + } + } + #ifdef WIN32 public: void RunHelperThread(void); private: - bool mDoSyncFlagOut, mSyncIsForcedOut, mReceivedCommandConn; + bool mDoSyncFlagOut, mSyncIsForcedOut; + HANDLE mhMessageToSendEvent, mhCommandReceivedEvent; + CRITICAL_SECTION mMessageQueueLock; + std::vector<std::string> mMessageList; #endif }; diff --git a/bin/bbackupd/ClientException.txt b/bin/bbackupd/ClientException.txt new file mode 100644 index 00000000..04f88620 --- /dev/null +++ b/bin/bbackupd/ClientException.txt @@ -0,0 +1,11 @@ + +# NOTE: Exception descriptions are for public distributions of Box Backup only -- do not rely for other applications. + + +EXCEPTION Client 13 + +Internal 0 +AssertFailed 1 +ClockWentBackwards 2 Invalid (negative) sync period: perhaps your clock is going backwards? +FailedToDeleteStoreObjectInfoFile 3 Failed to delete the StoreObjectInfoFile, backup cannot continue safely. +CorruptStoreObjectInfoFile 4 The store object info file contained an invalid value and is probably corrupt. Try deleting it. diff --git a/bin/bbackupd/Makefile.extra b/bin/bbackupd/Makefile.extra new file mode 100644 index 00000000..52e21366 --- /dev/null +++ b/bin/bbackupd/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_ClientException.h autogen_ClientException.cpp: $(MAKEEXCEPTION) ClientException.txt + $(PERL) $(MAKEEXCEPTION) ClientException.txt + diff --git a/bin/bbackupd/Win32BackupService.cpp b/bin/bbackupd/Win32BackupService.cpp index 7cbf4828..9f0ac867 100644 --- a/bin/bbackupd/Win32BackupService.cpp +++ b/bin/bbackupd/Win32BackupService.cpp @@ -29,33 +29,17 @@ void TerminateService(void) DWORD Win32BackupService::WinService(const char* pConfigFileName) { - char exepath[MAX_PATH]; - GetModuleFileName(NULL, exepath, sizeof(exepath)); + DWORD ret; - std::string configfile; - if (pConfigFileName != NULL) { - configfile = pConfigFileName; + ret = this->Main(pConfigFileName); } else { - // make the default config file name, - // based on the program path - configfile = exepath; - configfile = configfile.substr(0, - configfile.rfind(DIRECTORY_SEPARATOR_ASCHAR)); - configfile += DIRECTORY_SEPARATOR "bbackupd.conf"; + ret = this->Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE); } - const char *argv[] = {exepath, "-c", configfile.c_str()}; - int argc = sizeof(argv) / sizeof(*argv); - DWORD ret; - - MAINHELPER_START - ret = this->Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); - MAINHELPER_END - return ret; } diff --git a/bin/bbackupd/Win32ServiceFunctions.cpp b/bin/bbackupd/Win32ServiceFunctions.cpp index 3010cf3f..5acf5f67 100644 --- a/bin/bbackupd/Win32ServiceFunctions.cpp +++ b/bin/bbackupd/Win32ServiceFunctions.cpp @@ -44,7 +44,7 @@ void ErrorHandler(char *s, DWORD err) char buf[256]; memset(buf, 0, sizeof(buf)); _snprintf(buf, sizeof(buf)-1, "%s (%d)", s, err); - ::syslog(LOG_ERR, "%s", buf); + BOX_ERROR(buf); MessageBox(0, buf, "Error", MB_OK | MB_SETFOREGROUND | MB_DEFAULT_DESKTOP_ONLY); ExitProcess(err); @@ -161,7 +161,7 @@ VOID ServiceMain(DWORD argc, LPTSTR *argv) } } -void OurService(char* pConfigFileName) +int OurService(char* pConfigFileName) { spConfigFileName = pConfigFileName; @@ -180,7 +180,10 @@ void OurService(char* pConfigFileName) ErrorHandler("Failed to start service. Did you start " "Box Backup from the Service Control Manager? " "(StartServiceCtrlDispatcher)", GetLastError()); + return 1; } + + return 0; } int InstallService(const char* pConfigFileName) @@ -191,16 +194,16 @@ int InstallService(const char* pConfigFileName) if (emu_stat(pConfigFileName, &st) != 0) { - syslog(LOG_ERR, "Failed to open configuration file: " - "%s: %s", pConfigFileName, strerror(errno)); + BOX_ERROR("Failed to open configuration file '" << + pConfigFileName << "': " << strerror(errno)); return 1; } if (!(st.st_mode & S_IFREG)) { - syslog(LOG_ERR, "Failed to open configuration file: " - "%s: not a file", pConfigFileName); + BOX_ERROR("Failed to open configuration file '" << + pConfigFileName << "': not a file"); return 1; } } @@ -209,8 +212,8 @@ int InstallService(const char* pConfigFileName) if (!scm) { - syslog(LOG_ERR, "Failed to open service control manager: " - "error %d", GetLastError()); + BOX_ERROR("Failed to open service control manager: " << + GetErrorMessage(GetLastError())); return 1; } @@ -248,21 +251,21 @@ int InstallService(const char* pConfigFileName) { case ERROR_SERVICE_EXISTS: { - ::syslog(LOG_ERR, "Failed to create Box Backup " + BOX_ERROR("Failed to create Box Backup " "service: it already exists"); } break; case ERROR_SERVICE_MARKED_FOR_DELETE: { - ::syslog(LOG_ERR, "Failed to create Box Backup " + BOX_ERROR("Failed to create Box Backup " "service: it is waiting to be deleted"); } break; case ERROR_DUPLICATE_SERVICE_NAME: { - ::syslog(LOG_ERR, "Failed to create Box Backup " + BOX_ERROR("Failed to create Box Backup " "service: a service with this name " "already exists"); } @@ -270,15 +273,16 @@ int InstallService(const char* pConfigFileName) default: { - ::syslog(LOG_ERR, "Failed to create Box Backup " - "service: error %d", err); + BOX_ERROR("Failed to create Box Backup " + "service: error " << + GetErrorMessage(GetLastError())); } } return 1; } - ::syslog(LOG_INFO, "Created Box Backup service"); + BOX_INFO("Created Box Backup service"); SERVICE_DESCRIPTION desc; desc.lpDescription = "Backs up your data files over the Internet"; @@ -286,8 +290,8 @@ int InstallService(const char* pConfigFileName) if (!ChangeServiceConfig2(newService, SERVICE_CONFIG_DESCRIPTION, &desc)) { - ::syslog(LOG_WARNING, "Failed to set description for " - "Box Backup service: error %d", GetLastError()); + BOX_WARNING("Failed to set description for Box Backup " + "service: " << GetErrorMessage(GetLastError())); } CloseServiceHandle(newService); @@ -301,8 +305,8 @@ int RemoveService(void) if (!scm) { - syslog(LOG_ERR, "Failed to open service control manager: " - "error %d", GetLastError()); + BOX_ERROR("Failed to open service control manager: " << + GetErrorMessage(GetLastError())); return 1; } @@ -317,13 +321,13 @@ int RemoveService(void) err == ERROR_IO_PENDING) // hello microsoft? anyone home? { - syslog(LOG_ERR, "Failed to open Box Backup service: " + BOX_ERROR("Failed to open Box Backup service: " "not installed or not found"); } else { - syslog(LOG_ERR, "Failed to open Box Backup service: " - "error %d", err); + BOX_ERROR("Failed to open Box Backup service: " << + GetErrorMessage(err)); } return 1; } @@ -334,8 +338,8 @@ int RemoveService(void) err = GetLastError(); if (err != ERROR_SERVICE_NOT_ACTIVE) { - syslog(LOG_WARNING, "Failed to stop Box Backup " - "service: error %d", err); + BOX_WARNING("Failed to stop Box Backup service: " << + GetErrorMessage(err)); } } @@ -345,18 +349,18 @@ int RemoveService(void) if (deleted) { - syslog(LOG_INFO, "Box Backup service deleted"); + BOX_INFO("Box Backup service deleted"); return 0; } else if (err == ERROR_SERVICE_MARKED_FOR_DELETE) { - syslog(LOG_ERR, "Failed to remove Box Backup service: " + BOX_ERROR("Failed to remove Box Backup service: " "it is already being deleted"); } else { - syslog(LOG_ERR, "Failed to remove Box Backup service: " - "error %d", err); + BOX_ERROR("Failed to remove Box Backup service: " << + GetErrorMessage(err)); } return 1; diff --git a/bin/bbackupd/Win32ServiceFunctions.h b/bin/bbackupd/Win32ServiceFunctions.h index 70e1f085..cecd5c7b 100644 --- a/bin/bbackupd/Win32ServiceFunctions.h +++ b/bin/bbackupd/Win32ServiceFunctions.h @@ -12,8 +12,8 @@ #ifndef WIN32SERVICEFUNCTIONS_H #define WIN32SERVICEFUNCTIONS_H -int RemoveService (void); -int InstallService (const char* pConfigFilePath); -void OurService (char* pConfigFileName); +int RemoveService (void); +int InstallService (const char* pConfigFilePath); +int OurService (char* pConfigFileName); #endif diff --git a/bin/bbackupd/bbackupd-config.in b/bin/bbackupd/bbackupd-config.in index c06b818d..b5dc8be8 100755 --- a/bin/bbackupd/bbackupd-config.in +++ b/bin/bbackupd/bbackupd-config.in @@ -123,9 +123,9 @@ __E print ' ',$_,"\n" for(@tobackup); print <<__E; -Note: If other file systems are mounted inside these directories, then problems may occur -with files on the store server being renamed incorrectly. This will cause efficiency -problems, but not affect the integrity of the backups. +Note: If other file systems are mounted inside these directories, then +they will NOT be backed up. You will have to create separate locations for +any mounted filesystems inside your backup locations. WARNING: Directories not checked against mountpoints. Check mounted filesystems manually. @@ -211,12 +211,24 @@ $sendmail = 'sendmail' if $sendmail !~ m/\S/; print NOTIFY <<__EOS; #!/bin/sh +# This script is run whenever bbackupd changes state or encounters a +# problem which requires the system administrator to assist: +# +# 1) The store is full, and no more data can be uploaded. +# 2) Some files or directories were not readable. +# 3) A backup run starts or finishes. +# +# The default script emails the system administrator, except for backups +# starting and stopping, where it does nothing. + SUBJECT="BACKUP PROBLEM on host $hostname" SENDTO="$current_username" -if [ \$1 = store-full ] -then -$sendmail \$SENDTO <<EOM +if [ "\$1" = "" ]; then + echo "Usage: $0 <store-full|read-error|backup-error|backup-start|backup-finish>" >&2 + exit 2 +elif [ "\$1" = store-full ]; then + $sendmail \$SENDTO <<EOM Subject: \$SUBJECT (store full) To: \$SENDTO @@ -230,8 +242,7 @@ FILES ARE NOT BEING BACKED UP Please adjust the limits on account $account_num on server $server. EOM -elif [ \$1 = read-error ] -then +elif [ "\$1" = read-error ]; then $sendmail \$SENDTO <<EOM Subject: \$SUBJECT (read errors) To: \$SENDTO @@ -244,11 +255,14 @@ THESE FILES ARE NOT BEING BACKED UP =================================== Check the logs on $hostname for the files and directories which caused -these errors, and take appropraite action. +these errors, and take appropriate action. Other files are being backed up. EOM +elif [ "\$1" = backup-start -o "\$1" = backup-finish ]; then + # do nothing by default + true else $sendmail \$SENDTO <<EOM Subject: \$SUBJECT (unknown) @@ -287,11 +301,15 @@ TrustedCAsFile = $ca_root_cert DataDirectory = $working_dir -# This script is run whenever bbackupd encounters a problem which requires -# the system administrator to assist: +# This script is run whenever bbackupd changes state or encounters a +# problem which requires the system administrator to assist: +# # 1) The store is full, and no more data can be uploaded. # 2) Some files or directories were not readable. -# The default script emails the system administrator. +# 3) A backup run starts or finishes. +# +# The default script emails the system administrator, except for backups +# starting and stopping, where it does nothing. NotifyScript = $notify_script @@ -302,24 +320,46 @@ if($backup_mode eq 'lazy') # lazy mode configuration print CONFIG <<__E; -# A scan of the local discs will be made once an hour (approximately). -# To avoid cycles of load on the server, this time is randomly adjusted by a small +# The number of seconds between backup runs under normal conditions. To avoid +# cycles of load on the server, this time is randomly adjusted by a small # percentage as the daemon runs. UpdateStoreInterval = 3600 -# A file must have been modified at least 6 hours ago before it will be uploaded. +# The minimum age of a file, in seconds, that will be uploaded. Avoids +# repeated uploads of a file which is constantly being modified. MinimumFileAge = 21600 -# If a file is modified repeated, it won't be uploaded immediately in case it's modified again. -# However, it should be uploaded eventually. This is how long we should wait after first noticing -# a change. (1 day) +# If a file is modified repeated, it won't be uploaded immediately in case +# it's modified again, due to the MinimumFileAge specified above. However, it +# should be uploaded eventually even if it is being modified repeatedly. This +# is how long we should wait, in seconds, after first noticing a change. +# (86400 seconds = 1 day) MaxUploadWait = 86400 +# If the connection is idle for some time (e.g. over 10 minutes or 600 +# seconds, not sure exactly how long) then the server will give up and +# disconnect the client, resulting in Connection Protocol_Timeout errors +# on the server and TLSReadFailed or TLSWriteFailed errors on the client. +# Also, some firewalls and NAT gateways will kill idle connections after +# similar lengths of time. +# +# This can happen for example when most files are backed up already and +# don't need to be sent to the store again, while scanning a large +# directory, or while calculating diffs of a large file. To avoid this, +# KeepAliveTime specifies that special keep-alive messages should be sent +# when the connection is otherwise idle for a certain length of time, +# specified here in seconds. +# +# The default is that these messages are never sent, equivalent to setting +# this option to zero, but we recommend that all users enable this. + +KeepAliveTime = 120 + __E } else @@ -352,13 +392,17 @@ FileTrackingSizeThreshold = 65535 DiffingUploadSizeThreshold = 8192 -# The limit on how much time is spent diffing files. Most files shouldn't take very long, -# but if you have really big files you can use this to limit the time spent diffing them. +# The limit on how much time is spent diffing files, in seconds. Most files +# shouldn't take very long, but if you have really big files you can use this +# to limit the time spent diffing them. +# # * Reduce if you are having problems with processor usage. -# * Increase if you have large files, and think the upload of changes is too large and want -# to spend more time searching for unchanged blocks. +# +# * Increase if you have large files, and think the upload of changes is too +# large and you want bbackupd to spend more time searching for unchanged +# blocks. -MaximumDiffingTime = 20 +MaximumDiffingTime = 120 # Uncomment this line to see exactly what the daemon is going when it's connected to the server. @@ -366,14 +410,20 @@ MaximumDiffingTime = 20 # ExtendedLogging = yes -# Use this to temporarily stop bbackupd from syncronising or connecting to the store. -# This specifies a program or script script which is run just before each sync, and ideally -# the full path to the interpreter. It will be run as the same user bbackupd is running as, -# usually root. -# The script prints either "now" or a number to STDOUT (and a terminating newline, no quotes). -# If the result was "now", then the sync will happen. If it's a number, then the script will -# be asked again in that number of seconds. -# For example, you could use this on a laptop to only backup when on a specific network. +# This specifies a program or script script which is run just before each +# sync, and ideally the full path to the interpreter. It will be run as the +# same user bbackupd is running as, usually root. +# +# The script must output (print) either "now" or a number to STDOUT (and a +# terminating newline, no quotes). +# +# If the result was "now", then the sync will happen. If it's a number, then +# no backup will happen for that number of seconds (bbackupd will pause) and +# then the script will be run again. +# +# Use this to temporarily stop bbackupd from syncronising or connecting to the +# store. For example, you could use this on a laptop to only backup when on a +# specific network, or when it has a working Internet connection. # SyncAllowScript = /path/to/intepreter/or/exe script-name parameters etc @@ -396,7 +446,7 @@ Server PidFile = /var/run/bbackupd.pid } -# + # BackupLocations specifies which locations on disc should be backed up. Each # directory is in the format # @@ -425,15 +475,31 @@ Server # files, except one MP3 file in particular. # # In general, Exclude excludes a file or directory, unless the directory is -# explicitly mentioned in a AlwaysInclude directive. +# explicitly mentioned in a AlwaysInclude directive. However, Box Backup +# does NOT scan inside excluded directories and will never back up an +# AlwaysIncluded file or directory inside an excluded directory or any +# subdirectory thereof. +# +# To back up a directory inside an excluded directory, use a configuration +# like this, to ensure that each directory in the path to the important +# files is included, but none of their contents will be backed up except +# the directories futher down that path to the important one. +# +# ExcludeDirsRegex = /home/user/bigfiles/.* +# ExcludeFilesRegex = /home/user/bigfiles/.* +# AlwaysIncludeDir = /home/user/bigfiles/path +# AlwaysIncludeDir = /home/user/bigfiles/path/to +# AlwaysIncludeDir = /home/user/bigfiles/path/important +# AlwaysIncludeDir = /home/user/bigfiles/path/important/files +# AlwaysIncludeDirsRegex = /home/user/bigfiles/path/important/files/.* +# AlwaysIncludeFilesRegex = /home/user/bigfiles/path/important/files/.* # -# If a directive ends in Regex, then it is a regular expression rather than a +# If a directive ends in Regex, then it is a regular expression rather than a # explicit full pathname. See # # man 7 re_format # # for the regex syntax on your platform. -# BackupLocations { diff --git a/bin/bbackupd/bbackupd.cpp b/bin/bbackupd/bbackupd.cpp index e00d3628..e094d499 100644 --- a/bin/bbackupd/bbackupd.cpp +++ b/bin/bbackupd/bbackupd.cpp @@ -12,6 +12,7 @@ #include "MainHelper.h" #include "BoxPortsAndFiles.h" #include "BackupStoreException.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -24,12 +25,16 @@ int main(int argc, const char *argv[]) { + int ExitCode = 0; + MAINHELPER_START + Logging::SetProgramName("Box Backup (bbackupd)"); + Logging::ToConsole(true); + Logging::ToSyslog (true); + #ifdef WIN32 - ::openlog("Box Backup (bbackupd)", LOG_PID, LOG_LOCAL6); - if(argc == 2 && (::strcmp(argv[1], "--help") == 0 || ::strcmp(argv[1], "-h") == 0)) @@ -62,11 +67,9 @@ int main(int argc, const char *argv[]) EnableBackupRights(); - int ExitCode = 0; - if (runAsWin32Service) { - syslog(LOG_INFO, "Box Backup service starting"); + BOX_INFO("Box Backup service starting"); char* config = NULL; if (argc >= 3) @@ -74,33 +77,31 @@ int main(int argc, const char *argv[]) config = strdup(argv[2]); } - OurService(config); + ExitCode = OurService(config); if (config) { free(config); } - syslog(LOG_INFO, "Box Backup service shut down"); + BOX_INFO("Box Backup service shut down"); } else { ExitCode = gpDaemonService->Main( - BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); + BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE, argc, argv); } - ::closelog(); - delete gpDaemonService; - return ExitCode; - #else // !WIN32 BackupDaemon daemon; - return daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); + ExitCode = daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); #endif // WIN32 MAINHELPER_END + + return ExitCode; } diff --git a/bin/bbackupd/win32/ReadMe.txt b/bin/bbackupd/win32/ReadMe.txt deleted file mode 100644 index 3d260750..00000000 --- a/bin/bbackupd/win32/ReadMe.txt +++ /dev/null @@ -1,24 +0,0 @@ -Upgrade instructions - -Version 0.09g to 0.09h - -This version included patches to the server as well. The server for this -upgrade can be found at http://home.earthlink.net/~gniemcew/ but hopefully -will be merged into the core in the next release. - -New values in the bbackupd.conf can now be added: - -StoreObjectInfoFile = C:\Program Files\Box Backup\bbackupd\bbackupd.dat - -This stores the state when a backup daemon is shutdown. - -KeepAliveTime = 250 - -This is imperative if MaximumDiffingTime is larger than 300, this stops the ssl -layer timing out when a diff is performed. It is wise to set MaximumDiffingTime -long enough for the largest file you may have. If you do not wish to upgrade your -server then make KeepAliveTime greater than MaximumDiffingTime. - -Have fun - -Nick diff --git a/bin/bbackupquery/BackupQueries.cpp b/bin/bbackupquery/BackupQueries.cpp index be5eeb72..09003d4d 100644 --- a/bin/bbackupquery/BackupQueries.cpp +++ b/bin/bbackupquery/BackupQueries.cpp @@ -46,6 +46,8 @@ #include "BackupStoreException.h" #include "ExcludeList.h" #include "BackupClientMakeExcludeList.h" +#include "PathUtils.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -53,10 +55,10 @@ #undef min #undef max -#define COMPARE_RETURN_SAME 1 +#define COMPARE_RETURN_SAME 1 #define COMPARE_RETURN_DIFFERENT 2 #define COMPARE_RETURN_ERROR 3 - +#define COMMAND_RETURN_ERROR 4 // -------------------------------------------------------------------------- // @@ -179,7 +181,7 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) if (!ConvertEncoding(*i, CP_ACP, converted, GetConsoleCP())) { - printf("Failed to convert encoding"); + BOX_ERROR("Failed to convert encoding"); return; } *i = converted; @@ -206,7 +208,7 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) { "sh", "" }, { "getobject", "" }, { "get", "i" }, - { "compare", "alcqAE" }, + { "compare", "alcqAEQ" }, { "restore", "dri" }, { "help", "" }, { "usage", "" }, @@ -253,7 +255,7 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) // No such command if(alias[a] == 0) { - printf("Unrecognised command: %s\n", Command); + BOX_ERROR("Unrecognised command: " << Command); return; } } @@ -273,8 +275,8 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) // Valid option? if(::strchr(commands[cmd].opts, *c) == NULL) { - printf("Invalid option '%c' for command %s\n", - *c, commands[cmd].name); + BOX_ERROR("Invalid option '" << *c << "' for " + "command " << commands[cmd].name); return; } opts[(int)*c] = true; @@ -303,9 +305,8 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) case COMMAND_pwd: { // Simple implementation, so do it here - printf("%s (%08llx)\n", - GetCurrentDirectoryName().c_str(), - (long long)GetCurrentDirectoryID()); + BOX_INFO(GetCurrentDirectoryName() << " (" << + BOX_FORMAT_OBJECTID(GetCurrentDirectoryID())); } break; @@ -318,7 +319,7 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) break; case COMMAND_sh: - printf("The command to run must be specified as an argument.\n"); + BOX_ERROR("The command to run must be specified as an argument."); break; case COMMAND_GetObject: @@ -399,8 +400,8 @@ void BackupQueries::CommandList(const std::vector<std::string> &args, const bool if(rootDir == 0) { - printf("Directory '%s' not found on store\n", - args[0].c_str()); + BOX_ERROR("Directory '" << args[0] << "' not found " + "on store."); return; } } @@ -745,7 +746,7 @@ void BackupQueries::CommandChangeDir(const std::vector<std::string> &args, const { if(args.size() != 1 || args[0].size() == 0) { - printf("Incorrect usage.\ncd [-o] [-d] <directory>\n"); + BOX_ERROR("Incorrect usage. cd [-o] [-d] <directory>"); return; } @@ -762,7 +763,7 @@ void BackupQueries::CommandChangeDir(const std::vector<std::string> &args, const if(id == 0) { - printf("Directory '%s' not found\n", args[0].c_str()); + BOX_ERROR("Directory '" << args[0] << "' not found."); return; } @@ -783,22 +784,37 @@ void BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &args) { if(args.size() != 1 || args[0].size() == 0) { - printf("Incorrect usage.\nlcd <local-directory>\n"); + BOX_ERROR("Incorrect usage. lcd <local-directory>"); + SetReturnCode(COMMAND_RETURN_ERROR); return; } // Try changing directory #ifdef WIN32 std::string dirName; - if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) return; + if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) + { + BOX_ERROR("Failed to convert path from console encoding."); + SetReturnCode(COMMAND_RETURN_ERROR); + return; + } int result = ::chdir(dirName.c_str()); #else int result = ::chdir(args[0].c_str()); #endif if(result != 0) { - printf((errno == ENOENT || errno == ENOTDIR)?"Directory '%s' does not exist\n":"Error changing dir to '%s'\n", - args[0].c_str()); + if(errno == ENOENT || errno == ENOTDIR) + { + BOX_ERROR("Directory '" << args[0] << "' does not exist."); + } + else + { + BOX_ERROR("Error changing to directory '" << + args[0] << ": " << strerror(errno)); + } + + SetReturnCode(COMMAND_RETURN_ERROR); return; } @@ -806,15 +822,22 @@ void BackupQueries::CommandChangeLocalDir(const std::vector<std::string> &args) char wd[PATH_MAX]; if(::getcwd(wd, PATH_MAX) == 0) { - printf("Error getting current directory\n"); + BOX_ERROR("Error getting current directory: " << + strerror(errno)); + SetReturnCode(COMMAND_RETURN_ERROR); return; } #ifdef WIN32 - if(!ConvertUtf8ToConsole(wd, dirName)) return; - printf("Local current directory is now '%s'\n", dirName.c_str()); + if(!ConvertUtf8ToConsole(wd, dirName)) + { + BOX_ERROR("Failed to convert new path from console encoding."); + SetReturnCode(COMMAND_RETURN_ERROR); + return; + } + BOX_INFO("Local current directory is now '" << dirName << "'."); #else - printf("Local current directory is now '%s'\n", wd); + BOX_INFO("Local current directory is now '" << wd << "'."); #endif } @@ -832,14 +855,15 @@ void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const // Check args if(args.size() != 2) { - printf("Incorrect usage.\ngetobject <object-id> <local-filename>\n"); + BOX_ERROR("Incorrect usage. getobject <object-id> " + "<local-filename>"); return; } int64_t id = ::strtoll(args[0].c_str(), 0, 16); if(id == std::numeric_limits<long long>::min() || id == std::numeric_limits<long long>::max() || id == 0) { - printf("Not a valid object ID (specified in hex)\n"); + BOX_ERROR("Not a valid object ID (specified in hex)."); return; } @@ -847,7 +871,7 @@ void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const struct stat st; if(::stat(args[1].c_str(), &st) == 0 || errno != ENOENT) { - printf("The local file %s already exists\n", args[1].c_str()); + BOX_ERROR("The local file '" << args[1] << " already exists."); return; } @@ -865,18 +889,20 @@ void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); objectStream->CopyStreamTo(out); - printf("Object ID %08llx fetched successfully.\n", id); + BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(id) << + " fetched successfully."); } else { - printf("Object does not exist on store.\n"); + BOX_ERROR("Object ID " << BOX_FORMAT_OBJECTID(id) << + " does not exist on store."); ::unlink(args[1].c_str()); } } catch(...) { ::unlink(args[1].c_str()); - printf("Error occured fetching object.\n"); + BOX_ERROR("Error occured fetching object."); } } @@ -896,15 +922,17 @@ void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) // Check args if(args.size() < 1 || (opts['i'] && args.size() != 2) || args.size() > 2) { - printf("Incorrect usage.\n" + BOX_ERROR("Incorrect usage.\n" "get <remote-filename> [<local-filename>] or\n" - "get -i <object-id> <local-filename>\n"); + "get -i <object-id> <local-filename>"); return; } // Find object ID somehow - int64_t id; + int64_t fileId; + int64_t dirId = GetCurrentDirectoryID(); std::string localName; + // BLOCK { #ifdef WIN32 @@ -914,7 +942,7 @@ void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) std::string out; if(!ConvertConsoleToUtf8(i->c_str(), out)) { - fprintf(stderr, "failed to convert encoding\n"); + BOX_ERROR("Failed to convert encoding."); return; } *i = out; @@ -923,11 +951,30 @@ void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) std::string fileName(args[0]); + if(!opts['i']) + { + // does this remote filename include a path? + std::string::size_type index = fileName.rfind('/'); + if(index != std::string::npos) + { + std::string dirName(fileName.substr(0, index)); + fileName = fileName.substr(index + 1); + + dirId = FindDirectoryObjectID(dirName); + if(dirId == 0) + { + BOX_ERROR("Directory '" << dirName << + "' not found."); + return; + } + } + } + BackupStoreFilenameClear fn(fileName); // Need to look it up in the current directory mrConnection.QueryListDirectory( - GetCurrentDirectoryID(), + dirId, BackupProtocolClientListDirectory::Flags_File, // just files (opts['i'])?(BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING):(BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted), // only current versions false /* don't want attributes */); @@ -940,17 +987,24 @@ void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) if(opts['i']) { // Specified as ID. - id = ::strtoll(args[0].c_str(), 0, 16); - if(id == std::numeric_limits<long long>::min() || id == std::numeric_limits<long long>::max() || id == 0) + fileId = ::strtoll(args[0].c_str(), 0, 16); + if(fileId == std::numeric_limits<long long>::min() || + fileId == std::numeric_limits<long long>::max() || + fileId == 0) { - printf("Not a valid object ID (specified in hex)\n"); + BOX_ERROR("Not a valid object ID (specified in hex)."); return; } // Check that the item is actually in the directory - if(dir.FindEntryByID(id) == 0) + if(dir.FindEntryByID(fileId) == 0) { - printf("ID '%08llx' not found in current directory on store.\n(You can only download objects by ID from the current directory.)\n", id); + BOX_ERROR("File ID " << + BOX_FORMAT_OBJECTID(fileId) << + " not found in current " + "directory on store.\n" + "(You can only download files by ID " + "from the current directory.)"); return; } @@ -965,14 +1019,19 @@ void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) if(en == 0) { - printf("Filename '%s' not found in current directory on store.\n(Subdirectories in path not searched.)\n", args[0].c_str()); + BOX_ERROR("Filename '" << args[0] << "' " + "not found in current " + "directory on store.\n" + "(Subdirectories in path not " + "searched.)"); return; } - id = en->GetObjectID(); + fileId = en->GetObjectID(); - // Local name is the last argument, which is either the looked up filename, or - // a filename specified by the user. + // Local name is the last argument, which is either + // the looked up filename, or a filename specified + // by the user. localName = args[args.size() - 1]; } } @@ -981,7 +1040,9 @@ void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) struct stat st; if(::stat(localName.c_str(), &st) == 0 || errno != ENOENT) { - printf("The local file %s already exists, will not overwrite it.\n", localName.c_str()); + BOX_ERROR("The local file " << localName << " already exists, " + "will not overwrite it."); + SetReturnCode(COMMAND_RETURN_ERROR); return; } @@ -989,7 +1050,7 @@ void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) try { // Request object - mrConnection.QueryGetFile(GetCurrentDirectoryID(), id); + mrConnection.QueryGetFile(dirId, fileId); // Stream containing encoded file std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); @@ -998,12 +1059,25 @@ void BackupQueries::CommandGet(std::vector<std::string> args, const bool *opts) BackupStoreFile::DecodeFile(*objectStream, localName.c_str(), mrConnection.GetTimeout()); // Done. - printf("Object ID %08llx fetched sucessfully.\n", id); + BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(fileId) << + " fetched successfully."); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to fetch file: " << + e.what()); + ::unlink(localName.c_str()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to fetch file: " << + e.what()); + ::unlink(localName.c_str()); } catch(...) { + BOX_ERROR("Failed to fetch file: unknown error"); ::unlink(localName.c_str()); - printf("Error occured fetching file.\n"); } } @@ -1022,6 +1096,7 @@ BackupQueries::CompareParams::CompareParams() mIgnoreAttributes(false), mDifferences(0), mDifferencesExplainedByModTime(0), + mUncheckedFiles(0), mExcludedDirs(0), mExcludedFiles(0), mpExcludeFiles(0), @@ -1081,6 +1156,7 @@ void BackupQueries::CommandCompare(const std::vector<std::string> &args, const b // Parameters, including count of differences BackupQueries::CompareParams params; params.mQuickCompare = opts['q']; + params.mQuietCompare = opts['Q']; params.mIgnoreExcludes = opts['E']; params.mIgnoreAttributes = opts['A']; @@ -1098,14 +1174,16 @@ void BackupQueries::CommandCompare(const std::vector<std::string> &args, const b } else { - printf("Warning: couldn't determine the time of the last synchronisation -- checks not performed.\n"); + BOX_WARNING("Failed to determine the time of the last " + "synchronisation -- checks not performed."); } } // Quick compare? if(params.mQuickCompare) { - printf("WARNING: Quick compare used -- file attributes are not checked.\n"); + BOX_WARNING("Quick compare used -- file attributes are not " + "checked."); } if(!opts['l'] && opts['a'] && args.size() == 0) @@ -1130,7 +1208,7 @@ void BackupQueries::CommandCompare(const std::vector<std::string> &args, const b // Can't be bothered to do all the hard work to work out which location it's on, and hence which exclude list if(!params.mIgnoreExcludes) { - printf("Cannot use excludes on directory to directory comparison -- use -E flag to specify ignored excludes\n"); + BOX_ERROR("Cannot use excludes on directory to directory comparison -- use -E flag to specify ignored excludes."); return; } else @@ -1141,17 +1219,38 @@ void BackupQueries::CommandCompare(const std::vector<std::string> &args, const b } else { - printf("Incorrect usage.\ncompare -a\n or compare -l <location-name>\n or compare <store-dir-name> <local-dir-name>\n"); + BOX_ERROR("Incorrect usage.\ncompare -a\n or compare -l <location-name>\n or compare <store-dir-name> <local-dir-name>"); return; } - - printf("\n[ %d (of %d) differences probably due to file modifications after the last upload ]\nDifferences: %d (%d dirs excluded, %d files excluded)\n", - params.mDifferencesExplainedByModTime, params.mDifferences, params.mDifferences, params.mExcludedDirs, params.mExcludedFiles); + + if (!params.mQuietCompare) + { + BOX_INFO("[ " << + params.mDifferencesExplainedByModTime << " (of " << + params.mDifferences << ") differences probably " + "due to file modifications after the last upload ]"); + } + + BOX_INFO("Differences: " << params.mDifferences << " (" << + params.mExcludedDirs << " dirs excluded, " << + params.mExcludedFiles << " files excluded, " << + params.mUncheckedFiles << " files not checked)"); // Set return code? if(opts['c']) { - SetReturnCode((params.mDifferences == 0)?COMPARE_RETURN_SAME:COMPARE_RETURN_DIFFERENT); + if (params.mUncheckedFiles != 0) + { + SetReturnCode(COMPARE_RETURN_ERROR); + } + else if (params.mDifferences != 0) + { + SetReturnCode(COMPARE_RETURN_DIFFERENT); + } + else + { + SetReturnCode(COMPARE_RETURN_SAME); + } } } @@ -1170,10 +1269,23 @@ void BackupQueries::CompareLocation(const std::string &rLocation, BackupQueries: const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations")); if(!locations.SubConfigurationExists(rLocation.c_str())) { - printf("Location %s does not exist.\n", rLocation.c_str()); + BOX_ERROR("Location " << rLocation << " does not exist."); return; } const Configuration &loc(locations.GetSubConfiguration(rLocation.c_str())); + + #ifdef WIN32 + { + std::string path = loc.GetKeyValue("Path"); + if (path.size() > 0 && path[path.size()-1] == + DIRECTORY_SEPARATOR_ASCHAR) + { + BOX_WARNING("Location '" << rLocation << "' path ends " + "with '" DIRECTORY_SEPARATOR "', " + "compare may fail!"); + } + } + #endif try { @@ -1223,9 +1335,9 @@ void BackupQueries::Compare(const std::string &rStoreDir, const std::string &rLo // Found? if(dirID == 0) { - printf("Local directory '%s' exists, but " - "server directory '%s' does not exist\n", - rLocalDir.c_str(), rStoreDir.c_str()); + BOX_WARNING("Local directory '" << rLocalDir << "' exists, " + "but server directory '" << rStoreDir << "' does not " + "exist."); rParams.mDifferences ++; return; } @@ -1273,23 +1385,24 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s // What kind of error? if(errno == ENOTDIR) { - printf("Local object '%s' is a file, " - "server object '%s' is a directory\n", - localDirDisplay.c_str(), - storeDirDisplay.c_str()); + BOX_WARNING("Local object '" << localDirDisplay << "' " + "is a file, server object '" << + storeDirDisplay << "' is a directory."); rParams.mDifferences ++; } else if(errno == ENOENT) { - printf("Local directory '%s' does not exist " - "(compared to server directory '%s')\n", - localDirDisplay.c_str(), - storeDirDisplay.c_str()); + BOX_WARNING("Local directory '" << localDirDisplay << + "' does not exist (compared to server " + "directory '" << storeDirDisplay << "')."); + rParams.mDifferences ++; } else { - printf("ERROR: stat on local dir '%s'\n", - localDirDisplay.c_str()); + BOX_WARNING("Failed to access local directory '" << + localDirDisplay << ": " << strerror(errno) << + "'."); + rParams.mUncheckedFiles ++; } return; } @@ -1309,8 +1422,8 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s // Test out the attributes if(!dir.HasAttributes()) { - printf("Store directory '%s' doesn't have attributes.\n", - storeDirDisplay.c_str()); + BOX_WARNING("Store directory '" << storeDirDisplay << "' " + "doesn't have attributes."); } else { @@ -1325,10 +1438,9 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s if(!(attr.Compare(localAttr, true, true /* ignore modification times */))) { - printf("Local directory '%s' has different attributes " - "to store directory '%s'.\n", - localDirDisplay.c_str(), - storeDirDisplay.c_str()); + BOX_WARNING("Local directory '" << localDirDisplay << + "' has different attributes to store " + "directory '" << storeDirDisplay << "'."); rParams.mDifferences ++; } } @@ -1337,8 +1449,9 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s DIR *dirhandle = ::opendir(rLocalDir.c_str()); if(dirhandle == 0) { - printf("ERROR: opendir on local dir '%s'\n", - localDirDisplay.c_str()); + BOX_WARNING("Failed to open local directory '" << + localDirDisplay << "': " << strerror(errno)); + rParams.mUncheckedFiles ++; return; } try @@ -1358,9 +1471,9 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s #ifdef HAVE_VALID_DIRENT_D_TYPE if (localDirEn->d_type != DT_DIR) { - fprintf(stderr, "ERROR: d_type does " - "not really work on your " - "platform. Reconfigure Box!\n"); + BOX_ERROR("d_type does not really " + "work on your platform. " + "Reconfigure Box!"); return; } #endif @@ -1369,9 +1482,8 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s } #ifndef HAVE_VALID_DIRENT_D_TYPE - std::string fn(rLocalDir); - fn += DIRECTORY_SEPARATOR_ASCHAR; - fn += localDirEn->d_name; + std::string fn(MakeFullPath + (rLocalDir, localDirEn->d_name)); struct stat st; if(::lstat(fn.c_str(), &st) != 0) { @@ -1406,8 +1518,8 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s // Close directory if(::closedir(dirhandle) != 0) { - printf("ERROR: closedir on local dir '%s'\n", - localDirDisplay.c_str()); + BOX_ERROR("Failed to close local directory '" << + localDirDisplay << "': " << strerror(errno)); } dirhandle = 0; @@ -1455,29 +1567,28 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s const std::string& fileNameDisplay(i->first); #endif - std::string localPathDisplay = localDirDisplay + - DIRECTORY_SEPARATOR + fileNameDisplay; - std::string storePathDisplay = storeDirDisplay + - "/" + fileNameDisplay; + std::string localPath(MakeFullPath + (rLocalDir, fileName)); + std::string localPathDisplay(MakeFullPath + (localDirDisplay, fileNameDisplay)); + std::string storePathDisplay + (storeDirDisplay + "/" + fileNameDisplay); // Does the file exist locally? string_set_iter_t local(localFiles.find(fileName)); if(local == localFiles.end()) { // Not found -- report - printf("Local file '%s' does not exist, " - "but store file '%s' does.\n", - localPathDisplay.c_str(), - storePathDisplay.c_str()); + BOX_WARNING("Local file '" << + localDirDisplay << "' does not exist, " + "but store file '" << + storePathDisplay << "' does."); rParams.mDifferences ++; } else { try { - // make local name of file for comparison - std::string localPath(rLocalDir + DIRECTORY_SEPARATOR + fileName); - // Files the same flag? bool equal = true; @@ -1505,7 +1616,7 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s // Decode it std::auto_ptr<BackupStoreFile::DecodedStream> fileOnServerStream; - // Got additional attibutes? + // Got additional attributes? if(i->second->HasAttributes()) { // Use these attributes @@ -1539,24 +1650,28 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s #endif if(!rParams.mIgnoreAttributes && + #ifdef PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE + !fileOnServerStream->IsSymLink() && + #endif !localAttr.Compare(fileOnServerStream->GetAttributes(), ignoreAttrModTime, fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) { - printf("Local file '%s' " - "has different attributes " - "to store file '%s'.\n", - localPathDisplay.c_str(), - storePathDisplay.c_str()); + BOX_WARNING("Local file '" << + localPathDisplay << + "' has different attributes " + "to store file '" << + storePathDisplay << + "'."); rParams.mDifferences ++; if(modifiedAfterLastSync) { rParams.mDifferencesExplainedByModTime ++; - printf("(the file above was modified after the last sync time -- might be reason for difference)\n"); + BOX_INFO("(the file above was modified after the last sync time -- might be reason for difference)"); } else if(i->second->HasAttributes()) { - printf("(the file above has had new attributes applied)\n"); + BOX_INFO("(the file above has had new attributes applied)\n"); } } @@ -1595,7 +1710,7 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s equal = false; } - // Must always read the entire decoded string, if it's not a symlink + // Must always read the entire decoded stream, if it's not a symlink if(fileOnServerStream->StreamDataLeft()) { // Absorb all the data remaining @@ -1605,39 +1720,65 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s fileOnServerStream->Read(buffer, sizeof(buffer), mrConnection.GetTimeout()); } } + + // Must always read the entire encoded stream + if(objectStream->StreamDataLeft()) + { + // Absorb all the data remaining + char buffer[2048]; + while(objectStream->StreamDataLeft()) + { + objectStream->Read(buffer, sizeof(buffer), mrConnection.GetTimeout()); + } + } } } // Report if not equal. if(!equal) { - printf("Local file '%s' " + BOX_WARNING("Local file '" << + localPathDisplay << "' " "has different contents " - "to store file '%s'.\n", - localPathDisplay.c_str(), - storePathDisplay.c_str()); + "to store file '" << + storePathDisplay << + "'."); rParams.mDifferences ++; if(modifiedAfterLastSync) { rParams.mDifferencesExplainedByModTime ++; - printf("(the file above was modified after the last sync time -- might be reason for difference)\n"); + BOX_INFO("(the file above was modified after the last sync time -- might be reason for difference)"); } else if(i->second->HasAttributes()) { - printf("(the file above has had new attributes applied)\n"); + BOX_INFO("(the file above has had new attributes applied)\n"); } } } catch(BoxException &e) { - printf("ERROR: (%d/%d) during file fetch and comparison for '%s'\n", - e.GetType(), - e.GetSubType(), - storePathDisplay.c_str()); + BOX_ERROR("Failed to fetch and compare " + "'" << + storePathDisplay.c_str() << + "': error " << e.what() << + " (" << e.GetType() << + "/" << e.GetSubType() << ")"); + rParams.mUncheckedFiles ++; } - catch(...) + catch(std::exception &e) { - printf("ERROR: (unknown) during file fetch and comparison for '%s'\n", storePathDisplay.c_str()); + BOX_ERROR("Failed to fetch and compare " + "'" << + storePathDisplay.c_str() << + "': " << e.what()); + } + catch(...) + { + BOX_ERROR("Failed to fetch and compare " + "'" << + storePathDisplay.c_str() << + "': unknown error"); + rParams.mUncheckedFiles ++; } // Remove from set so that we know it's been compared @@ -1658,22 +1799,22 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s const std::string& fileNameDisplay(*i); #endif - std::string localPath(rLocalDir + - DIRECTORY_SEPARATOR + *i); - std::string localPathDisplay(localDirDisplay + - DIRECTORY_SEPARATOR + fileNameDisplay); - std::string storePathDisplay(storeDirDisplay + - "/" + fileNameDisplay); + std::string localPath(MakeFullPath + (rLocalDir, *i)); + std::string localPathDisplay(MakeFullPath + (localDirDisplay, fileNameDisplay)); + std::string storePathDisplay + (storeDirDisplay + "/" + fileNameDisplay); // Should this be ignored (ie is excluded)? if(rParams.mpExcludeFiles == 0 || !(rParams.mpExcludeFiles->IsExcluded(localPath))) { - printf("Local file '%s' exists, " - "but store file '%s' " - "does not exist.\n", - localPathDisplay.c_str(), - storePathDisplay.c_str()); + BOX_WARNING("Local file '" << + localPathDisplay << + "' exists, but store file '" << + storePathDisplay << + "' does not."); rParams.mDifferences ++; // Check the file modification time @@ -1684,7 +1825,7 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s if(FileModificationTime(st) > rParams.mLatestFileUploadTime) { rParams.mDifferencesExplainedByModTime ++; - printf("(the file above was modified after the last sync time -- might be reason for difference)\n"); + BOX_INFO("(the file above was modified after the last sync time -- might be reason for difference)"); } } } @@ -1699,7 +1840,7 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s localFiles.clear(); storeFiles.clear(); - // Now do the directories, recusively to check subdirectories + // Now do the directories, recursively to check subdirectories for(std::set<std::pair<std::string, BackupStoreDirectory::Entry *> >::const_iterator i = storeDirs.begin(); i != storeDirs.end(); ++i) { #ifdef WIN32 @@ -1713,26 +1854,44 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s const std::string& subdirNameDisplay(i->first); #endif - std::string localPathDisplay = localDirDisplay + - DIRECTORY_SEPARATOR + subdirNameDisplay; - std::string storePathDisplay = storeDirDisplay + - "/" + subdirNameDisplay; + std::string localPath(MakeFullPath + (rLocalDir, i->first)); + std::string localPathDisplay(MakeFullPath + (localDirDisplay, subdirNameDisplay)); + std::string storePathDisplay + (storeDirDisplay + "/" + subdirNameDisplay); // Does the directory exist locally? string_set_iter_t local(localDirs.find(i->first)); - if(local == localDirs.end()) + if(local == localDirs.end() && + rParams.mpExcludeDirs != NULL && + rParams.mpExcludeDirs->IsExcluded(localPath)) + { + // Not found -- report + BOX_WARNING("Local directory '" << + localPathDisplay << "' is excluded, " + "but store directory '" << + storePathDisplay << "' still exists."); + rParams.mDifferences ++; + } + else if(local == localDirs.end()) { // Not found -- report - printf("Local directory '%s' does not exist, " - "but store directory '%s' does.\n", - localPathDisplay.c_str(), - storePathDisplay.c_str()); + BOX_WARNING("Local directory '" << + localPathDisplay << "' does not exist, " + "but store directory '" << + storePathDisplay << "' does."); rParams.mDifferences ++; } + else if(rParams.mpExcludeDirs != NULL && + rParams.mpExcludeDirs->IsExcluded(localPath)) + { + // don't recurse into excluded directories + } else { // Compare directory - Compare(i->second->GetObjectID(), rStoreDir + "/" + i->first, rLocalDir + DIRECTORY_SEPARATOR + i->first, rParams); + Compare(i->second->GetObjectID(), rStoreDir + "/" + i->first, localPath, rParams); // Remove from set so that we know it's been compared localDirs.erase(local); @@ -1752,23 +1911,23 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s const std::string& fileNameDisplay(*i); #endif - std::string localPath = rLocalDir + - DIRECTORY_SEPARATOR + *i; - std::string storePath = rStoreDir + - "/" + *i; + std::string localPath(MakeFullPath + (rLocalDir, *i)); + std::string localPathDisplay(MakeFullPath + (localDirDisplay, fileNameDisplay)); - std::string localPathDisplay = localDirDisplay + - DIRECTORY_SEPARATOR + fileNameDisplay; - std::string storePathDisplay = storeDirDisplay + - "/" + fileNameDisplay; + std::string storePath + (rStoreDir + "/" + *i); + std::string storePathDisplay + (storeDirDisplay + "/" + fileNameDisplay); // Should this be ignored (ie is excluded)? if(rParams.mpExcludeDirs == 0 || !(rParams.mpExcludeDirs->IsExcluded(localPath))) { - printf("Local directory '%s' exists, but " - "store directory '%s' does not exist.\n", - localPathDisplay.c_str(), - storePathDisplay.c_str()); + BOX_WARNING("Local directory '" << + localPathDisplay << "' exists, but " + "store directory '" << + storePathDisplay << "' does not."); rParams.mDifferences ++; } else @@ -1801,7 +1960,7 @@ void BackupQueries::CommandRestore(const std::vector<std::string> &args, const b // Check arguments if(args.size() != 2) { - printf("Incorrect usage.\nrestore [-d] [-r] [-i] <directory-name> <local-directory-name>\n"); + BOX_ERROR("Incorrect usage. restore [-d] [-r] [-i] <remote-name> <local-name>"); return; } @@ -1816,7 +1975,7 @@ void BackupQueries::CommandRestore(const std::vector<std::string> &args, const b dirID = ::strtoll(args[0].c_str(), 0, 16); if(dirID == std::numeric_limits<long long>::min() || dirID == std::numeric_limits<long long>::max() || dirID == 0) { - printf("Not a valid object ID (specified in hex)\n"); + BOX_ERROR("Not a valid object ID (specified in hex)"); return; } } @@ -1839,12 +1998,12 @@ void BackupQueries::CommandRestore(const std::vector<std::string> &args, const b // Allowable? if(dirID == 0) { - printf("Directory '%s' not found on server\n", args[0].c_str()); + BOX_ERROR("Directory '" << args[0] << "' not found on server"); return; } if(dirID == BackupProtocolClientListDirectory::RootDirectory) { - printf("Cannot restore the root directory -- restore locations individually.\n"); + BOX_ERROR("Cannot restore the root directory -- restore locations individually."); return; } @@ -1856,25 +2015,55 @@ void BackupQueries::CommandRestore(const std::vector<std::string> &args, const b #endif // Go and restore... - switch(BackupClientRestore(mrConnection, dirID, localName.c_str(), - true /* print progress dots */, restoreDeleted, - false /* don't undelete after restore! */, - opts['r'] /* resume? */)) + int result; + + try + { + result = BackupClientRestore(mrConnection, dirID, + localName.c_str(), + true /* print progress dots */, restoreDeleted, + false /* don't undelete after restore! */, + opts['r'] /* resume? */); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to restore: " << e.what()); + return; + } + catch(...) + { + BOX_ERROR("Failed to restore: unknown exception"); + return; + } + + switch(result) { case Restore_Complete: - printf("Restore complete\n"); + BOX_INFO("Restore complete."); break; case Restore_ResumePossible: - printf("Resume possible -- repeat command with -r flag to resume\n"); + BOX_ERROR("Resume possible -- repeat command with -r flag to resume"); break; case Restore_TargetExists: - printf("The target directory exists. You cannot restore over an existing directory.\n"); + BOX_ERROR("The target directory exists. You cannot restore over an existing directory."); break; + #ifdef WIN32 + case Restore_TargetPathNotFound: + BOX_ERROR("The target directory path does not exist.\n" + "To restore to a directory whose parent " + "does not exist, create the parent first."); + break; + #endif + + case Restore_UnknownError: + BOX_ERROR("Unknown error during restore."); + break; + default: - printf("ERROR: Unknown restore result.\n"); + BOX_ERROR("Unknown restore result " << result << "."); break; } } @@ -1994,7 +2183,7 @@ void BackupQueries::CommandUndelete(const std::vector<std::string> &args, const // Check arguments if(args.size() != 1) { - printf("Incorrect usage.\nundelete <directory-name>\n"); + BOX_ERROR("Incorrect usage. undelete <directory-name>"); return; } @@ -2012,12 +2201,12 @@ void BackupQueries::CommandUndelete(const std::vector<std::string> &args, const // Allowable? if(dirID == 0) { - printf("Directory '%s' not found on server\n", args[0].c_str()); + BOX_ERROR("Directory '" << args[0] << "' not found on server."); return; } if(dirID == BackupProtocolClientListDirectory::RootDirectory) { - printf("Cannot undelete the root directory.\n"); + BOX_ERROR("Cannot undelete the root directory."); return; } diff --git a/bin/bbackupquery/BackupQueries.h b/bin/bbackupquery/BackupQueries.h index 17043b61..b2ef8cc2 100644 --- a/bin/bbackupquery/BackupQueries.h +++ b/bin/bbackupquery/BackupQueries.h @@ -67,10 +67,12 @@ private: ~CompareParams(); void DeleteExcludeLists(); bool mQuickCompare; + bool mQuietCompare; bool mIgnoreExcludes; bool mIgnoreAttributes; int mDifferences; int mDifferencesExplainedByModTime; + int mUncheckedFiles; int mExcludedDirs; int mExcludedFiles; const ExcludeList *mpExcludeFiles; diff --git a/bin/bbackupquery/bbackupquery.cpp b/bin/bbackupquery/bbackupquery.cpp index ef271af1..98726843 100644 --- a/bin/bbackupquery/bbackupquery.cpp +++ b/bin/bbackupquery/bbackupquery.cpp @@ -12,8 +12,14 @@ #ifdef HAVE_UNISTD_H #include <unistd.h> #endif + +#include <errno.h> #include <stdio.h> -#include <sys/types.h> + +#ifdef HAVE_SYS_TYPES_H + #include <sys/types.h> +#endif + #ifdef HAVE_LIBREADLINE #ifdef HAVE_READLINE_READLINE_H #include <readline/readline.h> @@ -45,6 +51,7 @@ #include "FdGetLine.h" #include "BackupClientCryptoKeys.h" #include "BannerText.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -63,7 +70,11 @@ void PrintUsageAndExit() int main(int argc, const char *argv[]) { - MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbackupquery.memleaks", "bbackupquery") + int returnCode = 0; + + MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbackupquery.memleaks", + "bbackupquery") + MAINHELPER_START #ifdef WIN32 WSADATA info; @@ -73,7 +84,7 @@ int main(int argc, const char *argv[]) if (WSAStartup(0x0101, &info) == SOCKET_ERROR) { - // throw error? perhaps give it its own id in the furture + // throw error? perhaps give it its own id in the future THROW_EXCEPTION(BackupStoreException, Internal) } #endif @@ -83,24 +94,34 @@ int main(int argc, const char *argv[]) BoxDebugTraceOn = false; #endif - int returnCode = 0; - - MAINHELPER_START - FILE *logFile = 0; // Filename for configuration file? - const char *configFilename = BOX_FILE_BBACKUPD_DEFAULT_CONFIG; + std::string configFilename; + + #ifdef WIN32 + configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; + #else + configFilename = BOX_FILE_BBACKUPD_DEFAULT_CONFIG; + #endif // Flags bool quiet = false; bool readWrite = false; + Logging::SetProgramName("Box Backup (bbackupquery)"); + + #ifdef NDEBUG + int masterLevel = Log::NOTICE; // need an int to do math with + #else + int masterLevel = Log::INFO; // need an int to do math with + #endif + #ifdef WIN32 - const char* validOpts = "qwuc:l:"; + const char* validOpts = "qvwuc:l:"; bool unicodeConsole = false; #else - const char* validOpts = "qwc:l:"; + const char* validOpts = "qvwc:l:"; #endif // See if there's another entry on the command line @@ -109,11 +130,35 @@ int main(int argc, const char *argv[]) { switch(c) { - case 'q': - // Quiet mode - quiet = true; + case 'q': + { + // Quiet mode + quiet = true; + + if(masterLevel == Log::NOTHING) + { + BOX_FATAL("Too many '-q': " + "Cannot reduce logging " + "level any more"); + return 2; + } + masterLevel--; + } break; - + + case 'v': + { + if(masterLevel == Log::EVERYTHING) + { + BOX_FATAL("Too many '-v': " + "Cannot increase logging " + "level any more"); + return 2; + } + masterLevel++; + } + break; + case 'w': // Read/write mode readWrite = true; @@ -129,7 +174,8 @@ int main(int argc, const char *argv[]) logFile = ::fopen(optarg, "w"); if(logFile == 0) { - printf("Can't open log file '%s'\n", optarg); + BOX_ERROR("Failed to open log file '" << + optarg << "': " << strerror(errno)); } break; @@ -148,11 +194,13 @@ int main(int argc, const char *argv[]) argc -= optind; argv += optind; + Logging::SetGlobalLevel((Log::Level)masterLevel); + // Print banner? if(!quiet) { const char *banner = BANNER_TEXT("Backup Query Tool"); - printf(banner); + BOX_NOTICE(banner); } #ifdef WIN32 @@ -160,14 +208,14 @@ int main(int argc, const char *argv[]) { if (!SetConsoleCP(CP_UTF8)) { - fprintf(stderr, "Failed to set input codepage: " - "error %d\n", GetLastError()); + BOX_ERROR("Failed to set input codepage: " << + GetErrorMessage(GetLastError())); } if (!SetConsoleOutputCP(CP_UTF8)) { - fprintf(stderr, "Failed to set output codepage: " - "error %d\n", GetLastError()); + BOX_ERROR("Failed to set output codepage: " << + GetErrorMessage(GetLastError())); } // enable input of Unicode characters @@ -181,12 +229,16 @@ int main(int argc, const char *argv[]) #endif // WIN32 // Read in the configuration file - if(!quiet) printf("Using configuration file %s\n", configFilename); + if(!quiet) BOX_INFO("Using configuration file " << configFilename); + std::string errs; - std::auto_ptr<Configuration> config(Configuration::LoadAndVerify(configFilename, &BackupDaemonConfigVerify, errs)); + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + (configFilename, &BackupDaemonConfigVerify, errs)); + if(config.get() == 0 || !errs.empty()) { - printf("Invalid configuration file:\n%s", errs.c_str()); + BOX_FATAL("Invalid configuration file: " << errs); return 1; } // Easier coding @@ -206,12 +258,12 @@ int main(int argc, const char *argv[]) BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str()); // 2. Connect to server - if(!quiet) printf("Connecting to store...\n"); + if(!quiet) BOX_INFO("Connecting to store..."); SocketStreamTLS socket; socket.Open(tlsContext, Socket::TypeINET, conf.GetKeyValue("StoreHostname").c_str(), BOX_PORT_BBSTORED); // 3. Make a protocol, and handshake - if(!quiet) printf("Handshake with store...\n"); + if(!quiet) BOX_INFO("Handshake with store..."); BackupProtocolClient connection(socket); connection.Handshake(); @@ -222,7 +274,7 @@ int main(int argc, const char *argv[]) } // 4. Log in to server - if(!quiet) printf("Login to store...\n"); + if(!quiet) BOX_INFO("Login to store..."); // Check the version of the server { std::auto_ptr<BackupProtocolClientVersion> serverVersion(connection.QueryVersion(BACKUP_STORE_SERVER_VERSION)); @@ -299,9 +351,9 @@ int main(int argc, const char *argv[]) #endif // Done... stop nicely - if(!quiet) printf("Logging off...\n"); + if(!quiet) BOX_INFO("Logging off..."); connection.QueryFinished(); - if(!quiet) printf("Session finished.\n"); + if(!quiet) BOX_INFO("Session finished."); // Return code returnCode = context.GetReturnCode(); @@ -314,13 +366,13 @@ int main(int argc, const char *argv[]) // Let everything be cleaned up on exit. - MAINHELPER_END - #ifdef WIN32 // Clean up our sockets WSACleanup(); #endif + MAINHELPER_END + return returnCode; } diff --git a/bin/bbackupquery/documentation.txt b/bin/bbackupquery/documentation.txt index 0fb0bacb..42217edc 100644 --- a/bin/bbackupquery/documentation.txt +++ b/bin/bbackupquery/documentation.txt @@ -123,7 +123,7 @@ compare <store-dir-name> <local-dir-name> The root cannot be restored -- restore locations individually. - -d -- restore a deleted directory. + -d -- restore a deleted directory or deleted files inside -r -- resume an interrupted restoration -i -- directory name is actually an ID diff --git a/bin/bbstoreaccounts/bbstoreaccounts.cpp b/bin/bbstoreaccounts/bbstoreaccounts.cpp index dd42458b..8a00e3b9 100644 --- a/bin/bbstoreaccounts/bbstoreaccounts.cpp +++ b/bin/bbstoreaccounts/bbstoreaccounts.cpp @@ -37,12 +37,13 @@ void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit) { if(SoftLimit >= HardLimit) { - printf("ERROR: Soft limit must be less than the hard limit.\n"); + BOX_FATAL("Soft limit must be less than the hard limit."); exit(1); } if(SoftLimit > ((HardLimit * MAX_SOFT_LIMIT_SIZE) / 100)) { - printf("ERROR: Soft limit must be no more than %d%% of the hard limit.\n", MAX_SOFT_LIMIT_SIZE); + BOX_FATAL("Soft limit must be no more than " << + MAX_SOFT_LIMIT_SIZE << "% of the hard limit."); exit(1); } } @@ -53,7 +54,7 @@ int BlockSizeOfDiscSet(int DiscSet) RaidFileController &controller(RaidFileController::GetController()); if(DiscSet < 0 || DiscSet >= controller.GetNumDiscSets()) { - printf("Disc set %d does not exist\n", DiscSet); + BOX_FATAL("Disc set " << DiscSet << " does not exist."); exit(1); } @@ -89,7 +90,7 @@ int64_t SizeStringToBlocks(const char *string, int DiscSet) int64_t number = strtol(string, &endptr, 0); if(endptr == string || number == LONG_MIN || number == LONG_MAX) { - printf("%s is an invalid number\n", string); + BOX_FATAL("'" << string << "' is not a valid number."); exit(1); } @@ -116,7 +117,8 @@ int64_t SizeStringToBlocks(const char *string, int DiscSet) break; default: - printf("%s has an invalid units specifier\nUse B for blocks, M for Mb, G for Gb, eg 2Gb\n", string); + BOX_FATAL(string << " has an invalid units specifier " + "(use B for blocks, M for Mb, G for Gb, eg 2Gb)"); exit(1); break; } @@ -143,7 +145,8 @@ bool GetWriteLockOnAccount(NamedLock &rLock, const std::string rRootDir, int Dis if(!gotLock) { // Couldn't lock the account -- just stop now - printf("Couldn't lock the account -- did not change the limits\nTry again later.\n"); + BOX_ERROR("Failed to lock the account, did not change limits. " + "Try again later."); return 1; } @@ -168,7 +171,8 @@ int SetLimit(Configuration &rConfig, const std::string &rUsername, int32_t ID, c // Already exists? if(!db->EntryExists(ID)) { - printf("Account %x does not exist\n", ID); + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " does not exist."); return 1; } @@ -198,7 +202,9 @@ int SetLimit(Configuration &rConfig, const std::string &rUsername, int32_t ID, c // Save info->Save(); - printf("Limits on account 0x%08x changed to %lld soft, %lld hard\n", ID, softlimit, hardlimit); + BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) << + " changed to " << softlimit << " soft, " << + hardlimit << " hard."); return 0; } @@ -211,7 +217,8 @@ int AccountInfo(Configuration &rConfig, int32_t ID) // Exists? if(!db->EntryExists(ID)) { - printf("Account %x does not exist\n", ID); + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " does not exist."); return 1; } @@ -241,11 +248,12 @@ int DeleteAccount(Configuration &rConfig, const std::string &rUsername, int32_t // Check user really wants to do this if(AskForConfirmation) { - ::printf("Really delete account %08x?\n(type 'yes' to confirm)\n", ID); + BOX_WARNING("Really delete account " << + BOX_FORMAT_ACCOUNT(ID) << "? (type 'yes' to confirm)"); char response[256]; if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0) { - printf("Deletion cancelled\n"); + BOX_NOTICE("Deletion cancelled."); return 0; } } @@ -256,7 +264,8 @@ int DeleteAccount(Configuration &rConfig, const std::string &rUsername, int32_t // Exists? if(!db->EntryExists(ID)) { - printf("Account %x does not exist\n", ID); + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " does not exist."); return 1; } @@ -318,24 +327,27 @@ int DeleteAccount(Configuration &rConfig, const std::string &rUsername, int32_t toDelete.push_back((*i) + DIRECTORY_SEPARATOR + rootDir); } } - + + int retcode = 0; + // Thirdly, delete the directories... for(std::vector<std::string>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) { - ::printf("Deleting store directory %s...\n", (*d).c_str()); + BOX_NOTICE("Deleting store directory " << (*d) << "..."); // Just use the rm command to delete the files std::string cmd("rm -rf "); cmd += *d; // Run command if(::system(cmd.c_str()) != 0) { - ::printf("ERROR: Deletion of %s failed.\n(when cleaning up, remember to delete all raid directories)\n", (*d).c_str()); - return 1; + BOX_ERROR("Failed to delete files in " << (*d) << + ", delete them manually."); + retcode = 1; } } // Success! - return 0; + return retcode; } int CheckAccount(Configuration &rConfig, const std::string &rUsername, int32_t ID, bool FixErrors, bool Quiet) @@ -346,7 +358,8 @@ int CheckAccount(Configuration &rConfig, const std::string &rUsername, int32_t I // Exists? if(!db->EntryExists(ID)) { - printf("Account %x does not exist\n", ID); + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " does not exist."); return 1; } @@ -381,7 +394,8 @@ int CreateAccount(Configuration &rConfig, const std::string &rUsername, int32_t // Already exists? if(db->EntryExists(ID)) { - printf("Account %x already exists\n", ID); + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " already exists."); return 1; } @@ -389,7 +403,7 @@ int CreateAccount(Configuration &rConfig, const std::string &rUsername, int32_t BackupStoreAccounts acc(*db); acc.Create(ID, DiscNumber, SoftLimit, HardLimit, rUsername); - printf("Account %x created\n", ID); + BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " created."); return 0; } @@ -402,12 +416,19 @@ void PrintUsageAndExit() int main(int argc, const char *argv[]) { - MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbstoreaccounts.memleaks", "bbstoreaccounts") + MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbstoreaccounts.memleaks", + "bbstoreaccounts") MAINHELPER_START - // Filename for configuraiton file? - const char *configFilename = BOX_FILE_BBSTORED_DEFAULT_CONFIG; + // Filename for configuration file? + std::string configFilename; + + #ifdef WIN32 + configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; + #else + configFilename = BOX_FILE_BBSTORED_DEFAULT_CONFIG; + #endif // See if there's another entry on the command line int c; @@ -431,10 +452,14 @@ int main(int argc, const char *argv[]) // Read in the configuration file std::string errs; - std::auto_ptr<Configuration> config(Configuration::LoadAndVerify(configFilename, &BackupConfigFileVerify, errs)); + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + (configFilename, &BackupConfigFileVerify, errs)); + if(config.get() == 0 || !errs.empty()) { - printf("Invalid configuration file:\n%s", errs.c_str()); + BOX_ERROR("Invalid configuration file " << configFilename << + ":" << errs); } // Get the user under which the daemon runs @@ -474,7 +499,8 @@ int main(int argc, const char *argv[]) if(argc < 5 || ::sscanf(argv[2], "%d", &discnum) != 1) { - printf("create requires raid file disc number, soft and hard limits\n"); + BOX_ERROR("create requires raid file disc number, " + "soft and hard limits."); return 1; } @@ -496,7 +522,7 @@ int main(int argc, const char *argv[]) // Change the limits on this account if(argc < 4) { - printf("setlimit requires soft and hard limits\n"); + BOX_ERROR("setlimit requires soft and hard limits."); return 1; } @@ -530,7 +556,7 @@ int main(int argc, const char *argv[]) } else { - ::printf("Unknown option %s.\n", argv[o]); + BOX_ERROR("Unknown option " << argv[o] << "."); return 2; } } @@ -540,7 +566,7 @@ int main(int argc, const char *argv[]) } else { - printf("Unknown command '%s'\n", argv[0]); + BOX_ERROR("Unknown command '" << argv[0] << "'."); return 1; } diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index 8c725b0f..16a1432a 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -11,10 +11,6 @@ #include <stdio.h> -#ifdef HAVE_SYSLOG_H - #include <syslog.h> -#endif - #include "BackupStoreDaemon.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreAccounts.h" @@ -50,7 +46,8 @@ void BackupStoreDaemon::HousekeepingProcess() { RunHousekeepingIfNeeded(); - // Calculate how long should wait before doing the next housekeeping run + // Calculate how long should wait before doing the next + // housekeeping run int64_t timeNow = GetCurrentBoxTime(); time_t secondsToGo = BoxTimeToSeconds( (mLastHousekeepingRun + housekeepingInterval) - @@ -72,6 +69,7 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() // Time now int64_t timeNow = GetCurrentBoxTime(); + // Do housekeeping if the time interval has elapsed since the last check if((timeNow - mLastHousekeepingRun) < housekeepingInterval) { @@ -80,7 +78,7 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() // Store the time mLastHousekeepingRun = timeNow; - ::syslog(LOG_INFO, "Starting housekeeping"); + BOX_INFO("Starting housekeeping"); // Get the list of accounts std::vector<int32_t> accounts; @@ -110,18 +108,25 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() } catch(BoxException &e) { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s (%d/%d) -- aborting housekeeping run for this account", - *i, e.what(), e.GetType(), e.GetSubType()); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(*i) << " threw exception, " + "aborting run for this account: " << + e.what() << " (" << + e.GetType() << "/" << e.GetSubType() << ")"); } catch(std::exception &e) { - ::syslog(LOG_ERR, "while housekeeping account %08X, exception %s -- aborting housekeeping run for this account", - *i, e.what()); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(*i) << " threw exception, " + "aborting run for this account: " << + e.what()); } catch(...) { - ::syslog(LOG_ERR, "while housekeeping account %08X, unknown exception -- aborting housekeeping run for this account", - *i); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(*i) << " threw exception, " + "aborting run for this account: " + "unknown exception"); } int64_t timeNow = GetCurrentBoxTime(); @@ -142,12 +147,25 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() } } - ::syslog(LOG_INFO, "Finished housekeeping"); + BOX_INFO("Finished housekeeping"); // Placed here for accuracy, if StopRun() is true, for example. SetProcessTitle("housekeeping, idle"); } +void BackupStoreDaemon::OnIdle() +{ + #ifdef WIN32 + if (!mHousekeepingInited) + { + HousekeepingInit(); + mHousekeepingInited = true; + } + + RunHousekeepingIfNeeded(); + #endif +} + // -------------------------------------------------------------------------- // // Function @@ -159,6 +177,11 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() // -------------------------------------------------------------------------- bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitTime) { + if(!mInterProcessCommsSocket.IsOpened()) + { + return false; + } + // First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate. if(mInterProcessComms.IsEOF()) { @@ -170,7 +193,7 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT std::string line; if(mInterProcessComms.GetLine(line, false /* no pre-processing */, MaximumWaitTime)) { - TRACE1("housekeeping received command '%s' over interprocess comms\n", line.c_str()); + TRACE1("Housekeeping received command '%s' over interprocess comms\n", line.c_str()); int account = 0; @@ -192,7 +215,9 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT if(account == AccountNum) { // Yes! -- need to stop now so when it retries to get the lock, it will succeed - ::syslog(LOG_INFO, "Housekeeping giving way to connection for account 0x%08x", AccountNum); + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(AccountNum) << + "giving way to client connection"); return true; } } diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index b199edbe..bca52c04 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -9,7 +9,7 @@ #include "Box.h" -#include <syslog.h> +#include <set> #include <sstream> #include "autogen_BackupProtocolServer.h" @@ -25,6 +25,8 @@ #include "BackupStoreInfo.h" #include "RaidFileController.h" #include "FileStream.h" +#include "InvisibleTempFileStream.h" +#include "BufferedStream.h" #include "MemLeakFindOn.h" @@ -82,11 +84,26 @@ std::auto_ptr<ProtocolObject> BackupProtocolServerLogin::DoCommand(BackupProtoco // Check given client ID against the ID in the certificate certificate // and that the client actually has an account on this machine - if(mClientID != rContext.GetClientID() || !rContext.GetClientHasAccount()) + if(mClientID != rContext.GetClientID()) { - ::syslog(LOG_INFO, "Failed login: Client ID presented was %08X", mClientID); - return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError( - BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_BadLogin)); + BOX_WARNING("Failed login from client ID " << + BOX_FORMAT_ACCOUNT(mClientID) << + ": wrong certificate for this account"); + return std::auto_ptr<ProtocolObject>( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_BadLogin)); + } + + if(!rContext.GetClientHasAccount()) + { + BOX_WARNING("Failed login from client ID " << + BOX_FORMAT_ACCOUNT(mClientID) << + ": no such account on this server"); + return std::auto_ptr<ProtocolObject>( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_BadLogin)); } // If we need to write, check that nothing else has got a write lock @@ -95,9 +112,12 @@ std::auto_ptr<ProtocolObject> BackupProtocolServerLogin::DoCommand(BackupProtoco // See if the context will get the lock if(!rContext.AttemptToGetWriteLock()) { - ::syslog(LOG_INFO, "Failed to get write lock (for Client ID %08X)", mClientID); - return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError( - BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_CannotLockStoreForWriting)); + BOX_WARNING("Failed to get write lock for Client ID " << + BOX_FORMAT_ACCOUNT(mClientID)); + return std::auto_ptr<ProtocolObject>( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_CannotLockStoreForWriting)); } // Debug: check we got the lock @@ -114,7 +134,11 @@ std::auto_ptr<ProtocolObject> BackupProtocolServerLogin::DoCommand(BackupProtoco rContext.SetPhase(BackupContext::Phase_Commands); // Log login - ::syslog(LOG_INFO, "Login: Client ID %08X, %s", mClientID, ((mFlags & Flags_ReadOnly) != Flags_ReadOnly)?"Read/Write":"Read-only"); + BOX_NOTICE("Login from Client ID " << + BOX_FORMAT_ACCOUNT(mClientID) << + " " << + (((mFlags & Flags_ReadOnly) != Flags_ReadOnly) + ?"Read/Write":"Read-only")); // Get the usage info for reporting to the client int64_t blocksUsed = 0, blocksSoftLimit = 0, blocksHardLimit = 0; @@ -134,7 +158,8 @@ std::auto_ptr<ProtocolObject> BackupProtocolServerLogin::DoCommand(BackupProtoco // -------------------------------------------------------------------------- std::auto_ptr<ProtocolObject> BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) { - ::syslog(LOG_INFO, "Session finished"); + BOX_NOTICE("Session finished for Client ID " << + BOX_FORMAT_ACCOUNT(rContext.GetClientID())); // Let the context know about it rContext.ReceivedFinishCommand(); @@ -305,13 +330,23 @@ std::auto_ptr<ProtocolObject> BackupProtocolServerGetFile::DoCommand(BackupProto en = rdir.FindEntryByID(id); if(en == 0) { - ::syslog(LOG_ERR, "Object %llx in dir %llx for account %x references object %llx which does not exist in dir", - mObjectID, mInDirectory, rContext.GetClientID(), id); - return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError( - BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_PatchConsistencyError)); + BOX_ERROR("Object " << + BOX_FORMAT_OBJECTID(mObjectID) << + " in dir " << + BOX_FORMAT_OBJECTID(mInDirectory) << + " for account " << + BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << + " references object " << + BOX_FORMAT_OBJECTID(id) << + " which does not exist in dir"); + return std::auto_ptr<ProtocolObject>( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_PatchConsistencyError)); } id = en->GetDependsNewer(); - } while(en != 0 && id != 0); + } + while(en != 0 && id != 0); // OK! The last entry in the chain is the full file, the others are patches back from it. // Open the last one, which is the current from file @@ -340,14 +375,14 @@ std::auto_ptr<ProtocolObject> BackupProtocolServerGetFile::DoCommand(BackupProto { { // Write nastily to allow this to work with gcc 2.x - std::auto_ptr<IOStream> t(new FileStream(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL)); + std::auto_ptr<IOStream> t( + new InvisibleTempFileStream( + tempFn.c_str(), + O_RDWR | O_CREAT | + O_EXCL | O_BINARY | + O_TRUNC)); combined = t; } - // Unlink immediately as it's a temporary file - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } } catch(...) { @@ -383,9 +418,10 @@ std::auto_ptr<ProtocolObject> BackupProtocolServerGetFile::DoCommand(BackupProto // Open the object std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID)); + BufferedStream buf(*object); // Verify it - if(!BackupStoreFile::VerifyEncodedFileFormat(*object)) + if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) { return std::auto_ptr<ProtocolObject>(new BackupProtocolServerError( BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_FileDoesNotVerify)); @@ -401,8 +437,9 @@ std::auto_ptr<ProtocolObject> BackupProtocolServerGetFile::DoCommand(BackupProto stream = t; } - // Object will be deleted when the stream is deleted, so can release the object auto_ptr here to - // avoid premature deletiong + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid + // premature deletion object.release(); } diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp index a3f614a5..16388099 100644 --- a/bin/bbstored/BackupContext.cpp +++ b/bin/bbstored/BackupContext.cpp @@ -24,6 +24,8 @@ #include "BackupStoreDaemon.h" #include "RaidFileController.h" #include "FileStream.h" +#include "InvisibleTempFileStream.h" +#include "BufferedStream.h" #include "MemLeakFindOn.h" @@ -125,7 +127,6 @@ void BackupContext::ReceivedFinishCommand() // -------------------------------------------------------------------------- bool BackupContext::AttemptToGetWriteLock() { -#ifndef WIN32 // Make the filename of the write lock file std::string writeLockFile; StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); @@ -151,7 +152,7 @@ bool BackupContext::AttemptToGetWriteLock() } while(!gotLock && tries > 0); } - + if(gotLock) { // Got the lock, mark as not read only @@ -159,10 +160,6 @@ bool BackupContext::AttemptToGetWriteLock() } return gotLock; -#else // WIN32 - // no housekeeping process, we do have the lock - return true; -#endif // !WIN32 } @@ -310,7 +307,8 @@ BackupStoreDirectory &BackupContext::GetDirectoryInternal(int64_t ObjectID) std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory); // Read it from the stream, then set it's revision ID - dir->ReadFromStream(*objectFile, IOStream::TimeOutInfinite); + BufferedStream buf(*objectFile); + dir->ReadFromStream(buf, IOStream::TimeOutInfinite); dir->SetRevisionID(revID); // Make sure the size of the directory is available for writing the dir back @@ -459,9 +457,9 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod { // Open it twice #ifdef WIN32 - FileStream diff(tempFn.c_str(), + InvisibleTempFileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_BINARY); - FileStream diff2(tempFn.c_str(), + InvisibleTempFileStream diff2(tempFn.c_str(), O_RDWR | O_BINARY); #else FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); @@ -521,14 +519,6 @@ int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t Mod ::unlink(tempFn.c_str()); throw; } - -#ifdef WIN32 - // we can't delete the file while it's open, above - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } -#endif } // Get the blocks used diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index ce7263da..c5d5fe40 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -42,6 +42,7 @@ BackupStoreDaemon::BackupStoreDaemon() mExtendedLogging(false), mHaveForkedHousekeeping(false), mIsHousekeepingProcess(false), + mHousekeepingInited(false), mInterProcessComms(mInterProcessCommsSocket) { } @@ -131,7 +132,23 @@ void BackupStoreDaemon::SetupInInitialProcess() // Initialise the raid files controller RaidFileController &rcontroller = RaidFileController::GetController(); - rcontroller.Initialise(config.GetKeyValue("RaidFileConf").c_str()); + + std::string raidFileConfig; + + #ifdef WIN32 + if (!config.KeyExists("RaidFileConf")) + { + raidFileConfig = BOX_GET_DEFAULT_RAIDFILE_CONFIG_FILE; + } + else + { + raidFileConfig = config.GetKeyValue("RaidFileConf"); + } + #else + raidFileConfig = config.GetKeyValue("RaidFileConf"); + #endif + + rcontroller.Initialise(raidFileConfig); // Load the account database std::auto_ptr<BackupStoreAccountDatabase> pdb(BackupStoreAccountDatabase::Read(config.GetKeyValue("AccountDatabase").c_str())); @@ -159,6 +176,9 @@ void BackupStoreDaemon::Run() const Configuration &config(GetConfiguration()); mExtendedLogging = config.GetKeyValueBool("ExtendedLogging"); +#ifdef WIN32 + // Housekeeping runs synchronously on Win32 +#else // Fork off housekeeping daemon -- must only do this the first time Run() is called if(!mHaveForkedHousekeeping) { @@ -188,7 +208,7 @@ void BackupStoreDaemon::Run() // Change the log name ::openlog("bbstored/hk", LOG_PID, LOG_LOCAL6); // Log that housekeeping started - ::syslog(LOG_INFO, "Housekeeping process started"); + BOX_INFO("Housekeeping process started"); // Ignore term and hup // Parent will handle these and alert the child via the socket, don't want to randomly die ::signal(SIGHUP, SIG_IGN); @@ -214,6 +234,7 @@ void BackupStoreDaemon::Run() THROW_EXCEPTION(ServerException, SocketCloseError) } } +#endif // WIN32 if(mIsHousekeepingProcess) { @@ -224,12 +245,18 @@ void BackupStoreDaemon::Run() { // In server process -- use the base class to do the magic ServerTLS<BOX_PORT_BBSTORED>::Run(); - + + if (!mInterProcessCommsSocket.IsOpened()) + { + return; + } + // Why did it stop? Tell the housekeeping process to do the same if(IsReloadConfigWanted()) { mInterProcessCommsSocket.Write("h\n", 2); } + if(IsTerminateWanted()) { mInterProcessCommsSocket.Write("t\n", 2); @@ -237,22 +264,54 @@ void BackupStoreDaemon::Run() } } - // -------------------------------------------------------------------------- // // Function // Name: BackupStoreDaemon::Connection(SocketStreamTLS &) -// Purpose: Handles a connection +// Purpose: Handles a connection, by catching exceptions and +// delegating to Connection2 // Created: 2003/08/20 // // -------------------------------------------------------------------------- void BackupStoreDaemon::Connection(SocketStreamTLS &rStream) { + try + { + Connection2(rStream); + } + catch(BoxException &e) + { + BOX_ERROR("Error in child process, terminating connection: " << + e.what() << " (" << e.GetType() << "/" << + e.GetSubType() << ")"); + } + catch(std::exception &e) + { + BOX_ERROR("Error in child process, terminating connection: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Error in child process, terminating connection: " << + "unknown exception"); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDaemon::Connection2(SocketStreamTLS &) +// Purpose: Handles a connection from bbackupd +// Created: 2006/11/12 +// +// -------------------------------------------------------------------------- +void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) +{ // Get the common name from the certificate std::string clientCommonName(rStream.GetPeerCommonName()); // Log the name - ::syslog(LOG_INFO, "Certificate CN: %s\n", clientCommonName.c_str()); + BOX_INFO("Client certificate CN: " << clientCommonName); // Check it int32_t id; @@ -298,10 +357,8 @@ void BackupStoreDaemon::LogConnectionStats(const char *commonName, const SocketStreamTLS &s) { // Log the amount of data transferred - ::syslog(LOG_INFO, "Connection statistics for %s: " - "IN=%lld OUT=%lld TOTAL=%lld\n", commonName, - (long long)s.GetBytesRead(), - (long long)s.GetBytesWritten(), - (long long)s.GetBytesRead() + - (long long)s.GetBytesWritten()); + BOX_INFO("Connection statistics for " << commonName << ":" + " IN=" << s.GetBytesRead() << + " OUT=" << s.GetBytesWritten() << + " TOTAL=" << (s.GetBytesRead() + s.GetBytesWritten())); } diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 0ce6f21f..eb665440 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -52,7 +52,8 @@ protected: virtual void Run(); - void Connection(SocketStreamTLS &rStream); + virtual void Connection(SocketStreamTLS &rStream); + void Connection2(SocketStreamTLS &rStream); virtual const char *DaemonName() const; virtual const char *DaemonBanner() const; @@ -71,10 +72,12 @@ private: bool mExtendedLogging; bool mHaveForkedHousekeeping; bool mIsHousekeepingProcess; + bool mHousekeepingInited; SocketStream mInterProcessCommsSocket; IOStreamGetLine mInterProcessComms; + virtual void OnIdle(); void HousekeepingInit(); void RunHousekeepingIfNeeded(); int64_t mLastHousekeepingRun; diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp index 91945306..9f4239e7 100644 --- a/bin/bbstored/HousekeepStoreAccount.cpp +++ b/bin/bbstored/HousekeepStoreAccount.cpp @@ -9,9 +9,10 @@ #include "Box.h" -#include <map> #include <stdio.h> +#include <map> + #include "HousekeepStoreAccount.h" #include "BackupStoreDaemon.h" #include "StoreStructure.h" @@ -23,6 +24,7 @@ #include "NamedLock.h" #include "autogen_BackupStoreException.h" #include "BackupStoreFile.h" +#include "BufferedStream.h" #include "MemLeakFindOn.h" @@ -136,11 +138,18 @@ void HousekeepStoreAccount::DoHousekeeping() || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles || usedDirectories != mBlocksInDirectories) { // Log this - ::syslog(LOG_ERR, "On housekeeping, sizes in store do not match calculated sizes, correcting"); - ::syslog(LOG_ERR, "different (store,calc): acc 0x%08x, used (%lld,%lld), old (%lld,%lld), deleted (%lld,%lld), dirs (%lld,%lld)", - mAccountID, - (used + mBlocksUsedDelta), mBlocksUsed, (usedOld + mBlocksInOldFilesDelta), mBlocksInOldFiles, - (usedDeleted + mBlocksInDeletedFilesDelta), mBlocksInDeletedFiles, usedDirectories, mBlocksInDirectories); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " found " + "and fixed wrong block counts: " + "used (" << + (used + mBlocksUsedDelta) << "," << + mBlocksUsed << "), old (" << + (usedOld + mBlocksInOldFilesDelta) << "," << + mBlocksInOldFiles << "), deleted (" << + (usedDeleted + mBlocksInDeletedFilesDelta) << + "," << mBlocksInDeletedFiles << "), dirs (" << + usedDirectories << "," << mBlocksInDirectories + << ")"); } // If the current values don't match, store them @@ -172,17 +181,33 @@ void HousekeepStoreAccount::DoHousekeeping() // Log deletion if anything was deleted if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) { - ::syslog(LOG_INFO, "Account 0x%08x, removed %lld blocks (%lld files, %lld dirs)%s", mAccountID, 0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta), - mFilesDeleted, mEmptyDirectoriesDeleted, - deleteInterrupted?" was interrupted":""); + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " " + "removed " << + (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << + " blocks (" << mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs)" << + (deleteInterrupted?" and was interrupted":"")); } // Make sure the delta's won't cause problems if the counts are really wrong, and // it wasn't fixed because the store was updated during the scan. - if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) mBlocksUsedDelta = (0 - info->GetBlocksUsed()); - if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles())) mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); - if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles())) mBlocksInDeletedFilesDelta =(0 - info->GetBlocksInDeletedFiles()); - if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories())) mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories()); + if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) + { + mBlocksUsedDelta = (0 - info->GetBlocksUsed()); + } + if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles())) + { + mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); + } + if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles())) + { + mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); + } + if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories())) + { + mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories()); + } // Update the usage counts in the store info->ChangeBlocksUsed(mBlocksUsedDelta); @@ -252,7 +277,8 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Read the directory in BackupStoreDirectory dir; - dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + BufferedStream buf(*dirStream); + dir.ReadFromStream(buf, IOStream::TimeOutInfinite); dirStream->Close(); // Is it empty? @@ -552,7 +578,14 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID); if(pentry == 0) { - ::syslog(LOG_ERR, "acc 0x%08x, object %lld not found in dir %lld, logic error/corruption? Run bbstoreaccounts check <accid> fix", mAccountID, ObjectID, InDirectory); + BOX_ERROR("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " " + "found error: object " << + BOX_FORMAT_OBJECTID(ObjectID) << " " + "not found in dir " << + BOX_FORMAT_OBJECTID(InDirectory) << ", " + "indicates logic error/corruption? Run " + "bbstoreaccounts check <accid> fix"); return; } diff --git a/bin/bbstored/bbstored.cpp b/bin/bbstored/bbstored.cpp index 3eaf2639..54858dd4 100644 --- a/bin/bbstored/bbstored.cpp +++ b/bin/bbstored/bbstored.cpp @@ -10,6 +10,7 @@ #include "Box.h" #include "BackupStoreDaemon.h" #include "MainHelper.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -17,8 +18,19 @@ int main(int argc, const char *argv[]) { MAINHELPER_START + Logging::SetProgramName("Box Backup (bbstored)"); + Logging::ToConsole(true); + Logging::ToSyslog (true); + BackupStoreDaemon daemon; - return daemon.Main(BOX_FILE_BBSTORED_DEFAULT_CONFIG, argc, argv); + + #ifdef WIN32 + return daemon.Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE, + argc, argv); + #else + return daemon.Main(BOX_FILE_BBSTORED_DEFAULT_CONFIG, + argc, argv); + #endif MAINHELPER_END } diff --git a/configure.ac b/configure.ac index edf221e0..cab2ee97 100644 --- a/configure.ac +++ b/configure.ac @@ -26,13 +26,21 @@ if test "x$GXX" = "xyes"; then # Use -rdynamic if we have gcc. This is needed for backtrace AC_SUBST([LDADD_RDYNAMIC], ['-rdynamic']) fi -AC_PATH_PROG([PERL], [perl], [no]) -if test "x$PERL" != "xno"; then - AC_DEFINE_UNQUOTED([PERL_EXECUTABLE], ["$PERL"], [Location of the perl executable]) -else - AC_MSG_ERROR([[perl executable was not found]]) -fi +AC_PATH_PROG([PERL], [perl], [AC_MSG_ERROR([[perl executable was not found]])]) + +case $target_os in +mingw*) + TARGET_PERL=perl + ;; +*) + TARGET_PERL=$PERL + ;; +esac + +AC_SUBST([TARGET_PERL]) +AC_DEFINE_UNQUOTED([PERL_EXECUTABLE], ["$TARGET_PERL"], + [Location of the perl executable]) ### Checks for libraries. @@ -94,16 +102,37 @@ esac AC_HEADER_STDC AC_HEADER_SYS_WAIT -AC_CHECK_HEADERS([execinfo.h process.h pwd.h regex.h signal.h]) +AC_CHECK_HEADERS([dlfcn.h execinfo.h getopt.h process.h pwd.h signal.h]) AC_CHECK_HEADERS([syslog.h time.h]) AC_CHECK_HEADERS([netinet/in.h]) AC_CHECK_HEADERS([sys/param.h sys/socket.h sys/time.h sys/types.h sys/wait.h]) AC_CHECK_HEADERS([sys/uio.h sys/xattr.h]) -if test "$ac_cv_header_regex_h" = "yes"; then - AC_SEARCH_LIBS([regcomp], [pcreposix]) +AC_CHECK_HEADER([regex.h], [have_regex_h=yes]) + +if test "$have_regex_h" = "yes"; then + AC_DEFINE([HAVE_REGEX_H], [1], [Define to 1 if regex.h is available]) +else + AC_CHECK_HEADER([pcreposix.h], [have_pcreposix_h=yes]) +fi + +if test "$have_pcreposix_h" = "yes"; then + AC_SEARCH_LIBS([regcomp], ["pcreposix -lpcre"],,[have_pcreposix_h=no_regcomp]) +fi + +if test "$have_pcreposix_h" = "yes"; then + AC_DEFINE([HAVE_PCREPOSIX_H], [1], [Define to 1 if pcreposix.h is available]) fi +if test "$have_regex_h" = "yes" -o "$have_pcreposix_h" = "yes"; then + have_regex_support=yes + AC_DEFINE([HAVE_REGEX_SUPPORT], [1], [Define to 1 if regular expressions are supported]) +else + have_regex_support=no +fi + +AC_SEARCH_LIBS([dlsym], ["dl"]) + ### Checks for typedefs, structures, and compiler characteristics. AC_CHECK_TYPES([u_int8_t, u_int16_t, u_int32_t, u_int64_t]) @@ -143,6 +172,7 @@ winnt*) ;; AX_CHECK_MOUNT_POINT(,[ AC_MSG_ERROR([[cannot work out how to discover mount points on your platform]]) ]) + AC_CHECK_MEMBERS([struct dirent.d_ino],,, [[#include <dirent.h>]]) ;; esac @@ -155,7 +185,7 @@ AC_FUNC_CLOSEDIR_VOID AC_FUNC_ERROR_AT_LINE AC_TYPE_SIGNAL AC_FUNC_STAT -AC_CHECK_FUNCS([getpeereid lchown setproctitle getpid]) +AC_CHECK_FUNCS([getpeereid lchown setproctitle getpid gettimeofday]) # NetBSD implements kqueue too differently for us to get it fixed by 0.10 # TODO: Remove this when NetBSD kqueue implementation is working netbsd_hack=`echo $target_os | sed 's/netbsd.*/netbsd/'` @@ -235,6 +265,7 @@ AX_CONFIG_SCRIPTS([bin/bbackupd/bbackupd-config bin/bbstored/bbstored-config infrastructure/makebuildenv.pl infrastructure/makeparcels.pl + infrastructure/makedistribution.pl lib/common/makeexception.pl lib/raidfile/raidfile-config lib/server/makeprotocol.pl @@ -259,7 +290,7 @@ A summary of the build configuration is below. Box Backup will function without these features, but will work better where they are present. Refer to the documentation for more information on each feature. -Regular expressions: $ac_cv_header_regex_h +Regular expressions: $have_regex_support Large files: $have_large_file_support Berkeley DB: $ax_path_bdb_ok Readline: $have_libreadline diff --git a/distribution/COMMON-MANIFEST.txt b/distribution/COMMON-MANIFEST.txt index be5ef416..cb3543da 100644 --- a/distribution/COMMON-MANIFEST.txt +++ b/distribution/COMMON-MANIFEST.txt @@ -17,9 +17,12 @@ docs/common/lib_crypto notes/lib_crypto docs/common/lib_server notes/lib_server MKDIR infrastructure infrastructure/buildenv-testmain-template.cpp -infrastructure/makebuildenv.pl +infrastructure/makebuildenv.pl.in +infrastructure/makedistribution.pl.in +infrastructure/makeparcels.pl.in infrastructure/BoxPlatform.pm.in -infrastructure/makeparcels.pl +infrastructure/mingw +infrastructure/msvc configure.ac NO-LICENSE config.sub config.sub @@ -27,7 +30,7 @@ NO-LICENSE config.guess config.guess bootstrap parcels.txt -runtest.pl +runtest.pl.in NO-LICENSE-IN-DIR infrastructure/m4 infrastructure/m4 configure diff --git a/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt index 96908264..b24a5527 100644 --- a/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt +++ b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt @@ -1,3 +1,4 @@ +lib/intercept lib/raidfile test/raidfile test/raidfile/testfiles @@ -37,4 +38,4 @@ infrastructure/msvc NO-LICENSE-IN-DIR infrastructure/msvc/2003 infrastructure/msvc/2003 NO-LICENSE-IN-DIR infrastructure/msvc/2005 -infrastructure/msvc/2005
\ No newline at end of file +infrastructure/msvc/2005 diff --git a/distribution/boxbackup/VERSION.txt b/distribution/boxbackup/VERSION.txt index 891c9b59..d46ff63a 100644 --- a/distribution/boxbackup/VERSION.txt +++ b/distribution/boxbackup/VERSION.txt @@ -1,2 +1,2 @@ -0.09_plus3 +0.10_plus_USE_SVN_VERSION boxbackup diff --git a/distribution/boxbackup/contrib/rpm/boxbackup.spec b/distribution/boxbackup/contrib/rpm/boxbackup.spec index 12a2bb9f..111a00ff 100644 --- a/distribution/boxbackup/contrib/rpm/boxbackup.spec +++ b/distribution/boxbackup/contrib/rpm/boxbackup.spec @@ -1,6 +1,20 @@ %define bb_user_id 171 %define ident %{name}-%{version} +# In official distribution tarballs, distribution files are copied to +# the base directory (where configure is), so distribution_dir should be empty. +# This is the default, overridden by the following block in non-distribution +# builds. +%define distribution_dir '' + +# BOX_PRIVATE_BEGIN +# In unofficial tarballs, made from svn, distribution files are still in +# distribution/boxbackup, so the following line overrides the default above: +# (this section will be removed automatically from distribution tarballs +# by infrastructure/makedistribution.pl) +%define distribution_dir distribution/boxbackup/ +# BOX_PRIVATE_END + # Detect distribution. So far we only special-case SUSE. If you need to make # any distro specific changes to get the package building on your system # please email them to boxbackup-dev@fluffy.co.uk @@ -72,6 +86,7 @@ This package contains the server. %setup -q %build +echo -e '%{version}\n%{name}' > VERSION.txt test -e configure || ./bootstrap %configure @@ -88,19 +103,28 @@ mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/box/bbackupd mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/box/bbstored mkdir -p $RPM_BUILD_ROOT%{_var}/lib/box -install -m 644 BUGS.txt $RPM_BUILD_ROOT%{_docdir}/%{ident} -install -m 644 LINUX.txt $RPM_BUILD_ROOT%{_docdir}/%{ident} -install -m 644 VERSION.txt $RPM_BUILD_ROOT%{_docdir}/%{ident} -install -m 644 CONTACT.txt $RPM_BUILD_ROOT%{_docdir}/%{ident} -install -m 644 DOCUMENTATION.txt $RPM_BUILD_ROOT%{_docdir}/%{ident} -install -m 644 ExceptionCodes.txt $RPM_BUILD_ROOT%{_docdir}/%{ident} -install -m 644 THANKS.txt $RPM_BUILD_ROOT%{_docdir}/%{ident} -install -m 644 LICENSE.txt $RPM_BUILD_ROOT%{_docdir}/%{ident} -install -m 644 TODO.txt $RPM_BUILD_ROOT%{_docdir}/%{ident} +install -m 644 BUGS.txt \ + $RPM_BUILD_ROOT%{_docdir}/%{ident} +install -m 644 VERSION.txt \ + $RPM_BUILD_ROOT%{_docdir}/%{ident} +install -m 644 ExceptionCodes.txt \ + $RPM_BUILD_ROOT%{_docdir}/%{ident} +install -m 644 LICENSE.txt \ + $RPM_BUILD_ROOT%{_docdir}/%{ident} + +install -m 644 %{distribution_dir}CONTACT.txt \ + $RPM_BUILD_ROOT%{_docdir}/%{ident} +install -m 644 %{distribution_dir}DOCUMENTATION.txt \ + $RPM_BUILD_ROOT%{_docdir}/%{ident} +install -m 644 %{distribution_dir}LINUX.txt \ + $RPM_BUILD_ROOT%{_docdir}/%{ident} +install -m 644 %{distribution_dir}THANKS.txt \ + $RPM_BUILD_ROOT%{_docdir}/%{ident} # Client touch $RPM_BUILD_ROOT%{_sysconfdir}/box/bbackupd.conf -install -m 755 contrib/%{dist}/bbackupd $RPM_BUILD_ROOT%{init_dir} +install -m 755 %{distribution_dir}contrib/%{dist}/bbackupd \ + $RPM_BUILD_ROOT%{init_dir} %if %{is_suse} ln -s ../../%{init_dir}/bbackupd $RPM_BUILD_ROOT%{_sbindir}/rcbbackupd %endif @@ -113,7 +137,8 @@ install %{client_dir}/bbackupd-config $RPM_BUILD_ROOT%{_sbindir} # Server touch $RPM_BUILD_ROOT%{_sysconfdir}/box/bbstored.conf touch $RPM_BUILD_ROOT%{_sysconfdir}/box/raidfile.conf -install -m 755 contrib/%{dist}/bbstored $RPM_BUILD_ROOT%{init_dir} +install -m 755 %{distribution_dir}contrib/%{dist}/bbstored \ + $RPM_BUILD_ROOT%{init_dir} %if %{is_suse} ln -s ../../%{init_dir}/bbstored $RPM_BUILD_ROOT%{_sbindir}/rcbbstored %endif @@ -195,6 +220,11 @@ rm -rf $RPM_BUILD_ROOT %{_sbindir}/raidfile-config %changelog +* Sat Jan 13 2006 Chris Wilson <chris+box@qwirx.com> +- Support building from an unofficial tarball (from svn) by changing + %{distribution_dir} at the top. +- Write our RPM version number into VERSION.txt and hence compile it in + * Wed Dec 28 2005 Martin Ebourne <martin@zepler.org> - Box now uses autoconf so use configure macro diff --git a/docs/backup/win32_build_on_cygwin_using_mingw.txt b/docs/backup/win32_build_on_cygwin_using_mingw.txt index c35764bb..e71b5764 100644 --- a/docs/backup/win32_build_on_cygwin_using_mingw.txt +++ b/docs/backup/win32_build_on_cygwin_using_mingw.txt @@ -1,13 +1,20 @@ How to build Box Backup on Win32 using Cygwin and MinGW -By Chris Wilson, 2005-12-07 +By Chris Wilson, 2007-05-26 + +(To read this document online with better formatting, browse to: +http://bbdev.fluffy.co.uk/trac/wiki/CompileWithMinGW) + +Start by installing Cygwin on your Windows machine [http://www.cygwin.org]. +Make sure to select the following packages during installation: -Install Cygwin on your Windows box [http://www.cygwin.org] -Make sure to include the following packages: * Devel/gcc-mingw * Devel/gcc-mingw-core * Devel/gcc-mingw-g++ * Mingw/mingw-zlib +If you already have Cygwin installed, please re-run the installer and +ensure that those packages are installed. + Download OpenSSL from [http://www.openssl.org/source/openssl-0.9.7i.tar.gz] @@ -35,20 +42,12 @@ Configure PCRE for MinGW compilation, and build and install it: export CFLAGS="-mno-cygwin" ./configure make winshared - cp .libs/pcreposix.dll /bin - cp .libs/pcreposix.dll.a /usr/i686-pc-mingw32/lib - cp pcreposix.h /usr/i686-pc-mingw32/include/regex.h + cp .libs/libpcre.a .libs/libpcreposix.a /usr/lib/mingw + cp pcreposix.h /usr/include/mingw Now unpack the Box Backup sources, enter the source directory, and configure like this: - export CXX="g++ -mno-cygwin" - export LD="g++ -mno-cygwin" - export CFLAGS="-mno-cygwin -mthreads" - export CXXFLAGS="-mno-cygwin -mthreads" - export LDFLAGS="-mno-cygwin -mthreads" - export LIBS="-lcrypto -lws2_32 -lgdi32" - (if you don't have a "configure" file, run "./bootstrap") - ./configure --target=i686-pc-mingw32 + ./infrastructure/mingw/configure.sh make diff --git a/docs/backup/win32_build_on_linux_using_mingw.txt b/docs/backup/win32_build_on_linux_using_mingw.txt index 12261d51..407096b1 100644 --- a/docs/backup/win32_build_on_linux_using_mingw.txt +++ b/docs/backup/win32_build_on_linux_using_mingw.txt @@ -28,11 +28,27 @@ Unpack and configure: tar xzvf openssl-0.9.8b.tar.gz cd openssl-0.9.8b - ./Configure mingw + ./Configure --prefix=/usr/local/i386-mingw32 mingw make makefile.one wget http://bbdev.fluffy.co.uk/svn/box/chris/win32/support/openssl-0.9.8b-mingw-cross.patch patch -p1 < openssl-0.9.8b-mingw-cross.patch make -f makefile.one + make -f makefile.one install + +Download PCRE from +[http://prdownloads.sourceforge.net/pcre/pcre-6.3.tar.bz2?download] + +Unpack: + + tar xjvf pcre-6.3.tar.bz2 + cd pcre-6.3 + +Configure and make: + + ./configure --host=i586-mingw32msvc --prefix=/usr/i386-mingw32/ + make winshared wininstall + cp .libs/libpcreposix.a /usr/i386-pc-mingw32/lib + cp pcreposix.h /usr/i386-pc-mingw32/include/regex.h Configure Box with: diff --git a/documentation/boxbackup/adminguide.xml b/documentation/boxbackup/adminguide.xml index c33e49d0..6ea4c7b5 100644 --- a/documentation/boxbackup/adminguide.xml +++ b/documentation/boxbackup/adminguide.xml @@ -9,7 +9,7 @@ <preface> <title>License</title> - <para>Copyright © 2003 - 2006, Ben Summers and contributors. All rights + <para>Copyright © 2003 - 2007, Ben Summers and contributors. All rights reserved.</para> <para>Redistribution and use in source and binary forms, with or without @@ -30,19 +30,28 @@ </listitem> <listitem> - <para>Neither the name of the authors nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission</para> + <para>All use of this software and associated advertising materials + must display the following acknowledgement: This product includes + software developed by Ben Summers and contributors.</para> + </listitem> + + <listitem> + <para>The names of the Authors may not be used to endorse or promote + products derived from this software without specific prior written + permission.</para> </listitem> </itemizedlist> - <para>THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + <para>[Where legally impermissible the Authors do not disclaim liability + for direct physical injury or death caused solely by defects in the + software unless it is modified by a third party.]</para> + + <para>THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN + NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS @@ -76,9 +85,13 @@ cd boxbackup-0.10-server-darwin8.5.0 <programlisting>useradd _bbstored</programlisting> - <para>BoxBackup supports Raid for the backup store. There are some - configuration options. Use the following command if you want to create - a simple server WITHOUT Raid protection:</para> + <para>Box Backup has a built-in software RAID facility (redundant + array of inexpensive disks) for the backup store. This allows you to + spread the store data over three disks, and recover from the loss of + any one disk without losing data. However, this is now deprecated, and + you are recommended to use the software or hardware RAID facilities of + your operating system instead. Use the following command if you want + to create a simple server without Box Backup RAID:</para> <programlisting>mkdir /tmp/boxbackupRepository # Create the directory chown _bbstored /tmp/boxbackupRepository/ # Change the owner to the new boxbackup daemon user @@ -95,7 +108,8 @@ chown _bbstored /tmp/boxbackupRepository/ # Change the owner to the Since you might be using NAT, might move the server around or the domain name might change, choose a name that describes the server. When the network address of the server changes, you need to update the - ListenAddresses directive in the /etc/box/bbstored.conf file.</para> + <literal>ListenAddresses</literal> directive in the + <filename>/etc/box/bbstored.conf</filename> file.</para> <programlisting>/usr/local/bin/bbstored-config /etc/box hostname _bbstored</programlisting> @@ -105,7 +119,7 @@ chown _bbstored /tmp/boxbackupRepository/ # Change the owner to the <para>TODO: Expand on this. Explain the 5 steps in detail.</para> - <para>If you want to run the system as a non-root user, look <link + <para>If you want to run the server as a non-root user, look <link linkend="WORoot">here</link>.</para> </section> @@ -120,47 +134,52 @@ chown _bbstored /tmp/boxbackupRepository/ # Change the owner to the not trivial. However, a script to does most of the work in a way which should be good enough for most deployments.</para> - <para><emphasis role="bold">Important Note:</emphasis> The certificate - authority directory is intended to be stored on another server. It - should not be kept on the backup server to limit the impact of a - server compromise. The instructions and the script assume that it will - be kept elsewhere, so will ask you to copy files to and from the - CA.</para> - - <para><emphasis role="bold">Clock time warning</emphasis>: SSL - certificates contain validity dates, including a "valid from" time. If - the clock on the machine which signs the certificates is not - syncronised to the clocks of the machines using these certificates, - you will probably get strange errors until the start time is reached - on all machines. If you get strange errors when attempting to use new - certificates, check the clocks! You will probably just need to wait a - while until the certificates become valid, rather than having to - regenerate them.</para> + <important> + <para>The certificate authority directory is intended to be stored + on another server. It should not be kept on the backup server, in + order to limit the impact of a server compromise. The instructions + and the script assume that it will be kept elsewhere, so will ask + you to copy files to and from the CA.</para> + </important> + + <warning> + <para>SSL certificates contain validity dates, including a "valid + from" time. If the clock on the machine which signs the certificates + is not syncronised to the clocks of the machines using these + certificates, you will probably get strange errors until the start + time is reached on all machines. If you get strange errors when + attempting to use new certificates, check the clocks on all machines + (client, store and CA). You will probably just need to wait a while + until the certificates become valid, rather than having to + regenerate them.</para> + </warning> <section> - <title>Set up a CA</title> + <title>Set up a Certificate Authority</title> - <para>It's best to do this on a machine other than your server, - probably without direct network access. The contents of this - directory control who can access your backup store server.</para> + <para>It is recommended that you keep your Certificate Authority on + a separate machine than either the client or the server, preferably + without direct network access. The contents of this directory + control who can access your backup store server.</para> <para>To setup the basic key structure, do the following:</para> <programlisting>/usr/local/bin/bbstored-certs ca init</programlisting> - <para>(See <link linkend="instguide.xml">OpenSSL notes</link> if you + <para>(See <ulink url="instguide.xml">OpenSSL notes</ulink> if you get an OpenSSL error)</para> - <para>This creates the directory 'ca' in the current directory, and - initialises it with basic keys.</para> + <para>This creates the directory called <filename>ca</filename> in + the current directory, and initialises it with basic keys.</para> </section> <section> <title>Sign a server certificate</title> - <para>When you use the bbstored-config script to set up a config - file for a server, it will generate a certificate request (CSR) for - you. Transfer it to the machine with your CA, then do:</para> + <para>When you use the <command>bbstored-config</command> script to + set up a config file for a server, it will generate a certificate + request (CSR) for you. Transfer it to the machine with your CA, then + do:</para> <programlisting>/usr/local/bin/bbstored-certs ca sign-server hostname-csr.pem</programlisting> @@ -270,12 +289,28 @@ local5.info /var/log/raidfile</programlisting> <programlisting>touch /var/log/box touch /var/log/raidfile</programlisting> - <para>Set up log rotation, by adding in /etc/newsyslog.conf:</para> + <para>Set up log rotation for these new log files. For example, if you + have <filename>/etc/newsyslog.conf</filename>, add the following lines + to it:</para> <programlisting>/var/log/box 644 7 2000 * Z /var/log/raidfile 644 7 2000 * Z</programlisting> - <para>Then restart syslogd.</para> + <para>If you have <filename>/etc/logrotate.d</filename>, create a new + file in there (for example + <filename>/etc/logrotate.d/boxbackup</filename>) containing the + following:</para> + + <programlisting>/var/log/box /var/log/raidfile { + weekly + create + compress + rotate 52 +}</programlisting> + + <para>Then restart syslogd, for example:</para> + + <programlisting>/etc/init.d/syslogd restart</programlisting> </section> <section> @@ -298,22 +333,24 @@ touch /var/log/raidfile</programlisting> <para>The backup client has to be run as root, because it needs to read all your files to back them up, although it is possible to back up a single user's files by running it as that user. (Tip: specify a - directory other than /etc/box, and then give the alternate config file - as the first argument to bbackupd). However, it will fall over if you - don't give yourself read access to one of your files.</para> + directory other than <filename>/etc/box</filename>, and then give the + alternate config file as the first argument to + <command>bbackupd</command>). However, it will fall over if you don't + give yourself read access to one of your files.</para> <section> <title id="BasicConfig">Basic configuration</title> - <para>Run the bbackupd-config script to generate the configuration - files and generate a private key and certificate request.</para> + <para>Run the <command>bbackupd-config</command> script to generate + the configuration files and generate a private key and certificate + request.</para> <programlisting>/usr/local/bin/bbackupd-config /etc/box lazy <emphasis role="bold">999 hostname</emphasis> /var/bbackupd <emphasis role="bold">/home</emphasis></programlisting> - <para>(See <link linkend="instguide.sgml">OpenSSL notes</link> if - you get an OpenSSL error)</para> + <para>(See <ulink url="instguide.xml">OpenSSL notes</ulink> if you + get an OpenSSL error)</para> <para>The items in bold need to be changed. In order, they are the account number, the hostname of the server you're using, and @@ -326,27 +363,34 @@ touch /var/log/raidfile</programlisting> catch bad configuration of this nature!</para> <para>You may also want to consider changing the mode from lazy to - snapshot, depending on what your system is used for.</para> - - <itemizedlist> - <listitem> - <para><emphasis role="bold">lazy</emphasis>. This mode regularly - scans the files, with only a rough schedule. It uploads files as - and when they are changed, if the latest version is more than a - set age. This is good for backing up user's documents stored on - a server, and spreads the load out over the day.</para> - </listitem> - - <listitem> - <para><emphasis role="bold">snapshot</emphasis>. This mode - emulates the traditional backup behaviour of taking a snapshot - of the filesystem. The backup daemon does absolutely nothing - until it is instructed to make a backup using the bbackupctl - utility (probably as a cron job), at which point it uploads all - files which have been changed since the last time it - uploaded.</para> - </listitem> - </itemizedlist> + snapshot, depending on what your system is used for:</para> + + <glosslist> + <glossentry> + <glossterm>Lazy Mode</glossterm> + + <glossdef> + <para>This mode regularly scans the files, with only a rough + schedule. It uploads files as and when they are changed, if + the latest version is more than a set age. This is good for + backing up user's documents stored on a server, and spreads + the load out over the day.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>Snapshot Mode</glossterm> + + <glossdef> + <para>This mode emulates the traditional backup behaviour of + taking a snapshot of the filesystem. The backup daemon does + absolutely nothing until it is instructed to make a backup + using the bbackupctl utility (probably as a cron job), at + which point it uploads all files which have been changed since + the last time it uploaded.</para> + </glossdef> + </glossentry> + </glosslist> <para>When you run the config script, it will tell you what you need to do next. Don't forget to read all the output. An example is shown @@ -362,11 +406,12 @@ touch /var/log/raidfile</programlisting> install them where instructed by the bbackupd-config script during basic bbackupd configuration.</para> - <para>You can then run the daemon (as root) by typing - /usr/local/bin/bbackupd, and of course, adding it to your system's - startup scripts. The first time it's run it will upload everything. - Interrupting it and restarting it will only upload files which were - not uploaded before - it's very tolerant.</para> + <para>You can then run the daemon (as root) by running + <command>/usr/local/bin/bbackupd</command>, and of course, adding it + to your system's startup scripts. The first time it's run it will + upload everything. Interrupting it and restarting it will only + upload files which were not uploaded before - it's very + tolerant.</para> <para>If you run in snapshot mode, you will need to add a cron job to schedule backups. The config script will tell you the exact @@ -384,11 +429,14 @@ touch /var/log/raidfile</programlisting> <para>If you want to see what it's doing in more detail (probably a good idea), follow the instructions in the server setup to create - new log files with syslog. Adding and removing backed up - locations.</para> + new log files with syslog. </para> + </section> + + <section> + <title>Adding and removing backed up locations</title> <para>By editing the /etc/box/bbackupd.conf file, you can add and - remove directories to back up -- see comments in this file for help. + remove directories to back up - see comments in this file for help. Send bbackupd a HUP signal after you modify it.</para> <para>When you remove a location, it will not be marked as deleted @@ -464,9 +512,10 @@ touch /var/log/raidfile</programlisting> <para>This is an example of output from the bbstored-config script.</para> - <para><emphasis role="bold">Important</emphasis>: Follow the - instructions output by your script, not the ones here -- they may be - different for your system.</para> + <important> + <para>Follow the instructions output by your script, not the ones + here -- they may be different for your system.</para> + </important> <programlisting>/usr/local/bin/bbackupd-config /etc/box lazy 51 server.example.com /var/bbackupd /home /etc/samba @@ -538,8 +587,690 @@ What you need to do now... <para>Remember to make a secure, offsite backup of your backup keys, as described in <link linkend="BasicConfig">Basic - configuration</link> above. If you do not,, and that key is lost, - you have no backups.</para> + configuration</link> above. If you do not, and that key is lost, you + have no backups.</para> + </section> + </section> + + <section> + <title>Configuration Options</title> + + <para>Box Backup has many options in its configuration file. We will + try to list them all here.</para> + + <para>First of all, here is an example configuration file, for + reference:</para> + + <example> + <title>Example Configuration File</title> + + <programlisting>StoreHostname = localhost +AccountNumber = 0x2 + +KeysFile = /etc/box/2-FileEncKeys.raw +CertificateFile = /etc/box/2-cert.pem +PrivateKeyFile = /etc/box/2-key.pem +TrustedCAsFile = /etc/box/serverCA.pem +DataDirectory = /var/run/boxbackup +NotifyScript = /etc/box/NotifySysadmin.sh +CommandSocket = /var/run/box/bbackupd.sock + +UpdateStoreInterval = 86400 +MinimumFileAge = 3600 +MaxUploadWait = 7200 +FileTrackingSizeThreshold = 65536 +DiffingUploadSizeThreshold = 65536 +MaximumDiffingTime = 20 +ExtendedLogging = no +LogAllFileAccess = yes + +Server +{ + PidFile = /var/run/bbackupd.pid +} +BackupLocations +{ + etc + { + Path = /etc + } + home + { + Path = /home + ExcludeDir = /home/shared + ExcludeDir = /home/chris/.ccache + ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache + } +}</programlisting> + </example> + + <para>As you can see from the example above, the configuration file + has a number of subsections, enclosed in curly braces {}. Some options + appear outside of any subsection, and we will refer to these as <link + linkend="RootOptions">root options</link>. The available options in + each section are described below.</para> + + <para>Every option has the form <quote>name = value</quote>. Names are + not case-sensitive, but values are. Depending on the option, the value + may be:</para> + + <itemizedlist> + <listitem> + <para>a path (to a file or directory);</para> + </listitem> + + <listitem> + <para>a number (usually in seconds or bytes);</para> + </listitem> + + <listitem> + <para>a boolean (the word Yes or No);</para> + </listitem> + + <listitem> + <para>a hostname (or IP address).</para> + </listitem> + </itemizedlist> + + <para>Paths are specified in native format, i.e. a full Windows path + with drive letter on Windows clients, or a full Unix path on Unix + clients.</para> + + <para><example> + <title>Example:</title> + + <para>StoreObjectInfoFile = + /var/state/boxbackup/bbackupd.dat</para> + + <para>StoreObjectInfoFile = C:\Program Files\Box + Backup\data\bbackupd.dat</para> + </example>The use of relative paths (which do not start with a + forward slash on Unix, or a drive specification on Windows) is + possible but not recommended, since they are interpreted relative to + the current working directory when bbackupd was started, which is + liable to change unexpectedly over time.</para> + + <para>Numbers which start with "0x" are interpreted as hexadecimal. + Numbers which do not start with "0x" are interpreted as + decimal.</para> + + <section> + <title id="RootOptions">Root Options</title> + + <para>These options appear outside of any subsection. By convention + they are at the beginning of the configuration file.</para> + + <para>Some options are required, and some are optional.</para> + + <glosslist> + <glossentry> + <glossterm>StoreHostname (required)</glossterm> + + <glossdef> + <para>The Internet host name (DNS name) or IP address of the + server. This is only used to connect to the server.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>AccountNumber (required)</glossterm> + + <glossdef> + <para>The number of the client's account on the server. This + must be provided by the server operator, and must match the + account number in the client's certificate, otherwise the + client will not be able to log into the server.</para> + + <para>The account number may be specified in hexadecimal + (starting with 0x, as in the example above) or in decimal, but + since the server operator works in hexadecimal, that format is + highly recommended and is the default.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>KeysFile (required)</glossterm> + + <glossdef> + <para>The path to the file containing the encryption key used + for data encryption of client file data and filenames. This is + the most important file to keep safe, since without it your + backups cannot be decrypted and are useless. Likewise, if an + attacker gets access to this key and to your encrypted + backups, he can decrypt them and read all your data. </para> + + <para>Do not change the encryption key without deleting all + files from the account on the server first. None of your old + files on the store will be readable if you do so, and if you + change it back, none of the files uploaded with the new key + will be readable.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>CertificateFile (required)</glossterm> + + <glossdef> + <para>The path to the OpenSSL client certificate in PEM + format. This is supplied by the server operator in response to + the certificate request which you send to them. Together with + the PrivateKeyFile, this provides access to the store server + and the encrypted data stored there.</para> + + <para>It is not critical to protect this file or to back it up + safely, since it can be regenerated by creating a new + certificate request, and asking the server operator to sign + it. You may wish to back it up, together with the + PrivateKeyFile, to avoid this inconvenience if you lose all + your data and need quick access to your backups.</para> + + <para>If you do back them up, you should keep them in a + separate location to the KeysFile, since any person holding + the KeysFile and the PrivateKeyFile can gain access to your + encrypted data and decrypt it.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>PrivateKeyFile (required)</glossterm> + + <glossdef> + <para>The path to the OpenSSL private key in PEM format. This + is generated at the same time as the certificate request, but + there is no need to send it to the server operator, and you + should not do so, in case the communication is intercepted by + an attacker. Together with the CertificateFile, this provides + access to the store server and the encrypted data stored + there.</para> + + <para>See the notes under CertificateFile for information + about backing up this file.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>TrustedCAsFile (required)</glossterm> + + <glossdef> + <para>The path to the OpenSSL certificate of the Client + Certificate Authority (CCA), in PEM format. This is supplied + by the server operator along with your account details, or + along with your signed client certificate. This is used to + verify that the server which you are connecting to is + authorised by the person who signed your certificate. It + protects you against DNS and ARP poisoning and IP spoofing + attacks.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>DataDirectory (required)</glossterm> + + <glossdef> + <para>The path to a directory where bbackupd will keep local + state information. This consists of timestamp files which + identify the last backup start and end times, used by + <command>bbackupquery</command> to determine whether files + have changed, and optionally a database of inode numbers, + which are used to check for files being renamed. The database + is only saved if Box Backup is built with Berkeley Database + (BDB) support.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>NotifyScript (optional)</glossterm> + + <glossdef> + <para>The path to the script or command to run when the Box + Backup client detects an error during the backup process. This + is normally used to notify the client system administrator by + e-mail when a backup fails for any reason.</para> + + <para>The script or command is called with one of the + following additional arguments to identify the cause of the + problem:</para> + + <glosslist> + <glossentry> + <glossterm>store-full</glossterm> + + <glossdef> + <para>The backup store is full. No new files are being + uploaded. If some files are marked as deleted, they + should be removed in due course by the server's + housekeeping process. Otherwise, you need to remove some + files from your backup set, or ask the store operator + for more space.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>read-error</glossterm> + + <glossdef> + <para>One or more files which were supposed to be backed + up could not be read. This could be due to:<itemizedlist> + <listitem> + <para>running the server as a non-root user;</para> + </listitem> + + <listitem> + <para>backing up a mounted filesystem such as + NFS;</para> + </listitem> + + <listitem> + <para>access control lists being applied to some + files;</para> + </listitem> + + <listitem> + <para>SELinux being enabled;</para> + </listitem> + + <listitem> + <para>trying to back up open files under + Windows;</para> + </listitem> + + <listitem> + <para>strange directory permissions such as 0000 or + 0400.</para> + </listitem> + </itemizedlist>Check the client logs, e.g. + /var/log/bbackupd on Unix, or the Windows Event Viewer + in Control Panel > Administrative Tools, for more + information about which files are not being backed up + and why.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>backup-error</glossterm> + + <glossdef> + <para>There was a communications error with the server, + or an unexpected exception was encountered during a + backup run. Check the client logs, e.g. + <filename>/var/log/box</filename> on Unix, or the + Windows Event Viewer in Control Panel > + Administrative Tools, for more information about the + problem.</para> + + <para>You may wish to check your Internet access to the + server, check that the server is running, and ask your + server operator to check your account on the + server.</para> + </glossdef> + </glossentry> + </glosslist> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>CommandSocket (optional)</glossterm> + + <glossdef> + <para>The path to the Unix socket which + <command>bbackupd</command> creates when running, and which + <command>bbackupctl</command> uses to communicate with it, for + example to force a sync or a configuration reload. If this + option is omitted, no socket will be created, and + <command>bbackupctl</command> will not function.</para> + + <para>Unix sockets appear within the filesystem on Unix, as a + special type of file, and must be created in a directory which + exists and to which bbackupd has write access, and bbackupctl + has read access. </para> + + <para>On Windows, the path is ignored, and a <glossterm>named + pipe</glossterm> is created instead. This does not currently + have any security attached, so it can be accessed by any user. + Unlike a Unix socket it can also be accessed remotely. Please + use this option with extreme caution on Windows, and only on + fully trusted networks.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>AutomaticBackup (optional)</glossterm> + + <glossdef> + <para>Enable or disable the client from connecting + automatically to the store every + <glossterm>UpdateStoreInterval</glossterm> seconds. When + enabled (set to <quote>Yes</quote>), the client is in + <glossterm>Lazy Mode</glossterm>. When disabled (set to + <quote>No</quote>), it is in <glossterm>Snapshot + Mode</glossterm>. This setting is optional, and the default + value is <quote>Yes</quote>.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>UpdateStoreInterval (required)</glossterm> + + <glossdef> + <para>The approximate time between successive connections to + the server, in seconds, when the client is in <glossterm>Lazy + Mode</glossterm>. The actual time is randomised slightly to + prevent "rush hour" traffic jams on the server, where many + clients try to connect at the same time.</para> + + <para>This value is ignored if the client is in + <glossterm>Snapshot Mode</glossterm>. However, it is still + required. It can be set to zero in this case.</para> + + <para>You will probably need to experiment with the value of + this option. A good value to start with is probably 86400 + seconds, which is one day.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>MinimumFileAge (required)</glossterm> + + <glossdef> + <para>The number of seconds since a file was last modified + before it will be backed up. The reason for this is to avoid + repeatedly backing up files which are repeatedly changing. A + good value is about 3600 seconds (one hour). If set to zero, + files which have changed will always be backed up on the next + backup run. </para> + + <para>The <glossterm>MaxUploadWait</glossterm> option + overrides this option in some circumstances.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>MaxUploadWait (required)</glossterm> + + <glossdef> + <para>The number of seconds since a file was last uploaded + before it will be uploaded again, even if it keeps changing. + The reason for this is to ensure that files which are + continuously modified are eventually uploaded anyway. This + should be no less than the value of + <glossterm>MinimumFileAge</glossterm>. A good value is about + 14400 seconds (4 hours).</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>MaxFileTimeInFuture (optional)</glossterm> + + <glossdef> + <para>The maximum time that a file's timestamp can be in the + future, before it will be backed up anyway. Due to clock + synchronisation problems, it is inevitable that you will + occasionally see files timestamped in the future. Normally, + for files which are dated only slightly in the future, you + will want to wait until after the file's date before backing + it up. However, for files whose dates are very wrong (more + than a few hours) you will normally prefer to back them up + immediately.</para> + + <para>A good value is about 7200 seconds (2 hours) to cope + with potential problems when moving in and out of daylight + saving time, if applicable in your timezone. The default + value, if this setting is not provided, is 172800 seconds (2 + days).</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>FileTrackingSizeThreshold (required)</glossterm> + + <glossdef> + <para>The minimum size of files which will be tracked by inode + number to detect renames. It is not worth detecting renames of + small files, since they are quick to upload again in full, and + keeping their inode numbers in memory increases the client's + memory usage and slows down searches. Larger files should be + tracked to avoid wasting space on the store and long + uploads.</para> + + <para>A good value is about 65536 bytes (64 kilobytes).</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>DiffingUploadSizeThreshold (required)</glossterm> + + <glossdef> + <para>The minimum size of files which will be compared to the + old file on the server, and for which only changes will be + uploaded. It is not worth comparing small files, since they + are quick to upload again in full, and sending the entire file + reduces the risk of data loss if the store is accidentally + corrupted. Larger files should have only their differences + uploaded to avoid wasting space on the store and long + uploads.</para> + + <para>A good value is about 65536 bytes (64 kilobytes).</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>MaximumDiffingTime (optional)</glossterm> + + <glossdef> + <para>The maximum time for which the client will attempt to + find differences between the current version and the old + version in the store, before giving up and uploading the + entire file again. Very large files (several gigabytes) may + take a very long time to scan for changes, but would also take + a very long time to upload again and use a lot of space on the + store, so it is normally worth omitting this value. </para> + + <para>Use this option only if, for some bizarre reason, you + prefer to upload really large files in full rather than spend + a long time scanning them for changes.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>KeepAliveTime (optional)</glossterm> + + <glossdef> + <para>The interval (in seconds) between sending Keep-Alive + messages to the server while performing long operations such + as finding differences in large files, or scanning large + directories. </para> + + <para>These messages ensure that the SSL connection is not + closed by the server, or an intervening firewall, due to lack + of activity.</para> + + <para>The server will normally wait up to 15 minutes (900 + seconds) before disconnecting the client, so the value should + be given and should be less than 900. Some firewalls may time + out inactive connections after 10 or 5 minutes. </para> + + <para>A good value is 300 seconds (5 minutes). You may need to + reduce this if you frequently see TLSReadFailed or + TLSWriteFailed errors on the client.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>StoreObjectInfoFile (optional)</glossterm> + + <glossdef> + <para>Enables the use of a state file, which stores the + client's internal state when the client is not running. This + is useful on clients machines which are frequently shut down, + for example desktop and laptop computers, because it removes + the need for the client to recontact the store and rescan all + directories on the first backup run, which may take some time. + This feature is somewhat experimental and not well tested. + </para> + + <para>This is option is disabled by default, in which case the + state is stored in memory only. The value is the path to the + state file.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>ExtendedLogging (optional)</glossterm> + + <glossdef> + <para>Enables the connection debugging mode of the client, + which writes all commands sent to or received from the server + to the system logs. This generates a <emphasis>lot</emphasis> + of output, so it should only be used when instructed, or when + you suspect a connection problem or client-server protocol + error (and you know how to interpret the output).</para> + + <para>This is a boolean value, which may be set to + <quote>Yes</quote> or <quote>No</quote>. The default is of + course <quote>No</quote>.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>ExtendedLogFile (optional, new in 0.11)</glossterm> + + <glossdef> + <para>Enables the same debugging output as + <glossterm>ExtendedLogging</glossterm>, but written to a file + instead of the system logs. This is useful if you need + extended logging, but do not have access to the system logs, + for example if you are not the administrator of the + computer.</para> + + <para>The value is the path to the file where these logs will + be written. If omitted, extended logs will not be written to a + file. This is entirely independent of the + <glossterm>ExtendedLogging</glossterm> option. It does not + make much sense to use both at the same time.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>LogAllFileAccess (optional, new in 0.11)</glossterm> + + <glossdef> + <para>Enables logging of all local file and directory access, + file uploads (full and differential), and excluded files. This + may be useful if the client is failing to upload a particular + file, or crashing while trying to upload it. The logs will be + sent to the system log or Windows Event Viewer.</para> + + <para>This is a boolean value, which may be set to + <quote>Yes</quote> or <quote>No</quote>. The default is of + course <quote>No</quote>.</para> + </glossdef> + </glossentry> + + <glossentry> + <glossterm>SyncAllowScript (optional)</glossterm> + + <glossdef> + <para>The path to the script or command to run when the client + is about to start an automatic backup run, and wishes to know + whether it is safe to do so. This is useful for clients which + do not always have access to the server, for example laptops + and computers on dial-up Internet connections.</para> + + <para>The script should either output the word + <quote>now</quote> if the backup should proceed, or else a + number, in seconds, which indicates how long the client should + wait before trying to connect again. Any other output will + result in an error on the client, and the backup will not + run.</para> + + <para>This value is optional, and by default no such script is + used.</para> + </glossdef> + </glossentry> + </glosslist> + </section> + + <section> + <title>Server Section</title> + + <para>These options appear within the Server subsection, which is at + the root level.</para> + + <glosslist> + <glossentry> + <glossterm>PidFile</glossterm> + + <glossdef> + <para>This option enables the client to write its processs + identifier (PID) to the specified file after starting. The + file will be deleted when the client daemon exits for any + reason. This is disabled by default, but is recommended + whenever you run the client daemon as a daemon (in the + background), which is usually the case. This file can be used + by scripts to determine whether the daemon is still running, + and to send it messages to reload its configuration or to + terminate.</para> + + <example> + <title>Example Server Section</title> + + <programlisting>Server +{ + PidFile = /var/state/boxbackup/bbackupd.pid +}</programlisting> + </example> + </glossdef> + </glossentry> + </glosslist> + </section> + + <section> + <title>Backup Locations Section</title> + + <para>This section serves only as a container for all defined backup + locations.</para> + + <example> + <title>Example Backup Locations Section</title> + + <programlisting>BackupLocations +{ + etc + { + Path = /etc + } + home + { + Path = /home + ExcludeDir = /home/shared + ExcludeDir = /home/chris/.ccache + ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache + } +}</programlisting> + </example> + + <para>Each subsection is a backup location. The name of the + subsection is the name that will be used on the server. The root + directory of the account on the server contains one subdirectory per + location. The name should be simple, not containing any spaces or + special characters.</para> + + <para>If you do not define any locations, the client will not back + up any files!</para> + + <para>It is currently not recommended to back up the root directory + of the filesystem on Unix. Box Backup is designed to back up + important data and configuration files, not full systems. + Nevertheless, nothing prevents you from doing so if you + desire.</para> + + <para>On Windows, it is currently not possible to back up files + which are open (currently in use), such as open documents in + Microsoft Office, and system files such as the registry and the + paging file. You will get an error for each open file which the + client attempts to back up. Once the file has been closed, it will + be backed up normally. System files will always be open, and should + be excluded from your backups.</para> </section> </section> </section> @@ -670,8 +1401,8 @@ What you need to do now... up the data it needs from /etc/box/bbackupd.conf. You should run it as root so it can find everything it needs.</para> - <para>Full documentation can be found in the <link - linkend="bbackupquery.xml">bbackupquery man page</link>. It follows + <para>Full documentation can be found in the <ulink + url="bbackupquery.xml">bbackupquery manual page</ulink>. It follows the model of a command line sftp client quite closely.</para> <para>TODO: Link to bbackupquery man-page here.</para> @@ -793,33 +1524,44 @@ query ></programlisting> <section> <title>Verify backups</title> - <para>Since this system is not yet 100% production-ready, you'll be - keen to verify that your backups are correct. This is easy:</para> + <para>As with any backup system, you should frequently check that + your backups are working properly by comparing them. Box Backup + makes this very easy and completely automatic. All you have to do is + schedule the <command>bbackupquery compare</command> command to run + regularly, and check its output. You can run the command manually as + follows:</para> <programlisting>/usr/local/bin/bbackupquery "compare -a" quit</programlisting> - <para>It will report all the differences between the store and the - files on disc. It will download everything, so may take a while. You - should expect to see some differences on a typical compare, because - files which have recently changed are unlikely to have been uploaded - yet. Consider checking the timestamps on the files, or keeping a - record of these messages and comparing them with a future - verification.</para> + <para>This command will report all the differences found between the + store and the files on disc. It will download everything, so may + take a while. You should expect to see some differences on a typical + compare, because files which have recently changed are unlikely to + have been uploaded yet. It will also tell you how many files have + been modified since the last backup run, since these will normally + have changed, and such failures are expected.</para> + + <para>You are strongly recommended to add this command as a + <command>cron</command> job, at least once a month, and to check the + output for anything suspicious, particularly a large number of + compare failures, failures on files that have not been modified, or + any error (anything except a compare mismatch) that occurs during + the compare operation.</para> + + <para>Consider keeping a record of these messages and comparing them + with a future verification.</para> <para>If you would like to do a "quick" check which just downloads - file checksums and compares against that, then do:</para> + file checksums and compares against that, then run:</para> <programlisting>/usr/local/bin/bbackupquery "compare -aq" quit</programlisting> <para>However, this does not check that the file attributes are correct, and since the checksums are generated on the client they may not reflect the data on the server if there is a problem -- the - server cannot check the encrypted contents. View this as a good + server cannot check the encrypted contents. View this as a quick indication, rather than a definite check that your backup verifies correctly.</para> - - <para>You may wish to run either one as a cron job while testing - this system.</para> </section> <section> @@ -936,8 +1678,8 @@ Object ID 0000437e fetched successfully.</programlisting> <title id="FixCorruptions">Fixing corruptions of store data</title> <para>This section gives help on what to do if your server has suffered - corruption, for example, after an unclean shutdown or other OS or - hardware problem.</para> + corruption, for example, after an unclean shutdown or other operating + system or hardware problem.</para> <para>In general, as updates to the store are made in an atomic manner, the most likely result is wasted disc space. However, if really bad @@ -959,34 +1701,34 @@ Object ID 0000437e fetched successfully.</programlisting> lost+found directories as deleted. Finally, those directories will be removed by the housekeeping process on the server.</para> - <para>These instructions assume you're working on account 1234, - subsitute this for whatever account you're actually working on. These - will need to be repeated for all affected accounts.</para> + <para>These instructions assume you're working on account 1234. Replace + this with the account number that you actually want to check (the one + that is experiencing errors). These steps will need to be repeated for + all affected accounts.</para> <section> <title>Stop bbackupd</title> <para>First, make sure that bbackupd is not running on the client - machine for the account you are going to recover. Use kill to - terminate it. This step is not strictly necessary, but is recommended. - During any checks on the account, bbackupd will be unable to log in, - and after they are complete, the account is marked as changed on the - server so bbackupd will perform a complete scan.</para> + machine for the account you are going to recover. Use + <command>bbackupctl terminate</command> to stop it. This step is not + strictly necessary, but is recommended. During any checks on the + account, bbackupd will be unable to log in, and after they are + complete, the account is marked as changed on the server so bbackupd + will perform a complete scan.</para> </section> <section> <title>Are you using RAID on the server?</title> - <para>At the moment, the raidfile recovery tools have not been - written. However, when two out of three files are available, the - server will run succesfully, even if it complains a lot in the logs. - So, your best bet here is to fix the accounts, if necessary, and - retrieve any files you need. Then move the old store directories aside - (in case you need them) and start afresh with new accounts, and let - the clients upload all their data again. These utilities will be - written shortly!</para> - - <para>TODO: Is this true anymore???</para> + <para>The raidfile recovery tools have not been written, and probably + will not be, since Box Backup RAID is deprecated. However, when two + out of three files are available, the server will successfully allow + access to your data, even if it complains a lot in the logs. The best + thing to do is to fix the accounts, if necessary, and retrieve any + files you need. Then move the old store directories aside (in case you + need them) and start afresh with new accounts, and let the clients + upload all their data again.</para> </section> <section> @@ -1038,7 +1780,7 @@ Object ID 0000437e fetched successfully.</programlisting> </section> <section> - <title>Troubleshooting</title> + <title id="Troubleshooting">Troubleshooting</title> <para>If you are trying to fix a store after your disc has been corrupted, see <link linkend="FixCorruptions">Fixing corruptions of diff --git a/documentation/boxbackup/instguide.xml b/documentation/boxbackup/instguide.xml index 2a5bc3e3..b66b7169 100644 --- a/documentation/boxbackup/instguide.xml +++ b/documentation/boxbackup/instguide.xml @@ -7,9 +7,8 @@ <preface> <title>License</title> - <para>Copyright (c) <YEAR>, <OWNER></para> - - <para>All rights reserved.</para> + <para>Copyright © 2003 - 2007, Ben Summers and contributors. + All rights reserved.</para> <para>Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -29,23 +28,34 @@ </listitem> <listitem> - <para>Neither the name of the <ORGANIZATION> nor the names of - its contributors may be used to endorse or promote products derived - from this software without specific prior written permission</para> + <para>All use of this software and associated advertising materials + must display the following acknowledgement: This product includes + software developed by Ben Summers.</para> + </listitem> + + <listitem> + <para>The names of the Authors may not be used to endorse or promote + products derived from this software without specific prior written + permission.</para> </listitem> </itemizedlist> - <para>THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</para> + + <para>[Where legally impermissible the Authors do not disclaim liability + for direct physical injury or death caused solely by defects in the + software unless it is modified by a third party.]</para> + + <para>THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS + OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE.</para> </preface> <chapter> @@ -747,4 +757,4 @@ bbackupd.exe -i</programlisting> <para>If you wish to customise the package you can find the spec file in the contrib/rpm directory.</para> </appendix> -</book>
\ No newline at end of file +</book> diff --git a/infrastructure/BoxPlatform.pm.in b/infrastructure/BoxPlatform.pm.in index c4522343..8c084090 100644 --- a/infrastructure/BoxPlatform.pm.in +++ b/infrastructure/BoxPlatform.pm.in @@ -1,8 +1,5 @@ package BoxPlatform; - use Exporter; -use English; - @ISA = qw/Exporter/; @EXPORT = qw/$build_os $target_os $make_command $bsd_make $platform_define $platform_cpu $gcc_v3 $product_version $product_name $install_into_dir $sub_make_options $platform_compile_line_extra $platform_link_line_extra $platform_lib_files $platform_exe_ext $target_windows/; @@ -15,7 +12,7 @@ BEGIN $target_windows = 1 if $target_os =~ m'^mingw32' or $target_os eq "winnt"; - if ($OSNAME eq "MSWin32" and not -x "/usr/bin/uname") + if ($^O eq "MSWin32" and not -x "/usr/bin/uname") { $build_os = "winnt"; } @@ -30,7 +27,9 @@ BEGIN $build_os = 'CYGWIN' if $build_os =~ m/CYGWIN/; $make_command = ($build_os eq 'Darwin') ? 'bsdmake' : ($build_os eq 'SunOS') ? 'gmake' : 'make'; - $bsd_make = ($build_os ne 'Linux' && $build_os ne 'CYGWIN' && $build_os ne "SunOS"); + + $bsd_make = ($build_os ne 'Linux' && $build_os ne 'CYGWIN' && + $build_os ne "SunOS" && $build_os ne 'GNU/kFreeBSD'); # blank extra flags by default $platform_compile_line_extra = '@CPPFLAGS@ @CXXFLAGS@ @CXXFLAGS_STRICT@'; @@ -54,7 +53,7 @@ BEGIN $product_name = <VERSION>; chomp $product_name; close VERSION; - if($product_version eq 'USE_SVN_VERSION') + if($product_version =~ /USE_SVN_VERSION/) { # for developers, use SVN version my $svnversion = `svnversion .`; @@ -73,7 +72,7 @@ BEGIN $svnurl =~ m!box/(.+)$!; my $svndir = $1; $svndir =~ tr/0-9A-Za-z/_/c; - $product_version = $svndir.'_'.$svnversion; + $product_version =~ s/USE_SVN_VERSION/$svndir.'_'.$svnversion/e; } # where to put the files diff --git a/infrastructure/buildenv-testmain-template.cpp b/infrastructure/buildenv-testmain-template.cpp index 62d2334b..4de30e2e 100644 --- a/infrastructure/buildenv-testmain-template.cpp +++ b/infrastructure/buildenv-testmain-template.cpp @@ -2,6 +2,9 @@ // AUTOMATICALLY GENERATED FILE // do not edit // +// Note that infrastructure/buildenv-testmain-template.cpp is NOT +// auto-generated, but test/*/_main.cpp are generated from it. +// // -------------------------------------------------------------------------- @@ -23,6 +26,11 @@ #include <stdarg.h> #include <fcntl.h> #include <errno.h> +#include <string> + +#ifdef HAVE_GETOPT_H + #include <getopt.h> +#endif #ifdef WIN32 #include "emu.h" @@ -32,6 +40,10 @@ #include <string> +#include "Logging.h" +#include "Test.h" +#include "Timer.h" + #include "MemLeakFindOn.h" int test(int argc, const char *argv[]); @@ -45,13 +57,14 @@ int test(int argc, const char *argv[]); int failures = 0; int first_fail_line; std::string first_fail_file; +std::string bbackupd_args, bbstored_args, bbackupquery_args; int filedes_open_at_beginning = -1; #ifdef WIN32 // any way to check for open file descriptors on Win32? -inline int count_filedes() { return 0; } +inline int count_filedes() { return 0; } inline bool checkfilesleftopen() { return false; } #else // !WIN32 @@ -77,7 +90,8 @@ bool checkfilesleftopen() { if(filedes_open_at_beginning == -1) { - // Not used correctly, pretend that there were things left open so this gets invesitgated + // Not used correctly, pretend that there were things + // left open so this gets investigated return true; } @@ -90,11 +104,72 @@ bool checkfilesleftopen() #endif -int main(int argc, const char *argv[]) +int main(int argc, char * const * argv) { // Start memory leak testing MEMLEAKFINDER_START +#ifdef HAVE_GETOPT_H + struct option longopts[] = + { + { "bbackupd-args", required_argument, NULL, 'c' }, + { "bbstored-args", required_argument, NULL, 's' }, + { NULL, 0, NULL, 0 } + }; + + int ch; + + while ((ch = getopt_long(argc, argv, "c:s:t:T", longopts, NULL)) + != -1) + { + switch(ch) + { + case 'c': + { + bbackupd_args += " "; + bbackupd_args += optarg; + } + break; + + case 's': + { + bbstored_args += " "; + bbstored_args += optarg; + } + break; + + case 't': + { + Console::SetTag(optarg); + } + break; + + case 'T': + { + Console::SetShowTime(true); + } + break; + + case '?': + { + fprintf(stderr, "Unknown option: %s\n", + optarg); + exit(2); + } + + default: + { + fprintf(stderr, "Unknown option code '%c'\n", + ch); + exit(2); + } + } + } + + argc -= optind - 1; + argv += optind - 1; +#endif // HAVE_GETOPT_H + // If there is more than one argument, then the test is doing something advanced, so leave it alone bool fulltestmode = (argc == 1); @@ -114,16 +189,23 @@ int main(int argc, const char *argv[]) TEST_THAT(WSAStartup(0x0101, &info) != SOCKET_ERROR) #endif } + try { - int returncode = test(argc, argv); + #ifdef BOX_MEMORY_LEAK_TESTING + memleakfinder_init(); + #endif + + Timers::Init(); + int returncode = test(argc, (const char **)argv); + Timers::Cleanup(); // check for memory leaks, if enabled #ifdef BOX_MEMORY_LEAK_TESTING if(memleakfinder_numleaks() != 0) { failures++; - printf("FAILURE: Memory leaks detected\n"); + printf("FAILURE: Memory leaks detected in test code\n"); printf("==== MEMORY LEAKS =================================\n"); memleakfinder_reportleaks(); printf("===================================================\n"); diff --git a/infrastructure/m4/ax_check_llong_minmax.m4 b/infrastructure/m4/ax_check_llong_minmax.m4 new file mode 100644 index 00000000..f3f99c53 --- /dev/null +++ b/infrastructure/m4/ax_check_llong_minmax.m4 @@ -0,0 +1,76 @@ +dnl @synopsis AX_CHECK_LLONG_MINMAX +dnl +dnl This macro will fix up LLONG_MIN and LLONG_MAX as appropriate. I'm finding +dnl it quite difficult to believe that so many hoops are necessary. The world +dnl seems to have gone quite mad. +dnl +dnl This gem is adapted from the OpenSSH configure script so here's +dnl the original copyright notice: +dnl +dnl Copyright (c) 1999-2004 Damien Miller +dnl +dnl Permission to use, copy, modify, and distribute this software for any +dnl purpose with or without fee is hereby granted, provided that the above +dnl copyright notice and this permission notice appear in all copies. +dnl +dnl @category C +dnl @author Martin Ebourne and Damien Miller +dnl @version 2005/07/07 + +AC_DEFUN([AX_CHECK_LLONG_MINMAX], [ + AC_CHECK_DECL([LLONG_MAX], [have_llong_max=1], , [[#include <limits.h>]]) + if test -z "$have_llong_max"; then + AC_MSG_CHECKING([[for max value of long long]]) + AC_RUN_IFELSE([AC_LANG_SOURCE([[ + #include <stdio.h> + /* Why is this so damn hard? */ + #undef __GNUC__ + #undef __USE_ISOC99 + #define __USE_ISOC99 + #include <limits.h> + #define DATA "conftest.llminmax" + int main(void) { + FILE *f; + long long i, llmin, llmax = 0; + + if((f = fopen(DATA,"w")) == NULL) + exit(1); + + #if defined(LLONG_MIN) && defined(LLONG_MAX) + fprintf(stderr, "Using system header for LLONG_MIN and LLONG_MAX\n"); + llmin = LLONG_MIN; + llmax = LLONG_MAX; + #else + fprintf(stderr, "Calculating LLONG_MIN and LLONG_MAX\n"); + /* This will work on one's complement and two's complement */ + for (i = 1; i > llmax; i <<= 1, i++) + llmax = i; + llmin = llmax + 1LL; /* wrap */ + #endif + + /* Sanity check */ + if (llmin + 1 < llmin || llmin - 1 < llmin || llmax + 1 > llmax || llmax - 1 > llmax) { + fprintf(f, "unknown unknown\n"); + exit(2); + } + + if (fprintf(f ,"%lld %lld", llmin, llmax) < 0) + exit(3); + + exit(0); + } + ]])], [ + read llong_min llong_max < conftest.llminmax + AC_MSG_RESULT([$llong_max]) + AC_DEFINE_UNQUOTED([LLONG_MAX], [${llong_max}LL], + [max value of long long calculated by configure]) + AC_MSG_CHECKING([[for min value of long long]]) + AC_MSG_RESULT([$llong_min]) + AC_DEFINE_UNQUOTED([LLONG_MIN], [${llong_min}LL], + [min value of long long calculated by configure]) + ], + [AC_MSG_RESULT(not found)], + [AC_MSG_WARN([[cross compiling: not checking]])] + ) + fi + ])dnl diff --git a/infrastructure/makebuildenv.pl.in b/infrastructure/makebuildenv.pl.in index 41623bc2..b5d462d9 100755 --- a/infrastructure/makebuildenv.pl.in +++ b/infrastructure/makebuildenv.pl.in @@ -401,31 +401,54 @@ for my $mod (@implicit_deps, @modules) sub writetestfile { my ($filename,$runcmd,$module) = @_; - open TESTFILE,">$filename" or die "Can't open test script file for $module for writing\n"; + + open TESTFILE,">$filename" or die "Can't open " . + "test script file for $module for writing\n"; print TESTFILE "#!/bin/sh\necho TEST: $module\n"; + if(-d "$module/testfiles") { print TESTFILE <<__E; +echo Killing any running daemons... +test -r testfiles/bbackupd.pid && kill `cat testfiles/bbackupd.pid` +test -r testfiles/bbstored.pid && kill `cat testfiles/bbstored.pid` + echo Removing old test files... +chmod -R a+rwx testfiles rm -rf testfiles + echo Copying new test files... cp -p -R ../../../$module/testfiles . + __E } + if(-e "$module/testextra") { - open FL,"$module/testextra" or die "Can't open $module/testextra"; + open FL,"$module/testextra" or die + "Can't open $module/testextra"; while(<FL>) {print TESTFILE} close FL; } + print TESTFILE "$runcmd\n"; + + if(-d "$module/testfiles") + { + print TESTFILE <<__E; +# echo Killing any running daemons... +test -r testfiles/bbackupd.pid && kill `cat testfiles/bbackupd.pid` +test -r testfiles/bbstored.pid && kill `cat testfiles/bbstored.pid` +__E + } + close TESTFILE; } - writetestfile("$mod/_t", - './test' . $platform_exe_ext . ' $1 $2 $3 $4 $5', $mod); - writetestfile("$mod/_t-gdb", - 'gdb ./test' . $platform_exe_ext, $mod); + writetestfile("$mod/_t", "GLIBCXX_FORCE_NEW=1 ". + './test' . $platform_exe_ext . ' "$@"', $mod); + writetestfile("$mod/_t-gdb", "GLIBCXX_FORCE_NEW=1 ". + 'gdb ./test' . $platform_exe_ext . ' "$@"', $mod); } @@ -522,6 +545,40 @@ VARIENT = DEBUG .endif __E + + if ($bsd_make) + { + print MAKE <<__E; +.ifdef V +HIDE = +_CXX = \$(CXX) +_LINK = \$(CXX) +_WINDRES = \$(WINDRES) +_AR = \$(AR) +_RANLIB = \$(RANLIB) +.else +HIDE = @ +_CXX = @ echo "[CXX] " \$(*F) && \$(CXX) +_LINK = @ echo "[LINK] " \$(*F) && \$(CXX) +_WINDRES = @ echo "[WINDRES]" \$(*F) && \$(WINDRES) +_AR = @ echo "[AR] " \$(*F) && \$(AR) +_RANLIB = @ echo "[RANLIB] " \$(*F) && \$(RANLIB) +.endif + +__E + } + else + { + print MAKE <<__E; +HIDE = \$(if \$(V),,@) +_CXX = \$(if \$(V),\$(CXX), @ echo "[CXX] \$<" && \$(CXX)) +_LINK = \$(if \$(V),\$(CXX), @ echo "[LINK] \$@" && \$(CXX)) +_WINDRES = \$(if \$(V),\$(WINDRES), @ echo "[WINDRES] \$<" && \$(WINDRES)) +_AR = \$(if \$(V),\$(AR), @ echo "[AR] \$@" && \$(AR)) +_RANLIB = \$(if \$(V),\$(RANLIB), @ echo "[RANLIB] \$@" && \$(RANLIB)) + +__E + } # read directory opendir DIR,$mod; @@ -623,12 +680,12 @@ __E if ($is_cpp) { - $make .= "\t\$(CXX) \$(CXXFLAGS) $compile_line_extra ". + $make .= "\t\$(_CXX) \$(CXXFLAGS) $compile_line_extra ". "-c $file -o $out_name\n\n"; } elsif ($is_rc) { - $make .= "\t\$(WINDRES) $file $out_name\n\n"; + $make .= "\t\$(_WINDRES) $file $out_name\n\n"; my $res_list = $module_resources_win32{$mod}; $res_list ||= []; push @$res_list, $base.'.o'; @@ -667,7 +724,7 @@ __E # run make for things we require for my $dep (@all_deps_for_module) { - $deps_makeinfo .= "\t\t(cd ../../$dep; \$(MAKE)$sub_make_options \$(DEPENDMAKEFLAGS) -D NODEPS)\n"; + $deps_makeinfo .= "\t\t\$(HIDE) (cd ../../$dep; \$(MAKE)$sub_make_options \$(DEPENDMAKEFLAGS) -D NODEPS)\n"; } $deps_makeinfo .= ".\tendif\n.endif\n\n"; } @@ -711,9 +768,9 @@ __E if($target_is_library) { # make a library archive... - print MAKE "\t(echo -n > $end_target; rm $end_target)\n"; - print MAKE "\t\$(AR) -q $end_target $o_file_list\n"; - print MAKE "\t\$(RANLIB) $end_target\n"; + print MAKE "\t\$(HIDE) (echo -n > $end_target; rm $end_target)\n"; + print MAKE "\t\$(_AR) -q $end_target $o_file_list\n"; + print MAKE "\t\$(_RANLIB) $end_target\n"; } else { @@ -737,7 +794,7 @@ __E } # link line... - print MAKE "\t\$(CXX) $link_line_extra -o $end_target $o_file_list $lib_files$lo $platform_lib_files\n"; + print MAKE "\t\$(_LINK) $link_line_extra -o $end_target $o_file_list $lib_files$lo $platform_lib_files\n"; } # tests need to copy the test file over if($type eq 'test') @@ -758,7 +815,7 @@ __E print MAKE "clean:\n\t-rm -rf \$(OUTDIR)/*\n.\tifndef SUBCLEAN\n"; for my $dep (@all_deps_for_module) { - print MAKE "\t(cd ../../$dep; \$(MAKE) \$(DEPENDMAKEFLAGS) -D SUBCLEAN clean)\n"; + print MAKE "\t\$(HIDE) (cd ../../$dep; \$(MAKE) \$(DEPENDMAKEFLAGS) -D SUBCLEAN clean)\n"; } print MAKE ".\tendif\n"; diff --git a/infrastructure/makedistribution.pl b/infrastructure/makedistribution.pl.in index 16f7409b..314259f6 100755 --- a/infrastructure/makedistribution.pl +++ b/infrastructure/makedistribution.pl.in @@ -1,3 +1,4 @@ +#!/usr/bin/perl #!@PERL@ use strict; use Symbol; @@ -6,7 +7,7 @@ use Symbol; my %comment_chars = ('cpp' => '// ', 'h' => '// ', 'pl' => '# ', 'pm' => '# ', '' => '# '); # other extensions which need text copying, just to remove the private stuff -my %text_files = ('txt' => 1); +my %text_files = ('txt' => 1, 'spec' => 1); # files which don't get the license added my %no_license = (); # 'filename' => 1 @@ -36,6 +37,29 @@ close VERSION; die "Archive name '$archive_name' is not equal to the distribution name '$distribution'" unless $archive_name eq $distribution; +my $svnversion = `svnversion .`; +chomp $svnversion; +$svnversion =~ tr/0-9A-Za-z/_/c; + +if($version =~ /USE_SVN_VERSION/) +{ + # for developers, use SVN version + open INFO,'svn info . |'; + my $svnurl; + while(<INFO>) + { + if(m/^URL: (.+?)[\n\r]+/) + { + $svnurl = $1; + } + } + close INFO; + $svnurl =~ m'box/(.+)$'; + my $svndir = $1; + $svndir =~ tr/0-9A-Za-z/_/c; + $version =~ s/USE_SVN_VERSION/$svndir.'_'.$svnversion/e; +} + # make initial directory my $base_name = "$archive_name-$version"; system "rm -rf $base_name"; @@ -47,8 +71,6 @@ open LICENSE,"$dist_root/LICENSE.txt" or die "Can't open $dist_root/LICENSE.txt" my $license_f; read LICENSE,$license_f,100000; close LICENSE; -my $svnversion = `svnversion .`; -chomp $svnversion; my @license = ('distribution '.$base_name.' (svn version: '.$svnversion.')',split(/\n/,$license_f)); # copy files, make a note of all the modules included @@ -114,6 +136,7 @@ copy_from_list("$dist_root/DISTRIBUTION-MANIFEST.txt"); or die "Copy of root extra files failed"; unlink "$base_name/DISTRIBUTION-MANIFEST.txt" or die "Delete of DISTRIBUTION-MANIFEST.txt file failed"; +replace_version_in("VERSION.txt"); # produce a new modules file my $modules = gensym; @@ -316,6 +339,7 @@ sub replace_version_in while(<IN>) { s/###DISTRIBUTION-VERSION-NUMBER###/$version/g; + s/.*USE_SVN_VERSION.*/$version/g; print OUT } diff --git a/infrastructure/makeparcels.pl.in b/infrastructure/makeparcels.pl.in index 5e226d1f..0846ef46 100755 --- a/infrastructure/makeparcels.pl.in +++ b/infrastructure/makeparcels.pl.in @@ -101,8 +101,12 @@ for my $parcel (@parcels) my $dir = parcel_dir($parcel); print MAKE "\ttest -d $dir || mkdir $dir\n"; - open SCRIPT,">parcels/scripts/install-$parcel" or die "Can't open installer script for $parcel for writing"; - print SCRIPT "#!/bin/sh\n\n"; + unless ($target_windows) + { + open SCRIPT,">parcels/scripts/install-$parcel" or die + "Can't open installer script for $parcel for writing"; + print SCRIPT "#!/bin/sh\n\n"; + } for(@{$parcel_contents{$parcel}}) { @@ -124,7 +128,8 @@ for my $parcel (@parcels) { if ($optional) { - print MAKE "\ttest -r $name && cp $name $dir\n"; + print MAKE "\ttest -r $name " . + "&& cp $name $dir || true\n"; } else { @@ -135,21 +140,34 @@ for my $parcel (@parcels) $name = $1; } - print SCRIPT "install $name $install_into_dir\n"; + unless ($target_windows) + { + print SCRIPT "install $name $install_into_dir\n"; + } + } + + unless ($target_windows) + { + close SCRIPT; + chmod 0755,"parcels/scripts/install-$parcel"; } - close SCRIPT; - - chmod 0755,"parcels/scripts/install-$parcel"; - my $root = parcel_root($parcel); - print MAKE "\tcp parcels/scripts/install-$parcel $dir\n"; + + unless ($target_windows) + { + print MAKE "\tcp parcels/scripts/install-$parcel $dir\n"; + } + print MAKE "\t(cd parcels; tar cf - $root | gzip -9 - > $root.tgz )\n"; print MAKE "\n"; - - print MAKE "install-$parcel:\n"; - print MAKE "\t(cd $dir; ./install-$parcel)\n\n"; + + unless ($target_windows) + { + print MAKE "install-$parcel:\n"; + print MAKE "\t(cd $dir; ./install-$parcel)\n\n"; + } } print MAKE <<__E; diff --git a/infrastructure/mingw/configure.sh b/infrastructure/mingw/configure.sh index 1dd1b014..f1ad353f 100755 --- a/infrastructure/mingw/configure.sh +++ b/infrastructure/mingw/configure.sh @@ -2,15 +2,15 @@ if [ ! -r "/usr/i686-pc-mingw32/lib/libssl.a" ]; then echo "Error: install OpenSSL as instructed by" \ - "docs/backup/mingw_build.txt" >&2 + "docs/backup/win32_build_on_cygwin_using_mingw.txt" >&2 exit 2 fi -if [ ! -r "/usr/i686-pc-mingw32/lib/libpcreposix.a" \ - -o ! -r "/usr/i686-pc-mingw32/lib/libpcre.a" \ - -o ! -r "/usr/i686-pc-mingw32/include/regex.h" ]; then +if [ ! -r "/usr/lib/mingw/libpcreposix.a" \ + -o ! -r "/usr/lib/mingw/libpcre.a" \ + -o ! -r "/usr/include/mingw/pcreposix.h" ]; then echo "Error: install PCRE as instructed by" \ - "docs/backup/mingw_build.txt" >&2 + "docs/backup/win32_build_on_cygwin_using_mingw.txt" >&2 exit 2 fi diff --git a/infrastructure/msvc/2003/bbackupctl.vcproj b/infrastructure/msvc/2003/bbackupctl.vcproj index b46d99f7..b2249ff8 100644 --- a/infrastructure/msvc/2003/bbackupctl.vcproj +++ b/infrastructure/msvc/2003/bbackupctl.vcproj @@ -20,7 +20,7 @@ Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\..\db-4.2.52.NC\build_win32";"$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING"
+ PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;NDEBUG "
MinimalRebuild="TRUE"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
@@ -149,6 +149,9 @@ Name="Resource Files"
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
+ <File
+ RelativePath="..\..\..\lib\win32\messages.rc">
+ </File>
</Filter>
</Files>
<Globals>
diff --git a/infrastructure/msvc/2003/bbackupd.vcproj b/infrastructure/msvc/2003/bbackupd.vcproj index 4ca3470e..182c10a2 100644 --- a/infrastructure/msvc/2003/bbackupd.vcproj +++ b/infrastructure/msvc/2003/bbackupd.vcproj @@ -20,7 +20,7 @@ Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""$(SolutionDir)..\..\..\..\db-4.2.52.NC\build_win32";"$(SolutionDir)..\..\..\..\boost_1_31_0";"$(SolutionDir)..\..\..\..\openssl\include";"$(SolutionDir)..\..\..\..\zlib\include";"$(SolutionDir)..\..\..\lib\backupclient";"$(SolutionDir)..\..\..\lib\server";"$(SolutionDir)..\..\..\lib\crypto";"$(SolutionDir)..\..\..\lib\compress";"$(SolutionDir)..\..\..\lib\win32";"$(SolutionDir)..\..\..\lib\common\""
- PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING"
+ PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;NDEBUG "
MinimalRebuild="TRUE"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
@@ -130,6 +130,9 @@ Name="bbackupd"
Filter="">
<File
+ RelativePath="..\..\..\bin\bbackupd\autogen_ClientException.cpp">
+ </File>
+ <File
RelativePath="..\..\..\bin\bbackupd\BackupClientContext.cpp">
</File>
<File
@@ -170,6 +173,9 @@ Name="bbackupd"
Filter="">
<File
+ RelativePath="..\..\..\bin\bbackupd\autogen_ClientException.h">
+ </File>
+ <File
RelativePath="..\..\..\bin\bbackupd\BackupClientContext.h">
</File>
<File
@@ -200,6 +206,9 @@ Name="Resource Files"
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
+ <File
+ RelativePath="..\..\..\lib\win32\messages.rc">
+ </File>
</Filter>
<File
RelativePath="..\..\..\ReadMe.txt">
diff --git a/infrastructure/msvc/2003/boxbackup.ncb b/infrastructure/msvc/2003/boxbackup.ncb Binary files differdeleted file mode 100644 index 8aacb715..00000000 --- a/infrastructure/msvc/2003/boxbackup.ncb +++ /dev/null diff --git a/infrastructure/msvc/2003/boxbackup.sln b/infrastructure/msvc/2003/boxbackup.sln index ede6faa1..d9a28041 100644 --- a/infrastructure/msvc/2003/boxbackup.sln +++ b/infrastructure/msvc/2003/boxbackup.sln @@ -1,7 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 8.00
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "boxquery", "boxquery.vcproj", "{FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}"
ProjectSection(ProjectDependencies) = postProject
- {98598F62-FEA7-4134-AA29-0AD2315A214F} = {98598F62-FEA7-4134-AA29-0AD2315A214F}
{A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
EndProjectSection
EndProject
@@ -11,7 +10,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "common", "common.vcproj", " EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbackupd", "bbackupd.vcproj", "{22D325FB-9131-4BD6-B390-968F0491D687}"
ProjectSection(ProjectDependencies) = postProject
- {98598F62-FEA7-4134-AA29-0AD2315A214F} = {98598F62-FEA7-4134-AA29-0AD2315A214F}
{A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
EndProjectSection
EndProject
@@ -25,10 +23,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbackupctl", "bbackupctl.vc {A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
EndProjectSection
EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "boost_regex", "lib\win32\boost_regex.vcproj", "{98598F62-FEA7-4134-AA29-0AD2315A214F}"
- ProjectSection(ProjectDependencies) = postProject
- EndProjectSection
-EndProject
Global
GlobalSection(SolutionConfiguration) = preSolution
Debug = Debug
@@ -55,10 +49,6 @@ Global {9FD51412-E945-4457-A17A-CA3C505CF431}.Debug.Build.0 = Debug|Win32
{9FD51412-E945-4457-A17A-CA3C505CF431}.Release.ActiveCfg = Release|Win32
{9FD51412-E945-4457-A17A-CA3C505CF431}.Release.Build.0 = Release|Win32
- {98598F62-FEA7-4134-AA29-0AD2315A214F}.Debug.ActiveCfg = Debug|Win32
- {98598F62-FEA7-4134-AA29-0AD2315A214F}.Debug.Build.0 = Debug|Win32
- {98598F62-FEA7-4134-AA29-0AD2315A214F}.Release.ActiveCfg = Release|Win32
- {98598F62-FEA7-4134-AA29-0AD2315A214F}.Release.Build.0 = Release|Win32
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
EndGlobalSection
diff --git a/infrastructure/msvc/2003/boxbackup.suo b/infrastructure/msvc/2003/boxbackup.suo Binary files differdeleted file mode 100644 index c461ff26..00000000 --- a/infrastructure/msvc/2003/boxbackup.suo +++ /dev/null diff --git a/infrastructure/msvc/2003/boxquery.vcproj b/infrastructure/msvc/2003/boxquery.vcproj index a2962c22..8d8cf20c 100644 --- a/infrastructure/msvc/2003/boxquery.vcproj +++ b/infrastructure/msvc/2003/boxquery.vcproj @@ -21,7 +21,7 @@ Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0""
- PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING"
+ PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;NDEBUG "
MinimalRebuild="TRUE"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
@@ -161,6 +161,9 @@ Name="Resource Files"
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
+ <File
+ RelativePath="..\..\..\lib\win32\messages.rc">
+ </File>
</Filter>
<File
RelativePath="..\..\..\ReadMe.txt">
diff --git a/infrastructure/msvc/2003/common.vcproj b/infrastructure/msvc/2003/common.vcproj index 553aaf1a..bdae0bb8 100644 --- a/infrastructure/msvc/2003/common.vcproj +++ b/infrastructure/msvc/2003/common.vcproj @@ -20,7 +20,7 @@ Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0\""
- PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING"
+ PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;NDEBUG "
MinimalRebuild="TRUE"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
@@ -171,15 +171,24 @@ RelativePath="..\..\..\lib\common\IOStreamGetLine.cpp">
</File>
<File
+ RelativePath="..\..\..\lib\common\Logging.cpp">
+ </File>
+ <File
RelativePath="..\..\..\lib\common\MemBlockStream.cpp">
</File>
<File
RelativePath="..\..\..\lib\common\PartialReadStream.cpp">
</File>
<File
+ RelativePath="..\..\..\lib\common\PathUtils.cpp">
+ </File>
+ <File
RelativePath="..\..\..\lib\common\ReadGatherStream.cpp">
</File>
<File
+ RelativePath="..\..\..\lib\common\ReadLoggingStream.cpp">
+ </File>
+ <File
RelativePath="..\..\..\lib\common\StreamableMemBlock.cpp">
</File>
<File
@@ -287,6 +296,9 @@ <File
RelativePath="..\..\..\lib\win32\emu.cpp">
</File>
+ <File
+ RelativePath="..\..\..\lib\win32\getopt_long.cxx">
+ </File>
</Filter>
<Filter
Name="server"
@@ -327,6 +339,10 @@ <File
RelativePath="..\..\..\lib\server\TLSContext.cpp">
</File>
+ <File
+ RelativePath="..\..\..\lib\server\WinNamedPipeStream.cpp">
+ </File>
+
</Filter>
</Filter>
</Filter>
@@ -435,6 +451,9 @@ RelativePath="..\..\..\lib\server\LocalProcessStream.h">
</File>
<File
+ RelativePath="..\..\..\lib\common\Logging.h">
+ </File>
+ <File
RelativePath="..\..\..\lib\common\MainHelper.h">
</File>
<File
@@ -456,6 +475,9 @@ RelativePath="..\..\..\lib\common\PartialReadStream.h">
</File>
<File
+ RelativePath="..\..\..\lib\common\PathUtils.h">
+ </File>
+ <File
RelativePath="..\..\..\lib\common\ReadGatherStream.h">
</File>
<File
@@ -468,6 +490,9 @@ RelativePath="..\..\..\lib\common\Test.h">
</File>
<File
+ RelativePath="..\..\..\lib\common\Timer.h">
+ </File>
+ <File
RelativePath="..\..\..\lib\common\UnixUser.h">
</File>
<File
@@ -476,6 +501,9 @@ <File
RelativePath="..\..\..\lib\common\WaitForEvent.h">
</File>
+ <File
+ RelativePath="..\..\..\lib\common\BoxVersion.h">
+ </File>
</Filter>
<Filter
Name="backupclient"
@@ -569,6 +597,12 @@ <File
RelativePath="..\..\..\lib\win32\emu.h">
</File>
+ <File
+ RelativePath="..\..\..\lib\win32\getopt.h">
+ </File>
+ <File
+ RelativePath="..\..\..\lib\win32\WinNamedPipeStream.h">
+ </File>
</Filter>
<Filter
Name="server"
@@ -621,6 +655,9 @@ <File
RelativePath="..\..\..\lib\server\TLSContext.h">
</File>
+ <File
+ RelativePath="..\..\..\lib\server\WinNamedPipeStream.h">
+ </File>
</Filter>
</Filter>
</Filter>
diff --git a/infrastructure/msvc/2005/bbackupctl.vcproj b/infrastructure/msvc/2005/bbackupctl.vcproj index 3da10702..ab75480c 100644 --- a/infrastructure/msvc/2005/bbackupctl.vcproj +++ b/infrastructure/msvc/2005/bbackupctl.vcproj @@ -4,6 +4,7 @@ Version="8.00"
Name="bbackupctl"
ProjectGUID="{9FD51412-E945-4457-A17A-CA3C505CF431}"
+ RootNamespace="bbackupctl"
Keyword="Win32Proj"
>
<Platforms>
@@ -41,7 +42,7 @@ Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;NDEBUG"
+ PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
@@ -61,7 +62,7 @@ />
<Tool
Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib"
+ AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\lib_pcre.lib"
OutputFile="$(OutDir)/bbackupctl.exe"
LinkIncremental="2"
GenerateDebugInformation="true"
@@ -121,8 +122,8 @@ <Tool
Name="VCCLCompilerTool"
EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING"
+ AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
+ PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
RuntimeLibrary="0"
BufferSecurityCheck="false"
UsePrecompiledHeader="0"
@@ -141,7 +142,7 @@ />
<Tool
Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib"
+ AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib"
OutputFile="$(OutDir)/bbackupctl.exe"
LinkIncremental="1"
IgnoreDefaultLibraryNames=""
@@ -210,6 +211,10 @@ Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
>
+ <File
+ RelativePath="..\..\..\lib\win32\messages.rc"
+ >
+ </File>
</Filter>
</Files>
<Globals>
diff --git a/infrastructure/msvc/2005/bbackupd.vcproj b/infrastructure/msvc/2005/bbackupd.vcproj index dbcf89d3..abed0f4c 100644 --- a/infrastructure/msvc/2005/bbackupd.vcproj +++ b/infrastructure/msvc/2005/bbackupd.vcproj @@ -4,6 +4,7 @@ Version="8.00"
Name="bbackupd"
ProjectGUID="{22D325FB-9131-4BD6-B390-968F0491D687}"
+ RootNamespace="bbackupd"
Keyword="Win32Proj"
>
<Platforms>
@@ -41,7 +42,7 @@ Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""$(SolutionDir)..\..\..\lib\backupclient";"$(SolutionDir)..\..\..\lib\common";"$(SolutionDir)..\..\..\lib\compress";"$(SolutionDir)..\..\..\lib\crypto";"$(SolutionDir)..\..\..\lib\server";"$(SolutionDir)..\..\..\lib\win32";"$(SolutionDir)..\..\..\..\openssl\inc32";"$(SolutionDir)..\..\..\..\zlib\include""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;NDEBUG"
+ PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
@@ -61,7 +62,7 @@ />
<Tool
Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib"
+ AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\lib_pcre.lib"
OutputFile="$(OutDir)/bbackupd.exe"
LinkIncremental="2"
IgnoreAllDefaultLibraries="false"
@@ -122,8 +123,8 @@ <Tool
Name="VCCLCompilerTool"
EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0""
- PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;BOOST_REGEX_NO_LIB"
+ AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(SolutionDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
+ PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
RuntimeLibrary="0"
BufferSecurityCheck="false"
UsePrecompiledHeader="0"
@@ -142,7 +143,7 @@ />
<Tool
Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\lib\win32\Release\boost_regex.lib"
+ AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib"
OutputFile="$(OutDir)/bbackupd.exe"
LinkIncremental="1"
IgnoreDefaultLibraryNames=""
@@ -194,6 +195,10 @@ Name="bbackupd"
>
<File
+ RelativePath="..\..\..\bin\bbackupd\autogen_ClientException.cpp"
+ >
+ </File>
+ <File
RelativePath="..\..\..\bin\bbackupd\BackupClientContext.cpp"
>
</File>
@@ -240,6 +245,10 @@ Name="bbackupd"
>
<File
+ RelativePath="..\..\..\bin\bbackupd\autogen_ClientException.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\bin\bbackupd\BackupClientContext.h"
>
</File>
@@ -275,6 +284,10 @@ Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
>
+ <File
+ RelativePath="..\..\..\lib\win32\messages.rc"
+ >
+ </File>
</Filter>
<File
RelativePath="..\..\..\ReadMe.txt"
diff --git a/infrastructure/msvc/2005/boxquery.vcproj b/infrastructure/msvc/2005/boxquery.vcproj index 414399cd..1c6be061 100644 --- a/infrastructure/msvc/2005/boxquery.vcproj +++ b/infrastructure/msvc/2005/boxquery.vcproj @@ -42,7 +42,7 @@ Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;NDEBUG"
+ PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
@@ -62,7 +62,7 @@ />
<Tool
Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib"
+ AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\lib_pcre.lib"
OutputFile="$(OutDir)/bbackupquery.exe"
LinkIncremental="2"
GenerateDebugInformation="true"
@@ -122,8 +122,8 @@ <Tool
Name="VCCLCompilerTool"
EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0""
- PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;BOOST_REGEX_NO_LIB"
+ AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
+ PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;PCRE_STATIC"
RuntimeLibrary="0"
BufferSecurityCheck="false"
UsePrecompiledHeader="0"
@@ -142,7 +142,7 @@ />
<Tool
Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\lib\win32\Release\boost_regex.lib"
+ AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib"
OutputFile="$(OutDir)/bbackupquery.exe"
LinkIncremental="1"
IgnoreDefaultLibraryNames=""
@@ -231,6 +231,10 @@ Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
>
+ <File
+ RelativePath="..\..\..\lib\win32\messages.rc"
+ >
+ </File>
</Filter>
<File
RelativePath="..\..\..\ReadMe.txt"
diff --git a/infrastructure/msvc/2005/common.vcproj b/infrastructure/msvc/2005/common.vcproj index 127d2617..5a715be4 100644 --- a/infrastructure/msvc/2005/common.vcproj +++ b/infrastructure/msvc/2005/common.vcproj @@ -25,6 +25,8 @@ >
<Tool
Name="VCPreBuildEventTool"
+ Description="Determining Version Number"
+ CommandLine="perl $(InputDir)..\getversion.pl"
/>
<Tool
Name="VCCustomBuildTool"
@@ -41,8 +43,8 @@ <Tool
Name="VCCLCompilerTool"
Optimization="0"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include";$(NOINHERIT)"
- PreprocessorDefinitions="WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;NDEBUG"
+ AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\..\pcre\pcre-6.7\";$(NOINHERIT)"
+ PreprocessorDefinitions="WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
@@ -108,8 +110,8 @@ <Tool
Name="VCCLCompilerTool"
EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0\""
- PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;NDEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING"
+ AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common\";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\..\pcre\pcre-6.7\""
+ PreprocessorDefinitions="WIN32;NDEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
RuntimeLibrary="0"
BufferSecurityCheck="false"
UsePrecompiledHeader="0"
@@ -242,6 +244,10 @@ >
</File>
<File
+ RelativePath="..\..\..\lib\common\Logging.cpp"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\MemBlockStream.cpp"
>
</File>
@@ -250,14 +256,26 @@ >
</File>
<File
+ RelativePath="..\..\..\lib\common\PathUtils.cpp"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\ReadGatherStream.cpp"
>
</File>
<File
+ RelativePath="..\..\..\lib\common\ReadLoggingStream.cpp"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\StreamableMemBlock.cpp"
>
</File>
<File
+ RelativePath="..\..\..\lib\common\Timer.cpp"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\UnixUser.cpp"
>
</File>
@@ -394,7 +412,7 @@ >
</File>
<File
- RelativePath="..\..\..\lib\win32\WinNamedPipeStream.cpp"
+ RelativePath="..\..\..\lib\win32\getopt_long.cxx"
>
</File>
</Filter>
@@ -449,6 +467,10 @@ RelativePath="..\..\..\lib\server\TLSContext.cpp"
>
</File>
+ <File
+ RelativePath="..\..\..\lib\server\WinNamedPipeStream.cpp"
+ >
+ </File>
</Filter>
</Filter>
</Filter>
@@ -504,7 +526,7 @@ >
</File>
<File
- RelativePath="..\..\..\lib\common\BoxConfig.h"
+ RelativePath="..\..\..\lib\common\BoxConfig-MSVC.h"
>
</File>
<File
@@ -532,6 +554,10 @@ >
</File>
<File
+ RelativePath="..\..\..\lib\common\BoxVersion.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\CollectInBufferStream.h"
>
</File>
@@ -588,6 +614,10 @@ >
</File>
<File
+ RelativePath="..\..\..\lib\common\Logging.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\MainHelper.h"
>
</File>
@@ -616,10 +646,18 @@ >
</File>
<File
+ RelativePath="..\..\..\lib\common\PathUtils.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\ReadGatherStream.h"
>
</File>
<File
+ RelativePath="..\..\..\lib\common\ReadLoggingStream.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\StreamableMemBlock.h"
>
</File>
@@ -632,6 +670,10 @@ >
</File>
<File
+ RelativePath="..\..\..\lib\common\Timer.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\UnixUser.h"
>
</File>
@@ -764,7 +806,7 @@ >
</File>
<File
- RelativePath="..\..\..\lib\win32\WinNamedPipeStream.h"
+ RelativePath="..\..\..\lib\win32\getopt.h"
>
</File>
</Filter>
@@ -835,6 +877,10 @@ RelativePath="..\..\..\lib\server\TLSContext.h"
>
</File>
+ <File
+ RelativePath="..\..\..\lib\server\WinNamedPipeStream.h"
+ >
+ </File>
</Filter>
</Filter>
</Filter>
diff --git a/infrastructure/msvc/2005/win32test.vcproj b/infrastructure/msvc/2005/win32test.vcproj index 91a7918e..460d7660 100644 --- a/infrastructure/msvc/2005/win32test.vcproj +++ b/infrastructure/msvc/2005/win32test.vcproj @@ -4,6 +4,7 @@ Version="8.00"
Name="win32test"
ProjectGUID="{28C29E72-76A2-4D0C-B35B-12D446733D2E}"
+ RootNamespace="win32test"
Keyword="Win32Proj"
>
<Platforms>
@@ -41,7 +42,7 @@ Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\bin\bbackupd";"$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;NDEBUG"
+ PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
@@ -61,7 +62,7 @@ />
<Tool
Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib"
+ AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\lib_pcre.lib"
OutputFile="$(OutDir)/win32test.exe"
LinkIncremental="2"
GenerateDebugInformation="true"
@@ -121,8 +122,8 @@ <Tool
Name="VCCLCompilerTool"
EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\bin\bbackupd";"$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING"
+ AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\bin\bbackupd";"$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common\";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include""
+ PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
RuntimeLibrary="0"
BufferSecurityCheck="false"
UsePrecompiledHeader="0"
@@ -141,7 +142,7 @@ />
<Tool
Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib"
+ AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib"
OutputFile="$(OutDir)/win32test.exe"
LinkIncremental="1"
IgnoreDefaultLibraryNames=""
diff --git a/infrastructure/msvc/getversion.pl b/infrastructure/msvc/getversion.pl index e39c2511..12554d01 100644 --- a/infrastructure/msvc/getversion.pl +++ b/infrastructure/msvc/getversion.pl @@ -1,15 +1,18 @@ #!perl $basedir = $0; -$basedir =~ s/(\\[^\\]*){5}$// or die "failed to remove last five ". - "directories from $basedir"; --d $basedir or die "$basedir: not a directory"; -chdir $basedir or die "cannot change to directory $basedir: $!"; +$basedir =~ s/\\[^\\]*$//; +$basedir =~ s/\\[^\\]*$//; +$basedir =~ s/\\[^\\]*$//; +$basedir =~ s/\\[^\\]*$//; +$basedir =~ s/\\[^\\]*$//; +-d $basedir or die "$basedir: $!"; +chdir $basedir or die "$basedir: $!"; require "$basedir\\infrastructure\\BoxPlatform.pm.in"; -my $verfile = "$basedir\\lib\\common\\BoxVersion.h"; -open VERSIONFILE, "> $verfile" or die "opening $verfile: $!"; +open VERSIONFILE, "> $basedir/lib/common/BoxVersion.h" + or die "BoxVersion.h: $!"; print VERSIONFILE "#define BOX_VERSION \"$BoxPlatform::product_version\"\n"; close VERSIONFILE; diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp index ba65d57a..925d1620 100644 --- a/lib/backupclient/BackupClientFileAttributes.cpp +++ b/lib/backupclient/BackupClientFileAttributes.cpp @@ -344,8 +344,8 @@ void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroM // to be true (still aborts), but it can at least hold 2^32. if (winTime >= 0x100000000LL || _gmtime64(&winTime) == 0) { - ::syslog(LOG_ERR, "Invalid Modification Time " - "caught for file: %s", Filename); + BOX_ERROR("Invalid Modification Time caught for " + "file: '" << Filename << "'"); pattr->ModificationTime = 0; } @@ -355,8 +355,8 @@ void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroM if (winTime > 0x100000000LL || _gmtime64(&winTime) == 0) { - ::syslog(LOG_ERR, "Invalid Attribute Modification " - "Time caught for file: %s", Filename); + BOX_ERROR("Invalid Attribute Modification Time " + "caught for file: '" << Filename << "'"); pattr->AttrModificationTime = 0; } #endif @@ -578,7 +578,8 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc // Created: 2003/10/07 // // -------------------------------------------------------------------------- -void BackupClientFileAttributes::WriteAttributes(const char *Filename) const +void BackupClientFileAttributes::WriteAttributes(const char *Filename, + bool MakeUserWritable) const { // Got something loaded if(GetSize() <= 0) @@ -626,9 +627,8 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename) const } #ifdef WIN32 - ::syslog(LOG_WARNING, - "Cannot create symbolic links on Windows: %s", - Filename); + BOX_WARNING("Cannot create symbolic links on Windows: '" << + Filename << "'"); #else // Make a symlink, first deleting anything in the way ::unlink(Filename); @@ -704,7 +704,12 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename) const THROW_EXCEPTION(CommonException, OSFileError) } } - + + if (MakeUserWritable) + { + mode |= S_IRWXU; + } + // Apply everything else... (allowable mode flags only) if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX)) != 0) // mode must be done last (think setuid) { diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h index 62ba2184..fa56ff65 100644 --- a/lib/backupclient/BackupClientFileAttributes.h +++ b/lib/backupclient/BackupClientFileAttributes.h @@ -45,7 +45,8 @@ public: void ReadAttributes(const char *Filename, bool ZeroModificationTimes = false, box_time_t *pModTime = 0, box_time_t *pAttrModTime = 0, int64_t *pFileSize = 0, InodeRefType *pInodeNumber = 0, bool *pHasMultipleLinks = 0); - void WriteAttributes(const char *Filename) const; + void WriteAttributes(const char *Filename, + bool MakeUserWritable = false) const; bool IsSymLink() const; diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp index fbb4ee47..3c03c87c 100644 --- a/lib/backupclient/BackupClientRestore.cpp +++ b/lib/backupclient/BackupClientRestore.cpp @@ -19,6 +19,7 @@ #include <set> #include <limits.h> #include <stdio.h> +#include <errno.h> #include "BackupClientRestore.h" #include "autogen_BackupProtocolClient.h" @@ -206,7 +207,7 @@ typedef struct // Created: 23/11/03 // // -------------------------------------------------------------------------- -static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName, +static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName, RestoreParams &Params, RestoreResumeInfo &rLevel) { // If we're resuming... check that we haven't got a next level to look at @@ -223,11 +224,35 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di rLevel.RemoveLevel(); } - // Save the resumption information - Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + // Create the local directory, if not already done. + // Path and owner set later, just use restrictive owner mode. - // Create the local directory (if not already done) -- path and owner set later, just use restrictive owner mode - switch(ObjectExists(rLocalDirectoryName.c_str())) + int exists; + + try + { + exists = ObjectExists(rLocalDirectoryName.c_str()); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to check existence for " << + rLocalDirectoryName << ": " << e.what()); + return Restore_UnknownError; + } + catch(std::exception &e) + { + BOX_ERROR("Failed to check existence for " << + rLocalDirectoryName << ": " << e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to check existence for " << + rLocalDirectoryName << ": unknown error"); + return Restore_UnknownError; + } + + switch(exists) { case ObjectExists_Dir: // Do nothing @@ -235,26 +260,152 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di case ObjectExists_File: { // File exists with this name, which is fun. Get rid of it. - ::printf("WARNING: File present with name '%s', removing out of the way of restored directory. Use specific restore with ID to restore this object.", rLocalDirectoryName.c_str()); + BOX_WARNING("File present with name '" << + rLocalDirectoryName << "', removing " << + "out of the way of restored directory. " + "Use specific restore with ID to " + "restore this object."); if(::unlink(rLocalDirectoryName.c_str()) != 0) { - THROW_EXCEPTION(CommonException, OSFileError); + BOX_ERROR("Failed to delete file " << + rLocalDirectoryName << ": " << + strerror(errno)); + return Restore_UnknownError; } - TRACE1("In restore, directory name collision with file %s", rLocalDirectoryName.c_str()); + BOX_TRACE("In restore, directory name " + "collision with file " << + rLocalDirectoryName); } - // follow through to... (no break) + break; case ObjectExists_NoObject: - if(::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } + // we'll create it in a second, after checking + // whether the parent directory exists break; default: ASSERT(false); break; } - - // Fetch the directory listing from the server -- getting a list of files which is approparite to the restore type + + std::string parentDirectoryName(rLocalDirectoryName); + if(parentDirectoryName[parentDirectoryName.size() - 1] == + DIRECTORY_SEPARATOR_ASCHAR) + { + parentDirectoryName.resize(parentDirectoryName.size() - 1); + } + + size_t lastSlash = parentDirectoryName.rfind(DIRECTORY_SEPARATOR_ASCHAR); + + if(lastSlash == std::string::npos) + { + // might be a forward slash separator, + // especially in the unit tests! + lastSlash = parentDirectoryName.rfind('/'); + } + + if(lastSlash != std::string::npos) + { + // the target directory is a deep path, remove the last + // directory name and check that the resulting parent + // exists, otherwise the restore should fail. + parentDirectoryName.resize(lastSlash); + + #ifdef WIN32 + // if the path is a drive letter, then we need to + // add a a backslash to query the root directory. + if (lastSlash == 2 && parentDirectoryName[1] == ':') + { + parentDirectoryName += '\\'; + } + else if (lastSlash == 0) + { + parentDirectoryName += '\\'; + } + #endif + + int parentExists; + + try + { + parentExists = ObjectExists(parentDirectoryName.c_str()); + } + catch (BoxException &e) + { + BOX_ERROR("Failed to check existence for " << + parentDirectoryName << ": " << e.what()); + return Restore_UnknownError; + } + catch(std::exception &e) + { + BOX_ERROR("Failed to check existence for " << + parentDirectoryName << ": " << e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to check existence for " << + parentDirectoryName << ": unknown error"); + return Restore_UnknownError; + } + + switch(parentExists) + { + case ObjectExists_Dir: + // this is fine, do nothing + break; + + case ObjectExists_File: + BOX_ERROR("Failed to restore: '" << + parentDirectoryName << "' " + "is a file, but should be a " + "directory."); + return Restore_TargetPathNotFound; + + case ObjectExists_NoObject: + BOX_ERROR("Failed to restore: parent '" << + parentDirectoryName << "' of target " + "directory does not exist."); + return Restore_TargetPathNotFound; + + default: + BOX_ERROR("Failed to restore: unknown " + "result from ObjectExists('" << + parentDirectoryName << "')"); + return Restore_UnknownError; + } + } + + if((exists == ObjectExists_NoObject || + exists == ObjectExists_File) && + ::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0) + { + BOX_ERROR("Failed to create directory '" << + rLocalDirectoryName << "': " << + strerror(errno)); + return Restore_UnknownError; + } + + // Save the restore info, in case it's needed later + try + { + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << "': " << + e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << + "': unknown error"); + return Restore_UnknownError; + } + + // Fetch the directory listing from the server -- getting a + // list of files which is appropriate to the restore type rConnection.QueryListDirectory( DirectoryID, Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING), @@ -269,7 +420,23 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di // Apply attributes to the directory const StreamableMemBlock &dirAttrBlock(dir.GetAttributes()); BackupClientFileAttributes dirAttr(dirAttrBlock); - dirAttr.WriteAttributes(rLocalDirectoryName.c_str()); + + try + { + dirAttr.WriteAttributes(rLocalDirectoryName.c_str(), true); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to restore attributes for '" << + rLocalDirectoryName << "': " << e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to restore attributes for '" << + rLocalDirectoryName << "': unknown error"); + return Restore_UnknownError; + } int64_t bytesWrittenSinceLastRestoreInfoSave = 0; @@ -287,7 +454,13 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di std::string localFilename(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename()); // Unlink anything which already exists -- for resuming restores, we can't overwrite files already there. - ::unlink(localFilename.c_str()); + if(::unlink(localFilename.c_str()) == 0) + { + BOX_ERROR("Failed to delete file '" << + localFilename << "': " << + strerror(errno)); + return Restore_UnknownError; + } // Request it from the store rConnection.QueryGetFile(DirectoryID, en->GetObjectID()); @@ -297,17 +470,34 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di // Decode the file -- need to do different things depending on whether // the directory entry has additional attributes - if(en->HasAttributes()) + try { - // Use these attributes - const StreamableMemBlock &storeAttr(en->GetAttributes()); - BackupClientFileAttributes attr(storeAttr); - BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout(), &attr); + if(en->HasAttributes()) + { + // Use these attributes + const StreamableMemBlock &storeAttr(en->GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout(), &attr); + } + else + { + // Use attributes stored in file + BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout()); + } } - else + catch(std::exception &e) { - // Use attributes stored in file - BackupStoreFile::DecodeFile(*objectStream, localFilename.c_str(), rConnection.GetTimeout()); + BOX_ERROR("Failed to restore file '" << + localFilename << "': " << + e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to restore file '" << + localFilename << + "': unknown error"); + return Restore_UnknownError; } // Progress display? @@ -322,7 +512,34 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di // Save restore info? int64_t fileSize; - if(FileExists(localFilename.c_str(), &fileSize, true /* treat links as not existing */)) + int exists; + + try + { + exists = FileExists( + localFilename.c_str(), + &fileSize, + true /* treat links as not + existing */); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to determine " + "whether file exists: '" << + localFilename << "': " << + e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to determine " + "whether file exists: '" << + localFilename << "': " + "unknown error"); + return Restore_UnknownError; + } + + if(exists) { // File exists... bytesWrittenSinceLastRestoreInfoSave += fileSize; @@ -330,7 +547,25 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES) { // Save the restore info, in case it's needed later - Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + try + { + Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << + "': " << e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << + "': unknown error"); + return Restore_UnknownError; + } + bytesWrittenSinceLastRestoreInfoSave = 0; } } @@ -342,12 +577,31 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di if(bytesWrittenSinceLastRestoreInfoSave != 0) { // Save the restore info, in case it's needed later - Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); + try + { + Params.mResumeInfo.Save( + Params.mRestoreResumeInfoFilename); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << "': " << + e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to save resume info file '" << + Params.mRestoreResumeInfoFilename << + "': unknown error"); + return Restore_UnknownError; + } + bytesWrittenSinceLastRestoreInfoSave = 0; } - // Recuse to directories + // Recurse to directories { BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; @@ -364,7 +618,14 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di RestoreResumeInfo &rnextLevel(rLevel.AddLevel(en->GetObjectID(), nm.GetClearFilename())); // Recurse - BackupClientRestoreDir(rConnection, en->GetObjectID(), localDirname, Params, rnextLevel); + int result = BackupClientRestoreDir( + rConnection, en->GetObjectID(), + localDirname, Params, rnextLevel); + + if (result != Restore_Complete) + { + return result; + } // Remove the level for the above call rLevel.RemoveLevel(); @@ -373,7 +634,27 @@ static void BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Di rLevel.mRestoredObjects.insert(en->GetObjectID()); } } - } + } + + // now remove the user writable flag, if we added it earlier + try + { + dirAttr.WriteAttributes(rLocalDirectoryName.c_str(), false); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to restore attributes for '" << + rLocalDirectoryName << "': " << e.what()); + return Restore_UnknownError; + } + catch(...) + { + BOX_ERROR("Failed to restore attributes for '" << + rLocalDirectoryName << "': unknown error"); + return Restore_UnknownError; + } + + return Restore_Complete; } @@ -444,7 +725,12 @@ int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, // Restore the directory std::string localName(LocalDirectoryName); - BackupClientRestoreDir(rConnection, DirectoryID, localName, params, params.mResumeInfo); + int result = BackupClientRestoreDir(rConnection, DirectoryID, + localName, params, params.mResumeInfo); + if (result != Restore_Complete) + { + return result; + } // Undelete the directory on the server? if(RestoreDeleted && UndeleteAfterRestoreDeleted) diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h index 4b382771..f7724030 100644 --- a/lib/backupclient/BackupClientRestore.h +++ b/lib/backupclient/BackupClientRestore.h @@ -16,7 +16,9 @@ enum { Restore_Complete = 0, Restore_ResumePossible = 1, - Restore_TargetExists = 2 + Restore_TargetExists = 2, + Restore_TargetPathNotFound = 3, + Restore_UnknownError = 4, }; int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName, diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index 89ad4d54..9000ec6d 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -81,7 +81,9 @@ static const ConfigurationVerifyKey verifyrootkeys[] = {"FileTrackingSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, {"DiffingUploadSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, {"StoreHostname", 0, ConfigTest_Exists, 0}, - {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file + {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // extended log to syslog + {"ExtendedLogFile", NULL, 0, 0}, // extended log to a file + {"LogAllFileAccess", "no", ConfigTest_IsBool, 0}, {"CommandSocket", 0, 0, 0}, // not compulsory to have this {"KeepAliveTime", 0, ConfigTest_IsInt, 0}, // optional diff --git a/lib/backupclient/BackupStoreDirectory.cpp b/lib/backupclient/BackupStoreDirectory.cpp index 61e6461d..0d06da34 100644 --- a/lib/backupclient/BackupStoreDirectory.cpp +++ b/lib/backupclient/BackupStoreDirectory.cpp @@ -149,6 +149,11 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) int count = ntohl(hdr.mNumEntries); // Clear existing list + for(std::vector<Entry*>::iterator i = mEntries.begin(); + i != mEntries.end(); i++) + { + delete (*i); + } mEntries.clear(); // Read them in! diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp index 278bf50a..7e93d59d 100644 --- a/lib/backupclient/BackupStoreFile.cpp +++ b/lib/backupclient/BackupStoreFile.cpp @@ -17,10 +17,8 @@ #include <string.h> #include <new> #include <string.h> + #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - #ifndef WIN32 - #include <syslog.h> - #endif #include <stdio.h> #endif @@ -46,6 +44,7 @@ #include "ReadGatherStream.h" #include "Random.h" #include "BackupStoreFileEncodeStream.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -291,6 +290,20 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile } out.Close(); + + // The stream might have uncertain size, in which case + // we need to drain it to get the + // Protocol::ProtocolStreamHeader_EndOfStream byte + // out of our connection stream. + char buffer[1]; + int drained = rEncodedFile.Read(buffer, 1); + + // The Read will return 0 if we are actually at the end + // of the stream, but some tests decode files directly, + // in which case we are actually positioned at the start + // of the block index. I hope that reading an extra byte + // doesn't hurt! + // ASSERT(drained == 0); // Write the attributes stream->GetAttributes().WriteAttributes(DecodedFilename); @@ -743,8 +756,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // Warn and log this issue if(!sWarnedAboutBackwardsCompatiblity) { - ::printf("WARNING: Decoded one or more files using backwards compatibility mode for block index.\n"); - ::syslog(LOG_ERR, "WARNING: Decoded one or more files using backwards compatibility mode for block index.\n"); + BOX_WARNING("WARNING: Decoded one or more files using backwards compatibility mode for block index."); sWarnedAboutBackwardsCompatiblity = true; } } @@ -1485,9 +1497,8 @@ void BackupStoreFile::EncodingBuffer::Allocate(int Size) // -------------------------------------------------------------------------- void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) { -#ifndef WIN32 - TRACE2("Reallocating EncodingBuffer from %d to %d\n", mBufferSize, NewSize); -#endif + BOX_TRACE("Reallocating EncodingBuffer from " << mBufferSize << + " to " << NewSize); ASSERT(mpBuffer != 0); uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize); if(buffer == 0) diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h index 437b4232..3ee5ddb0 100644 --- a/lib/backupclient/BackupStoreFile.h +++ b/lib/backupclient/BackupStoreFile.h @@ -50,17 +50,16 @@ public: DiffTimer(); virtual ~DiffTimer(); public: - virtual void DoKeepAlive() = 0; - virtual time_t GetTimeMgmtEpoch() = 0; - virtual int GetMaximumDiffingTime() = 0; - virtual int GetKeepaliveTime() = 0; + virtual void DoKeepAlive() = 0; + virtual int GetMaximumDiffingTime() = 0; + virtual bool IsManaged() = 0; }; // -------------------------------------------------------------------------- // // Class // Name: BackupStoreFile -// Purpose: Class to hold together utils for maniplating files. +// Purpose: Class to hold together utils for manipulating files. // Created: 2003/08/28 // // -------------------------------------------------------------------------- diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp index ee09f1c8..3a3f1a5e 100644 --- a/lib/backupclient/BackupStoreFileDiff.cpp +++ b/lib/backupclient/BackupStoreFileDiff.cpp @@ -29,6 +29,7 @@ #include "RollingChecksum.h" #include "MD5Digest.h" #include "CommonException.h" +#include "Timer.h" #include "MemLeakFindOn.h" @@ -52,31 +53,6 @@ static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChec BlocksAvailableEntry *pIndex, std::map<int64_t, int64_t> &rFoundBlocks); static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksAvailableEntry *pIndex, int64_t NumBlocks, std::map<int64_t, int64_t> &rFoundBlocks, int64_t SizeOfInputFile); -// sDiffTimerExpired flags when the diff timer has expired. When true, the -// diff routine should check the wall clock as soon as possible, to determine -// whether it's time for a keepalive to be sent, or whether the diff has been -// running for too long and should be terminated. -static bool sDiffTimerExpired = false; - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreFile::DiffTimerExpired() -// Purpose: Notifies BackupStoreFile object that the diff operation -// timer has expired, which may mean that a keepalive should -// be sent, or the diff should be terminated. Called from an -// external timer, so it should not do more than set a flag. -// -// Created: 19/1/06 -// -// -------------------------------------------------------------------------- -void BackupStoreFile::DiffTimerExpired() -{ - sDiffTimerExpired = true; -} - - // -------------------------------------------------------------------------- // // Function @@ -483,15 +459,11 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer) { - time_t TimeMgmtEpoch = 0; - int MaximumDiffingTime = 0; - int KeepAliveTime = 0; + Timer maximumDiffingTime(0); - if (pDiffTimer) + if(pDiffTimer && pDiffTimer->IsManaged()) { - TimeMgmtEpoch = pDiffTimer->GetTimeMgmtEpoch(); - MaximumDiffingTime = pDiffTimer->GetMaximumDiffingTime(); - KeepAliveTime = pDiffTimer->GetKeepaliveTime(); + maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime()); } std::map<int64_t, int32_t> goodnessOfFit; @@ -577,31 +549,20 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> int rollOverInitialBytes = 0; while(true) { - if(sDiffTimerExpired) + if(maximumDiffingTime.HasExpired()) { - ASSERT(TimeMgmtEpoch > 0); ASSERT(pDiffTimer != NULL); - - time_t tTotalRunIntvl = time(NULL) - TimeMgmtEpoch; - - if(MaximumDiffingTime > 0 && - tTotalRunIntvl >= MaximumDiffingTime) - { - TRACE0("MaximumDiffingTime reached - " - "suspending file diff\n"); - abortSearch = true; - break; - } - else if(KeepAliveTime > 0) - { - TRACE0("KeepAliveTime reached - " - "initiating keep-alive\n"); - pDiffTimer->DoKeepAlive(); - } - - sDiffTimerExpired = false; + TRACE0("MaximumDiffingTime reached - " + "suspending file diff\n"); + abortSearch = true; + break; } - + + if(pDiffTimer) + { + pDiffTimer->DoKeepAlive(); + } + // Load in another block of data, and record how big it is int bytesInEndings = rFile.Read(endings, Sizes[s]); int tmp; diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp index c692f18e..423c11a3 100644 --- a/lib/backupclient/BackupStoreFileEncodeStream.cpp +++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp @@ -38,6 +38,7 @@ using namespace BackupStoreFileCryptVar; BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() : mpRecipe(0), mpFile(0), + mpLogging(0), mStatus(Status_Header), mSendData(true), mTotalBlocks(0), @@ -79,6 +80,13 @@ BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() mpFile = 0; } + // Clear up logging stream + if(mpLogging) + { + delete mpLogging; + mpLogging = 0; + } + // Free the recipe if(mpRecipe != 0) { @@ -199,6 +207,9 @@ void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEnc { // Open the file mpFile = new FileStream(Filename); + + // Create logging stream + mpLogging = new ReadLoggingStream(*mpFile); // Work out the largest possible block required for the encoded data mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); @@ -267,7 +278,7 @@ void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; - } while(rBlockSizeOut <= BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); + } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); // Last block size rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); @@ -474,7 +485,7 @@ void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() } // Move forward in the stream - mpFile->Seek(sizeToSkip, IOStream::SeekType_Relative); + mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative); } @@ -518,14 +529,14 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() ASSERT(blockRawSize < mAllocatedBufferSize); // Check file open - if(mpFile == 0) + if(mpFile == 0 || mpLogging == 0) { // File should be open, but isn't. So logical error. THROW_EXCEPTION(BackupStoreException, Internal) } // Read the data in - if(!mpFile->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */)) + if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */)) { // TODO: Do something more intelligent, and abort this upload because the file // has changed diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h index 1c748798..fb5d0851 100644 --- a/lib/backupclient/BackupStoreFileEncodeStream.h +++ b/lib/backupclient/BackupStoreFileEncodeStream.h @@ -17,6 +17,7 @@ #include "CollectInBufferStream.h" #include "MD5Digest.h" #include "BackupStoreFile.h" +#include "ReadLoggingStream.h" namespace BackupStoreFileCreation { @@ -100,6 +101,7 @@ private: Recipe *mpRecipe; IOStream *mpFile; // source file CollectInBufferStream mData; // buffer for header and index entries + ReadLoggingStream *mpLogging; int mStatus; bool mSendData; // true if there's file data to send (ie not a symlink) int64_t mTotalBlocks; // Total number of blocks in the file diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp index c415b9bb..9114fdd1 100644 --- a/lib/backupclient/BackupStoreFilenameClear.cpp +++ b/lib/backupclient/BackupStoreFilenameClear.cpp @@ -13,6 +13,7 @@ #include "CipherContext.h" #include "CipherBlowfish.h" #include "Guards.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -203,9 +204,9 @@ static void EnsureEncDecBufferSize(int BufSize) { if(sEncDecBufferSize < BufSize) { -#ifndef WIN32 - TRACE2("Reallocating filename encoding/decoding buffer from %d to %d\n", sEncDecBufferSize, BufSize); -#endif + BOX_TRACE("Reallocating filename encoding/decoding " + "buffer from " << sEncDecBufferSize << + " to " << BufSize); spEncDecBuffer->Resize(BufSize); sEncDecBufferSize = BufSize; MEMLEAKFINDER_NOT_A_LEAK(*spEncDecBuffer); diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index 16eeecf9..176ece8f 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -102,7 +102,7 @@ void BackupStoreCheck::Check() // Couldn't lock the account -- just stop now if(!mQuiet) { - ::printf("Couldn't lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server.\n"); + BOX_ERROR("Failed to lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server."); } THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount) } @@ -110,41 +110,43 @@ void BackupStoreCheck::Check() if(!mQuiet && mFixErrors) { - ::printf("NOTE: Will fix errors encountered during checking.\n"); + BOX_NOTICE("Will fix errors encountered during checking."); } // Phase 1, check objects if(!mQuiet) { - ::printf("Check store account ID %08x\nPhase 1, check objects...\n", mAccountID); + BOX_INFO("Checking store account ID " << + BOX_FORMAT_ACCOUNT(mAccountID) << "..."); + BOX_INFO("Phase 1, check objects..."); } CheckObjects(); // Phase 2, check directories if(!mQuiet) { - ::printf("Phase 2, check directories...\n"); + BOX_INFO("Phase 2, check directories..."); } CheckDirectories(); // Phase 3, check root if(!mQuiet) { - ::printf("Phase 3, check root...\n"); + BOX_INFO("Phase 3, check root..."); } CheckRoot(); // Phase 4, check unattached objects if(!mQuiet) { - ::printf("Phase 4, fix unattached objects...\n"); + BOX_INFO("Phase 4, fix unattached objects..."); } CheckUnattachedObjects(); // Phase 5, fix bad info if(!mQuiet) { - ::printf("Phase 5, fix unrecovered inconsistencies...\n"); + BOX_INFO("Phase 5, fix unrecovered inconsistencies..."); } FixDirsWithWrongContainerID(); FixDirsWithLostDirs(); @@ -152,7 +154,7 @@ void BackupStoreCheck::Check() // Phase 6, regenerate store info if(!mQuiet) { - ::printf("Phase 6, regenerate store info...\n"); + BOX_INFO("Phase 6, regenerate store info..."); } WriteNewStoreInfo(); @@ -160,29 +162,40 @@ void BackupStoreCheck::Check() if(mNumberErrorsFound > 0) { - ::printf("%lld errors found\n", mNumberErrorsFound); + BOX_WARNING("Finished checking store account ID " << + BOX_FORMAT_ACCOUNT(mAccountID) << ": " << + mNumberErrorsFound << " errors found"); if(!mFixErrors) { - ::printf("NOTE: No changes to the store account have been made.\n"); + BOX_WARNING("No changes to the store account " + "have been made."); } if(!mFixErrors && mNumberErrorsFound > 0) { - ::printf("Run again with fix option to fix these errors\n"); + BOX_WARNING("Run again with fix option to " + "fix these errors"); } - if(mNumberErrorsFound > 0) + if(mFixErrors && mNumberErrorsFound > 0) { - ::printf("You should now use bbackupquery on the client machine to examine the store.\n"); + BOX_WARNING("You should now use bbackupquery " + "on the client machine to examine the store."); if(mLostAndFoundDirectoryID != 0) { - ::printf("A lost+found directory was created in the account root.\n"\ - "This contains files and directories which could not be matched to existing directories.\n"\ - "bbackupd will delete this directory in a few days time.\n"); + BOX_WARNING("A lost+found directory was " + "created in the account root.\n" + "This contains files and directories " + "which could not be matched to " + "existing directories.\n"\ + "bbackupd will delete this directory " + "in a few days time."); } } } else { - ::printf("Store account checked, no errors found.\n"); + BOX_NOTICE("Finished checking store account ID " << + BOX_FORMAT_ACCOUNT(mAccountID) << ": " + "no errors found"); } } @@ -304,7 +317,10 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const } else { - ::printf("Spurious or invalid directory %s/%s found%s -- delete manually\n", rDirName.c_str(), (*i).c_str(), mFixErrors?", deleting":""); + BOX_WARNING("Spurious or invalid directory " << + rDirName << DIRECTORY_SEPARATOR << + (*i) << " found, " << + (mFixErrors?"deleting":"delete manually")); ++mNumberErrorsFound; } } @@ -336,7 +352,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) // Check directory exists if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName)) { - TRACE1("RaidFile dir %s does not exist\n", dirName.c_str()); + BOX_WARNING("RaidFile dir " << dirName << " does not exist"); return; } @@ -378,9 +394,9 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) if(!fileOK) { // Unexpected or bad file, delete it - ::printf("Spurious file %s" DIRECTORY_SEPARATOR "%s " - "found%s\n", dirName.c_str(), (*i).c_str(), - mFixErrors?", deleting":""); + BOX_WARNING("Spurious file " << dirName << + DIRECTORY_SEPARATOR << (*i) << " found" << + (mFixErrors?", deleting":"")); ++mNumberErrorsFound; if(mFixErrors) { @@ -401,7 +417,9 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) if(!CheckAndAddObject(StartID | i, dirName + leaf)) { // File was bad, delete it - ::printf("Corrupted file %s%s found%s\n", dirName.c_str(), leaf, mFixErrors?", deleting":""); + BOX_WARNING("Corrupted file " << dirName << + leaf << " found" << + (mFixErrors?", deleting":"")); ++mNumberErrorsFound; if(mFixErrors) { @@ -509,7 +527,7 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) { // Get that dodgy thing deleted! - ::printf("Have file as root directory. This is bad.\n"); + BOX_ERROR("Have file as root directory. This is bad."); return -1; } @@ -596,7 +614,9 @@ void BackupStoreCheck::CheckDirectories() if(dir.CheckAndFix()) { // Wasn't quite right, and has been modified - ::printf("Directory ID %llx has bad structure\n", pblock->mID[e]); + BOX_WARNING("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " has bad structure"); ++mNumberErrorsFound; isModified = true; } @@ -622,7 +642,11 @@ void BackupStoreCheck::CheckDirectories() != ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir)) { // Entry is of wrong type - ::printf("Directory ID %llx references object %llx which has a different type than expected.\n", pblock->mID[e], en->GetObjectID()); + BOX_WARNING("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " references object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " which has a different type than expected."); badEntry = true; } else @@ -630,8 +654,12 @@ void BackupStoreCheck::CheckDirectories() // Check that the entry is not already contained. if(iflags & Flags_IsContained) { + BOX_WARNING("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " references object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " which is already contained."); badEntry = true; - ::printf("Directory ID %llx references object %llx which is already contained.\n", pblock->mID[e], en->GetObjectID()); } else { @@ -645,13 +673,13 @@ void BackupStoreCheck::CheckDirectories() if(iflags & Flags_IsDir) { // Add to will fix later list - ::printf("Directory ID %llx has wrong container ID.\n", en->GetObjectID()); + BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " has wrong container ID."); mDirsWithWrongContainerID.push_back(en->GetObjectID()); } else { // This is OK for files, they might move - ::printf("File ID %llx has different container ID, probably moved\n", en->GetObjectID()); + BOX_WARNING("File ID " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " has different container ID, probably moved"); } // Fix entry for now @@ -670,7 +698,7 @@ void BackupStoreCheck::CheckDirectories() // Mark as changed isModified = true; // Tell user - ::printf("Directory ID %llx has wrong size for object %llx\n", pblock->mID[e], en->GetObjectID()); + BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " has wrong size for object " << BOX_FORMAT_OBJECTID(en->GetObjectID())); } } } @@ -686,7 +714,7 @@ void BackupStoreCheck::CheckDirectories() { // Just remove the entry badEntry = true; - ::printf("Directory ID %llx references object %llx which does not exist.\n", pblock->mID[e], en->GetObjectID()); + BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist."); } } @@ -729,7 +757,7 @@ void BackupStoreCheck::CheckDirectories() if(isModified && mFixErrors) { - ::printf("Fixing directory ID %llx\n", pblock->mID[e]); + BOX_WARNING("Fixing directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e])); // Save back to disc RaidFileWrite fixed(mDiscSetNumber, filename); diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 7bc9a109..9c6f2452 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -47,7 +47,7 @@ void BackupStoreCheck::CheckRoot() } else { - ::printf("Root directory doesn't exist\n"); + BOX_WARNING("Root directory doesn't exist"); ++mNumberErrorsFound; @@ -118,7 +118,7 @@ void BackupStoreCheck::CheckUnattachedObjects() if((flags & Flags_IsContained) == 0) { // Unattached object... - ::printf("Object %llx is unattached.\n", pblock->mID[e]); + BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached."); ++mNumberErrorsFound; // What's to be done? @@ -147,7 +147,7 @@ void BackupStoreCheck::CheckUnattachedObjects() // Just delete it to be safe. if(diffFromObjectID != 0) { - ::printf("Object %llx is unattached, and is a patch. Deleting, cannot reliably recover.\n", pblock->mID[e]); + BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached, and is a patch. Deleting, cannot reliably recover."); // Delete this object instead if(mFixErrors) @@ -229,11 +229,15 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) // Can recreate this! Wooo! if(!mFixErrors) { - ::printf("Missing directory %llx could be recreated\n", MissingDirectoryID); + BOX_WARNING("Missing directory " << + BOX_FORMAT_OBJECTID(MissingDirectoryID) << + " could be recreated."); mDirsAdded.insert(MissingDirectoryID); return true; } - ::printf("Recreating missing directory %llx\n", MissingDirectoryID); + + BOX_WARNING("Recreating missing directory " << + BOX_FORMAT_OBJECTID(MissingDirectoryID)); // Create a blank directory BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */); @@ -300,7 +304,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() if(!dir.NameInUse(lostAndFound)) { // Found a name which can be used - ::printf("Lost and found dir has name %s\n", name); + BOX_WARNING("Lost and found dir has name " << name); break; } } @@ -524,7 +528,7 @@ void BackupStoreCheck::WriteNewStoreInfo() } catch(...) { - ::printf("Load of existing store info failed, regenerating.\n"); + BOX_WARNING("Load of existing store info failed, regenerating."); ++mNumberErrorsFound; } @@ -551,7 +555,7 @@ void BackupStoreCheck::WriteNewStoreInfo() } else { - ::printf("NOTE: Soft limit for account changed to ensure housekeeping doesn't delete files on next run\n"); + BOX_WARNING("Soft limit for account changed to ensure housekeeping doesn't delete files on next run."); } if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard) { @@ -559,7 +563,7 @@ void BackupStoreCheck::WriteNewStoreInfo() } else { - ::printf("NOTE: Hard limit for account changed to ensure housekeeping doesn't delete files on next run\n"); + BOX_WARNING("Hard limit for account changed to ensure housekeeping doesn't delete files on next run."); } // Object ID @@ -586,7 +590,7 @@ void BackupStoreCheck::WriteNewStoreInfo() if(mFixErrors) { info->Save(); - ::printf("New store info file written successfully.\n"); + BOX_NOTICE("New store info file written successfully."); } } diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp index 6fa05d06..784adfb8 100644 --- a/lib/backupstore/BackupStoreConfigVerify.cpp +++ b/lib/backupstore/BackupStoreConfigVerify.cpp @@ -35,7 +35,13 @@ static const ConfigurationVerifyKey verifyrootkeys[] = {"AccountDatabase", 0, ConfigTest_Exists, 0}, {"TimeBetweenHousekeeping", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file - {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0} + + #ifdef WIN32 + {"RaidFileConf", "", ConfigTest_LastEntry, 0} + #else + {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0} + #endif + }; const ConfigurationVerify BackupConfigFileVerify = diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index ce785dde..3588cc00 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -124,7 +124,8 @@ void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, }; // Generate the filename - ASSERT(rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + ASSERT(rRootDir[rRootDir.size() - 1] == '/' || + rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); std::string fn(rRootDir + INFO_FILENAME); // Open the file for writing diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h index 7e28fa16..c9d85d5c 100644 --- a/lib/common/BannerText.h +++ b/lib/common/BannerText.h @@ -11,7 +11,7 @@ #define BANNERTEXT__H #define BANNER_TEXT(UtilityName) \ - "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and contributors 2003-2006\n" + "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and contributors 2003-2007" #endif // BANNERTEXT__H diff --git a/lib/common/Box.h b/lib/common/Box.h index eb33673c..abaad9f2 100644 --- a/lib/common/Box.h +++ b/lib/common/Box.h @@ -27,13 +27,14 @@ #endif #ifdef SHOW_BACKTRACE_ON_EXCEPTION - #include "Utils.h" + #include "Utils.h" #define OPTIONAL_DO_BACKTRACE DumpStackBacktrace(); #else #define OPTIONAL_DO_BACKTRACE #endif #include "CommonException.h" +#include "Logging.h" #ifndef NDEBUG @@ -94,22 +95,27 @@ #ifdef BOX_MEMORY_LEAK_TESTING // Memory leak testing #include "MemLeakFinder.h" + #define DEBUG_NEW new(__FILE__,__LINE__) #define MEMLEAKFINDER_NOT_A_LEAK(x) memleakfinder_notaleak(x); + #define MEMLEAKFINDER_NO_LEAKS MemLeakSuppressionGuard _guard; + #define MEMLEAKFINDER_INIT memleakfinder_init(); #define MEMLEAKFINDER_START {memleakfinder_global_enable = true;} - #define MEMLEAKFINDER_STOP {memleakfinder_global_enable = false;} + #define MEMLEAKFINDER_STOP {memleakfinder_global_enable = false;} #else #define DEBUG_NEW new #define MEMLEAKFINDER_NOT_A_LEAK(x) + #define MEMLEAKFINDER_NO_LEAKS + #define MEMLEAKFINDER_INIT #define MEMLEAKFINDER_START #define MEMLEAKFINDER_STOP #endif - -#define THROW_EXCEPTION(type, subtype) \ - { \ - OPTIONAL_DO_BACKTRACE \ - TRACE1("Exception thrown: " #type "(" #subtype ") at " __FILE__ "(%d)\n", __LINE__) \ - throw type(type::subtype); \ +#define THROW_EXCEPTION(type, subtype) \ + { \ + OPTIONAL_DO_BACKTRACE \ + BOX_WARNING("Exception thrown: " #type "(" #subtype ") at " \ + __FILE__ "(" << __LINE__ << ")") \ + throw type(type::subtype); \ } // extra macros for converting to network byte order diff --git a/lib/win32/config.h.win32 b/lib/common/BoxConfig-MSVC.h index edc44a75..6ce496f5 100644 --- a/lib/win32/config.h.win32 +++ b/lib/common/BoxConfig-MSVC.h @@ -56,6 +56,10 @@ you don't. */ #define HAVE_DECL_XATTR_NOFOLLOW 0 +/* Define to 1 if you have the declaration of `O_BINARY', and to 0 if you + don't. */ +#define HAVE_DECL_O_BINARY 1 + /* Define to 1 if #define of pragmas works */ /* #undef HAVE_DEFINE_PRAGMA */ @@ -82,7 +86,7 @@ /* #undef HAVE_GETPEEREID */ /* Define to 1 if you have the `getpid' function. */ -#define HAVE_GETPID 1 +// #define HAVE_GETPID 1 /* Define to 1 if you have the `getxattr' function. */ /* #undef HAVE_GETXATTR */ @@ -172,7 +176,9 @@ /* #undef HAVE_READLINE_READLINE_H */ /* Define to 1 if you have the <regex.h> header file. */ -// #define HAVE_REGEX_H 1 +/* #undef HAVE_REGEX_H */ +#define HAVE_PCREPOSIX_H 1 +#define HAVE_REGEX_SUPPORT 1 /* Define to 1 if you have the `setproctitle' function. */ /* #undef HAVE_SETPROCTITLE */ @@ -268,7 +274,7 @@ // #define HAVE_SYS_TIME_H 1 /* Define to 1 if you have the <sys/types.h> header file. */ -#define HAVE_SYS_TYPES_H 1 +// #define HAVE_SYS_TYPES_H 1 /* Define to 1 if you have the <sys/wait.h> header file. */ /* #undef HAVE_SYS_WAIT_H */ @@ -318,6 +324,12 @@ /* Define to 1 if __syscall is available but needs a definition */ /* #undef HAVE___SYSCALL_NEED_DEFN */ +/* max value of long long calculated by configure */ +/* #undef LLONG_MAX */ + +/* min value of long long calculated by configure */ +/* #undef LLONG_MIN */ + /* Define to 1 if `lstat' dereferences a symlink specified with a trailing slash. */ /* #undef LSTAT_FOLLOWS_SLASHED_SYMLINK */ diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h index cb83f7a9..4b58d31d 100644 --- a/lib/common/BoxPlatform.h +++ b/lib/common/BoxPlatform.h @@ -21,7 +21,12 @@ #define PLATFORM_DEV_NULL "/dev/null" +#ifdef _MSC_VER +#include "BoxConfig-MSVC.h" +#include "BoxVersion.h" +#else #include "BoxConfig.h" +#endif #ifdef WIN32 // need msvcrt version 6.1 or higher for _gmtime64() @@ -52,6 +57,14 @@ #define PLATFORM_DISABLE_MEM_LEAK_TESTING #endif +// Darwin also has a weird idea of permissions and dates on symlinks: +// perms are fixed at creation time by your umask, and dates can't be +// changed. This breaks unit tests if we try to compare these things. +// See: http://lists.apple.com/archives/darwin-kernel/2006/Dec/msg00057.html +#ifdef __APPLE__ + #define PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE +#endif + // Find out if credentials on UNIX sockets can be obtained #ifndef HAVE_GETPEEREID #if !HAVE_DECL_SO_PEERCRED @@ -97,8 +110,6 @@ #define HAVE_U_INT16_T #define HAVE_U_INT32_T #define HAVE_U_INT64_T - - typedef int pid_t; #endif // WIN32 && !__MINGW32__ // Define missing types diff --git a/lib/common/BoxPortsAndFiles.h b/lib/common/BoxPortsAndFiles.h index 562c6724..a6ca9f6d 100644 --- a/lib/common/BoxPortsAndFiles.h +++ b/lib/common/BoxPortsAndFiles.h @@ -14,20 +14,27 @@ // Backup store daemon -#define BOX_PORT_BBSTORED (BOX_PORT_BASE+1) -#define BOX_FILE_BBSTORED_DEFAULT_CONFIG "/etc/box/bbstored.conf" +#define BOX_PORT_BBSTORED (BOX_PORT_BASE+1) + // directory within the RAIDFILE root for the backup store daemon -#define BOX_RAIDFILE_ROOT_BBSTORED "backup" +#define BOX_RAIDFILE_ROOT_BBSTORED "backup" -// Backup client daemon +// configuration file paths #ifdef WIN32 -#define BOX_FILE_BBACKUPD_DEFAULT_CONFIG "C:\\Program Files\\Box Backup\\bbackupd.conf" + // no default config file path, use these macros to call + // GetDefaultConfigFilePath() instead. + + #define BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE \ + GetDefaultConfigFilePath("bbackupd.conf").c_str() + #define BOX_GET_DEFAULT_RAIDFILE_CONFIG_FILE \ + GetDefaultConfigFilePath("raidfile.conf").c_str() + #define BOX_GET_DEFAULT_BBSTORED_CONFIG_FILE \ + GetDefaultConfigFilePath("bbstored.conf").c_str() #else #define BOX_FILE_BBACKUPD_DEFAULT_CONFIG "/etc/box/bbackupd.conf" -#endif - -// RaidFile conf location default #define BOX_FILE_RAIDFILE_DEFAULT_CONFIG "/etc/box/raidfile.conf" +#define BOX_FILE_BBSTORED_DEFAULT_CONFIG "/etc/box/bbstored.conf" +#endif // Default name of the named pipe #define BOX_NAMED_PIPE_NAME L"\\\\.\\pipe\\boxbackup" diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp index 960fc329..1ddcffd4 100644 --- a/lib/common/BoxTime.cpp +++ b/lib/common/BoxTime.cpp @@ -9,7 +9,16 @@ #include "Box.h" -#include <time.h> +#ifdef HAVE_SYS_TIME_H + #include <sys/time.h> +#endif + +#ifdef HAVE_TIME_H + #include <time.h> +#endif + +#include <errno.h> +#include <string.h> #include "BoxTime.h" @@ -19,13 +28,27 @@ // // Function // Name: GetCurrentBoxTime() -// Purpose: Returns the current time as a box time. (1 sec precision) +// Purpose: Returns the current time as a box time. +// (1 sec precision, or better if supported by system) // Created: 2003/10/08 // // -------------------------------------------------------------------------- box_time_t GetCurrentBoxTime() { + #ifdef HAVE_GETTIMEOFDAY + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) + { + BOX_ERROR("Failed to gettimeofday(), dropping " + "precision: " << strerror(errno)); + } + else + { + box_time_t timeNow = (tv.tv_sec * MICRO_SEC_IN_SEC_LL) + + tv.tv_usec; + return timeNow; + } + #endif + return SecondsToBoxTime(time(0)); } - - diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h index 398e6b1c..e62a77ab 100644 --- a/lib/common/BoxTime.h +++ b/lib/common/BoxTime.h @@ -35,6 +35,9 @@ inline uint64_t BoxTimeToMilliSeconds(box_time_t Time) { return Time / MILLI_SEC_IN_NANO_SEC_LL; } +inline uint64_t BoxTimeToMicroSeconds(box_time_t Time) +{ + return Time; +} #endif // BOXTIME__H - diff --git a/lib/common/BufferedStream.cpp b/lib/common/BufferedStream.cpp new file mode 100644 index 00000000..288e1ed1 --- /dev/null +++ b/lib/common/BufferedStream.cpp @@ -0,0 +1,207 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BufferedStream.cpp +// Purpose: Buffering wrapper around IOStreams +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BufferedStream.h" +#include "CommonException.h" + +#include <string.h> + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::BufferedStream(const char *, int, int) +// Purpose: Constructor, set up buffer +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +BufferedStream::BufferedStream(IOStream& rSource) +: mrSource(rSource), mBufferSize(0), mBufferPosition(0) +{ } + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::Read(void *, int) +// Purpose: Reads bytes from the file +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +int BufferedStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + if (mBufferSize == mBufferPosition) + { + // buffer is empty, fill it. + + int numBytesRead = mrSource.Read(mBuffer, sizeof(mBuffer), + Timeout); + + if (numBytesRead < 0) + { + return numBytesRead; + } + + mBufferSize = numBytesRead; + } + + int sizeToReturn = mBufferSize - mBufferPosition; + + if (sizeToReturn > NBytes) + { + sizeToReturn = NBytes; + } + + memcpy(pBuffer, mBuffer + mBufferPosition, sizeToReturn); + mBufferPosition += sizeToReturn; + + if (mBufferPosition == mBufferSize) + { + // clear out the buffer + mBufferSize = 0; + mBufferPosition = 0; + } + + return sizeToReturn; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +IOStream::pos_type BufferedStream::BytesLeftToRead() +{ + return mrSource.BytesLeftToRead() + mBufferSize - mBufferPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::Write(void *, int) +// Purpose: Writes bytes to the underlying stream (not supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type BufferedStream::GetPosition() const +{ + return mrSource.GetPosition() - mBufferSize + mBufferPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek, invalidate buffer +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + switch (SeekType) + { + case SeekType_Absolute: + { + // just go there + mrSource.Seek(Offset, SeekType); + } + break; + + case SeekType_Relative: + { + // Actual underlying file position is + // (mBufferSize - mBufferPosition) ahead of us. + // Need to subtract that amount from the seek + // to seek forward that much less, putting the + // real pointer in the right place. + mrSource.Seek(Offset - mBufferSize + mBufferPosition, + SeekType); + } + break; + + case SeekType_End: + { + // Actual underlying file position is + // (mBufferSize - mBufferPosition) ahead of us. + // Need to add that amount to the seek + // to seek backwards that much more, putting the + // real pointer in the right place. + mrSource.Seek(Offset + mBufferSize - mBufferPosition, + SeekType); + } + } + + // always clear the buffer for now (may be slightly wasteful) + mBufferSize = 0; + mBufferPosition = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::Close() +// Purpose: Closes the underlying stream (not needed) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void BufferedStream::Close() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool BufferedStream::StreamDataLeft() +{ + return mrSource.StreamDataLeft(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BufferedStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool BufferedStream::StreamClosed() +{ + return mrSource.StreamClosed(); +} + diff --git a/lib/common/BufferedStream.h b/lib/common/BufferedStream.h new file mode 100644 index 00000000..234061c4 --- /dev/null +++ b/lib/common/BufferedStream.h @@ -0,0 +1,43 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BufferedStream.h +// Purpose: Buffering wrapper around IOStreams +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- + +#ifndef BUFFEREDSTREAM__H +#define BUFFEREDSTREAM__H + +#include "IOStream.h" + +class BufferedStream : public IOStream +{ +private: + IOStream& mrSource; + char mBuffer[4096]; + int mBufferSize; + int mBufferPosition; + +public: + BufferedStream(IOStream& rSource); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + BufferedStream(const BufferedStream &rToCopy) + : mrSource(rToCopy.mrSource) { /* do not call */ } +}; + +#endif // BUFFEREDSTREAM__H + + diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt index 5fa443d0..b2819886 100644 --- a/lib/common/CommonException.txt +++ b/lib/common/CommonException.txt @@ -44,3 +44,4 @@ KQueueNotSupportedOnThisPlatform 36 IOStreamGetLineNotEnoughDataToIgnore 37 Bad value passed to IOStreamGetLine::IgnoreBufferedData() TempDirPathTooLong 38 Your temporary directory path is too long. Check the TMP and TEMP environment variables. ArchiveBlockIncompleteRead 39 The Store Object Info File is too short or corrupted, and will be rewritten automatically when the next backup completes. +AccessDenied 40 Access to the file or directory was denied. Please check the permissions. diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp index def93571..5f9376c5 100644 --- a/lib/common/Configuration.cpp +++ b/lib/common/Configuration.cpp @@ -83,19 +83,16 @@ Configuration::~Configuration() // Created: 2003/07/23 // // -------------------------------------------------------------------------- -std::auto_ptr<Configuration> Configuration::LoadAndVerify(const char *Filename, const ConfigurationVerify *pVerify, std::string &rErrorMsg) +std::auto_ptr<Configuration> Configuration::LoadAndVerify( + const std::string& rFilename, + const ConfigurationVerify *pVerify, + std::string &rErrorMsg) { - // Check arguments - if(Filename == 0) - { - THROW_EXCEPTION(CommonException, BadArguments) - } - // Just to make sure rErrorMsg.erase(); // Open the file - FileHandleGuard<O_RDONLY> file(Filename); + FileHandleGuard<O_RDONLY> file(rFilename); // GetLine object FdGetLine getline(file); diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h index 4c455b0f..64e7568e 100644 --- a/lib/common/Configuration.h +++ b/lib/common/Configuration.h @@ -69,8 +69,15 @@ public: MultiValueSeparator = '\x01' }; - static std::auto_ptr<Configuration> LoadAndVerify(const char *Filename, const ConfigurationVerify *pVerify, std::string &rErrorMsg); - static std::auto_ptr<Configuration> Load(const char *Filename, std::string &rErrorMsg) { return LoadAndVerify(Filename, 0, rErrorMsg); } + static std::auto_ptr<Configuration> LoadAndVerify( + const std::string& rFilename, + const ConfigurationVerify *pVerify, + std::string &rErrorMsg); + + static std::auto_ptr<Configuration> Load( + const std::string& rFilename, + std::string &rErrorMsg) + { return LoadAndVerify(rFilename, 0, rErrorMsg); } bool KeyExists(const char *pKeyName) const; const std::string &GetKeyValue(const char *pKeyName) const; diff --git a/lib/common/ConversionString.cpp b/lib/common/ConversionString.cpp index b86bad4f..2d0a8d58 100644 --- a/lib/common/ConversionString.cpp +++ b/lib/common/ConversionString.cpp @@ -123,7 +123,7 @@ int32_t BoxConvert::_ConvertStringToInt(const char *pString, int Size) void BoxConvert::_ConvertIntToString(std::string &rTo, int32_t From) { char text[64]; // size more than enough - ::sprintf(text, "%d", From); + ::sprintf(text, "%d", (int)From); rTo = text; } diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp index 17a20a6d..e9b0e681 100644 --- a/lib/common/DebugMemLeakFinder.cpp +++ b/lib/common/DebugMemLeakFinder.cpp @@ -25,6 +25,9 @@ #include <string.h> #include <set> +#include "MemLeakFinder.h" + +static bool memleakfinder_initialised = false; bool memleakfinder_global_enable = false; typedef struct @@ -46,6 +49,17 @@ namespace { static std::map<void *, MallocBlockInfo> sMallocBlocks; static std::map<void *, ObjectInfo> sObjectBlocks; + static bool sTrackingDataDestroyed = false; + + static class DestructionWatchdog + { + public: + ~DestructionWatchdog() + { + sTrackingDataDestroyed = true; + } + } + sWatchdog; static bool sTrackMallocInSection = false; static std::set<void *> sSectionMallocBlocks; @@ -55,11 +69,41 @@ namespace static std::set<void *> sNotLeaks; void *sNotLeaksPre[1024]; - int sNotLeaksPreNum = 0; + size_t sNotLeaksPreNum = 0; +} + +void memleakfinder_init() +{ + ASSERT(!memleakfinder_initialised); + memleakfinder_initialised = true; } +MemLeakSuppressionGuard::MemLeakSuppressionGuard() +{ + ASSERT(memleakfinder_global_enable); + memleakfinder_global_enable = false; +} + +MemLeakSuppressionGuard::~MemLeakSuppressionGuard() +{ + ASSERT(!memleakfinder_global_enable); + memleakfinder_global_enable = true; +} + +// these functions may well allocate memory, which we don't want to track. +static int sInternalAllocDepth = 0; + +class InternalAllocGuard +{ + public: + InternalAllocGuard () { sInternalAllocDepth++; } + ~InternalAllocGuard() { sInternalAllocDepth--; } +}; + void memleakfinder_malloc_add_block(void *b, size_t size, const char *file, int line) { + InternalAllocGuard guard; + if(b != 0) { MallocBlockInfo i; @@ -75,11 +119,13 @@ void memleakfinder_malloc_add_block(void *b, size_t size, const char *file, int } } - void *memleakfinder_malloc(size_t size, const char *file, int line) { + InternalAllocGuard guard; + void *b = ::malloc(size); if(!memleakfinder_global_enable) return b; + if(!memleakfinder_initialised) return b; memleakfinder_malloc_add_block(b, size, file, line); @@ -89,7 +135,9 @@ void *memleakfinder_malloc(size_t size, const char *file, int line) void *memleakfinder_realloc(void *ptr, size_t size) { - if(!memleakfinder_global_enable) + InternalAllocGuard guard; + + if(!memleakfinder_global_enable || !memleakfinder_initialised) { return ::realloc(ptr, size); } @@ -133,7 +181,9 @@ void *memleakfinder_realloc(void *ptr, size_t size) void memleakfinder_free(void *ptr) { - if(memleakfinder_global_enable) + InternalAllocGuard guard; + + if(memleakfinder_global_enable && memleakfinder_initialised) { // Check it's been allocated std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); @@ -143,7 +193,7 @@ void memleakfinder_free(void *ptr) } else { - TRACE1("Block %x freed, but not known. Error? Or allocated in startup static allocation?\n", ptr); + TRACE1("Block %p freed, but not known. Error? Or allocated in startup static allocation?\n", ptr); } if(sTrackMallocInSection) @@ -160,30 +210,44 @@ void memleakfinder_free(void *ptr) void memleakfinder_notaleak_insert_pre() { + InternalAllocGuard guard; + if(!memleakfinder_global_enable) return; - for(int l = 0; l < sNotLeaksPreNum; l++) + if(!memleakfinder_initialised) return; + + for(size_t l = 0; l < sNotLeaksPreNum; l++) { sNotLeaks.insert(sNotLeaksPre[l]); } + sNotLeaksPreNum = 0; } bool is_leak(void *ptr) { + InternalAllocGuard guard; + + ASSERT(memleakfinder_initialised); memleakfinder_notaleak_insert_pre(); return sNotLeaks.find(ptr) == sNotLeaks.end(); } void memleakfinder_notaleak(void *ptr) { + InternalAllocGuard guard; + + ASSERT(!sTrackingDataDestroyed); + memleakfinder_notaleak_insert_pre(); - if(memleakfinder_global_enable) + if(memleakfinder_global_enable && memleakfinder_initialised) { sNotLeaks.insert(ptr); } else { - sNotLeaksPre[sNotLeaksPreNum++] = ptr; + if ( sNotLeaksPreNum < + sizeof(sNotLeaksPre)/sizeof(*sNotLeaksPre) ) + sNotLeaksPre[sNotLeaksPreNum++] = ptr; } /* { std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); @@ -204,6 +268,11 @@ void memleakfinder_notaleak(void *ptr) // start monitoring a section of code void memleakfinder_startsectionmonitor() { + InternalAllocGuard guard; + + ASSERT(memleakfinder_initialised); + ASSERT(!sTrackingDataDestroyed); + sTrackMallocInSection = true; sSectionMallocBlocks.clear(); sTrackObjectsInSection = true; @@ -213,6 +282,11 @@ void memleakfinder_startsectionmonitor() // trace all blocks allocated and still allocated since memleakfinder_startsectionmonitor() called void memleakfinder_traceblocksinsection() { + InternalAllocGuard guard; + + ASSERT(memleakfinder_initialised); + ASSERT(!sTrackingDataDestroyed); + std::set<void *>::iterator s(sSectionMallocBlocks.begin()); for(; s != sSectionMallocBlocks.end(); ++s) { @@ -223,17 +297,22 @@ void memleakfinder_traceblocksinsection() } else { - TRACE4("Block 0x%08p size %d allocated at %s:%d\n", i->first, i->second.size, i->second.file, i->second.line); + TRACE4("Block %p size %d allocated at %s:%d\n", i->first, i->second.size, i->second.file, i->second.line); } } for(std::map<void *, ObjectInfo>::const_iterator i(sSectionObjectBlocks.begin()); i != sSectionObjectBlocks.end(); ++i) { - TRACE5("Object%s 0x%08p size %d allocated at %s:%d\n", i->second.array?" []":"", i->first, i->second.size, i->second.file, i->second.line); + TRACE5("Object%s %p size %d allocated at %s:%d\n", i->second.array?" []":"", i->first, i->second.size, i->second.file, i->second.line); } } int memleakfinder_numleaks() { + InternalAllocGuard guard; + + ASSERT(memleakfinder_initialised); + ASSERT(!sTrackingDataDestroyed); + int n = 0; for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) @@ -243,6 +322,7 @@ int memleakfinder_numleaks() for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) { + const ObjectInfo& rInfo = i->second; if(is_leak(i->first)) ++n; } @@ -251,24 +331,32 @@ int memleakfinder_numleaks() void memleakfinder_reportleaks_file(FILE *file) { + InternalAllocGuard guard; + + ASSERT(!sTrackingDataDestroyed); + for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) { - if(is_leak(i->first)) ::fprintf(file, "Block 0x%08p size %d allocated at %s:%d\n", i->first, i->second.size, i->second.file, i->second.line); + if(is_leak(i->first)) ::fprintf(file, "Block 0x%p size %d allocated at %s:%d\n", i->first, i->second.size, i->second.file, i->second.line); } for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) { - if(is_leak(i->first)) ::fprintf(file, "Object%s 0x%08p size %d allocated at %s:%d\n", i->second.array?" []":"", i->first, i->second.size, i->second.file, i->second.line); + if(is_leak(i->first)) ::fprintf(file, "Object%s 0x%p size %d allocated at %s:%d\n", i->second.array?" []":"", i->first, i->second.size, i->second.file, i->second.line); } } void memleakfinder_reportleaks() { + InternalAllocGuard guard; + // report to stdout memleakfinder_reportleaks_file(stdout); } void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext) { + InternalAllocGuard guard; + FILE *file = ::fopen(filename, "a"); if(file != 0) { @@ -286,7 +374,8 @@ void memleakfinder_reportleaks_appendfile(const char *filename, const char *mark } else { - printf("WARNING: Couldn't open memory leak results file %s for appending\n", filename); + BOX_WARNING("Couldn't open memory leak results file " << + filename << " for appending"); } } @@ -315,7 +404,11 @@ void memleakfinder_setup_exit_report(const char *filename, const char *markertex void add_object_block(void *block, size_t size, const char *file, int line, bool array) { + InternalAllocGuard guard; + if(!memleakfinder_global_enable) return; + if(!memleakfinder_initialised) return; + ASSERT(!sTrackingDataDestroyed); if(block != 0) { @@ -335,7 +428,11 @@ void add_object_block(void *block, size_t size, const char *file, int line, bool void remove_object_block(void *block) { + InternalAllocGuard guard; + if(!memleakfinder_global_enable) return; + if(!memleakfinder_initialised) return; + if(sTrackingDataDestroyed) return; std::map<void *, ObjectInfo>::iterator i(sObjectBlocks.find(block)); if(i != sObjectBlocks.end()) @@ -355,34 +452,68 @@ void remove_object_block(void *block) // If it's not in the list, just ignore it, as lots of stuff goes this way... } -void *operator new(size_t size, const char *file, int line) +static void *internal_new(size_t size, const char *file, int line) { - void *r = ::malloc(size); - add_object_block(r, size, file, line, false); - //TRACE4("new(), %d, %s, %d, %08x\n", size, file, line, r); + void *r; + + { + InternalAllocGuard guard; + r = ::malloc(size); + } + + if (sInternalAllocDepth == 0) + { + InternalAllocGuard guard; + add_object_block(r, size, file, line, false); + //TRACE4("new(), %d, %s, %d, %08x\n", size, file, line, r); + } + return r; } +void *operator new(size_t size, const char *file, int line) +{ + return internal_new(size, file, line); +} + void *operator new[](size_t size, const char *file, int line) { - void *r = ::malloc(size); - add_object_block(r, size, file, line, true); - //TRACE4("new[](), %d, %s, %d, %08x\n", size, file, line, r); - return r; + return internal_new(size, file, line); } -void operator delete[](void *ptr) throw () +// where there is no doctor... need to override standard new() too +// http://www.relisoft.com/book/tech/9new.html +// disabled because it causes hangs on FC2 in futex() in test/common +// while reading files. reason unknown. +/* +void *operator new(size_t size) { + return internal_new(size, "standard libraries", 0); +} +*/ + +void *operator new[](size_t size) +{ + return internal_new(size, "standard libraries", 0); +} + +void internal_delete(void *ptr) +{ + InternalAllocGuard guard; + ::free(ptr); remove_object_block(ptr); //TRACE1("delete[]() called, %08x\n", ptr); } +void operator delete[](void *ptr) throw () +{ + internal_delete(ptr); +} + void operator delete(void *ptr) throw () { - ::free(ptr); - remove_object_block(ptr); - //TRACE1("delete() called, %08x\n", ptr); + internal_delete(ptr); } #endif // NDEBUG diff --git a/lib/common/EventWatchFilesystemObject.cpp b/lib/common/EventWatchFilesystemObject.cpp index a9508c22..84781113 100644 --- a/lib/common/EventWatchFilesystemObject.cpp +++ b/lib/common/EventWatchFilesystemObject.cpp @@ -9,6 +9,7 @@ #include "Box.h" +#include <errno.h> #include <fcntl.h> #ifdef HAVE_UNISTD_H @@ -17,6 +18,7 @@ #include "EventWatchFilesystemObject.h" #include "autogen_CommonException.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -37,6 +39,9 @@ EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) #ifdef HAVE_KQUEUE if(mDescriptor == -1) { + BOX_ERROR("EventWatchFilesystemObject: " + "Failed to open file '" << Filename << "': " << + strerror(errno)); THROW_EXCEPTION(CommonException, OSFileOpenError) } #else diff --git a/lib/common/ExcludeList.cpp b/lib/common/ExcludeList.cpp index 43991594..b9f41634 100644 --- a/lib/common/ExcludeList.cpp +++ b/lib/common/ExcludeList.cpp @@ -9,8 +9,12 @@ #include "Box.h" -#ifdef HAVE_REGEX_H - #include <regex.h> +#ifdef HAVE_REGEX_SUPPORT + #ifdef HAVE_PCREPOSIX_H + #include <pcreposix.h> + #else + #include <regex.h> + #endif #define EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED #endif @@ -18,6 +22,7 @@ #include "Utils.h" #include "Configuration.h" #include "Archive.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -45,7 +50,7 @@ ExcludeList::ExcludeList() // -------------------------------------------------------------------------- ExcludeList::~ExcludeList() { -#ifdef HAVE_REGEX_H +#ifdef HAVE_REGEX_SUPPORT // free regex memory while(mRegex.size() > 0) { @@ -65,6 +70,45 @@ ExcludeList::~ExcludeList() } } +#ifdef WIN32 +std::string ExcludeList::ReplaceSlashesDefinite(const std::string& input) const +{ + std::string output = input; + + for (std::string::size_type pos = output.find("/"); + pos != std::string::npos; + pos = output.find("/")) + { + output.replace(pos, 1, DIRECTORY_SEPARATOR); + } + + for (std::string::iterator i = output.begin(); i != output.end(); i++) + { + *i = tolower(*i); + } + + return output; +} + +std::string ExcludeList::ReplaceSlashesRegex(const std::string& input) const +{ + std::string output = input; + + for (std::string::size_type pos = output.find("/"); + pos != std::string::npos; + pos = output.find("/")) + { + output.replace(pos, 1, "\\" DIRECTORY_SEPARATOR); + } + + for (std::string::iterator i = output.begin(); i != output.end(); i++) + { + *i = tolower(*i); + } + + return output; +} +#endif // -------------------------------------------------------------------------- // @@ -88,7 +132,24 @@ void ExcludeList::AddDefiniteEntries(const std::string &rEntries) { if(i->size() > 0) { - mDefinite.insert(*i); + std::string entry = *i; + + // Convert any forward slashes in the string + // to backslashes + + #ifdef WIN32 + entry = ReplaceSlashesDefinite(entry); + #endif + + if (entry.size() > 0 && entry[entry.size() - 1] == + DIRECTORY_SEPARATOR_ASCHAR) + { + BOX_WARNING("Exclude entry ends in path " + "separator, will never match: " + << entry); + } + + mDefinite.insert(entry); } } } @@ -107,7 +168,7 @@ void ExcludeList::AddDefiniteEntries(const std::string &rEntries) // -------------------------------------------------------------------------- void ExcludeList::AddRegexEntries(const std::string &rEntries) { -#ifdef HAVE_REGEX_H +#ifdef HAVE_REGEX_SUPPORT // Split strings up std::vector<std::string> ens; @@ -123,8 +184,18 @@ void ExcludeList::AddRegexEntries(const std::string &rEntries) try { + std::string entry = *i; + + // Convert any forward slashes in the string + // to appropriately escaped backslashes + + #ifdef WIN32 + entry = ReplaceSlashesRegex(entry); + #endif + // Compile - if(::regcomp(pregex, i->c_str(), REG_EXTENDED | REG_NOSUB) != 0) + if(::regcomp(pregex, entry.c_str(), + REG_EXTENDED | REG_NOSUB) != 0) { THROW_EXCEPTION(CommonException, BadRegularExpression) } @@ -132,7 +203,7 @@ void ExcludeList::AddRegexEntries(const std::string &rEntries) // Store in list of regular expressions mRegex.push_back(pregex); // Store in list of regular expression string for Serialize - mRegexStr.push_back(i->c_str()); + mRegexStr.push_back(entry.c_str()); } catch(...) { @@ -158,10 +229,16 @@ void ExcludeList::AddRegexEntries(const std::string &rEntries) // -------------------------------------------------------------------------- bool ExcludeList::IsExcluded(const std::string &rTest) const { + std::string test = rTest; + + #ifdef WIN32 + test = ReplaceSlashesDefinite(test); + #endif + // Check against the always include list if(mpAlwaysInclude != 0) { - if(mpAlwaysInclude->IsExcluded(rTest)) + if(mpAlwaysInclude->IsExcluded(test)) { // Because the "always include" list says it's 'excluded' // this means it should actually be included. @@ -170,17 +247,17 @@ bool ExcludeList::IsExcluded(const std::string &rTest) const } // Is it in the set of definite entries? - if(mDefinite.find(rTest) != mDefinite.end()) + if(mDefinite.find(test) != mDefinite.end()) { return true; } // Check against regular expressions -#ifdef HAVE_REGEX_H +#ifdef HAVE_REGEX_SUPPORT for(std::vector<regex_t *>::const_iterator i(mRegex.begin()); i != mRegex.end(); ++i) { // Test against this expression - if(regexec(*i, rTest.c_str(), 0, 0 /* no match information required */, 0 /* no flags */) == 0) + if(regexec(*i, test.c_str(), 0, 0 /* no match information required */, 0 /* no flags */) == 0) { // match happened return true; @@ -232,7 +309,7 @@ void ExcludeList::Deserialize(Archive & rArchive) // mDefinite.clear(); -#ifdef HAVE_REGEX_H +#ifdef HAVE_REGEX_SUPPORT // free regex memory while(mRegex.size() > 0) { @@ -273,7 +350,7 @@ void ExcludeList::Deserialize(Archive & rArchive) // // // -#ifdef HAVE_REGEX_H +#ifdef HAVE_REGEX_SUPPORT rArchive.Read(iCount); if (iCount > 0) @@ -310,7 +387,7 @@ void ExcludeList::Deserialize(Archive & rArchive) } } } -#endif // HAVE_REGEX_H +#endif // HAVE_REGEX_SUPPORT // // @@ -365,7 +442,7 @@ void ExcludeList::Serialize(Archive & rArchive) const // // // -#ifdef HAVE_REGEX_H +#ifdef HAVE_REGEX_SUPPORT // don't even try to save compiled regular expressions, // use string copies instead. ASSERT(mRegex.size() == mRegexStr.size()); @@ -378,7 +455,7 @@ void ExcludeList::Serialize(Archive & rArchive) const { rArchive.Write(*i); } -#endif // HAVE_REGEX_H +#endif // HAVE_REGEX_SUPPORT // // diff --git a/lib/common/ExcludeList.h b/lib/common/ExcludeList.h index 720b6788..3c41bd11 100644 --- a/lib/common/ExcludeList.h +++ b/lib/common/ExcludeList.h @@ -50,7 +50,7 @@ public: // Mainly for tests unsigned int SizeOfDefiniteList() const {return mDefinite.size();} unsigned int SizeOfRegexList() const -#ifdef HAVE_REGEX_H +#ifdef HAVE_REGEX_SUPPORT {return mRegex.size();} #else {return 0;} @@ -58,11 +58,16 @@ public: private: std::set<std::string> mDefinite; -#ifdef HAVE_REGEX_H +#ifdef HAVE_REGEX_SUPPORT std::vector<regex_t *> mRegex; std::vector<std::string> mRegexStr; // save original regular expression string-based source for Serialize #endif +#ifdef WIN32 + std::string ReplaceSlashesDefinite(const std::string& input) const; + std::string ReplaceSlashesRegex (const std::string& input) const; +#endif + // For exceptions to the excludes ExcludeList *mpAlwaysInclude; }; diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp index b6ae083e..e0806e10 100644 --- a/lib/common/FileStream.cpp +++ b/lib/common/FileStream.cpp @@ -10,6 +10,9 @@ #include "Box.h" #include "FileStream.h" #include "CommonException.h" +#include "Logging.h" + +#include <errno.h> #include "MemLeakFindOn.h" @@ -36,7 +39,15 @@ FileStream::FileStream(const char *Filename, int flags, int mode) #endif { MEMLEAKFINDER_NOT_A_LEAK(this); - THROW_EXCEPTION(CommonException, OSFileOpenError) + + if(errno == EACCES) + { + THROW_EXCEPTION(CommonException, AccessDenied) + } + else + { + THROW_EXCEPTION(CommonException, OSFileOpenError) + } } #ifdef WIN32 this->fileName = Filename; @@ -47,7 +58,7 @@ FileStream::FileStream(const char *Filename, int flags, int mode) // -------------------------------------------------------------------------- // // Function -// Name: FileStream::FileStream(int) +// Name: FileStream::FileStream(tOSFileHandle) // Purpose: Constructor, using existing file descriptor // Created: 2003/08/28 // @@ -63,8 +74,12 @@ FileStream::FileStream(tOSFileHandle FileDescriptor) #endif { MEMLEAKFINDER_NOT_A_LEAK(this); + BOX_ERROR("FileStream: called with invalid file handle"); THROW_EXCEPTION(CommonException, OSFileOpenError) } +#ifdef WIN32 + this->fileName = "HANDLE"; +#endif } #if 0 @@ -87,6 +102,7 @@ FileStream::FileStream(const FileStream &rToCopy) #endif { MEMLEAKFINDER_NOT_A_LEAK(this); + BOX_ERROR("FileStream: copying unopened file"); THROW_EXCEPTION(CommonException, OSFileOpenError) } } @@ -138,8 +154,14 @@ int FileStream::Read(void *pBuffer, int NBytes, int Timeout) { r = numBytesRead; } + else if (GetLastError() == ERROR_BROKEN_PIPE) + { + r = 0; + } else { + BOX_ERROR("Failed to read from file: " << + GetErrorMessage(GetLastError())); r = -1; } #else @@ -203,7 +225,7 @@ void FileStream::Write(const void *pBuffer, int NBytes) NULL ); - if ( (res == 0) || (numBytesWritten != NBytes)) + if ((res == 0) || (numBytesWritten != (DWORD)NBytes)) { // DWORD err = GetLastError(); THROW_EXCEPTION(CommonException, OSFileWriteError) diff --git a/lib/common/Guards.h b/lib/common/Guards.h index b1bca0fa..d2fb84e0 100644 --- a/lib/common/Guards.h +++ b/lib/common/Guards.h @@ -15,12 +15,16 @@ #include <unistd.h> #endif +#include <errno.h> #include <fcntl.h> #include <stdlib.h> +#include <string.h> #include <sys/stat.h> + #include <new> #include "CommonException.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -28,11 +32,13 @@ template <int flags = O_RDONLY | O_BINARY, int mode = (S_IRUSR | S_IWUSR | S_IRG class FileHandleGuard { public: - FileHandleGuard(const char *filename) - : mOSFileHandle(::open(filename, flags, mode)) + FileHandleGuard(const std::string& rFilename) + : mOSFileHandle(::open(rFilename.c_str(), flags, mode)) { if(mOSFileHandle < 0) { + BOX_ERROR("FileHandleGuard: failed to open file '" << + rFilename << "': " << strerror(errno)); THROW_EXCEPTION(CommonException, OSFileOpenError) } } diff --git a/lib/common/InvisibleTempFileStream.cpp b/lib/common/InvisibleTempFileStream.cpp new file mode 100644 index 00000000..a7e19ad3 --- /dev/null +++ b/lib/common/InvisibleTempFileStream.cpp @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: InvisibleTempFileStream.cpp +// Purpose: IOStream interface to temporary files that +// delete themselves +// Created: 2006/10/13 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "InvisibleTempFileStream.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: InvisibleTempFileStream::InvisibleTempFileStream +// (const char *, int, int) +// Purpose: Constructor, opens invisible file +// Created: 2006/10/13 +// +// -------------------------------------------------------------------------- +InvisibleTempFileStream::InvisibleTempFileStream(const char *Filename, int flags, int mode) +#ifdef WIN32 + : FileStream(::openfile(Filename, flags | O_TEMPORARY, mode)) +#else + : FileStream(::open(Filename, flags, mode)) +#endif +{ + #ifndef WIN32 + if(unlink(Filename) != 0) + { + MEMLEAKFINDER_NOT_A_LEAK(this); + THROW_EXCEPTION(CommonException, OSFileOpenError) + } + #endif +} diff --git a/lib/common/InvisibleTempFileStream.h b/lib/common/InvisibleTempFileStream.h new file mode 100644 index 00000000..a77d05e2 --- /dev/null +++ b/lib/common/InvisibleTempFileStream.h @@ -0,0 +1,35 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: InvisibleTempFileStream.h +// Purpose: FileStream interface to temporary files that +// delete themselves +// Created: 2006/10/13 +// +// -------------------------------------------------------------------------- + +#ifndef INVISIBLETEMPFILESTREAM__H +#define INVISIBLETEMPFILESTREAM__H + +#include "FileStream.h" + +class InvisibleTempFileStream : public FileStream +{ +public: + InvisibleTempFileStream(const char *Filename, +#ifdef WIN32 + int flags = (O_RDONLY | O_BINARY), +#else + int flags = O_RDONLY, +#endif + int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + +private: + InvisibleTempFileStream(const InvisibleTempFileStream &rToCopy) + : FileStream(INVALID_FILE) + { /* do not call */ } +}; + +#endif // INVISIBLETEMPFILESTREAM__H + + diff --git a/lib/common/Logging.cpp b/lib/common/Logging.cpp new file mode 100644 index 00000000..d22db238 --- /dev/null +++ b/lib/common/Logging.cpp @@ -0,0 +1,334 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Logging.cpp +// Purpose: Generic logging core routines implementation +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <errno.h> +#include <time.h> + +#ifdef HAVE_SYSLOG_H + #include <syslog.h> +#endif + +#include "Logging.h" + +#include <iomanip> + +bool Logging::sLogToSyslog = false; +bool Logging::sLogToConsole = false; +bool Logging::sContextSet = false; + +std::vector<Logger*> Logging::sLoggers; +std::string Logging::sContext; +Console* Logging::spConsole = NULL; +Syslog* Logging::spSyslog = NULL; +Log::Level Logging::sGlobalLevel = Log::EVERYTHING; +Logging Logging::sGlobalLogging; //automatic initialisation + +Logging::Logging() +{ + ASSERT(!spConsole); + ASSERT(!spSyslog); + spConsole = new Console(); + spSyslog = new Syslog(); + sLogToConsole = true; + sLogToSyslog = true; +} + +Logging::~Logging() +{ + sLogToConsole = false; + sLogToSyslog = false; + delete spConsole; + delete spSyslog; + spConsole = NULL; + spSyslog = NULL; +} + +void Logging::ToSyslog(bool enabled) +{ + if (!sLogToSyslog && enabled) + { + Add(spSyslog); + } + + if (sLogToSyslog && !enabled) + { + Remove(spSyslog); + } + + sLogToSyslog = enabled; +} + +void Logging::ToConsole(bool enabled) +{ + if (!sLogToConsole && enabled) + { + Add(spConsole); + } + + if (sLogToConsole && !enabled) + { + Remove(spConsole); + } + + sLogToConsole = enabled; +} + +void Logging::FilterConsole(Log::Level level) +{ + spConsole->Filter(level); +} + +void Logging::FilterSyslog(Log::Level level) +{ + spSyslog->Filter(level); +} + +void Logging::Add(Logger* pNewLogger) +{ + for (std::vector<Logger*>::iterator i = sLoggers.begin(); + i != sLoggers.end(); i++) + { + if (*i == pNewLogger) + { + return; + } + } + + sLoggers.insert(sLoggers.begin(), pNewLogger); +} + +void Logging::Remove(Logger* pOldLogger) +{ + for (std::vector<Logger*>::iterator i = sLoggers.begin(); + i != sLoggers.end(); i++) + { + if (*i == pOldLogger) + { + sLoggers.erase(i); + return; + } + } +} + +void Logging::Log(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage) +{ + if (level > sGlobalLevel) + { + return; + } + + std::string newMessage; + + if (sContextSet) + { + newMessage += "[" + sContext + "] "; + } + + newMessage += rMessage; + + for (std::vector<Logger*>::iterator i = sLoggers.begin(); + i != sLoggers.end(); i++) + { + bool result = (*i)->Log(level, rFile, line, newMessage); + if (!result) + { + return; + } + } +} + +void Logging::SetContext(std::string context) +{ + sContext = context; + sContextSet = true; +} + +void Logging::ClearContext() +{ + sContextSet = false; +} + +void Logging::SetProgramName(const std::string& rProgramName) +{ + for (std::vector<Logger*>::iterator i = sLoggers.begin(); + i != sLoggers.end(); i++) + { + (*i)->SetProgramName(rProgramName); + } +} + +Logger::Logger() +: mCurrentLevel(Log::EVERYTHING) +{ + Logging::Add(this); +} + +Logger::~Logger() +{ + Logging::Remove(this); +} + +bool Console::sShowTime = false; +bool Console::sShowTag = false; +std::string Console::sTag; + +void Console::SetTag(const std::string& rTag) +{ + sTag = rTag; + sShowTag = true; +} + +void Console::SetShowTime(bool enabled) +{ + sShowTime = enabled; +} + +bool Console::Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage) +{ + if (level > GetLevel()) + { + return true; + } + + FILE* target = stdout; + + if (level <= Log::WARNING) + { + target = stderr; + } + + std::string msg; + + if (sShowTime) + { + struct tm time_now, *tm_ptr = &time_now; + time_t time_t_now = time(NULL); + + if (time_t_now == ((time_t)-1)) + { + msg += strerror(errno); + msg += " "; + } + #ifdef WIN32 + else if ((tm_ptr = localtime(&time_t_now)) != NULL) + #else + else if (localtime_r(&time_t_now, &time_now) != NULL) + #endif + { + std::ostringstream buf; + buf << std::setfill('0') << + std::setw(2) << tm_ptr->tm_hour << ":" << + std::setw(2) << tm_ptr->tm_min << ":" << + std::setw(2) << tm_ptr->tm_sec << " "; + msg += buf.str(); + } + else + { + msg += strerror(errno); + msg += " "; + } + } + + if (sShowTag) + { + msg += "[" + sTag + "] "; + } + + if (level <= Log::FATAL) + { + msg += "FATAL: "; + } + else if (level <= Log::ERROR) + { + msg += "ERROR: "; + } + else if (level <= Log::WARNING) + { + msg += "WARNING: "; + } + else if (level <= Log::NOTICE) + { + msg += "NOTICE: "; + } + + msg += rMessage; + + fprintf(target, "%s\n", msg.c_str()); + + return true; +} + +bool Syslog::Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage) +{ + if (level > GetLevel()) + { + return true; + } + + int syslogLevel = LOG_ERR; + + switch(level) + { + case Log::NOTHING: /* fall through */ + case Log::FATAL: syslogLevel = LOG_CRIT; break; + case Log::ERROR: syslogLevel = LOG_ERR; break; + case Log::WARNING: syslogLevel = LOG_WARNING; break; + case Log::NOTICE: syslogLevel = LOG_NOTICE; break; + case Log::INFO: syslogLevel = LOG_INFO; break; + case Log::TRACE: /* fall through */ + case Log::EVERYTHING: syslogLevel = LOG_DEBUG; break; + } + + std::string msg; + + if (level <= Log::FATAL) + { + msg = "FATAL: "; + } + else if (level <= Log::ERROR) + { + msg = "ERROR: "; + } + else if (level <= Log::WARNING) + { + msg = "WARNING: "; + } + else if (level <= Log::NOTICE) + { + msg = "NOTICE: "; + } + + msg += rMessage; + + syslog(syslogLevel, "%s", msg.c_str()); + + return true; +} + +Syslog::Syslog() +{ + ::openlog("Box Backup", LOG_PID, LOG_LOCAL6); +} + +Syslog::~Syslog() +{ + ::closelog(); +} + +void Syslog::SetProgramName(const std::string& rProgramName) +{ + mName = rProgramName; + ::closelog(); + ::openlog(mName.c_str(), LOG_PID, LOG_LOCAL6); +} diff --git a/lib/common/Logging.h b/lib/common/Logging.h new file mode 100644 index 00000000..9601a495 --- /dev/null +++ b/lib/common/Logging.h @@ -0,0 +1,197 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Logging.h +// Purpose: Generic logging core routines declarations and macros +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +#ifndef LOGGING__H +#define LOGGING__H + +#include <iomanip> +#include <sstream> +#include <vector> + +/* +#define BOX_LOG(level, stuff) \ +{ \ + if(Log::sMaxLoggingLevelForAnyOutput >= level) \ + std::ostringstream line; \ + line << stuff; \ + Log::Write(level, __FILE__, __LINE__, line.str()); \ + } \ +} +*/ + +#define BOX_LOG(level, stuff) \ +{ \ + std::ostringstream line; \ + line << stuff; \ + Logging::Log(level, __FILE__, __LINE__, line.str()); \ +} + +#define BOX_FATAL(stuff) BOX_LOG(Log::FATAL, stuff) +#define BOX_ERROR(stuff) BOX_LOG(Log::ERROR, stuff) +#define BOX_WARNING(stuff) BOX_LOG(Log::WARNING, stuff) +#define BOX_NOTICE(stuff) BOX_LOG(Log::NOTICE, stuff) +#define BOX_INFO(stuff) BOX_LOG(Log::INFO, stuff) +#if defined NDEBUG && ! defined COMPILE_IN_TRACES + #define BOX_TRACE(stuff) +#else + #define BOX_TRACE(stuff) BOX_LOG(Log::TRACE, stuff) +#endif + +#define BOX_FORMAT_ACCOUNT(accno) \ + std::hex << \ + std::showbase << \ + std::internal << \ + std::setw(10) << \ + std::setfill('0') << \ + (accno) + +#define BOX_FORMAT_OBJECTID(objectid) \ + std::hex << \ + std::showbase << \ + (objectid) + +#undef ERROR + +namespace Log +{ + enum Level + { + NOTHING = 1, + FATAL, + ERROR, + WARNING, + NOTICE, + INFO, + TRACE, + EVERYTHING + }; +} + +// -------------------------------------------------------------------------- +// +// Class +// Name: Logger +// Purpose: Abstract base class for log targets +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +class Logger +{ + private: + Log::Level mCurrentLevel; + + public: + Logger(); + virtual ~Logger(); + + virtual bool Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage) = 0; + + void Filter(Log::Level level) + { + mCurrentLevel = level; + } + + virtual const char* GetType() = 0; + Log::Level GetLevel() { return mCurrentLevel; } + + virtual void SetProgramName(const std::string& rProgramName) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Console +// Purpose: Console logging target +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +class Console : public Logger +{ + private: + static bool sShowTime; + static bool sShowTag; + static std::string sTag; + + public: + virtual bool Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage); + virtual const char* GetType() { return "Console"; } + virtual void SetProgramName(const std::string& rProgramName) { } + + static void SetTag(const std::string& rTag); + static void SetShowTime(bool enabled); +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Syslog +// Purpose: Syslog (or Windows Event Viewer) logging target +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +class Syslog : public Logger +{ + private: + std::string mName; + + public: + Syslog(); + virtual ~Syslog(); + + virtual bool Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage); + virtual const char* GetType() { return "Syslog"; } + virtual void SetProgramName(const std::string& rProgramName); +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Logging +// Purpose: Static logging helper, keeps track of enabled loggers +// and distributes log messages to them. +// Created: 2006/12/16 +// +// -------------------------------------------------------------------------- + +class Logging +{ + private: + static std::vector<Logger*> sLoggers; + static bool sLogToSyslog, sLogToConsole; + static std::string sContext; + static bool sContextSet; + static Console* spConsole; + static Syslog* spSyslog; + static Log::Level sGlobalLevel; + static Logging sGlobalLogging; + + public: + Logging (); + ~Logging(); + static void ToSyslog (bool enabled); + static void ToConsole (bool enabled); + static void FilterSyslog (Log::Level level); + static void FilterConsole (Log::Level level); + static void Add (Logger* pNewLogger); + static void Remove (Logger* pOldLogger); + static void Log(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage); + static void SetContext(std::string context); + static void ClearContext(); + static void SetGlobalLevel(Log::Level level) { sGlobalLevel = level; } + static void SetProgramName(const std::string& rProgramName); +}; + +#endif // LOGGING__H diff --git a/lib/common/MainHelper.h b/lib/common/MainHelper.h index ab75af96..d91bc2f9 100644 --- a/lib/common/MainHelper.h +++ b/lib/common/MainHelper.h @@ -17,6 +17,7 @@ #define MAINHELPER_START \ if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \ { printf(BOX_VERSION "\n"); return 0; } \ + MEMLEAKFINDER_INIT \ MEMLEAKFINDER_START \ try { #define MAINHELPER_END \ diff --git a/lib/common/MemLeakFinder.h b/lib/common/MemLeakFinder.h index f5887dac..450d42f8 100644 --- a/lib/common/MemLeakFinder.h +++ b/lib/common/MemLeakFinder.h @@ -10,8 +10,6 @@ #ifndef MEMLEAKFINDER__H #define MEMLEAKFINDER__H -#define DEBUG_NEW new(__FILE__,__LINE__) - #ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING // include stdlib now, to avoid problems with having the macros defined already #include <stdlib.h> @@ -20,6 +18,13 @@ // global enable flag extern bool memleakfinder_global_enable; +class MemLeakSuppressionGuard +{ + public: + MemLeakSuppressionGuard(); + ~MemLeakSuppressionGuard(); +}; + extern "C" { void *memleakfinder_malloc(size_t size, const char *file, int line); @@ -27,6 +32,8 @@ extern "C" void memleakfinder_free(void *ptr); } +void memleakfinder_init(); + int memleakfinder_numleaks(); void memleakfinder_reportleaks(); @@ -41,12 +48,9 @@ void memleakfinder_traceblocksinsection(); void memleakfinder_notaleak(void *ptr); -void *operator new(size_t size, const char *file, int line); +void *operator new (size_t size, const char *file, int line); void *operator new[](size_t size, const char *file, int line); -void operator delete(void *ptr) throw (); -void operator delete[](void *ptr) throw (); - // define the malloc functions now, if required #ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) @@ -55,6 +59,5 @@ void operator delete[](void *ptr) throw (); #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED #endif - #endif // MEMLEAKFINDER__H diff --git a/lib/common/PartialReadStream.cpp b/lib/common/PartialReadStream.cpp index 0b5c4cf6..76096738 100644 --- a/lib/common/PartialReadStream.cpp +++ b/lib/common/PartialReadStream.cpp @@ -16,13 +16,15 @@ // -------------------------------------------------------------------------- // // Function -// Name: PartialReadStream::PartialReadStream(IOStream &, int) -// Purpose: Constructor, taking another stream and the number of bytes -// to be read from it. +// Name: PartialReadStream::PartialReadStream(IOStream &, +// pos_type) +// Purpose: Constructor, taking another stream and the number of +// bytes to be read from it. // Created: 2003/08/26 // // -------------------------------------------------------------------------- -PartialReadStream::PartialReadStream(IOStream &rSource, int BytesToRead) +PartialReadStream::PartialReadStream(IOStream &rSource, + pos_type BytesToRead) : mrSource(rSource), mBytesLeft(BytesToRead) { diff --git a/lib/common/PartialReadStream.h b/lib/common/PartialReadStream.h index 42cb7aeb..1b46b0bd 100644 --- a/lib/common/PartialReadStream.h +++ b/lib/common/PartialReadStream.h @@ -23,7 +23,7 @@ class PartialReadStream : public IOStream { public: - PartialReadStream(IOStream &rSource, int BytesToRead); + PartialReadStream(IOStream &rSource, pos_type BytesToRead); ~PartialReadStream(); private: // no copying allowed @@ -39,7 +39,7 @@ public: private: IOStream &mrSource; - int mBytesLeft; + pos_type mBytesLeft; }; #endif // PARTIALREADSTREAM__H diff --git a/lib/common/PathUtils.cpp b/lib/common/PathUtils.cpp new file mode 100644 index 00000000..924d47d2 --- /dev/null +++ b/lib/common/PathUtils.cpp @@ -0,0 +1,34 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: PathUtils.cpp +// Purpose: Platform-independent path manipulation +// Created: 2007/01/17 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include <string> + +// -------------------------------------------------------------------------- +// +// Function +// Name: MakeFullPath(const std::string& rDir, const std::string& rFile) +// Purpose: Combine directory and file name +// Created: 2006/08/10 +// +// -------------------------------------------------------------------------- +std::string MakeFullPath(const std::string& rDir, const std::string& rEntry) +{ + std::string result(rDir); + + if (result.size() > 0 && + result[result.size()-1] != DIRECTORY_SEPARATOR_ASCHAR) + { + result += DIRECTORY_SEPARATOR; + } + + result += rEntry; + + return result; +} diff --git a/lib/common/PathUtils.h b/lib/common/PathUtils.h new file mode 100644 index 00000000..1cf2e507 --- /dev/null +++ b/lib/common/PathUtils.h @@ -0,0 +1,26 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: PathUtils.h +// Purpose: Platform-independent path manipulation +// Created: 2007/01/17 +// +// -------------------------------------------------------------------------- + +#ifndef PATHUTILS_H +#define PATHUTILS_H + +#include <string> + +// -------------------------------------------------------------------------- +// +// Function +// Name: MakeFullPath(const std::string& rDir, const std::string& rFile) +// Purpose: Combine directory and file name +// Created: 2006/08/10 +// +// -------------------------------------------------------------------------- + +std::string MakeFullPath(const std::string& rDir, const std::string& rEntry); + +#endif // !PATHUTILS_H diff --git a/lib/common/ReadGatherStream.cpp b/lib/common/ReadGatherStream.cpp index 9ccc3a54..f50e6664 100644 --- a/lib/common/ReadGatherStream.cpp +++ b/lib/common/ReadGatherStream.cpp @@ -59,8 +59,9 @@ ReadGatherStream::~ReadGatherStream() // // Function // Name: ReadGatherStream::AddComponent(IOStream *) -// Purpose: Add a component to this stream, returning the index of this component -// in the internal list. Use this with AddBlock() +// Purpose: Add a component to this stream, returning the index +// of this component in the internal list. Use this +// with AddBlock() // Created: 10/12/03 // // -------------------------------------------------------------------------- @@ -145,10 +146,10 @@ int ReadGatherStream::Read(void *pBuffer, int NBytes, int Timeout) if(mPositionInCurrentBlock < mBlocks[mCurrentBlock].mLength) { // Read! - int s = mBlocks[mCurrentBlock].mLength - mPositionInCurrentBlock; + pos_type s = mBlocks[mCurrentBlock].mLength - mPositionInCurrentBlock; if(s > bytesToRead) s = bytesToRead; - int r = mComponents[mBlocks[mCurrentBlock].mComponent]->Read(buffer, s, Timeout); + pos_type r = mComponents[mBlocks[mCurrentBlock].mComponent]->Read(buffer, s, Timeout); // update variables mPositionInCurrentBlock += r; diff --git a/lib/common/ReadLoggingStream.cpp b/lib/common/ReadLoggingStream.cpp new file mode 100644 index 00000000..9023f827 --- /dev/null +++ b/lib/common/ReadLoggingStream.cpp @@ -0,0 +1,207 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ReadLoggingStream.cpp +// Purpose: Buffering wrapper around IOStreams +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> + +#include "ReadLoggingStream.h" +#include "CommonException.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::ReadLoggingStream(const char *, int, int) +// Purpose: Constructor, set up buffer +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +ReadLoggingStream::ReadLoggingStream(IOStream& rSource) +: mrSource(rSource), + mOffset(0), + mLength(mrSource.BytesLeftToRead()), + mTotalRead(0), + mStartTime(GetCurrentBoxTime()) +{ } + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::Read(void *, int) +// Purpose: Reads bytes from the file +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +int ReadLoggingStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + int numBytesRead = mrSource.Read(pBuffer, NBytes, Timeout); + + if (numBytesRead > 0) + { + mTotalRead += numBytesRead; + mOffset += numBytesRead; + } + + if (mLength >= 0 && mTotalRead > 0) + { + box_time_t timeNow = GetCurrentBoxTime(); + box_time_t elapsed = timeNow - mStartTime; + box_time_t finish = (elapsed * mLength) / mTotalRead; + box_time_t remain = finish - elapsed; + + BOX_TRACE("Read " << numBytesRead << " bytes at " << mOffset << + ", " << (mLength - mOffset) << " remain, eta " << + BoxTimeToSeconds(remain) << "s"); + } + else if (mLength >= 0 && mTotalRead == 0) + { + BOX_TRACE("Read " << numBytesRead << " bytes at " << mOffset << + ", " << (mLength - mOffset) << " remain"); + } + else + { + BOX_TRACE("Read " << numBytesRead << " bytes at " << mOffset << + ", unknown bytes remaining"); + } + + return numBytesRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ReadLoggingStream::BytesLeftToRead() +{ + return mLength - mOffset; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::Write(void *, int) +// Purpose: Writes bytes to the underlying stream (not supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ReadLoggingStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ReadLoggingStream::GetPosition() const +{ + return mOffset; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek, invalidate buffer +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ReadLoggingStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + mrSource.Seek(Offset, SeekType); + + switch (SeekType) + { + case SeekType_Absolute: + { + // just go there + mOffset = Offset; + } + break; + + case SeekType_Relative: + { + // Actual underlying file position is + // (mBufferSize - mBufferPosition) ahead of us. + // Need to subtract that amount from the seek + // to seek forward that much less, putting the + // real pointer in the right place. + mOffset += Offset; + } + break; + + case SeekType_End: + { + // Actual underlying file position is + // (mBufferSize - mBufferPosition) ahead of us. + // Need to add that amount to the seek + // to seek backwards that much more, putting the + // real pointer in the right place. + mOffset = mLength - Offset; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::Close() +// Purpose: Closes the underlying stream (not needed) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ReadLoggingStream::Close() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool ReadLoggingStream::StreamDataLeft() +{ + return mrSource.StreamDataLeft(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ReadLoggingStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool ReadLoggingStream::StreamClosed() +{ + return mrSource.StreamClosed(); +} + diff --git a/lib/common/ReadLoggingStream.h b/lib/common/ReadLoggingStream.h new file mode 100644 index 00000000..15c3ef48 --- /dev/null +++ b/lib/common/ReadLoggingStream.h @@ -0,0 +1,43 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ReadLoggingStream.h +// Purpose: Wrapper around IOStreams that logs read progress +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- + +#ifndef READLOGGINGSTREAM__H +#define READLOGGINGSTREAM__H + +#include "IOStream.h" +#include "BoxTime.h" + +class ReadLoggingStream : public IOStream +{ +private: + IOStream& mrSource; + IOStream::pos_type mOffset, mLength, mTotalRead; + box_time_t mStartTime; + +public: + ReadLoggingStream(IOStream& rSource); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + ReadLoggingStream(const ReadLoggingStream &rToCopy) + : mrSource(rToCopy.mrSource) { /* do not call */ } +}; + +#endif // READLOGGINGSTREAM__H + + diff --git a/lib/common/Test.h b/lib/common/Test.h index 56a6c261..f6704db4 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -10,27 +10,42 @@ #ifndef TEST__H #define TEST__H -#include <sys/types.h> -#include <sys/stat.h> -#include <stdlib.h> +#include <errno.h> #include <signal.h> +#include <stdio.h> +#include <stdlib.h> + +#include <sys/stat.h> +#include <sys/types.h> #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -#include <stdio.h> #include <string> extern int failures; +extern int first_fail_line; +extern std::string first_fail_file; +extern std::string bbackupd_args, bbstored_args, bbackupquery_args; + +#define TEST_FAIL_WITH_MESSAGE(msg) \ +{ \ + if (failures == 0) \ + { \ + first_fail_file = __FILE__; \ + first_fail_line = __LINE__; \ + } \ + failures++; \ + printf("FAILURE: " msg " at " __FILE__ "(%d)\n", __LINE__); \ +} -#define TEST_FAIL_WITH_MESSAGE(msg) {failures++; printf("FAILURE: " msg " at " __FILE__ "(%d)\n", __LINE__);} -#define TEST_ABORT_WITH_MESSAGE(msg) {failures++; printf("FAILURE: " msg " at " __FILE__ "(%d)\n", __LINE__); return 1;} +#define TEST_ABORT_WITH_MESSAGE(msg) {TEST_FAIL_WITH_MESSAGE(msg); return 1;} #define TEST_THAT(condition) {if(!(condition)) TEST_FAIL_WITH_MESSAGE("Condition [" #condition "] failed")} #define TEST_THAT_ABORTONFAIL(condition) {if(!(condition)) TEST_ABORT_WITH_MESSAGE("Condition [" #condition "] failed")} -// NOTE: The 0- bit it to allow this to work with stuff which has negative constants for flags (eg ConnectionException) +// NOTE: The 0- bit is to allow this to work with stuff which has negative constants for flags (eg ConnectionException) #define TEST_CHECK_THROWS(statement, excepttype, subtype) \ { \ bool didthrow = false; \ @@ -80,55 +95,74 @@ inline int TestGetFileSize(const char *Filename) return -1; } -inline int RunCommand(const char *pCommandLine) +inline std::string ConvertPaths(const std::string& rOriginal) { #ifdef WIN32 // convert UNIX paths to native - std::string command; - for (int i = 0; pCommandLine[i] != 0; i++) + std::string converted; + for (size_t i = 0; i < rOriginal.size(); i++) { - if (pCommandLine[i] == '/') + if (rOriginal[i] == '/') { - command += '\\'; + converted += '\\'; } else { - command += pCommandLine[i]; + converted += rOriginal[i]; } } + return converted; #else // !WIN32 - std::string command = pCommandLine; + return rOriginal; #endif +} - return ::system(command.c_str()); +inline int RunCommand(const std::string& rCommandLine) +{ + return ::system(ConvertPaths(rCommandLine).c_str()); } -inline int LaunchServer(const char *pCommandLine, const char *pidFile) +#ifdef WIN32 +#include <windows.h> +#endif + +inline bool ServerIsAlive(int pid) { - if(RunCommand(pCommandLine) != 0) +#ifdef WIN32 + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, false, pid); + if (hProcess == NULL) { - printf("Server: %s\n", pCommandLine); - TEST_FAIL_WITH_MESSAGE("Couldn't start server"); - return -1; + if (GetLastError() != ERROR_INVALID_PARAMETER) + { + printf("Failed to open process %d: error %d\n", + pid, (int)GetLastError()); + } + return false; } - // time for it to start up - ::sleep(1); - - // read pid file + CloseHandle(hProcess); + return true; +#else // !WIN32 + if(pid == 0) return false; + return ::kill(pid, 0) != -1; +#endif // WIN32 +} + +inline int ReadPidFile(const char *pidFile) +{ if(!TestFileExists(pidFile)) { - printf("Server: %s\n", pCommandLine); - TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file " + "(perhaps one was already running?)"); return -1; } - FILE *f = fopen(pidFile, "r"); int pid = -1; + + FILE *f = fopen(pidFile, "r"); if(f == NULL || fscanf(f, "%d", &pid) != 1) { - printf("Server: %s (pidfile %s)\n", pCommandLine, pidFile); TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); return -1; } @@ -137,160 +171,177 @@ inline int LaunchServer(const char *pCommandLine, const char *pidFile) return pid; } -#ifdef WIN32 - -#include "WinNamedPipeStream.h" -#include "IOStreamGetLine.h" -#include "BoxPortsAndFiles.h" - -bool SendCommands(const std::string& rCmd) +inline int LaunchServer(const std::string& rCommandLine, const char *pidFile) { - WinNamedPipeStream connection; +#ifdef WIN32 - try + PROCESS_INFORMATION procInfo; + + STARTUPINFO startInfo; + startInfo.cb = sizeof(startInfo); + startInfo.lpReserved = NULL; + startInfo.lpDesktop = NULL; + startInfo.lpTitle = NULL; + startInfo.dwFlags = 0; + startInfo.cbReserved2 = 0; + startInfo.lpReserved2 = NULL; + + std::string cmd = ConvertPaths(rCommandLine); + CHAR* tempCmd = strdup(cmd.c_str()); + + DWORD result = CreateProcess + ( + NULL, // lpApplicationName, naughty! + tempCmd, // lpCommandLine + NULL, // lpProcessAttributes + NULL, // lpThreadAttributes + false, // bInheritHandles + 0, // dwCreationFlags + NULL, // lpEnvironment + NULL, // lpCurrentDirectory + &startInfo, // lpStartupInfo + &procInfo // lpProcessInformation + ); + + free(tempCmd); + + if (result == 0) { - connection.Connect(BOX_NAMED_PIPE_NAME); - } - catch(...) - { - printf("Failed to connect to daemon control socket.\n"); - return false; + DWORD err = GetLastError(); + printf("Launch failed: %s: error %d\n", rCommandLine.c_str(), + (int)err); + return -1; } - // For receiving data - IOStreamGetLine getLine(connection); - - // Wait for the configuration summary - std::string configSummary; - if(!getLine.GetLine(configSummary)) - { - printf("Failed to receive configuration summary from daemon\n"); - return false; - } + CloseHandle(procInfo.hProcess); + CloseHandle(procInfo.hThread); - // Was the connection rejected by the server? - if(getLine.IsEOF()) - { - printf("Server rejected the connection.\n"); - return false; - } +#else // !WIN32 - // Decode it - int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait; - if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", - &autoBackup, &updateStoreInterval, - &minimumFileAge, &maxUploadWait) != 4) + if(RunCommand(rCommandLine) != 0) { - printf("Config summary didn't decode\n"); - return false; + printf("Server: %s\n", rCommandLine.c_str()); + TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + return -1; } - std::string cmds; - bool expectResponse; +#endif // WIN32 - if (rCmd != "") - { - cmds = rCmd; - cmds += "\nquit\n"; - expectResponse = true; - } - else + #ifdef WIN32 + // on other platforms there is no other way to get + // the PID, so a NULL pidFile doesn't make sense. + + if (pidFile == NULL) { - cmds = "quit\n"; - expectResponse = false; + return (int)procInfo.dwProcessId; } - - connection.Write(cmds.c_str(), cmds.size()); - - // Read the response - std::string line; - bool statusOk = !expectResponse; + #endif + + // time for it to start up + ::fprintf(stdout, "Starting server: %s\n", rCommandLine.c_str()); + ::fprintf(stdout, "Waiting for server to start: "); - while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line)) + for (int i = 0; i < 15; i++) { - // Is this an OK or error line? - if (line == "ok") + if (TestFileExists(pidFile)) { - statusOk = true; - } - else if (line == "error") - { - printf("ERROR (%s)\n", rCmd.c_str()); break; } - else + + #ifdef WIN32 + if (!ServerIsAlive((int)procInfo.dwProcessId)) { - printf("WARNING: Unexpected response to command '%s': " - "%s", rCmd.c_str(), line.c_str()); + break; } + #endif + + ::fprintf(stdout, "."); + ::fflush(stdout); + ::sleep(1); } - - return statusOk; -} -inline bool ServerIsAlive() -{ - return SendCommands(""); -} + #ifdef WIN32 + // on Win32 we can check whether the process is alive + // without even checking the PID file -inline bool HUPServer(int pid) -{ - return SendCommands("reload"); -} + if (!ServerIsAlive((int)procInfo.dwProcessId)) + { + ::fprintf(stdout, "server died!\n"); + TEST_FAIL_WITH_MESSAGE("Server died!"); + return -1; + } + #endif -inline bool KillServer(int pid) -{ - TEST_THAT(SendCommands("terminate")); + if (!TestFileExists(pidFile)) + { + ::fprintf(stdout, "timed out!\n"); + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); + return -1; + } + + ::fprintf(stdout, "done.\n"); + + // wait a second for the pid to be written to the file ::sleep(1); - return !ServerIsAlive(); -} -#else // !WIN32 + // read pid file + int pid = ReadPidFile(pidFile); -inline bool ServerIsAlive(int pid) -{ - if(pid == 0) return false; - return ::kill(pid, 0) != -1; -} + #ifdef WIN32 + // On Win32 we can check whether the PID in the pidFile matches + // the one returned by the system, which it always should. -inline bool HUPServer(int pid) -{ - if(pid == 0) return false; - return ::kill(pid, SIGHUP) != -1; -} + if (pid != (int)procInfo.dwProcessId) + { + printf("Server wrote wrong pid to file (%s): expected %d " + "but found %d\n", pidFile, + (int)procInfo.dwProcessId, pid); + TEST_FAIL_WITH_MESSAGE("Server wrote wrong pid to file"); + return -1; + } + #endif -inline bool KillServer(int pid) -{ - if(pid == 0 || pid == -1) return false; - bool KilledOK = ::kill(pid, SIGTERM) != -1; - TEST_THAT(KilledOK); - ::sleep(1); - return !ServerIsAlive(pid); + return pid; } -#endif // WIN32 +#define TestRemoteProcessMemLeaks(filename) \ + TestRemoteProcessMemLeaksFunc(filename, __FILE__, __LINE__) -inline void TestRemoteProcessMemLeaks(const char *filename) +inline void TestRemoteProcessMemLeaksFunc(const char *filename, + const char* file, int line) { #ifdef BOX_MEMORY_LEAK_TESTING // Does the file exist? if(!TestFileExists(filename)) { + if (failures == 0) + { + first_fail_file = file; + first_fail_line = line; + } ++failures; - printf("FAILURE: MemLeak report not available (file %s)\n", filename); + printf("FAILURE: MemLeak report not available (file %s) " + "at %s:%d\n", filename, file, line); } else { // Is it empty? if(TestGetFileSize(filename) > 0) { + if (failures == 0) + { + first_fail_file = file; + first_fail_line = line; + } ++failures; - printf("FAILURE: Memory leaks found in other process (file %s)\n==========\n", filename); + printf("FAILURE: Memory leaks found in other process " + "(file %s) at %s:%d\n==========\n", + filename, file, line); FILE *f = fopen(filename, "r"); - char line[512]; - while(::fgets(line, sizeof(line), f) != 0) + char linebuf[512]; + while(::fgets(linebuf, sizeof(linebuf), f) != 0) { - printf("%s", line); + printf("%s", linebuf); } fclose(f); printf("==========\n"); @@ -302,5 +353,67 @@ inline void TestRemoteProcessMemLeaks(const char *filename) #endif } -#endif // TEST__H +#ifdef WIN32 +#define BBACKUPCTL "..\\..\\bin\\bbackupctl\\bbackupctl.exe" +#define BBACKUPD "..\\..\\bin\\bbackupd\\bbackupd.exe" +#define BBSTORED "..\\..\\bin\\bbstored\\bbstored.exe" +#define BBACKUPQUERY "..\\..\\bin\\bbackupquery\\bbackupquery.exe" +#define BBSTOREACCOUNTS "..\\..\\bin\\bbstoreaccounts\\bbstoreaccounts.exe" +#define TEST_RETURN(actual, expected) TEST_THAT(actual == expected); +#else +#define BBACKUPCTL "../../bin/bbackupctl/bbackupctl" +#define BBACKUPD "../../bin/bbackupd/bbackupd" +#define BBSTORED "../../bin/bbstored/bbstored" +#define BBACKUPQUERY "../../bin/bbackupquery/bbackupquery" +#define BBSTOREACCOUNTS "../../bin/bbstoreaccounts/bbstoreaccounts" +#define TEST_RETURN(actual, expected) TEST_THAT(actual == expected*256); +#endif + +inline void terminate_bbackupd(int pid) +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "terminate") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + + for (int i = 0; i < 20; i++) + { + if (!ServerIsAlive(pid)) break; + fprintf(stdout, "."); + fflush(stdout); + sleep(1); + } + + TEST_THAT(!ServerIsAlive(pid)); + TestRemoteProcessMemLeaks("bbackupd.memleaks"); +} + + +// Wait a given number of seconds for something to complete +inline void wait_for_operation(int seconds) +{ + printf("waiting: "); + fflush(stdout); + for(int l = 0; l < seconds; ++l) + { + sleep(1); + printf("."); + fflush(stdout); + } + printf("\n"); + fflush(stdout); +} + +inline void safe_sleep(int seconds) +{ +#ifdef WIN32 + Sleep(seconds * 1000); +#else + struct timespec ts; + ts.tv_sec = seconds; + ts.tv_nsec = 0; + while (nanosleep(&ts, &ts) == -1 && errno == EINTR) + { /* sleep again */ } +#endif +} +#endif // TEST__H diff --git a/lib/common/Timer.cpp b/lib/common/Timer.cpp new file mode 100644 index 00000000..a032f330 --- /dev/null +++ b/lib/common/Timer.cpp @@ -0,0 +1,386 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Timer.cpp +// Purpose: Generic timers which execute arbitrary code when +// they expire. +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <signal.h> + +#include "Timer.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +std::vector<Timer*>* Timers::spTimers = NULL; +bool Timers::sRescheduleNeeded = false; + +typedef void (*sighandler_t)(int); + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Init() +// Purpose: Initialise timers, prepare signal handler +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Init() +{ + ASSERT(!spTimers); + + #if defined WIN32 && ! defined PLATFORM_CYGWIN + // no support for signals at all + InitTimer(); + SetTimerHandler(Timers::SignalHandler); + #else + sighandler_t oldHandler = ::sigset(SIGALRM, + Timers::SignalHandler); + ASSERT(oldHandler == 0); + #endif // WIN32 && !PLATFORM_CYGWIN + + spTimers = new std::vector<Timer*>; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Cleanup() +// Purpose: Clean up timers, stop signal handler +// Created: 6/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Cleanup() +{ + ASSERT(spTimers); + + #if defined WIN32 && ! defined PLATFORM_CYGWIN + // no support for signals at all + FiniTimer(); + SetTimerHandler(NULL); + #else + struct itimerval timeout; + memset(&timeout, 0, sizeof(timeout)); + + int result = ::setitimer(ITIMER_REAL, &timeout, NULL); + ASSERT(result == 0); + + sighandler_t oldHandler = ::sigset(SIGALRM, NULL); + ASSERT(oldHandler == Timers::SignalHandler); + #endif // WIN32 && !PLATFORM_CYGWIN + + spTimers->clear(); + delete spTimers; + spTimers = NULL; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Add(Timer&) +// Purpose: Add a new timer to the set, and reschedule next wakeup +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Add(Timer& rTimer) +{ + ASSERT(spTimers); + ASSERT(&rTimer); + spTimers->push_back(&rTimer); + Reschedule(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Remove(Timer&) +// Purpose: Removes the timer from the set (preventing it from +// being called) and reschedule next wakeup +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Remove(Timer& rTimer) +{ + ASSERT(spTimers); + ASSERT(&rTimer); + + bool restart = true; + while (restart) + { + restart = false; + + for (std::vector<Timer*>::iterator i = spTimers->begin(); + i != spTimers->end(); i++) + { + if (&rTimer == *i) + { + spTimers->erase(i); + restart = true; + break; + } + } + } + + Reschedule(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::Reschedule() +// Purpose: Recalculate when the next wakeup is due +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::Reschedule() +{ + ASSERT(spTimers); + if (spTimers == NULL) + { + THROW_EXCEPTION(CommonException, Internal) + } + + #ifndef WIN32 + void (*oldhandler)(int) = ::sigset(SIGALRM, Timers::SignalHandler); + if (oldhandler != Timers::SignalHandler) + { + printf("Signal handler was %p, expected %p\n", + oldhandler, Timers::SignalHandler); + THROW_EXCEPTION(CommonException, Internal) + } + #endif + + // Clear the reschedule-needed flag to false before we start. + // If a timer event occurs while we are scheduling, then we + // may or may not need to reschedule again, but this way + // we will do it anyway. + sRescheduleNeeded = false; + + box_time_t timeNow = GetCurrentBoxTime(); + + // scan for, trigger and remove expired timers. Removal requires + // us to restart the scan each time, due to std::vector semantics. + bool restart = true; + while (restart) + { + restart = false; + + for (std::vector<Timer*>::iterator i = spTimers->begin(); + i != spTimers->end(); i++) + { + Timer& rTimer = **i; + int64_t timeToExpiry = rTimer.GetExpiryTime() - timeNow; + + if (timeToExpiry <= 0) + { + BOX_TRACE((int)(timeNow / 1000000) << "." << + (int)(timeNow % 1000000) << + ": timer " << *i << " has expired, " + "triggering it"); + rTimer.OnExpire(); + spTimers->erase(i); + restart = true; + break; + } + else + { + BOX_TRACE((int)(timeNow / 1000000) << "." << + (int)(timeNow % 1000000) << + ": timer " << *i << " has not " + "expired, triggering in " << + (int)(timeToExpiry / 1000000) << "." << + (int)(timeToExpiry % 1000000) << + " seconds"); + } + } + } + + // Now the only remaining timers should all be in the future. + // Scan to find the next one to fire (earliest deadline). + + int64_t timeToNextEvent = 0; + + for (std::vector<Timer*>::iterator i = spTimers->begin(); + i != spTimers->end(); i++) + { + Timer& rTimer = **i; + int64_t timeToExpiry = rTimer.GetExpiryTime() - timeNow; + + if (timeToExpiry <= 0) + { + timeToExpiry = 1; + } + + if (timeToNextEvent == 0 || timeToNextEvent > timeToExpiry) + { + timeToNextEvent = timeToExpiry; + } + } + + ASSERT(timeToNextEvent >= 0); + + struct itimerval timeout; + memset(&timeout, 0, sizeof(timeout)); + + timeout.it_value.tv_sec = BoxTimeToSeconds(timeToNextEvent); + timeout.it_value.tv_usec = (int) + (BoxTimeToMicroSeconds(timeToNextEvent) % MICRO_SEC_IN_SEC); + + if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) + { + BOX_ERROR("Failed to initialise timer\n"); + THROW_EXCEPTION(CommonException, Internal) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: static void Timers::SignalHandler(unused) +// Purpose: Called as signal handler. Nothing is safe in a signal +// handler, not even traversing the list of timers, so +// just request a reschedule in future, which will do +// that for us, and trigger any expired timers at that +// time. +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- +void Timers::SignalHandler(int iUnused) +{ + // ASSERT(spTimers); + Timers::RequestReschedule(); +} + +Timer::Timer(size_t timeoutSecs) +: mExpires(GetCurrentBoxTime() + SecondsToBoxTime(timeoutSecs)), + mExpired(false) +{ + #if !defined NDEBUG && !defined WIN32 + struct timeval tv; + gettimeofday(&tv, NULL); + if (timeoutSecs == 0) + { + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " initialised for " << + timeoutSecs << " secs, will not fire"); + } + else + { + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " initialised for " << + timeoutSecs << " secs, to fire at " << + (int)(mExpires / 1000000) << "." << + (int)(mExpires % 1000000)); + } + #endif + + if (timeoutSecs == 0) + { + mExpires = 0; + } + else + { + Timers::Add(*this); + } +} + +Timer::~Timer() +{ + #if !defined NDEBUG && !defined WIN32 + struct timeval tv; + gettimeofday(&tv, NULL); + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " destroyed"); + #endif + + Timers::Remove(*this); +} + +Timer::Timer(const Timer& rToCopy) +: mExpires(rToCopy.mExpires), + mExpired(rToCopy.mExpired) +{ + #if !defined NDEBUG && !defined WIN32 + struct timeval tv; + gettimeofday(&tv, NULL); + if (mExpired) + { + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " initialised from timer " << + &rToCopy << ", already expired, will not fire"); + } + else if (mExpires == 0) + { + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " initialised from timer " << + &rToCopy << ", no expiry, will not fire"); + } + else + { + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " initialised from timer " << + &rToCopy << " to fire at " << + (int)(mExpires / 1000000) << "." << + (int)(mExpires % 1000000)); + } + #endif + + if (!mExpired && mExpires != 0) + { + Timers::Add(*this); + } +} + +Timer& Timer::operator=(const Timer& rToCopy) +{ + #if !defined NDEBUG && !defined WIN32 + struct timeval tv; + gettimeofday(&tv, NULL); + if (rToCopy.mExpired) + { + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " initialised from timer " << + &rToCopy << ", already expired, will not fire"); + } + else if (rToCopy.mExpires == 0) + { + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " initialised from timer " << + &rToCopy << ", no expiry, will not fire"); + } + else + { + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " initialised from timer " << + &rToCopy << " to fire at " << + (int)(rToCopy.mExpires / 1000000) << "." << + (int)(rToCopy.mExpires % 1000000)); + } + #endif + + Timers::Remove(*this); + mExpires = rToCopy.mExpires; + mExpired = rToCopy.mExpired; + if (!mExpired && mExpires != 0) + { + Timers::Add(*this); + } + return *this; +} + +void Timer::OnExpire() +{ + #if !defined NDEBUG && !defined WIN32 + struct timeval tv; + gettimeofday(&tv, NULL); + BOX_TRACE(tv.tv_sec << "." << tv.tv_usec << + ": timer " << this << " fired"); + #endif + + mExpired = true; +} diff --git a/lib/common/Timer.h b/lib/common/Timer.h new file mode 100644 index 00000000..ba6d71f4 --- /dev/null +++ b/lib/common/Timer.h @@ -0,0 +1,87 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Timer.h +// Purpose: Generic timers which execute arbitrary code when +// they expire. +// Created: 5/11/2006 +// +// -------------------------------------------------------------------------- + +#ifndef TIMER__H +#define TIMER__H + +#ifdef HAVE_SYS_TIME_H + #include <sys/time.h> +#endif + +#include <vector> + +#include "BoxTime.h" + +#include "MemLeakFindOn.h" + +class Timer; + +// -------------------------------------------------------------------------- +// +// Class +// Name: Timers +// Purpose: Static class to manage all timers and arrange +// efficient delivery of wakeup signals +// Created: 19/3/04 +// +// -------------------------------------------------------------------------- +class Timers +{ + private: + static std::vector<Timer*>* spTimers; + static void Reschedule(); + + static bool sRescheduleNeeded; + static void SignalHandler(int iUnused); + + public: + static void Init(); + static void Cleanup(); + static void Add (Timer& rTimer); + static void Remove(Timer& rTimer); + static void RequestReschedule() + { + sRescheduleNeeded = true; + } + + static void RescheduleIfNeeded() + { + if (sRescheduleNeeded) + { + Reschedule(); + } + } +}; + +class Timer +{ +public: + Timer(size_t timeoutSecs); + virtual ~Timer(); + Timer(const Timer &); + Timer &operator=(const Timer &); + +public: + box_time_t GetExpiryTime() { return mExpires; } + virtual void OnExpire(); + bool HasExpired() + { + Timers::RescheduleIfNeeded(); + return mExpired; + } + +private: + box_time_t mExpires; + bool mExpired; +}; + +#include "MemLeakFindOff.h" + +#endif // TIMER__H diff --git a/lib/common/UnixUser.cpp b/lib/common/UnixUser.cpp index 5419d8f8..f81b474c 100644 --- a/lib/common/UnixUser.cpp +++ b/lib/common/UnixUser.cpp @@ -69,7 +69,7 @@ UnixUser::UnixUser(uid_t UID, gid_t GID) // // Function // Name: UnixUser::~UnixUser() -// Purpose: Destructor -- reverts to previous user if the change wasn't permanant +// Purpose: Destructor -- reverts to previous user if the change wasn't perminant // Created: 21/1/04 // // -------------------------------------------------------------------------- @@ -78,8 +78,7 @@ UnixUser::~UnixUser() if(mRevertOnDestruction) { // Revert to "real" user and group id of the process - if(::setegid(::getgid()) != 0 - || ::seteuid(::getuid()) != 0) + if(::setegid(::getgid()) != 0 || ::seteuid(::getuid()) != 0) { THROW_EXCEPTION(CommonException, CouldNotRestoreProcessUser) } @@ -101,8 +100,7 @@ void UnixUser::ChangeProcessUser(bool Temporary) if(Temporary) { // Change temporarily (change effective only) - if(::setegid(mGID) != 0 - || ::seteuid(mUID) != 0) + if(::setegid(mGID) != 0 || ::seteuid(mUID) != 0) { THROW_EXCEPTION(CommonException, CouldNotChangeProcessUser) } @@ -113,8 +111,7 @@ void UnixUser::ChangeProcessUser(bool Temporary) else { // Change permanently (change all UIDs and GIDs) - if(::setgid(mGID) != 0 - || ::setuid(mUID) != 0) + if(::setgid(mGID) != 0 || ::setuid(mUID) != 0) { THROW_EXCEPTION(CommonException, CouldNotChangeProcessUser) } diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp index 63d45b5a..5dc03124 100644 --- a/lib/common/Utils.cpp +++ b/lib/common/Utils.cpp @@ -20,6 +20,7 @@ #include "Utils.h" #include "CommonException.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -71,14 +72,16 @@ void DumpStackBacktrace() size = backtrace (array, 10); strings = backtrace_symbols (array, size); - printf ("Obtained %zd stack frames.\n", size); + BOX_TRACE("Obtained " << size << " stack frames."); for(i = 0; i < size; i++) - printf("%s\n", strings[i]); + { + BOX_TRACE(strings[i]); + } -#ifndef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED +#include "MemLeakFindOff.h" free (strings); -#endif +#include "MemLeakFindOn.h" } #endif diff --git a/lib/common/ZeroStream.cpp b/lib/common/ZeroStream.cpp new file mode 100644 index 00000000..9d87d76a --- /dev/null +++ b/lib/common/ZeroStream.cpp @@ -0,0 +1,170 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ZeroStream.cpp +// Purpose: An IOStream which returns all zeroes up to a certain size +// Created: 2007/04/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "ZeroStream.h" +#include "CommonException.h" + +#include <string.h> + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::ZeroStream(IOStream::pos_type) +// Purpose: Constructor +// Created: 2007/04/28 +// +// -------------------------------------------------------------------------- +ZeroStream::ZeroStream(IOStream::pos_type size) +: mSize(size), mPosition(0) +{ } + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::Read(void *, int) +// Purpose: Reads bytes from the file +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +int ZeroStream::Read(void *pBuffer, int NBytes, int Timeout) +{ + ASSERT(NBytes > 0); + + int bytesToRead = NBytes; + + if (bytesToRead > mSize - mPosition) + { + bytesToRead = mSize - mPosition; + } + + memset(pBuffer, 0, bytesToRead); + mPosition += bytesToRead; + + return bytesToRead; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::BytesLeftToRead() +// Purpose: Returns number of bytes to read (may not be most efficient function ever) +// Created: 2007/01/16 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ZeroStream::BytesLeftToRead() +{ + return mSize - mPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::Write(void *, int) +// Purpose: Writes bytes to the underlying stream (not supported) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ZeroStream::Write(const void *pBuffer, int NBytes) +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::GetPosition() +// Purpose: Get position in stream +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +IOStream::pos_type ZeroStream::GetPosition() const +{ + return mPosition; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::Seek(pos_type, int) +// Purpose: Seeks within file, as lseek, invalidate buffer +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ZeroStream::Seek(IOStream::pos_type Offset, int SeekType) +{ + switch (SeekType) + { + case SeekType_Absolute: + { + mPosition = Offset; + } + break; + + case SeekType_Relative: + { + mPosition += Offset; + } + break; + + case SeekType_End: + { + mPosition = mSize - Offset; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::Close() +// Purpose: Closes the underlying stream (not needed) +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +void ZeroStream::Close() +{ + THROW_EXCEPTION(CommonException, NotSupported); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::StreamDataLeft() +// Purpose: Any data left to write? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool ZeroStream::StreamDataLeft() +{ + return false; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ZeroStream::StreamClosed() +// Purpose: Is the stream closed? +// Created: 2003/08/02 +// +// -------------------------------------------------------------------------- +bool ZeroStream::StreamClosed() +{ + return false; +} + diff --git a/lib/common/ZeroStream.h b/lib/common/ZeroStream.h new file mode 100644 index 00000000..0119045b --- /dev/null +++ b/lib/common/ZeroStream.h @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: ZeroStream.h +// Purpose: An IOStream which returns all zeroes up to a certain size +// Created: 2007/04/28 +// +// -------------------------------------------------------------------------- + +#ifndef ZEROSTREAM__H +#define ZEROSTREAM__H + +#include "IOStream.h" + +class ZeroStream : public IOStream +{ +private: + IOStream::pos_type mSize, mPosition; + +public: + ZeroStream(IOStream::pos_type mSize); + + virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); + virtual pos_type BytesLeftToRead(); + virtual void Write(const void *pBuffer, int NBytes); + virtual pos_type GetPosition() const; + virtual void Seek(IOStream::pos_type Offset, int SeekType); + virtual void Close(); + + virtual bool StreamDataLeft(); + virtual bool StreamClosed(); + +private: + ZeroStream(const ZeroStream &rToCopy); +}; + +#endif // ZEROSTREAM__H + + diff --git a/lib/crypto/Random.cpp b/lib/crypto/Random.cpp index 30049ff7..1d6a07f0 100644 --- a/lib/crypto/Random.cpp +++ b/lib/crypto/Random.cpp @@ -34,7 +34,8 @@ void Random::Initialise() THROW_EXCEPTION(CipherException, RandomInitFailed) } #else - ::fprintf(stderr, "No random device -- additional seeding of random number generator not performed.\n"); + BOX_ERROR("No random device -- additional seeding of random number " + "generator not performed."); #endif } diff --git a/test/raidfile/intercept.cpp b/lib/intercept/intercept.cpp index 6df344e1..a93a5a3b 100644 --- a/test/raidfile/intercept.cpp +++ b/lib/intercept/intercept.cpp @@ -9,14 +9,24 @@ #include "Box.h" +#include "intercept.h" + #ifdef HAVE_SYS_SYSCALL_H #include <sys/syscall.h> #endif #include <sys/types.h> #include <unistd.h> -#include <sys/uio.h> + +#ifdef HAVE_SYS_UIO_H + #include <sys/uio.h> +#endif + #include <errno.h> +#ifdef HAVE_DLFCN_H +#include <dlfcn.h> +#endif + #ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE #if !defined(HAVE_SYSCALL) && !defined(HAVE___SYSCALL) && !defined(HAVE___SYSCALL_NEED_DEFN) @@ -55,43 +65,61 @@ #include "MemLeakFindOn.h" -bool intercept_enabled = false; +int intercept_count = 0; const char *intercept_filename = 0; int intercept_filedes = -1; off_t intercept_errorafter = 0; int intercept_errno = 0; int intercept_syscall = 0; off_t intercept_filepos = 0; +int intercept_delay_ms = 0; #define SIZE_ALWAYS_ERROR -773 void intercept_clear_setup() { - intercept_enabled = false; + intercept_count = 0; intercept_filename = 0; intercept_filedes = -1; intercept_errorafter = 0; intercept_syscall = 0; intercept_filepos = 0; + intercept_delay_ms = 0; } bool intercept_triggered() { - return !intercept_enabled; + return intercept_count == 0; } void intercept_setup_error(const char *filename, unsigned int errorafter, int errortoreturn, int syscalltoerror) { TRACE4("Setup for error: %s, after %d, err %d, syscall %d\n", filename, errorafter, errortoreturn, syscalltoerror); - intercept_enabled = true; + intercept_count = 1; intercept_filename = filename; intercept_filedes = -1; intercept_errorafter = errorafter; intercept_syscall = syscalltoerror; intercept_errno = errortoreturn; intercept_filepos = 0; + intercept_delay_ms = 0; } +void intercept_setup_delay(const char *filename, unsigned int delay_after, + int delay_ms, int syscall_to_delay, int num_delays) +{ + TRACE5("Setup for delay: %s, after %d, wait %d ms, times %d, " + "syscall %d\n", filename, delay_after, delay_ms, + num_delays, syscall_to_delay); + intercept_count = num_delays; + intercept_filename = filename; + intercept_filedes = -1; + intercept_errorafter = delay_after; + intercept_syscall = syscall_to_delay; + intercept_errno = 0; + intercept_filepos = 0; + intercept_delay_ms = delay_ms; +} bool intercept_errornow(int d, int size, int syscallnum) { if(intercept_filedes != -1 && d == intercept_filedes && syscallnum == intercept_syscall) @@ -106,7 +134,19 @@ bool intercept_errornow(int d, int size, int syscallnum) // where are we in the file? if(intercept_filepos >= intercept_errorafter || intercept_filepos >= ((off_t)intercept_errorafter - size)) { - TRACE3("Returning error %d for syscall %d, file pos %d\n", intercept_errno, syscallnum, (int)intercept_filepos); + if (intercept_errno != 0) + { + TRACE3("Returning error %d for syscall %d, " + "file pos %d\n", intercept_errno, + syscallnum, (int)intercept_filepos); + } + else if (intercept_delay_ms != 0) + { + TRACE3("Delaying %d ms for syscall %d, " + "file pos %d\n", intercept_delay_ms, + syscallnum, (int)intercept_filepos); + } + return true; } } @@ -115,28 +155,42 @@ bool intercept_errornow(int d, int size, int syscallnum) int intercept_reterr() { - intercept_enabled = false; - intercept_filename = 0; - intercept_filedes = -1; - intercept_errorafter = 0; - intercept_syscall = 0; - return intercept_errno; + int err = intercept_errno; + intercept_clear_setup(); + return err; } #define CHECK_FOR_FAKE_ERROR_COND(D, S, CALL, FAILRES) \ - if(intercept_enabled) \ + if(intercept_count > 0) \ { \ if(intercept_errornow(D, S, CALL)) \ { \ - errno = intercept_reterr(); \ - return FAILRES; \ + if(intercept_delay_ms > 0) \ + { \ + struct timespec tm; \ + tm.tv_sec = intercept_delay_ms / 1000; \ + tm.tv_nsec = (intercept_delay_ms % 1000) \ + * 1000000; \ + while (nanosleep(&tm, &tm) != 0 && \ + errno == EINTR) { } \ + intercept_count --; \ + if (intercept_count == 0) \ + { \ + intercept_clear_setup(); \ + } \ + } \ + else \ + { \ + errno = intercept_reterr(); \ + return FAILRES; \ + } \ } \ } extern "C" int open(const char *path, int flags, mode_t mode) { - if(intercept_enabled) + if(intercept_count > 0) { if(intercept_syscall == SYS_open && strcmp(path, intercept_filename) == 0) { @@ -149,7 +203,7 @@ open(const char *path, int flags, mode_t mode) #else int r = syscall(SYS_open, path, flags, mode); #endif - if(intercept_enabled && intercept_filedes == -1) + if(intercept_count > 0 && intercept_filedes == -1) { // Right file? if(strcmp(intercept_filename, path) == 0) @@ -269,4 +323,173 @@ lseek(int fildes, off_t offset, int whence) return r; } +static opendir_t* opendir_real = NULL; +static readdir_t* readdir_real = NULL; +static readdir_t* readdir_hook = NULL; +static closedir_t* closedir_real = NULL; +static lstat_t* lstat_real = NULL; +static lstat_t* lstat_hook = NULL; +static const char* lstat_file = NULL; + +void intercept_setup_readdir_hook(const char *dirname, readdir_t hookfn) +{ + if (hookfn != NULL && dirname == NULL) + { + dirname = intercept_filename; + } + + if (hookfn != NULL) + { + TRACE2("readdir hooked to %p for %s\n", hookfn, dirname); + } + else + { + TRACE2("readdir unhooked from %p for %s\n", readdir_hook, + intercept_filename); + } + + intercept_filename = dirname; + readdir_hook = hookfn; +} + +void intercept_setup_lstat_hook(const char *filename, lstat_t hookfn) +{ + /* + if (hookfn != NULL) + { + TRACE2("lstat hooked to %p for %s\n", hookfn, filename); + } + else + { + TRACE2("lstat unhooked from %p for %s\n", lstat_hook, + lstat_file); + } + */ + + lstat_file = filename; + lstat_hook = hookfn; +} + +extern "C" +DIR *opendir(const char *dirname) +{ + if (opendir_real == NULL) + { + opendir_real = (opendir_t*)(dlsym(RTLD_NEXT, "opendir")); + } + + if (opendir_real == NULL) + { + perror("cannot find real opendir"); + return NULL; + } + + DIR* r = opendir_real(dirname); + + if(readdir_hook != NULL && intercept_filedes == -1 && + strcmp(intercept_filename, dirname) == 0) + { + intercept_filedes = dirfd(r); + //printf("Found file to intercept, h = %d\n", r); + } + + return r; +} + +extern "C" +struct dirent *readdir(DIR *dir) +{ + if (readdir_hook != NULL && dirfd(dir) == intercept_filedes) + { + return readdir_hook(dir); + } + + if (readdir_real == NULL) + { + #if readdir == readdir64 + readdir_real = (readdir_t*)(dlsym(RTLD_NEXT, "readdir64")); + #else + readdir_real = (readdir_t*)(dlsym(RTLD_NEXT, "readdir")); + #endif + } + + if (readdir_real == NULL) + { + perror("cannot find real readdir"); + return NULL; + } + + return readdir_real(dir); +} + +extern "C" +int closedir(DIR *dir) +{ + if (dirfd(dir) == intercept_filedes) + { + intercept_filedes = -1; + } + + if (closedir_real == NULL) + { + closedir_real = (closedir_t*)(dlsym(RTLD_NEXT, "closedir")); + } + + if (closedir_real == NULL) + { + perror("cannot find real closedir"); + errno = ENOSYS; + return -1; + } + + return closedir_real(dir); +} + +extern "C" int +#ifdef LINUX_WEIRD_LSTAT +__lxstat(int ver, const char *file_name, STAT_STRUCT *buf) +#else +lstat(const char *file_name, STAT_STRUCT *buf) +#endif +{ + if (lstat_real == NULL) + { + #ifdef LINUX_WEIRD_LSTAT + #if __lxstat == __lxstat64 + lstat_real = (lstat_t*)(dlsym(RTLD_NEXT, "__lxstat64")); + #else + lstat_real = (lstat_t*)(dlsym(RTLD_NEXT, "__lxstat")); + #endif + #else + #if lstat == lstat64 + lstat_real = (lstat_t*)(dlsym(RTLD_NEXT, "lstat64")); + #else + lstat_real = (lstat_t*)(dlsym(RTLD_NEXT, "lstat")); + #endif + #endif + } + + if (lstat_real == NULL) + { + perror("cannot find real lstat"); + errno = ENOSYS; + return -1; + } + + if (lstat_hook == NULL || strcmp(file_name, lstat_file) != 0) + { + #ifdef LINUX_WEIRD_LSTAT + return lstat_real(ver, file_name, buf); + #else + return lstat_real(file_name, buf); + #endif + } + + #ifdef LINUX_WEIRD_LSTAT + return lstat_hook(ver, file_name, buf); + #else + return lstat_hook(file_name, buf); + #endif +} + #endif // n PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE diff --git a/lib/intercept/intercept.h b/lib/intercept/intercept.h new file mode 100644 index 00000000..bc6557f3 --- /dev/null +++ b/lib/intercept/intercept.h @@ -0,0 +1,47 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: intercept.h +// Purpose: Syscall interception code for unit tests +// Created: 2006/11/29 +// +// -------------------------------------------------------------------------- + +#ifndef INTERCEPT_H +#define INTERCEPT_H +#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + +#include <dirent.h> + +#include <sys/types.h> +#include <sys/stat.h> + +extern "C" +{ + typedef DIR *(opendir_t) (const char *name); + typedef struct dirent *(readdir_t) (DIR *dir); + typedef struct dirent *(readdir_t) (DIR *dir); + typedef int (closedir_t)(DIR *dir); +#if defined __GNUC__ && __GNUC__ >= 2 +#define LINUX_WEIRD_LSTAT +#define STAT_STRUCT struct stat /* should be stat64 */ + typedef int (lstat_t) (int ver, const char *file_name, + STAT_STRUCT *buf); +#else +#define STAT_STRUCT struct stat + typedef int (lstat_t) (const char *file_name, + STAT_STRUCT *buf); +#endif +} + +void intercept_setup_error(const char *filename, unsigned int errorafter, + int errortoreturn, int syscalltoerror); +void intercept_setup_delay(const char *filename, unsigned int delay_after, + int delay_ms, int syscall_to_delay, int num_delays); +bool intercept_triggered(); + +void intercept_setup_readdir_hook(const char *dirname, readdir_t hookfn); +void intercept_setup_lstat_hook (const char *filename, lstat_t hookfn); + +#endif // !PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE +#endif // !INTERCEPT_H diff --git a/lib/raidfile/RaidFileController.cpp b/lib/raidfile/RaidFileController.cpp index 89ae6791..0cc2ede7 100644 --- a/lib/raidfile/RaidFileController.cpp +++ b/lib/raidfile/RaidFileController.cpp @@ -59,13 +59,15 @@ RaidFileController::RaidFileController(const RaidFileController &rController) // -------------------------------------------------------------------------- // // Function -// Name: RaidFileController::Initialise(const char *) +// Name: RaidFileController::Initialise(const std::string&) // Purpose: Initialises the system, loading the configuration file. // Created: 2003/07/08 // // -------------------------------------------------------------------------- -void RaidFileController::Initialise(const char *ConfigFilename) +void RaidFileController::Initialise(const std::string& rConfigFilename) { + MEMLEAKFINDER_NO_LEAKS; + static const ConfigurationVerifyKey verifykeys[] = { {"SetNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, @@ -95,11 +97,12 @@ void RaidFileController::Initialise(const char *ConfigFilename) // Load the configuration std::string err; - std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(ConfigFilename, &verify, err); + std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify( + rConfigFilename, &verify, err); if(pconfig.get() == 0 || !err.empty()) { - fprintf(stderr, "RaidFile configuation file errors:\n%s", err.c_str()); + BOX_ERROR("RaidFile configuration file errors: " << err); THROW_EXCEPTION(RaidFileException, BadConfigFile) } diff --git a/lib/raidfile/RaidFileController.h b/lib/raidfile/RaidFileController.h index 4962d236..783cb055 100644 --- a/lib/raidfile/RaidFileController.h +++ b/lib/raidfile/RaidFileController.h @@ -81,7 +81,8 @@ public: ~RaidFileController(); public: - void Initialise(const char *ConfigFilename = "/etc/box/raidfile.conf"); + void Initialise(const std::string& rConfigFilename = + "/etc/box/raidfile.conf"); int GetNumDiscSets() {return mSetList.size();} // -------------------------------------------------------------------------- diff --git a/lib/raidfile/RaidFileException.txt b/lib/raidfile/RaidFileException.txt index 6ad74563..c69dc2a2 100644 --- a/lib/raidfile/RaidFileException.txt +++ b/lib/raidfile/RaidFileException.txt @@ -11,7 +11,7 @@ NotOpen 7 OSError 8 Error when accessing an underlying file. Check file permissions allow files to be read and written in the configured raid directories. WriteFileOpenOnTransform 9 WrongNumberOfDiscsInSet 10 There should be three directories in each disc set. -RaidFileDoesntExist 11 +RaidFileDoesntExist 11 Error when accessing a file on the store. Check the store with bbstoreaccounts check. ErrorOpeningFileForRead 12 FileIsDamagedNotRecoverable 13 InvalidRaidFile 14 diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp index ad040c22..187270f9 100644 --- a/lib/raidfile/RaidFileRead.cpp +++ b/lib/raidfile/RaidFileRead.cpp @@ -9,21 +9,18 @@ #include "Box.h" -#include <sys/types.h> -#include <unistd.h> -#include <fcntl.h> #include <errno.h> -#include <sys/stat.h> +#include <fcntl.h> #include <stdarg.h> +#include <unistd.h> + +#include <sys/stat.h> +#include <sys/types.h> #ifdef HAVE_SYS_UIO_H #include <sys/uio.h> #endif -#ifdef HAVE_SYSLOG_H - #include <syslog.h> -#endif - #ifdef HAVE_DIRENT_H #include <dirent.h> #endif @@ -41,7 +38,10 @@ #include "MemLeakFindOn.h" #define READ_NUMBER_DISCS_REQUIRED 3 -#define READV_MAX_BLOCKS 64 +#define READV_MAX_BLOCKS 64 + +// We want to use POSIX fstat() for now, not the emulated one +#undef fstat // -------------------------------------------------------------------------- // @@ -544,8 +544,8 @@ void RaidFileRead_Raid::MoveDamagedFileAlertDaemon(int SetNumber, const std::str // -------------------------------------------------------------------------- void RaidFileRead_Raid::AttemptToRecoverFromIOError(bool Stripe1) { - TRACE3("Attempting to recover from I/O error: %d %s, on stripe %d\n", mSetNumber, mFilename.c_str(), Stripe1?1:2); - ::syslog(LOG_ERR | LOG_LOCAL5, "Attempting to recover from I/O error: %d %s, on stripe %d\n", mSetNumber, mFilename.c_str(), Stripe1?1:2); + BOX_WARNING("Attempting to recover from I/O error: " << mSetNumber << + " " << mFilename << ", on stripe " << (Stripe1?1:2)); // Close offending file if(Stripe1) @@ -857,8 +857,10 @@ void RaidFileRead_Raid::SetPosition(pos_type FilePosition) { if(errno == EIO) { - TRACE3("I/O error when seeking in %d %s (to %d), stripe 1\n", mSetNumber, mFilename.c_str(), (int)FilePosition); - ::syslog(LOG_ERR | LOG_LOCAL5, "I/O error when seeking in %d %s (to %d), stripe 1\n", mSetNumber, mFilename.c_str(), (int)FilePosition); + BOX_ERROR("I/O error when seeking in " << + mSetNumber << " " << mFilename << + " (to " << FilePosition << "), " << + "stripe 1"); // Attempt to recover AttemptToRecoverFromIOError(true /* is stripe 1 */); ASSERT(mStripe1Handle == -1); @@ -875,8 +877,10 @@ void RaidFileRead_Raid::SetPosition(pos_type FilePosition) { if(errno == EIO) { - TRACE3("I/O error when seeking in %d %s (to %d), stripe 2\n", mSetNumber, mFilename.c_str(), (int)FilePosition); - ::syslog(LOG_ERR | LOG_LOCAL5, "I/O error when seeking in %d %s (to %d), stripe 2\n", mSetNumber, mFilename.c_str(), (int)FilePosition); + BOX_ERROR("I/O error when seeking in " << + mSetNumber << " " << mFilename << + " (to " << FilePosition << "), " << + "stripe 2"); // Attempt to recover AttemptToRecoverFromIOError(false /* is stripe 2 */); ASSERT(mStripe2Handle == -1); @@ -1050,8 +1054,9 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string { if(existance != RaidFileUtil::AsRaid) { - TRACE2("Opening %d %s in normal mode, but parity file doesn't exist\n", SetNumber, Filename.c_str()); - ::syslog(LOG_ERR | LOG_LOCAL5, "Opening %d %s in normal mode, but parity file doesn't exist\n", SetNumber, Filename.c_str()); + BOX_ERROR("Opening " << SetNumber << " " << + Filename << " in normal mode, but " + "parity file doesn't exist"); // TODO: Alert recovery daemon } @@ -1126,8 +1131,9 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string bool oktotryagain = true; if(stripe1errno == EIO) { - TRACE2("I/O error on opening %d %s stripe 1, trying recovery mode\n", SetNumber, Filename.c_str()); - ::syslog(LOG_ERR | LOG_LOCAL5, "I/O error on opening %d %s stripe 1, trying recovery mode\n", SetNumber, Filename.c_str()); + BOX_ERROR("I/O error on opening " << + SetNumber << " " << Filename << + " stripe 1, trying recovery mode"); RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, true /* is stripe 1 */); existingFiles = existingFiles & ~RaidFileUtil::Stripe1Exists; @@ -1142,8 +1148,9 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string if(stripe2errno == EIO) { - TRACE2("I/O error on opening %d %s stripe 2, trying recovery mode\n", SetNumber, Filename.c_str()); - ::syslog(LOG_ERR | LOG_LOCAL5, "I/O error on opening %d %s stripe 2, trying recovery mode\n", SetNumber, Filename.c_str()); + BOX_ERROR("I/O error on opening " << + SetNumber << " " << Filename << + " stripe 2, trying recovery mode"); RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, false /* is stripe 2 */); existingFiles = existingFiles & ~RaidFileUtil::Stripe2Exists; @@ -1165,8 +1172,10 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string if(existance == RaidFileUtil::AsRaidWithMissingReadable) { - TRACE3("Attempting to open RAID file %d %s in recovery mode (stripe %d present)\n", SetNumber, Filename.c_str(), (existingFiles & RaidFileUtil::Stripe1Exists)?1:2); - ::syslog(LOG_ERR | LOG_LOCAL5, "Attempting to open RAID file %d %s in recovery mode (stripe %d present)\n", SetNumber, Filename.c_str(), (existingFiles & RaidFileUtil::Stripe1Exists)?1:2); + BOX_ERROR("Attempting to open RAID file " << SetNumber << + " " << Filename << " in recovery mode (stripe " << + ((existingFiles & RaidFileUtil::Stripe1Exists)?1:2) << + " present)"); // Generate the filenames of all the lovely files std::string stripe1Filename(RaidFileUtil::MakeRaidComponentName(rdiscSet, Filename, (0 + startDisc) % READ_NUMBER_DISCS_REQUIRED)); diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp index e30162fa..66ab81c8 100644 --- a/lib/raidfile/RaidFileWrite.cpp +++ b/lib/raidfile/RaidFileWrite.cpp @@ -35,6 +35,9 @@ // Must have this number of discs in the set #define TRANSFORM_NUMBER_DISCS_REQUIRED 3 +// we want to use POSIX fstat() for now, not the emulated one +#undef fstat + // -------------------------------------------------------------------------- // // Function @@ -104,7 +107,8 @@ void RaidFileWrite::Open(bool AllowOverwrite) writeFilename += 'X'; // Attempt to open - mOSFileHandle = ::open(writeFilename.c_str(), O_WRONLY | O_CREAT, + mOSFileHandle = ::open(writeFilename.c_str(), + O_WRONLY | O_CREAT | O_BINARY, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); if(mOSFileHandle == -1) { @@ -115,7 +119,7 @@ void RaidFileWrite::Open(bool AllowOverwrite) #ifdef HAVE_FLOCK int errnoBlock = EWOULDBLOCK; if(::flock(mOSFileHandle, LOCK_EX | LOCK_NB) != 0) -#else +#elif HAVE_DECL_F_SETLK int errnoBlock = EAGAIN; struct flock desc; desc.l_type = F_WRLCK; @@ -123,6 +127,9 @@ void RaidFileWrite::Open(bool AllowOverwrite) desc.l_start = 0; desc.l_len = 0; if(::fcntl(mOSFileHandle, F_SETLK, &desc) != 0) +#else + int errnoBlock = ENOSYS; + if (0) #endif { // Lock was not obtained. @@ -242,23 +249,46 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) } // Rename it into place -- BEFORE it's closed so lock remains + +#ifdef WIN32 + // Except on Win32 which doesn't allow renaming open files + // Close file... + if(::close(mOSFileHandle) != 0) + { + THROW_EXCEPTION(RaidFileException, OSError) + } + mOSFileHandle = -1; +#endif // WIN32 + RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); // Get the filename for the write file std::string renameTo(RaidFileUtil::MakeWriteFileName(rdiscSet, mFilename)); // And the current name std::string renameFrom(renameTo + 'X'); + +#ifdef WIN32 + // need to delete the target first + if(::unlink(renameTo.c_str()) != 0 && + GetLastError() != ERROR_FILE_NOT_FOUND) + { + THROW_EXCEPTION(RaidFileException, OSError) + } +#endif + if(::rename(renameFrom.c_str(), renameTo.c_str()) != 0) { THROW_EXCEPTION(RaidFileException, OSError) } +#ifndef WIN32 // Close file... if(::close(mOSFileHandle) != 0) { THROW_EXCEPTION(RaidFileException, OSError) } mOSFileHandle = -1; +#endif // !WIN32 // Raid it? if(ConvertToRaidNow) @@ -292,8 +322,15 @@ void RaidFileWrite::Discard() writeFilename += 'X'; // Unlink and close it - if((::unlink(writeFilename.c_str()) != 0) - || (::close(mOSFileHandle) != 0)) + +#ifdef WIN32 + // On Win32 we must close it first + if (::close(mOSFileHandle) != 0 || + ::unlink(writeFilename.c_str()) != 0) +#else // !WIN32 + if (::unlink(writeFilename.c_str()) != 0 || + ::close(mOSFileHandle) != 0) +#endif // !WIN32 { THROW_EXCEPTION(RaidFileException, OSError) } @@ -388,13 +425,13 @@ void RaidFileWrite::TransformToRaidStorage() try { #if HAVE_DECL_O_EXLOCK - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> stripe1(stripe1FilenameW.c_str()); - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> stripe2(stripe2FilenameW.c_str()); - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK)> parity(parityFilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK | O_BINARY)> stripe1(stripe1FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK | O_BINARY)> stripe2(stripe2FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_EXLOCK | O_BINARY)> parity(parityFilenameW.c_str()); #else - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> stripe1(stripe1FilenameW.c_str()); - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> stripe2(stripe2FilenameW.c_str()); - FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL)> parity(parityFilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_BINARY)> stripe1(stripe1FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_BINARY)> stripe2(stripe2FilenameW.c_str()); + FileHandleGuard<(O_WRONLY | O_CREAT | O_EXCL | O_BINARY)> parity(parityFilenameW.c_str()); #endif // Then... read in data... @@ -530,6 +567,21 @@ void RaidFileWrite::TransformToRaidStorage() parity.Close(); stripe2.Close(); stripe1.Close(); + +#ifdef WIN32 + // Must delete before renaming + #define CHECK_UNLINK(file) \ + { \ + if (::unlink(file) != 0 && errno != ENOENT) \ + { \ + THROW_EXCEPTION(RaidFileException, OSError); \ + } \ + } + CHECK_UNLINK(stripe1Filename.c_str()); + CHECK_UNLINK(stripe2Filename.c_str()); + CHECK_UNLINK(parityFilename.c_str()); + #undef CHECK_UNLINK +#endif // Rename them into place if(::rename(stripe1FilenameW.c_str(), stripe1Filename.c_str()) != 0 diff --git a/lib/server/ConnectionException.txt b/lib/server/ConnectionException.txt index 5056754f..c3429116 100644 --- a/lib/server/ConnectionException.txt +++ b/lib/server/ConnectionException.txt @@ -10,7 +10,7 @@ SocketConnectError 15 Probably a network issue between client and server, bad TLSHandshakeFailed 30 TLSShutdownFailed 32 TLSWriteFailed 33 Probably a network issue between client and server. -TLSReadFailed 34 Probably a network issue between client and server. +TLSReadFailed 34 Probably a network issue between client and server, or a problem with the server. TLSNoPeerCertificate 36 TLSPeerCertificateInvalid 37 Check certification process TLSClosedWhenWriting 38 diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp index 2131bdcb..a9eb5bf5 100644 --- a/lib/server/Daemon.cpp +++ b/lib/server/Daemon.cpp @@ -19,10 +19,6 @@ #include <string.h> #include <stdarg.h> -#ifdef HAVE_SYSLOG_H - #include <syslog.h> -#endif - #ifdef WIN32 #include <ws2tcpip.h> #endif @@ -33,6 +29,7 @@ #include "Guards.h" #include "UnixUser.h" #include "FileModificationTime.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -48,11 +45,14 @@ Daemon *Daemon::spDaemon = 0; // // -------------------------------------------------------------------------- Daemon::Daemon() - : mpConfiguration(0), + : mpConfiguration(NULL), mReloadConfigWanted(false), - mTerminateWanted(false) + mTerminateWanted(false), + mSingleProcess(false), + mRunInForeground(false), + mKeepConsoleOpenAfterFork(false) { - if(spDaemon != 0) + if(spDaemon != NULL) { THROW_EXCEPTION(ServerException, AlreadyDaemonConstructed) } @@ -79,56 +79,184 @@ Daemon::~Daemon() delete mpConfiguration; mpConfiguration = 0; } + + ASSERT(spDaemon == this); + spDaemon = NULL; } // -------------------------------------------------------------------------- // // Function // Name: Daemon::Main(const char *, int, const char *[]) -// Purpose: Starts the daemon off -- equivalent of C main() function +// Purpose: Parses command-line options, and then calls +// Main(std::string& configFile, bool singleProcess) +// to start the daemon. // Created: 2003/07/29 // // -------------------------------------------------------------------------- int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) { - // Banner (optional) + // Find filename of config file + mConfigFileName = DefaultConfigFile; + bool haveConfigFile = false; + + #ifdef NDEBUG + int masterLevel = Log::NOTICE; // need an int to do math with + #else + int masterLevel = Log::INFO; // need an int to do math with + #endif + + signed int c; + + // reset getopt, just in case anybody used it before. + // unfortunately glibc and BSD differ on this point! + // http://www.ussg.iu.edu/hypermail/linux/kernel/0305.3/0262.html + #ifdef __GLIBC__ + optind = 0; + #else + optind = 1; + optreset = 1; + #endif + + while((c = getopt(argc, (char * const *)argv, "c:DFqvVt:Tk")) != -1) { - const char *banner = DaemonBanner(); - if(banner != 0) + switch(c) { - printf("%s", banner); - } - } + case 'c': + { + mConfigFileName = optarg; + haveConfigFile = true; + } + break; - std::string pidFileName; + case 'D': + { + mSingleProcess = true; + } + break; - try - { - // Find filename of config file - mConfigFileName = DefaultConfigFile; - if(argc >= 2) - { - // First argument is config file, or it's -c and the next arg is the config file - if(::strcmp(argv[1], "-c") == 0 && argc >= 3) + case 'F': { - mConfigFileName = argv[2]; + mRunInForeground = true; } - else + break; + + case 'q': { - mConfigFileName = argv[1]; + if(masterLevel == Log::NOTHING) + { + BOX_FATAL("Too many '-q': " + "Cannot reduce logging " + "level any more"); + return 2; + } + masterLevel--; } - } - - // Test mode with no daemonisation? - bool asDaemon = true; - if(argc >= 3) - { - if(::strcmp(argv[2], "SINGLEPROCESS") == 0) + break; + + case 'v': + { + if(masterLevel == Log::EVERYTHING) + { + BOX_FATAL("Too many '-v': " + "Cannot increase logging " + "level any more"); + return 2; + } + masterLevel++; + } + break; + + case 'V': + { + masterLevel = Log::EVERYTHING; + } + break; + + case 't': + { + Console::SetTag(optarg); + } + break; + + case 'T': + { + Console::SetShowTime(true); + } + break; + + case 'k': + { + mKeepConsoleOpenAfterFork = true; + } + break; + + case '?': + { + BOX_FATAL("Unknown option on command line: " + << "'" << (char)optopt << "'"); + return 2; + } + break; + + default: { - asDaemon = false; + BOX_FATAL("Unknown error in getopt: returned " + << "'" << c << "'"); + return 1; } } + } + + if (argc > optind && !haveConfigFile) + { + mConfigFileName = argv[optind]; optind++; + } + + if (argc > optind && ::strcmp(argv[optind], "SINGLEPROCESS") == 0) + { + mSingleProcess = true; optind++; + } + + if (argc > optind) + { + BOX_FATAL("Unknown parameter on command line: " + << "'" << std::string(argv[optind]) << "'"); + return 2; + } + + Logging::SetGlobalLevel((Log::Level)masterLevel); + + return Main(mConfigFileName); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Main(const std::string& rConfigFileName) +// Purpose: Starts the daemon off -- equivalent of C main() function +// Created: 2003/07/29 +// +// -------------------------------------------------------------------------- +int Daemon::Main(const std::string &rConfigFileName) +{ + // Banner (optional) + { + const char *banner = DaemonBanner(); + if(banner != 0) + { + BOX_NOTICE(banner); + } + } + + std::string pidFileName; + mConfigFileName = rConfigFileName; + + bool asDaemon = !mSingleProcess && !mRunInForeground; + + try + { // Load the configuration file. std::string errors; std::auto_ptr<Configuration> pconfig; @@ -144,16 +272,9 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) if(e.GetType() == CommonException::ExceptionType && e.GetSubType() == CommonException::OSFileOpenError) { - fprintf(stderr, "%s: failed to start: " - "failed to open configuration file: " - "%s\n", DaemonName(), - mConfigFileName.c_str()); -#ifdef WIN32 - ::syslog(LOG_ERR, "%s: failed to start: " - "failed to open configuration file: " - "%s", DaemonName(), - mConfigFileName.c_str()); -#endif + BOX_FATAL("Failed to start: failed to open " + "configuration file: " + << mConfigFileName); return 1; } @@ -164,14 +285,8 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) if(pconfig.get() == 0 || !errors.empty()) { // Tell user about errors - fprintf(stderr, "%s: Errors in config file %s:\n%s", - DaemonName(), mConfigFileName.c_str(), - errors.c_str()); -#ifdef WIN32 - ::syslog(LOG_ERR, "%s: Errors in config file %s:\n%s", - DaemonName(), mConfigFileName.c_str(), - errors.c_str()); -#endif + BOX_FATAL("Failed to start: errors in configuration " + "file: " << mConfigFileName << ": " << errors); // And give up return 1; } @@ -183,18 +298,6 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) // Let the derived class have a go at setting up stuff in the initial process SetupInInitialProcess(); -#ifndef WIN32 - // Set signal handler - struct sigaction sa; - sa.sa_handler = SignalHandler; - sa.sa_flags = 0; - sigemptyset(&sa.sa_mask); // macro - if(::sigaction(SIGHUP, &sa, NULL) != 0 || ::sigaction(SIGTERM, &sa, NULL) != 0) - { - THROW_EXCEPTION(ServerException, DaemoniseFailed) - } -#endif // !WIN32 - // Server configuration const Configuration &serverConfig( mpConfiguration->GetSubConfiguration("Server")); @@ -232,7 +335,7 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) default: // parent - _exit(0); + // _exit(0); return 0; break; @@ -246,7 +349,8 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) // Set new session if(::setsid() == -1) { - ::syslog(LOG_ERR, "can't setsid"); + BOX_ERROR("Failed to setsid(): " << + strerror(errno)); THROW_EXCEPTION(ServerException, DaemoniseFailed) } @@ -269,14 +373,24 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) break; } } -#endif // ! WIN32 - // open the log - ::openlog(DaemonName(), LOG_PID, LOG_LOCAL6); + // Set signal handler + // Don't do this in the parent, since it might be anything + // (e.g. test/bbackupd) + + struct sigaction sa; + sa.sa_handler = SignalHandler; + sa.sa_flags = 0; + sigemptyset(&sa.sa_mask); // macro + if(::sigaction(SIGHUP, &sa, NULL) != 0 || ::sigaction(SIGTERM, &sa, NULL) != 0) + { + THROW_EXCEPTION(ServerException, DaemoniseFailed) + } +#endif // !WIN32 // Log the start message - ::syslog(LOG_INFO, "Starting daemon (config: %s) (version " - BOX_VERSION ")", mConfigFileName.c_str()); + BOX_NOTICE("Starting daemon, version " << BOX_VERSION + << ", config: " << mConfigFileName); // Write PID to file char pid[32]; @@ -289,7 +403,7 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) if(::write(pidFile, pid, pidsize) != pidsize) { - ::syslog(LOG_ERR, "can't write pid file"); + BOX_FATAL("can't write pid file"); THROW_EXCEPTION(ServerException, DaemoniseFailed) } @@ -302,7 +416,7 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) } #endif // BOX_MEMORY_LEAK_TESTING - if(asDaemon) + if(asDaemon && !mKeepConsoleOpenAfterFork) { #ifndef WIN32 // Close standard streams @@ -330,37 +444,24 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) // And definitely don't try and send anything to those file descriptors // -- this has in the past sent text to something which isn't expecting it. TRACE_TO_STDOUT(false); + Logging::ToConsole(false); } } catch(BoxException &e) { - fprintf(stderr, "%s: failed to start: exception %s (%d/%d)\n", - DaemonName(), e.what(), e.GetType(), e.GetSubType()); -#ifdef WIN32 - ::syslog(LOG_ERR, "%s: failed to start: " - "exception %s (%d/%d)\n", DaemonName(), - e.what(), e.GetType(), e.GetSubType()); -#endif + BOX_FATAL("Failed to start: exception " << e.what() + << " (" << e.GetType() + << "/" << e.GetSubType() << ")"); return 1; } catch(std::exception &e) { - fprintf(stderr, "%s: failed to start: exception %s\n", - DaemonName(), e.what()); -#ifdef WIN32 - ::syslog(LOG_ERR, "%s: failed to start: exception %s\n", - DaemonName(), e.what()); -#endif + BOX_FATAL("Failed to start: exception " << e.what()); return 1; } catch(...) { - fprintf(stderr, "%s: failed to start: unknown exception\n", - DaemonName()); -#ifdef WIN32 - ::syslog(LOG_ERR, "%s: failed to start: unknown exception\n", - DaemonName()); -#endif + BOX_FATAL("Failed to start: unknown error"); return 1; } @@ -373,7 +474,7 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) if (WSAStartup(0x0101, &info) == SOCKET_ERROR) { // will not run without sockets - ::syslog(LOG_ERR, "Failed to initialise Windows Sockets"); + BOX_FATAL("Failed to initialise Windows Sockets"); THROW_EXCEPTION(CommonException, Internal) } #endif @@ -390,9 +491,8 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) if(mReloadConfigWanted && !mTerminateWanted) { // Need to reload that config file... - ::syslog(LOG_INFO, "Reloading configuration " - "(config: %s)", - mConfigFileName.c_str()); + BOX_NOTICE("Reloading configuration file: " + << mConfigFileName); std::string errors; std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify( @@ -403,10 +503,9 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) if(pconfig.get() == 0 || !errors.empty()) { // Tell user about errors - ::syslog(LOG_ERR, "Errors in config " - "file %s:\n%s", - mConfigFileName.c_str(), - errors.c_str()); + BOX_FATAL("Error in configuration " + << "file: " << mConfigFileName + << ": " << errors); // And give up retcode = 1; break; @@ -430,25 +529,23 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) ::unlink(pidFileName.c_str()); // Log - ::syslog(LOG_INFO, "Terminating daemon"); + BOX_NOTICE("Terminating daemon"); } catch(BoxException &e) { - ::syslog(LOG_ERR, "%s: terminating due to exception %s " - "(%d/%d)", DaemonName(), e.what(), e.GetType(), - e.GetSubType()); + BOX_FATAL("Terminating due to exception " << e.what() + << " (" << e.GetType() + << "/" << e.GetSubType() << ")"); retcode = 1; } catch(std::exception &e) { - ::syslog(LOG_ERR, "%s: terminating due to exception %s", - DaemonName(), e.what()); + BOX_FATAL("Terminating due to exception " << e.what()); retcode = 1; } catch(...) { - ::syslog(LOG_ERR, "%s: terminating due to unknown exception", - DaemonName()); + BOX_FATAL("Terminating due to unknown exception"); retcode = 1; } diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h index dbc83e98..bce6c3c1 100644 --- a/lib/server/Daemon.h +++ b/lib/server/Daemon.h @@ -41,6 +41,7 @@ private: public: int Main(const char *DefaultConfigFile, int argc, const char *argv[]); + int Main(const std::string &rConfigFile); virtual void Run(); const Configuration &GetConfiguration() const; @@ -65,6 +66,7 @@ public: protected: box_time_t GetLoadedConfigModifiedTime() const; + bool IsSingleProcess() { return mSingleProcess; } private: static void SignalHandler(int sigraised); @@ -76,6 +78,9 @@ private: box_time_t mLoadedConfigModifiedTime; bool mReloadConfigWanted; bool mTerminateWanted; + bool mSingleProcess; + bool mRunInForeground; + bool mKeepConsoleOpenAfterFork; static Daemon *spDaemon; }; diff --git a/lib/server/LocalProcessStream.cpp b/lib/server/LocalProcessStream.cpp index f1b6b2b8..af24de1b 100644 --- a/lib/server/LocalProcessStream.cpp +++ b/lib/server/LocalProcessStream.cpp @@ -18,10 +18,15 @@ #endif #include "LocalProcessStream.h" -#include "SocketStream.h" #include "autogen_ServerException.h" #include "Utils.h" +#ifdef WIN32 + #include "FileStream.h" +#else + #include "SocketStream.h" +#endif + #include "MemLeakFindOn.h" #define MAX_ARGUMENTS 64 @@ -30,19 +35,22 @@ // // Function // Name: LocalProcessStream(const char *, pid_t &) -// Purpose: Run a new process, and return a stream giving access to it's -// stdin and stdout. Returns the PID of the new process -- this -// must be waited on at some point to avoid zombies. +// Purpose: Run a new process, and return a stream giving access +// to its stdin and stdout (stdout and stderr on +// Win32). Returns the PID of the new process -- this +// must be waited on at some point to avoid zombies +// (except on Win32). // Created: 12/3/04 // // -------------------------------------------------------------------------- std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidOut) { +#ifndef WIN32 + // Split up command std::vector<std::string> command; SplitString(std::string(CommandLine), ' ', command); -#ifndef WIN32 // Build arguments char *args[MAX_ARGUMENTS + 4]; { @@ -101,10 +109,68 @@ std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidO // Return the stream object and PID rPidOut = pid; return stream; + #else // WIN32 - ::syslog(LOG_ERR, "vfork not implemented - LocalProcessStream.cpp"); - std::auto_ptr<IOStream> stream; + + SECURITY_ATTRIBUTES secAttr; + secAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + secAttr.bInheritHandle = TRUE; + secAttr.lpSecurityDescriptor = NULL; + + HANDLE writeInChild, readFromChild; + if(!CreatePipe(&readFromChild, &writeInChild, &secAttr, 0)) + { + BOX_ERROR("Failed to CreatePipe for child process: " << + GetErrorMessage(GetLastError())); + THROW_EXCEPTION(ServerException, SocketPairFailed) + } + SetHandleInformation(readFromChild, HANDLE_FLAG_INHERIT, 0); + + PROCESS_INFORMATION procInfo; + STARTUPINFO startupInfo; + + ZeroMemory(&procInfo, sizeof(procInfo)); + ZeroMemory(&startupInfo, sizeof(startupInfo)); + startupInfo.cb = sizeof(startupInfo); + startupInfo.hStdError = writeInChild; + startupInfo.hStdOutput = writeInChild; + startupInfo.hStdInput = INVALID_HANDLE_VALUE; + startupInfo.dwFlags |= STARTF_USESTDHANDLES; + + CHAR* commandLineCopy = (CHAR*)malloc(strlen(CommandLine) + 1); + strcpy(commandLineCopy, CommandLine); + + BOOL result = CreateProcess(NULL, + commandLineCopy, // command line + NULL, // process security attributes + NULL, // primary thread security attributes + TRUE, // handles are inherited + 0, // creation flags + NULL, // use parent's environment + NULL, // use parent's current directory + &startupInfo, // STARTUPINFO pointer + &procInfo); // receives PROCESS_INFORMATION + + free(commandLineCopy); + + if(!result) + { + BOX_ERROR("Failed to CreateProcess: '" << CommandLine << + "': " << GetErrorMessage(GetLastError())); + CloseHandle(writeInChild); + CloseHandle(readFromChild); + THROW_EXCEPTION(ServerException, ServerForkError) + } + + CloseHandle(procInfo.hProcess); + CloseHandle(procInfo.hThread); + CloseHandle(writeInChild); + + rPidOut = (int)(procInfo.dwProcessId); + + std::auto_ptr<IOStream> stream(new FileStream(readFromChild)); return stream; + #endif // ! WIN32 } diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp index 988d44c8..4398a58f 100644 --- a/lib/server/Protocol.cpp +++ b/lib/server/Protocol.cpp @@ -22,6 +22,7 @@ #include "ServerException.h" #include "PartialReadStream.h" #include "ProtocolUncertainStream.h" +#include "Logging.h" #include "MemLeakFindOn.h" @@ -55,7 +56,8 @@ Protocol::Protocol(IOStream &rStream) mLastErrorType(NoError), mLastErrorSubType(NoError) { - TRACE1("Send block allocation size is %d\n", PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); + BOX_TRACE("Send block allocation size is " << + PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK); } // -------------------------------------------------------------------------- diff --git a/lib/server/SSLLib.cpp b/lib/server/SSLLib.cpp index 2a5bdbde..e9c990b9 100644 --- a/lib/server/SSLLib.cpp +++ b/lib/server/SSLLib.cpp @@ -14,10 +14,6 @@ #include <openssl/err.h> #include <openssl/rand.h> -#ifndef WIN32 -#include <syslog.h> -#endif - #include "SSLLib.h" #include "ServerException.h" @@ -53,7 +49,8 @@ void SSLLib::Initialise() THROW_EXCEPTION(ServerException, SSLRandomInitFailed) } #else - ::fprintf(stderr, "No random device -- additional seeding of random number generator not performed.\n"); + BOX_WARNING("No random device -- additional seeding of " + "random number generator not performed."); #endif } @@ -73,13 +70,8 @@ void SSLLib::LogError(const char *ErrorDuringAction) while((errcode = ERR_get_error()) != 0) { ::ERR_error_string_n(errcode, errname, sizeof(errname)); - #ifndef NDEBUG - if(SSLLib__TraceErrors) - { - TRACE2("SSL err during %s: %s\n", ErrorDuringAction, errname); - } - #endif - ::syslog(LOG_ERR, "SSL err during %s: %s", ErrorDuringAction, errname); + BOX_ERROR("SSL error during " << ErrorDuringAction << ": " << + errname); } } diff --git a/lib/server/ServerControl.h b/lib/server/ServerControl.h new file mode 100644 index 00000000..c51dd4e2 --- /dev/null +++ b/lib/server/ServerControl.h @@ -0,0 +1,178 @@ +#ifndef SERVER_CONTROL_H +#define SERVER_CONTROL_H + +#include "Test.h" + +#ifdef WIN32 + +#include "WinNamedPipeStream.h" +#include "IOStreamGetLine.h" +#include "BoxPortsAndFiles.h" + +static bool SendCommands(const std::string& rCmd) +{ + WinNamedPipeStream connection; + + try + { + connection.Connect(BOX_NAMED_PIPE_NAME); + } + catch(...) + { + printf("Failed to connect to daemon control socket.\n"); + return false; + } + + // For receiving data + IOStreamGetLine getLine(connection); + + // Wait for the configuration summary + std::string configSummary; + if(!getLine.GetLine(configSummary)) + { + printf("Failed to receive configuration summary from daemon\n"); + return false; + } + + // Was the connection rejected by the server? + if(getLine.IsEOF()) + { + printf("Server rejected the connection.\n"); + return false; + } + + // Decode it + int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait; + if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", + &autoBackup, &updateStoreInterval, + &minimumFileAge, &maxUploadWait) != 4) + { + printf("Config summary didn't decode\n"); + return false; + } + + std::string cmds; + bool expectResponse; + + if (rCmd != "") + { + cmds = rCmd; + cmds += "\nquit\n"; + expectResponse = true; + } + else + { + cmds = "quit\n"; + expectResponse = false; + } + + connection.Write(cmds.c_str(), cmds.size()); + + // Read the response + std::string line; + bool statusOk = !expectResponse; + + while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line)) + { + // Is this an OK or error line? + if (line == "ok") + { + statusOk = true; + } + else if (line == "error") + { + printf("ERROR (%s)\n", rCmd.c_str()); + break; + } + else + { + printf("WARNING: Unexpected response to command '%s': " + "%s", rCmd.c_str(), line.c_str()); + } + } + + return statusOk; +} + +inline bool HUPServer(int pid) +{ + return SendCommands("reload"); +} + +inline bool KillServerInternal(int pid) +{ + HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, false, pid); + if (hProcess == NULL) + { + printf("Failed to open process %d: error %d\n", + pid, (int)GetLastError()); + return false; + } + + if (!TerminateProcess(hProcess, 1)) + { + printf("Failed to terminate process %d: error %d\n", + pid, (int)GetLastError()); + CloseHandle(hProcess); + return false; + } + + CloseHandle(hProcess); + return true; +} + +#else // !WIN32 + +inline bool HUPServer(int pid) +{ + if(pid == 0) return false; + return ::kill(pid, SIGHUP) == 0; +} + +inline bool KillServerInternal(int pid) +{ + if(pid == 0 || pid == -1) return false; + bool killed = (::kill(pid, SIGTERM) == 0); + TEST_THAT(killed); + return killed; +} + +#endif // WIN32 + +inline bool KillServer(int pid) +{ + if (!KillServerInternal(pid)) + { + return false; + } + + for (int i = 0; i < 30; i++) + { + if (!ServerIsAlive(pid)) break; + ::sleep(1); + if (!ServerIsAlive(pid)) break; + + if (i == 0) + { + printf("waiting for server to die"); + } + + printf("."); + fflush(stdout); + } + + if (!ServerIsAlive(pid)) + { + printf("done.\n"); + } + else + { + printf("failed!\n"); + } + + fflush(stdout); + + return !ServerIsAlive(pid); +} + +#endif // SERVER_CONTROL_H diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h index 998a266e..5f615336 100644 --- a/lib/server/ServerStream.h +++ b/lib/server/ServerStream.h @@ -14,7 +14,6 @@ #include <errno.h> #ifndef WIN32 - #include <syslog.h> #include <sys/wait.h> #endif @@ -73,7 +72,10 @@ public: { if(childExit) { - ::syslog(LOG_ERR, "in server child, exception %s (%d/%d) -- terminating child", e.what(), e.GetType(), e.GetSubType()); + BOX_ERROR("Error in child process, " + "terminating connection: exception " << + e.what() << "(" << e.GetType() << + "/" << e.GetSubType() << ")"); _exit(1); } else throw; @@ -82,7 +84,9 @@ public: { if(childExit) { - ::syslog(LOG_ERR, "in server child, exception %s -- terminating child", e.what()); + BOX_ERROR("Error in child process, " + "terminating connection: exception " << + e.what()); _exit(1); } else throw; @@ -91,7 +95,9 @@ public: { if(childExit) { - ::syslog(LOG_ERR, "in server child, unknown exception -- terminating child"); + BOX_ERROR("Error in child process, " + "terminating connection: " + "unknown exception"); _exit(1); } else throw; @@ -170,16 +176,22 @@ public: } else if(c[0] == "unix") { - // Check arguments size - if(c.size() != 2) - { - THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) - } + #ifdef WIN32 + BOX_WARNING("Ignoring request to listen on a Unix socket on Windows: " << addrlist[a]); + delete psocket; + psocket = NULL; + #else + // Check arguments size + if(c.size() != 2) + { + THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) + } - // unlink anything there - ::unlink(c[1].c_str()); - - psocket->Listen(Socket::TypeUNIX, c[1].c_str()); + // unlink anything there + ::unlink(c[1].c_str()); + + psocket->Listen(Socket::TypeUNIX, c[1].c_str()); + #endif // WIN32 } else { @@ -187,8 +199,11 @@ public: THROW_EXCEPTION(ServerException, ServerStreamBadListenAddrs) } - // Add to list of sockets - mSockets.push_back(psocket); + if (psocket != NULL) + { + // Add to list of sockets + mSockets.push_back(psocket); + } } catch(...) { @@ -196,8 +211,11 @@ public: throw; } - // Add to the list of things to wait on - connectionWait.Add(psocket); + if (psocket != NULL) + { + // Add to the list of things to wait on + connectionWait.Add(psocket); + } } } @@ -217,7 +235,8 @@ public: if(connection.get()) { // Since this is a template parameter, the if() will be optimised out by the compiler - if(WillForkToHandleRequests()) + #ifndef WIN32 // no fork on Win32 + if(ForkToHandleRequests && !IsSingleProcess()) { pid_t pid = ::fork(); switch(pid) @@ -253,22 +272,26 @@ public: } // Log it - ::syslog(LOG_INFO, "%s (handling in child %d)", logMessage.c_str(), pid); + BOX_WARNING("Error message from child process " << pid << ": " << logMessage); } else { - // Just handle in this connection + #endif // !WIN32 + // Just handle in this process SetProcessTitle("handling"); HandleConnection(*connection); SetProcessTitle("idle"); + #ifndef WIN32 } + #endif // !WIN32 } } OnIdle(); + #ifndef WIN32 // Clean up child processes (if forking daemon) - if(WillForkToHandleRequests()) + if(ForkToHandleRequests && !IsSingleProcess()) { int status = 0; int p = 0; @@ -281,6 +304,7 @@ public: } } while(p > 0); } + #endif // !WIN32 } } catch(...) @@ -301,14 +325,14 @@ public: virtual void Connection(StreamType &rStream) = 0; protected: - // For checking code in dervied classes -- use if you have an algorithm which + // For checking code in derived classes -- use if you have an algorithm which // depends on the forking model in case someone changes it later. bool WillForkToHandleRequests() { #ifdef WIN32 return false; #else - return ForkToHandleRequests; + return ForkToHandleRequests && !IsSingleProcess(); #endif // WIN32 } diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp index afed4888..28dae69f 100644 --- a/lib/server/Socket.cpp +++ b/lib/server/Socket.cpp @@ -17,7 +17,6 @@ #ifndef WIN32 #include <sys/socket.h> #include <netdb.h> -#include <syslog.h> #include <netinet/in.h> #include <arpa/inet.h> #endif @@ -124,18 +123,20 @@ void Socket::LogIncomingConnection(const struct sockaddr *addr, socklen_t addrle switch(addr->sa_family) { case AF_UNIX: - ::syslog(LOG_INFO, "Incoming connection from local (UNIX socket)"); + BOX_INFO("Incoming connection from local (UNIX socket)"); break; case AF_INET: { sockaddr_in *a = (sockaddr_in*)addr; - ::syslog(LOG_INFO, "Incoming connection from %s port %d", inet_ntoa(a->sin_addr), ntohs(a->sin_port)); + BOX_INFO("Incoming connection from " << + inet_ntoa(a->sin_addr) << " port " << + ntohs(a->sin_port)); } break; default: - ::syslog(LOG_INFO, "Incoming connection of unknown type"); + BOX_WARNING("Incoming connection of unknown type"); break; } } diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp index 61d3846f..ebc57041 100644 --- a/lib/server/SocketStream.cpp +++ b/lib/server/SocketStream.cpp @@ -15,6 +15,7 @@ #include <sys/types.h> #include <errno.h> +#include <string.h> #ifndef WIN32 #include <poll.h> @@ -164,6 +165,10 @@ void SocketStream::Open(int Type, const char *Name, int Port) #else ::close(mSocketHandle); #endif + BOX_ERROR("Failed to connect to socket (type " << Type << + ", name " << Name << ", port " << Port << "): " << + "error " << errno << " (" << strerror(errno) << + ")"); mSocketHandle = INVALID_SOCKET_VALUE; THROW_EXCEPTION(ConnectionException, Conn_SocketConnectError) } diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h index 7d5e6d93..51f2e306 100644 --- a/lib/server/SocketStream.h +++ b/lib/server/SocketStream.h @@ -14,7 +14,7 @@ #ifdef WIN32 typedef SOCKET tOSSocketHandle; - #define INVALID_SOCKET_VALUE INVALID_SOCKET + #define INVALID_SOCKET_VALUE (tOSSocketHandle)(-1) #else typedef int tOSSocketHandle; #define INVALID_SOCKET_VALUE -1 @@ -67,6 +67,7 @@ public: off_t GetBytesRead() const {return mBytesRead;} off_t GetBytesWritten() const {return mBytesWritten;} void ResetCounters() {mBytesRead = mBytesWritten = 0;} + bool IsOpened() { return mSocketHandle != INVALID_SOCKET_VALUE; } }; #endif // SOCKETSTREAM__H diff --git a/lib/server/SocketStreamTLS.cpp b/lib/server/SocketStreamTLS.cpp index af4ad460..58dc5754 100644 --- a/lib/server/SocketStreamTLS.cpp +++ b/lib/server/SocketStreamTLS.cpp @@ -23,6 +23,7 @@ #include "SSLLib.h" #include "ServerException.h" #include "TLSContext.h" +#include "BoxTime.h" #include "MemLeakFindOn.h" @@ -244,20 +245,30 @@ bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout) break; } p.revents = 0; - switch(::poll(&p, 1, (Timeout == IOStream::TimeOutInfinite)?INFTIM:Timeout)) + + int64_t start, end; + start = BoxTimeToMilliSeconds(GetCurrentBoxTime()); + end = start + Timeout; + int result; + + do { - case -1: - // error - if(errno == EINTR) - { - // Signal. Do "time out" - return false; - } - else + int64_t now = BoxTimeToMilliSeconds(GetCurrentBoxTime()); + int poll_timeout = (int)(end - now); + if (poll_timeout < 0) poll_timeout = 0; + if (Timeout == IOStream::TimeOutInfinite) { - // Bad! - THROW_EXCEPTION(ServerException, SocketPollError) + poll_timeout = INFTIM; } + result = ::poll(&p, 1, poll_timeout); + } + while(result == -1 && errno == EINTR); + + switch(result) + { + case -1: + // error - Bad! + THROW_EXCEPTION(ServerException, SocketPollError) break; case 0: diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp index cc125d00..49143801 100644 --- a/lib/server/TLSContext.cpp +++ b/lib/server/TLSContext.cpp @@ -61,6 +61,11 @@ TLSContext::~TLSContext() // -------------------------------------------------------------------------- void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const char *PrivateKeyFile, const char *TrustedCAsFile) { + if(mpContext != 0) + { + ::SSL_CTX_free(mpContext); + } + mpContext = ::SSL_CTX_new(AsServer?TLSv1_server_method():TLSv1_client_method()); if(mpContext == NULL) { diff --git a/lib/server/WinNamedPipeStream.cpp b/lib/server/WinNamedPipeStream.cpp index c5b7eaa5..d7e90793 100644 --- a/lib/server/WinNamedPipeStream.cpp +++ b/lib/server/WinNamedPipeStream.cpp @@ -35,7 +35,9 @@ // // -------------------------------------------------------------------------- WinNamedPipeStream::WinNamedPipeStream() - : mSocketHandle(NULL), + : mSocketHandle(INVALID_HANDLE_VALUE), + mReadableEvent(INVALID_HANDLE_VALUE), + mBytesInBuffer(0), mReadClosed(false), mWriteClosed(false), mIsServer(false), @@ -53,9 +55,17 @@ WinNamedPipeStream::WinNamedPipeStream() // -------------------------------------------------------------------------- WinNamedPipeStream::~WinNamedPipeStream() { - if (mSocketHandle != NULL) + if (mSocketHandle != INVALID_HANDLE_VALUE) { - Close(); + try + { + Close(); + } + catch (std::exception &e) + { + BOX_ERROR("Caught exception while destroying " + "named pipe, ignored: " << e.what()); + } } } @@ -70,16 +80,17 @@ WinNamedPipeStream::~WinNamedPipeStream() // -------------------------------------------------------------------------- void WinNamedPipeStream::Accept(const wchar_t* pName) { - if (mSocketHandle != NULL || mIsConnected) + if (mSocketHandle != INVALID_HANDLE_VALUE || mIsConnected) { THROW_EXCEPTION(ServerException, SocketAlreadyOpen) } mSocketHandle = CreateNamedPipeW( pName, // pipe name - PIPE_ACCESS_DUPLEX, // read/write access - PIPE_TYPE_MESSAGE | // message type pipe - PIPE_READMODE_MESSAGE | // message-read mode + PIPE_ACCESS_DUPLEX | // read/write access + FILE_FLAG_OVERLAPPED, // enabled overlapped I/O + PIPE_TYPE_BYTE | // message type pipe + PIPE_READMODE_BYTE | // message-read mode PIPE_WAIT, // blocking mode 1, // max. instances 4096, // output buffer size @@ -87,10 +98,10 @@ void WinNamedPipeStream::Accept(const wchar_t* pName) NMPWAIT_USE_DEFAULT_WAIT, // client time-out NULL); // default security attribute - if (mSocketHandle == NULL) + if (mSocketHandle == INVALID_HANDLE_VALUE) { - ::syslog(LOG_ERR, "CreateNamedPipeW failed: %d", - GetLastError()); + BOX_ERROR("Failed to CreateNamedPipeW(" << pName << "): " << + GetErrorMessage(GetLastError())); THROW_EXCEPTION(ServerException, SocketOpenError) } @@ -98,16 +109,48 @@ void WinNamedPipeStream::Accept(const wchar_t* pName) if (!connected) { - ::syslog(LOG_ERR, "ConnectNamedPipe failed: %d", - GetLastError()); + BOX_ERROR("Failed to ConnectNamedPipe(" << pName << "): " << + GetErrorMessage(GetLastError())); Close(); THROW_EXCEPTION(ServerException, SocketOpenError) } + mBytesInBuffer = 0; mReadClosed = false; mWriteClosed = false; mIsServer = true; // must flush and disconnect before closing mIsConnected = true; + + // create the Readable event + mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + + if (mReadableEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to create the Readable event: " << + GetErrorMessage(GetLastError())); + Close(); + THROW_EXCEPTION(CommonException, Internal) + } + + // initialise the OVERLAPPED structure + memset(&mReadOverlap, 0, sizeof(mReadOverlap)); + mReadOverlap.hEvent = mReadableEvent; + + // start the first overlapped read + if (!ReadFile(mSocketHandle, mReadBuffer, sizeof(mReadBuffer), + NULL, &mReadOverlap)) + { + DWORD err = GetLastError(); + + if (err != ERROR_IO_PENDING) + { + BOX_ERROR("Failed to start overlapped read: " << + GetErrorMessage(err)); + Close(); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } } // -------------------------------------------------------------------------- @@ -120,7 +163,7 @@ void WinNamedPipeStream::Accept(const wchar_t* pName) // -------------------------------------------------------------------------- void WinNamedPipeStream::Connect(const wchar_t* pName) { - if (mSocketHandle != NULL || mIsConnected) + if (mSocketHandle != INVALID_HANDLE_VALUE || mIsConnected) { THROW_EXCEPTION(ServerException, SocketAlreadyOpen) } @@ -137,8 +180,17 @@ void WinNamedPipeStream::Connect(const wchar_t* pName) if (mSocketHandle == INVALID_HANDLE_VALUE) { - ::syslog(LOG_ERR, "Failed to connect to server's named pipe: " - "error %d", GetLastError()); + DWORD err = GetLastError(); + if (err == ERROR_PIPE_BUSY) + { + BOX_ERROR("Failed to connect to backup daemon: " + "it is busy with another connection"); + } + else + { + BOX_ERROR("Failed to connect to backup daemon: " << + GetErrorMessage(err)); + } THROW_EXCEPTION(ServerException, SocketOpenError) } @@ -159,33 +211,192 @@ void WinNamedPipeStream::Connect(const wchar_t* pName) int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) { // TODO no support for timeouts yet - ASSERT(Timeout == IOStream::TimeOutInfinite) + if (Timeout != IOStream::TimeOutInfinite) + { + THROW_EXCEPTION(CommonException, AssertFailed) + } - if (mSocketHandle == NULL || !mIsConnected) + if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) { THROW_EXCEPTION(ServerException, BadSocketHandle) } + if (mReadClosed) + { + THROW_EXCEPTION(ConnectionException, SocketShutdownError) + } + + // ensure safe to cast NBytes to unsigned + if (NBytes < 0) + { + THROW_EXCEPTION(CommonException, AssertFailed) + } + DWORD NumBytesRead; - - bool Success = ReadFile( - mSocketHandle, // pipe handle - pBuffer, // buffer to receive reply - NBytes, // size of buffer - &NumBytesRead, // number of bytes read - NULL); // not overlapped - - if (!Success) + + if (mIsServer) { - THROW_EXCEPTION(ConnectionException, Conn_SocketReadError) + // satisfy from buffer if possible, to avoid + // blocking on read. + bool needAnotherRead = false; + if (mBytesInBuffer == 0) + { + // overlapped I/O completed successfully? + // (wait if needed) + + if (GetOverlappedResult(mSocketHandle, + &mReadOverlap, &NumBytesRead, TRUE)) + { + needAnotherRead = true; + } + else + { + DWORD err = GetLastError(); + + if (err == ERROR_HANDLE_EOF) + { + mReadClosed = true; + } + else + { + if (err == ERROR_BROKEN_PIPE) + { + BOX_ERROR("Control client " + "disconnected"); + } + else + { + BOX_ERROR("Failed to wait for " + "ReadFile to complete: " + << GetErrorMessage(err)); + } + + Close(); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } + } + else + { + NumBytesRead = 0; + } + + size_t BytesToCopy = NumBytesRead + mBytesInBuffer; + size_t BytesRemaining = 0; + + if (BytesToCopy > (size_t)NBytes) + { + BytesRemaining = BytesToCopy - NBytes; + BytesToCopy = NBytes; + } + + memcpy(pBuffer, mReadBuffer, BytesToCopy); + memmove(mReadBuffer, mReadBuffer + BytesToCopy, BytesRemaining); + + mBytesInBuffer = BytesRemaining; + NumBytesRead = BytesToCopy; + + if (needAnotherRead) + { + // reinitialise the OVERLAPPED structure + memset(&mReadOverlap, 0, sizeof(mReadOverlap)); + mReadOverlap.hEvent = mReadableEvent; + } + + // start the next overlapped read + if (needAnotherRead && !ReadFile(mSocketHandle, + mReadBuffer + mBytesInBuffer, + sizeof(mReadBuffer) - mBytesInBuffer, + NULL, &mReadOverlap)) + { + DWORD err = GetLastError(); + if (err == ERROR_IO_PENDING) + { + // Don't reset yet, there might be data + // in the buffer waiting to be read, + // will check below. + // ResetEvent(mReadableEvent); + } + else if (err == ERROR_HANDLE_EOF) + { + mReadClosed = true; + } + else if (err == ERROR_BROKEN_PIPE) + { + BOX_ERROR("Control client disconnected"); + mReadClosed = true; + } + else + { + BOX_ERROR("Failed to start overlapped read: " + << GetErrorMessage(err)); + Close(); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } + + // If the read succeeded immediately, leave the event + // signaled, so that we will be called again to process + // the newly read data and start another overlapped read. + if (needAnotherRead && !mReadClosed) + { + // leave signalled + } + else if (!needAnotherRead && mBytesInBuffer > 0) + { + // leave signalled + } + else + { + // nothing left to read, reset the event + ResetEvent(mReadableEvent); + // FIXME: a pending read could have signalled + // the event (again) while we were busy reading. + // that signal would be lost, and the reading + // thread would block. Should be pretty obvious + // if this happens in practice: control client + // hangs. + } } - - // Closed for reading at EOF? - if (NumBytesRead == 0) + else { - mReadClosed = true; + if (!ReadFile( + mSocketHandle, // pipe handle + pBuffer, // buffer to receive reply + NBytes, // size of buffer + &NumBytesRead, // number of bytes read + NULL)) // not overlapped + { + DWORD err = GetLastError(); + + Close(); + + // ERROR_NO_DATA is a strange name for + // "The pipe is being closed". No exception wanted. + + if (err == ERROR_NO_DATA || + err == ERROR_PIPE_NOT_CONNECTED) + { + NumBytesRead = 0; + } + else + { + BOX_ERROR("Failed to read from control socket: " + << GetErrorMessage(err)); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } + + // Closed for reading at EOF? + if (NumBytesRead == 0) + { + mReadClosed = true; + } } - + return NumBytesRead; } @@ -199,7 +410,7 @@ int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) // -------------------------------------------------------------------------- void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) { - if (mSocketHandle == NULL || !mIsConnected) + if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) { THROW_EXCEPTION(ServerException, BadSocketHandle) } @@ -223,9 +434,23 @@ void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) if (!Success) { - mWriteClosed = true; // assume can't write again - THROW_EXCEPTION(ConnectionException, - Conn_SocketWriteError) + DWORD err = GetLastError(); + BOX_ERROR("Failed to write to control socket: " << + GetErrorMessage(err)); + Close(); + + // ERROR_NO_DATA is a strange name for + // "The pipe is being closed". No exception wanted. + + if (err == ERROR_NO_DATA) + { + return; + } + else + { + THROW_EXCEPTION(ConnectionException, + Conn_SocketWriteError) + } } NumBytesWrittenTotal += NumBytesWrittenThisTime; @@ -242,30 +467,52 @@ void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) // -------------------------------------------------------------------------- void WinNamedPipeStream::Close() { - if (mSocketHandle == NULL && mIsConnected) + if (mSocketHandle == INVALID_HANDLE_VALUE && mIsConnected) { - fprintf(stderr, "Inconsistent connected state\n"); - ::syslog(LOG_ERR, "Inconsistent connected state"); + BOX_ERROR("Named pipe: inconsistent connected state"); mIsConnected = false; } - if (mSocketHandle == NULL) + if (mSocketHandle == INVALID_HANDLE_VALUE) { THROW_EXCEPTION(ServerException, BadSocketHandle) } if (mIsServer) - { + { + if (!CancelIo(mSocketHandle)) + { + BOX_ERROR("Failed to cancel outstanding I/O: " << + GetErrorMessage(GetLastError())); + } + + if (mReadableEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to destroy Readable event: " + "invalid handle"); + } + else if (!CloseHandle(mReadableEvent)) + { + BOX_ERROR("Failed to destroy Readable event: " << + GetErrorMessage(GetLastError())); + } + + mReadableEvent = INVALID_HANDLE_VALUE; + if (!FlushFileBuffers(mSocketHandle)) { - ::syslog(LOG_INFO, "FlushFileBuffers failed: %d", - GetLastError()); + BOX_ERROR("Failed to FlushFileBuffers: " << + GetErrorMessage(GetLastError())); } if (!DisconnectNamedPipe(mSocketHandle)) { - ::syslog(LOG_ERR, "DisconnectNamedPipe failed: %d", - GetLastError()); + DWORD err = GetLastError(); + if (err != ERROR_PIPE_NOT_CONNECTED) + { + BOX_ERROR("Failed to DisconnectNamedPipe: " << + GetErrorMessage(err)); + } } mIsServer = false; @@ -273,12 +520,15 @@ void WinNamedPipeStream::Close() bool result = CloseHandle(mSocketHandle); - mSocketHandle = NULL; + mSocketHandle = INVALID_HANDLE_VALUE; mIsConnected = false; + mReadClosed = true; + mWriteClosed = true; if (!result) { - ::syslog(LOG_ERR, "CloseHandle failed: %d", GetLastError()); + BOX_ERROR("Failed to CloseHandle: " << + GetErrorMessage(GetLastError())); THROW_EXCEPTION(ServerException, SocketCloseError) } } @@ -309,4 +559,27 @@ bool WinNamedPipeStream::StreamClosed() return mWriteClosed; } +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::WriteAllBuffered() +// Purpose: Ensures that any data which has been buffered is written to the stream +// Created: 2003/08/26 +// +// -------------------------------------------------------------------------- +void WinNamedPipeStream::WriteAllBuffered() +{ + if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) + { + THROW_EXCEPTION(ServerException, BadSocketHandle) + } + + if (!FlushFileBuffers(mSocketHandle)) + { + BOX_ERROR("Failed to FlushFileBuffers: " << + GetErrorMessage(GetLastError())); + } +} + + #endif // WIN32 diff --git a/lib/server/WinNamedPipeStream.h b/lib/server/WinNamedPipeStream.h index 5a800371..aded2d59 100644 --- a/lib/server/WinNamedPipeStream.h +++ b/lib/server/WinNamedPipeStream.h @@ -36,13 +36,15 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual void Write(const void *pBuffer, int NBytes); + virtual void WriteAllBuffered(); virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); bool IsConnected() { return mIsConnected; } + HANDLE GetSocketHandle() { return mSocketHandle; } + HANDLE GetReadableEvent() { return mReadableEvent; } protected: - HANDLE GetSocketHandle(); void MarkAsReadClosed() {mReadClosed = true;} void MarkAsWriteClosed() {mWriteClosed = true;} @@ -51,6 +53,10 @@ private: { /* do not call */ } HANDLE mSocketHandle; + HANDLE mReadableEvent; + OVERLAPPED mReadOverlap; + uint8_t mReadBuffer[4096]; + size_t mBytesInBuffer; bool mReadClosed; bool mWriteClosed; bool mIsServer; diff --git a/lib/server/makeprotocol.pl.in b/lib/server/makeprotocol.pl.in index f89c780d..f4561168 100755 --- a/lib/server/makeprotocol.pl.in +++ b/lib/server/makeprotocol.pl.in @@ -1,5 +1,4 @@ #!@PERL@ - use strict; use lib "../../infrastructure"; @@ -830,8 +829,8 @@ if($implement_filelog || $implement_syslog) } if($implement_filelog) { - $fR .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain":"Receiving stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; - $fS .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain":"Sending stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; + $fR .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain\\n":"Receiving stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; + $fS .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain\\n":"Sending stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; } print CPP <<__E; @@ -916,6 +915,7 @@ __E close H; close CPP; + sub obj_is_type { my ($c,$ty) = @_; diff --git a/lib/win32/MSG00001.bin b/lib/win32/MSG00001.bin Binary files differnew file mode 100755 index 00000000..b4377b85 --- /dev/null +++ b/lib/win32/MSG00001.bin diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index 91684f1c..071dc788 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -9,17 +9,16 @@ #include <assert.h> #include <fcntl.h> +#include <process.h> #include <windows.h> #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -#ifdef HAVE_PROCESS_H - #include <process.h> -#endif #include <string> #include <list> +#include <sstream> // message resource definitions for syslog() @@ -28,6 +27,7 @@ // our implementation for a timer, based on a // simple thread which sleeps for a period of time +static bool gTimerInitialised = false; static bool gFinishTimer; static CRITICAL_SECTION gLock; @@ -43,24 +43,31 @@ static void (__cdecl *gTimerFunc) (int) = NULL; int setitimer(int type, struct itimerval *timeout, void *arg) { - if (ITIMER_VIRTUAL == type) + assert(gTimerInitialised); + + if (ITIMER_REAL != type) { - EnterCriticalSection(&gLock); - // we only need seconds for the mo! - if (timeout->it_value.tv_sec == 0 && - timeout->it_value.tv_usec == 0) - { - gTimerList.clear(); - } - else - { - Timer_t ourTimer; - ourTimer.countDown = timeout->it_value.tv_sec; - ourTimer.interval = timeout->it_interval.tv_sec; - gTimerList.push_back(ourTimer); - } - LeaveCriticalSection(&gLock); + errno = ENOSYS; + return -1; + } + + EnterCriticalSection(&gLock); + + // we only need seconds for the mo! + if (timeout->it_value.tv_sec == 0 && + timeout->it_value.tv_usec == 0) + { + gTimerList.clear(); } + else + { + Timer_t ourTimer; + ourTimer.countDown = timeout->it_value.tv_sec; + ourTimer.interval = timeout->it_interval.tv_sec; + gTimerList.push_back(ourTimer); + } + + LeaveCriticalSection(&gLock); // indicate success return 0; @@ -131,20 +138,26 @@ int SetTimerHandler(void (__cdecl *func ) (int)) void InitTimer(void) { - InitializeCriticalSection(&gLock); + assert(!gTimerInitialised); + InitializeCriticalSection(&gLock); + // create our thread HANDLE ourThread = (HANDLE)_beginthreadex(NULL, 0, RunTimer, 0, CREATE_SUSPENDED, NULL); SetThreadPriority(ourThread, THREAD_PRIORITY_LOWEST); ResumeThread(ourThread); + + gTimerInitialised = true; } void FiniTimer(void) { + assert(gTimerInitialised); gFinishTimer = true; EnterCriticalSection(&gLock); DeleteCriticalSection(&gLock); + gTimerInitialised = false; } //Our constants we need to keep track of @@ -205,6 +218,44 @@ bool EnableBackupRights( void ) } } +// forward declaration +char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage); + +// -------------------------------------------------------------------------- +// +// Function +// Name: GetDefaultConfigFilePath(std::string name) +// Purpose: Calculates the default configuration file name, +// by using the directory location of the currently +// executing program, and appending the provided name. +// In case of fire, returns an empty string. +// Created: 26th May 2007 +// +// -------------------------------------------------------------------------- +std::string GetDefaultConfigFilePath(const std::string& rName) +{ + WCHAR exePathWide[MAX_PATH]; + GetModuleFileNameW(NULL, exePathWide, MAX_PATH-1); + + char* exePathUtf8 = ConvertFromWideString(exePathWide, CP_UTF8); + if (exePathUtf8 == NULL) + { + return ""; + } + + std::string configfile = exePathUtf8; + delete [] exePathUtf8; + + // make the default config file name, + // based on the program path + configfile = configfile.substr(0, + configfile.rfind('\\')); + configfile += "\\"; + configfile += rName; + + return configfile; +} + // -------------------------------------------------------------------------- // // Function @@ -362,27 +413,34 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) // -------------------------------------------------------------------------- // // Function -// Name: ConvertUtf8ToConsole -// Purpose: Converts a string from UTF-8 to the console -// code page. On success, replaces contents of rDest -// and returns true. In case of fire, logs the error -// and returns false. -// Created: 4th February 2006 +// Name: ConvertEncoding(const std::string&, int, +// std::string&, int) +// Purpose: Converts a string from one code page to another. +// On success, replaces contents of rDest and returns +// true. In case of fire, logs the error and returns +// false. +// Created: 15th October 2006 // // -------------------------------------------------------------------------- -bool ConvertUtf8ToConsole(const char* pString, std::string& rDest) +bool ConvertEncoding(const std::string& rSource, int sourceCodePage, + std::string& rDest, int destCodePage) { - WCHAR* pWide = ConvertUtf8ToWideString(pString); + WCHAR* pWide = ConvertToWideString(rSource.c_str(), sourceCodePage); if (pWide == NULL) { + ::syslog(LOG_ERR, "Failed to convert string '%s' from " + "current code page %d to wide string: %s", + rSource.c_str(), sourceCodePage, + GetErrorMessage(GetLastError()).c_str()); return false; } - char* pConsole = ConvertFromWideString(pWide, GetConsoleOutputCP()); + char* pConsole = ConvertFromWideString(pWide, destCodePage); delete [] pWide; if (!pConsole) { + // Error should have been logged by ConvertFromWideString return false; } @@ -392,39 +450,25 @@ bool ConvertUtf8ToConsole(const char* pString, std::string& rDest) return true; } -// -------------------------------------------------------------------------- -// -// Function -// Name: ConvertConsoleToUtf8 -// Purpose: Converts a string from the console code page -// to UTF-8. On success, replaces contents of rDest -// and returns true. In case of fire, logs the error -// and returns false. -// Created: 4th February 2006 -// -// -------------------------------------------------------------------------- -bool ConvertConsoleToUtf8(const char* pString, std::string& rDest) +bool ConvertToUtf8(const char* pString, std::string& rDest, int sourceCodePage) { - WCHAR* pWide = ConvertToWideString(pString, GetConsoleCP()); - if (pWide == NULL) - { - return false; - } - - char* pConsole = ConvertFromWideString(pWide, CP_UTF8); - delete [] pWide; - - if (!pConsole) - { - return false; - } + return ConvertEncoding(pString, sourceCodePage, rDest, CP_UTF8); +} - rDest = pConsole; - delete [] pConsole; +bool ConvertFromUtf8(const char* pString, std::string& rDest, int destCodePage) +{ + return ConvertEncoding(pString, CP_UTF8, rDest, destCodePage); +} - return true; +bool ConvertConsoleToUtf8(const char* pString, std::string& rDest) +{ + return ConvertEncoding(pString, GetConsoleCP(), rDest, CP_UTF8); } +bool ConvertUtf8ToConsole(const char* pString, std::string& rDest) +{ + return ConvertEncoding(pString, CP_UTF8, rDest, GetConsoleOutputCP()); +} // -------------------------------------------------------------------------- // @@ -454,23 +498,36 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName) // Is the path relative or absolute? // Absolute paths on Windows are always a drive letter // followed by ':' - - if (filename.length() >= 2 && filename[1] != ':') + + char wd[PATH_MAX]; + if (::getcwd(wd, PATH_MAX) == 0) + { + ::syslog(LOG_WARNING, + "Failed to open '%s': path too long", + pFileName); + errno = ENAMETOOLONG; + tmpStr = ""; + return tmpStr; + } + + if (filename.length() > 2 && filename[0] == '\\' && + filename[1] == '\\') + { + tmpStr += "UNC\\"; + filename.replace(0, 2, ""); + // \\?\UNC\<server>\<share> + // see http://msdn2.microsoft.com/en-us/library/aa365247.aspx + } + else if (filename.length() >= 1 && filename[0] == '\\') + { + // root directory of current drive. + tmpStr = wd; + tmpStr.resize(2); // drive letter and colon + } + else if (filename.length() >= 2 && filename[1] != ':') { // Must be relative. We need to get the // current directory to make it absolute. - - char wd[PATH_MAX]; - if (::getcwd(wd, PATH_MAX) == 0) - { - ::syslog(LOG_WARNING, - "Failed to open '%s': path too long", - pFileName); - errno = ENAMETOOLONG; - tmpStr = ""; - return tmpStr; - } - tmpStr += wd; if (tmpStr[tmpStr.length()] != '\\') { @@ -482,6 +539,37 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName) return tmpStr; } +std::string GetErrorMessage(DWORD errorCode) +{ + char* pMsgBuf = NULL; + + DWORD chars = FormatMessage + ( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + errorCode, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (char *)(&pMsgBuf), + 0, NULL + ); + + if (chars == 0 || pMsgBuf == NULL) + { + return std::string("failed to get error message"); + } + + // remove embedded newline + pMsgBuf[chars - 1] = 0; + pMsgBuf[chars - 2] = 0; + + std::ostringstream line; + line << pMsgBuf << " (" << errorCode << ")"; + LocalFree(pMsgBuf); + + return line.str(); +} + // -------------------------------------------------------------------------- // // Function @@ -513,48 +601,62 @@ HANDLE openfile(const char *pFileName, int flags, int mode) // flags could be O_WRONLY | O_CREAT | O_RDONLY DWORD createDisposition = OPEN_EXISTING; - DWORD shareMode = FILE_SHARE_READ; - DWORD accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY | FILE_READ_EA; + DWORD shareMode = FILE_SHARE_READ | FILE_SHARE_WRITE + | FILE_SHARE_DELETE; + DWORD accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY + | FILE_READ_EA; if (flags & O_WRONLY) { accessRights = FILE_WRITE_DATA; - shareMode = FILE_SHARE_WRITE; } - else if (flags & (O_RDWR | O_CREAT)) + else if (flags & O_RDWR) { accessRights |= FILE_WRITE_ATTRIBUTES | FILE_WRITE_DATA | FILE_WRITE_EA; - shareMode |= FILE_SHARE_WRITE; } if (flags & O_CREAT) { createDisposition = OPEN_ALWAYS; } + if (flags & O_TRUNC) { createDisposition = CREATE_ALWAYS; } - if (flags & O_EXCL) + + if ((flags & O_CREAT) && (flags & O_EXCL)) + { + createDisposition = CREATE_NEW; + } + + if (flags & O_LOCK) { shareMode = 0; } + DWORD winFlags = FILE_FLAG_BACKUP_SEMANTICS; + if (flags & O_TEMPORARY) + { + winFlags |= FILE_FLAG_DELETE_ON_CLOSE; + } + HANDLE hdir = CreateFileW(pBuffer, accessRights, shareMode, NULL, createDisposition, - FILE_FLAG_BACKUP_SEMANTICS, + winFlags, NULL); delete [] pBuffer; if (hdir == INVALID_HANDLE_VALUE) { - ::syslog(LOG_WARNING, "Failed to open file %s: " - "error %i", pFileName, GetLastError()); + ::syslog(LOG_WARNING, "Failed to open file '%s': " + "%s", pFileName, + GetErrorMessage(GetLastError()).c_str()); return INVALID_HANDLE_VALUE; } @@ -582,7 +684,7 @@ int emu_fstat(HANDLE hdir, struct stat * st) if (!GetFileInformationByHandle(hdir, &fi)) { ::syslog(LOG_WARNING, "Failed to read file information: " - "error %d", GetLastError()); + "%s", GetErrorMessage(GetLastError()).c_str()); errno = EACCES; return -1; } @@ -590,7 +692,7 @@ int emu_fstat(HANDLE hdir, struct stat * st) if (INVALID_FILE_ATTRIBUTES == fi.dwFileAttributes) { ::syslog(LOG_WARNING, "Failed to get file attributes: " - "error %d", GetLastError()); + "%s", GetErrorMessage(GetLastError()).c_str()); errno = EACCES; return -1; } @@ -621,7 +723,7 @@ int emu_fstat(HANDLE hdir, struct stat * st) if (!GetFileSizeEx(hdir, &st_size)) { ::syslog(LOG_WARNING, "Failed to get file size: " - "error %d", GetLastError()); + "%s", GetErrorMessage(GetLastError()).c_str()); errno = EACCES; return -1; } @@ -659,6 +761,22 @@ int emu_fstat(HANDLE hdir, struct stat * st) st->st_mode |= S_IWRITE; } + // st_dev is normally zero, regardless of the drive letter, + // since backup locations can't normally span drives. However, + // a reparse point does allow all kinds of weird stuff to happen. + // We set st_dev to 1 for a reparse point, so that Box will detect + // a change of device number (from 0) and refuse to recurse down + // the reparse point (which could lead to havoc). + + if (fi.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + { + st->st_dev = 1; + } + else + { + st->st_dev = 0; + } + return 0; } @@ -728,8 +846,9 @@ HANDLE OpenFileByNameUtf8(const char* pFileName, DWORD flags) } else { - ::syslog(LOG_WARNING, - "Failed to open '%s': error %d", pFileName, err); + ::syslog(LOG_WARNING, "Failed to open '%s': " + "%s", pFileName, + GetErrorMessage(err).c_str()); errno = EACCES; } @@ -797,7 +916,8 @@ int statfs(const char * pName, struct statfs * s) if (!GetFileInformationByHandle(handle, &fi)) { ::syslog(LOG_WARNING, "Failed to get file information " - "for '%s': error %d", pName, GetLastError()); + "for '%s': %s", pName, + GetErrorMessage(GetLastError()).c_str()); CloseHandle(handle); errno = EACCES; return -1; @@ -807,7 +927,7 @@ int statfs(const char * pName, struct statfs * s) _ui64toa(fi.dwVolumeSerialNumber, s->f_mntonname + 1, 16); // pseudo unix mount point - s->f_mntonname[0] = DIRECTORY_SEPARATOR_ASCHAR; + s->f_mntonname[0] = '\\'; CloseHandle(handle); // close the handle @@ -850,8 +970,8 @@ int emu_utimes(const char * pName, const struct timeval times[]) if (!SetFileTime(handle, &creationTime, NULL, &modificationTime)) { - ::syslog(LOG_ERR, "Failed to set times on '%s': error %d", - pName, GetLastError()); + ::syslog(LOG_ERR, "Failed to set times on '%s': %s", pName, + GetErrorMessage(GetLastError()).c_str()); CloseHandle(handle); return 1; } @@ -893,8 +1013,8 @@ int emu_chmod(const char * pName, mode_t mode) DWORD attribs = GetFileAttributesW(pBuffer); if (attribs == INVALID_FILE_ATTRIBUTES) { - ::syslog(LOG_ERR, "Failed to get file attributes of '%s': " - "error %d", pName, GetLastError()); + ::syslog(LOG_ERR, "Failed to get file attributes of '%s': %s", + pName, GetErrorMessage(GetLastError()).c_str()); errno = EACCES; free(pBuffer); return -1; @@ -911,14 +1031,14 @@ int emu_chmod(const char * pName, mode_t mode) if (!SetFileAttributesW(pBuffer, attribs)) { - ::syslog(LOG_ERR, "Failed to set file attributes of '%s': " - "error %d", pName, GetLastError()); + ::syslog(LOG_ERR, "Failed to set file attributes of '%s': %s", + pName, GetErrorMessage(GetLastError()).c_str()); errno = EACCES; free(pBuffer); return -1; } - free(pBuffer); + delete [] pBuffer; return 0; } @@ -1178,15 +1298,15 @@ BOOL AddEventSource // Work out the executable file name, to register ourselves // as the event source - char cmd[MAX_PATH]; - if (GetModuleFileName(NULL, cmd, sizeof(cmd)-1) == 0) + WCHAR cmd[MAX_PATH]; + DWORD len = GetModuleFileNameW(NULL, cmd, MAX_PATH); + + if (len == 0) { - ::syslog(LOG_ERR, "Failed to get the program file name: " - "error %d", GetLastError()); + ::syslog(LOG_ERR, "Failed to get the program file name: %s", + GetErrorMessage(GetLastError()).c_str()); return FALSE; } - cmd[sizeof(cmd)-1] = 0; - std::string exepath(cmd); // Create the event source as a subkey of the log. @@ -1201,22 +1321,22 @@ BOOL AddEventSource 0, NULL, REG_OPTION_NON_VOLATILE, KEY_WRITE, NULL, &hk, &dwDisp)) { - ::syslog(LOG_ERR, "Failed to create the registry key: " - "error %d", GetLastError()); + ::syslog(LOG_ERR, "Failed to create the registry key: %s", + GetErrorMessage(GetLastError()).c_str()); return FALSE; } // Set the name of the message file. - if (RegSetValueEx(hk, // subkey handle - "EventMessageFile", // value name - 0, // must be zero - REG_EXPAND_SZ, // value type - (LPBYTE) exepath.c_str(), // pointer to value data - (DWORD) (exepath.size()))) // data size - { - ::syslog(LOG_ERR, "Failed to set the event message file: " - "error %d", GetLastError()); + if (RegSetValueExW(hk, // subkey handle + L"EventMessageFile", // value name + 0, // must be zero + REG_EXPAND_SZ, // value type + (LPBYTE)cmd, // pointer to value data + len*sizeof(WCHAR))) // data size + { + ::syslog(LOG_ERR, "Failed to set the event message file: %s", + GetErrorMessage(GetLastError()).c_str()); RegCloseKey(hk); return FALSE; } @@ -1233,23 +1353,23 @@ BOOL AddEventSource (LPBYTE) &dwData, // pointer to value data sizeof(DWORD))) // length of value data { - ::syslog(LOG_ERR, "Failed to set the supported types: " - "error %d", GetLastError()); + ::syslog(LOG_ERR, "Failed to set the supported types: %s", + GetErrorMessage(GetLastError()).c_str()); RegCloseKey(hk); return FALSE; } // Set the category message file and number of categories. - if (RegSetValueEx(hk, // subkey handle - "CategoryMessageFile", // value name - 0, // must be zero - REG_EXPAND_SZ, // value type - (LPBYTE) exepath.c_str(), // pointer to value data - (DWORD) (exepath.size()))) // data size + if (RegSetValueExW(hk, // subkey handle + L"CategoryMessageFile", // value name + 0, // must be zero + REG_EXPAND_SZ, // value type + (LPBYTE)cmd, // pointer to value data + len*sizeof(WCHAR))) // data size { ::syslog(LOG_ERR, "Failed to set the category message file: " - "error %d", GetLastError()); + "%s", GetErrorMessage(GetLastError()).c_str()); RegCloseKey(hk); return FALSE; } @@ -1261,8 +1381,8 @@ BOOL AddEventSource (LPBYTE) &dwNum, // pointer to value data sizeof(DWORD))) // length of value data { - ::syslog(LOG_ERR, "Failed to set the category count: " - "error %d", GetLastError()); + ::syslog(LOG_ERR, "Failed to set the category count: %s", + GetErrorMessage(GetLastError()).c_str()); RegCloseKey(hk); return FALSE; } @@ -1285,17 +1405,21 @@ void openlog(const char * daemonName, int, int) { } - if (!AddEventSource("Box Backup", 0)) + char* name = strdup(daemonName); + BOOL success = AddEventSource(name, 0); + free(name); + + if (!success) { ::syslog(LOG_ERR, "Failed to add our own event source"); return; } - HANDLE newSyslogH = RegisterEventSource(NULL, "Box Backup"); + HANDLE newSyslogH = RegisterEventSource(NULL, daemonName); if (newSyslogH == NULL) { ::syslog(LOG_ERR, "Failed to register our own event source: " - "error %d", GetLastError()); + "%s", GetErrorMessage(GetLastError()).c_str()); return; } @@ -1394,8 +1518,8 @@ void syslog(int loglevel, const char *frmt, ...) } else { - printf("Unable to send message to Event Log: " - "error %i:\r\n", (int)err); + printf("Unable to send message to Event Log: %s:\r\n", + GetErrorMessage(err).c_str()); fflush(stdout); } } @@ -1404,12 +1528,13 @@ void syslog(int loglevel, const char *frmt, ...) sHaveWarnedEventLogFull = false; } - printf("%s\r\n", buffer); - fflush(stdout); + // printf("%s\r\n", buffer); + // fflush(stdout); } int emu_chdir(const char* pDirName) { + /* std::string AbsPathWithUnicode = ConvertPathToAbsoluteUnicode(pDirName); @@ -1420,11 +1545,19 @@ int emu_chdir(const char* pDirName) } WCHAR* pBuffer = ConvertUtf8ToWideString(AbsPathWithUnicode.c_str()); + */ + + WCHAR* pBuffer = ConvertUtf8ToWideString(pDirName); if (!pBuffer) return -1; + int result = SetCurrentDirectoryW(pBuffer); delete [] pBuffer; + if (result != 0) return 0; + errno = EACCES; + fprintf(stderr, "Failed to change directory to '%s': %s\n", + pDirName, GetErrorMessage(GetLastError()).c_str()); return -1; } @@ -1454,6 +1587,7 @@ char* emu_getcwd(char* pBuffer, int BufSize) if (result <= 0 || result >= len) { errno = EACCES; + delete [] pWide; return NULL; } @@ -1539,7 +1673,74 @@ int emu_unlink(const char* pFileName) else { ::syslog(LOG_WARNING, "Failed to delete file " - "'%s': error %d", pFileName, (int)err); + "'%s': %s", pFileName, + GetErrorMessage(err).c_str()); + errno = ENOSYS; + } + return -1; + } + + return 0; +} + +int emu_rename(const char* pOldFileName, const char* pNewFileName) +{ + std::string OldPathWithUnicode = + ConvertPathToAbsoluteUnicode(pOldFileName); + + if (OldPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + WCHAR* pOldBuffer = ConvertUtf8ToWideString(OldPathWithUnicode.c_str()); + if (!pOldBuffer) + { + return -1; + } + + std::string NewPathWithUnicode = + ConvertPathToAbsoluteUnicode(pNewFileName); + + if (NewPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + delete [] pOldBuffer; + return -1; + } + + WCHAR* pNewBuffer = ConvertUtf8ToWideString(NewPathWithUnicode.c_str()); + if (!pNewBuffer) + { + delete [] pOldBuffer; + return -1; + } + + BOOL result = MoveFileW(pOldBuffer, pNewBuffer); + DWORD err = GetLastError(); + delete [] pOldBuffer; + delete [] pNewBuffer; + + if (!result) + { + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + { + errno = ENOENT; + } + else if (err == ERROR_SHARING_VIOLATION) + { + errno = EBUSY; + } + else if (err == ERROR_ACCESS_DENIED) + { + errno = EACCES; + } + else + { + ::syslog(LOG_WARNING, "Failed to rename file " + "'%s' to '%s': %s", pOldFileName, pNewFileName, + GetErrorMessage(err).c_str()); errno = ENOSYS; } return -1; @@ -1555,12 +1756,12 @@ int console_read(char* pBuffer, size_t BufferSize) if (hConsole == INVALID_HANDLE_VALUE) { ::fprintf(stderr, "Failed to get a handle on standard input: " - "error %d\n", GetLastError()); + "%s", GetErrorMessage(GetLastError()).c_str()); return -1; } size_t WideSize = BufferSize / 5; - WCHAR* pWideBuffer = new WCHAR [WideSize]; + WCHAR* pWideBuffer = new WCHAR [WideSize + 1]; if (!pWideBuffer) { @@ -1578,14 +1779,16 @@ int console_read(char* pBuffer, size_t BufferSize) NULL // reserved )) { - ::fprintf(stderr, "Failed to read from console: error %d\n", - GetLastError()); + ::fprintf(stderr, "Failed to read from console: %s\n", + GetErrorMessage(GetLastError()).c_str()); return -1; } pWideBuffer[numCharsRead] = 0; char* pUtf8 = ConvertFromWideString(pWideBuffer, GetConsoleCP()); + delete [] pWideBuffer; + strncpy(pBuffer, pUtf8, BufferSize); delete [] pUtf8; @@ -1675,13 +1878,13 @@ bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo) // Convert the last-write time to local time. if (!SystemTimeToFileTime(&stUTC, pTo)) { - syslog(LOG_ERR, "Failed to convert between time formats: " - "error %d", GetLastError()); + syslog(LOG_ERR, "Failed to convert between time formats: %s", + GetErrorMessage(GetLastError()).c_str()); return false; } return true; } - #endif // WIN32 + diff --git a/lib/win32/emu.h b/lib/win32/emu.h index d898968a..8ab74130 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -8,25 +8,24 @@ #ifdef __MINGW32__ #include <stdint.h> - typedef uint32_t u_int32_t; #else // MSVC - typedef __int64 int64_t; - typedef __int32 int32_t; - typedef __int16 int16_t; - typedef __int8 int8_t; - typedef unsigned __int64 u_int64_t; - typedef unsigned __int32 u_int32_t; - typedef unsigned __int64 uint64_t; + typedef __int64 int64_t; typedef unsigned __int32 uint32_t; + typedef unsigned __int32 u_int32_t; + typedef __int32 int32_t; typedef unsigned __int16 uint16_t; + typedef __int16 int16_t; typedef unsigned __int8 uint8_t; + typedef __int8 int8_t; #endif // emulated types, present on MinGW but not MSVC or vice versa -#ifdef _MSC_VER +#ifdef __MINGW32__ + typedef uint32_t u_int32_t; +#else typedef unsigned int mode_t; typedef unsigned int pid_t; @@ -66,7 +65,7 @@ ( *(_result) = *gmtime( (_clock) ), \ (_result) ) -#define ITIMER_VIRTUAL 0 +#define ITIMER_REAL 0 #ifdef _MSC_VER // Microsoft decided to deprecate the standard POSIX functions. Great! @@ -84,12 +83,6 @@ int setitimer(int type, struct itimerval *timeout, void *arg); void InitTimer(void); void FiniTimer(void); -inline int geteuid(void) -{ - //lets pretend to be root! - return 0; -} - struct passwd { char *pw_name; char *pw_passwd; @@ -148,43 +141,9 @@ inline int chown(const char * Filename, u_int32_t uid, u_int32_t gid) return 0; } -int emu_chdir (const char* pDirName); -int emu_unlink(const char* pFileName); -char* emu_getcwd(char* pBuffer, int BufSize); -int emu_utimes(const char* pName, const struct timeval[]); -int emu_chmod (const char* pName, mode_t mode); - -#define utimes(buffer, times) emu_utimes(buffer, times) - -#ifdef _MSC_VER - #define chmod(file, mode) emu_chmod(file, mode) - #define chdir(directory) emu_chdir(directory) - #define unlink(file) emu_unlink(file) - #define getcwd(buffer, size) emu_getcwd(buffer, size) -#else - inline int chmod(const char * pName, mode_t mode) - { - return emu_chmod(pName, mode); - } - - inline int chdir(const char* pDirName) - { - return emu_chdir(pDirName); - } - - inline char* getcwd(char* pBuffer, int BufSize) - { - return emu_getcwd(pBuffer, BufSize); - } - - inline int unlink(const char* pFileName) - { - return emu_unlink(pFileName); - } -#endif - -//I do not perceive a need to change the user or group on a backup client -//at any rate the owner of a service can be set in the service settings +// Windows and Unix owners and groups are pretty fundamentally different. +// Ben prefers that we kludge here rather than litter the code with #ifdefs. +// Pretend to be root, and pretend that set...() operations succeed. inline int setegid(int) { return true; @@ -209,6 +168,10 @@ inline int getuid(void) { return 0; } +inline int geteuid(void) +{ + return 0; +} #ifndef PATH_MAX #define PATH_MAX MAX_PATH @@ -235,11 +198,6 @@ struct itimerval typedef int socklen_t; #endif -// I (re-)defined here for the moment; has to be removed later !!! -#ifndef BOX_VERSION -#define BOX_VERSION "0.09hWin32" -#endif - #define S_IRGRP S_IWRITE #define S_IWGRP S_IREAD #define S_IROTH S_IWRITE | S_IREAD @@ -252,13 +210,6 @@ struct itimerval #define vsnprintf _vsnprintf -int emu_mkdir(const char* pPathName); - -inline int mkdir(const char *pPathName, mode_t mode) -{ - return emu_mkdir(pPathName); -} - #ifndef __MINGW32__ inline int strcasecmp(const char *s1, const char *s2) { @@ -290,11 +241,17 @@ DIR *opendir(const char *name); struct dirent *readdir(DIR *dp); int closedir(DIR *dp); +// local constant to open file exclusively without shared access +#define O_LOCK 0x10000 + HANDLE openfile(const char *filename, int flags, int mode); +#define LOG_DEBUG LOG_INFO #define LOG_INFO 6 +#define LOG_NOTICE LOG_INFO #define LOG_WARNING 4 #define LOG_ERR 3 +#define LOG_CRIT LOG_ERR #define LOG_PID 0 #define LOG_LOCAL5 0 #define LOG_LOCAL6 0 @@ -366,43 +323,36 @@ struct stat { time_t st_mtime; time_t st_ctime; }; -#endif - -int emu_stat(const char * name, struct stat * st); -int emu_fstat(HANDLE file, struct stat * st); -int statfs(const char * name, struct statfs * s); +#endif // 0 // need this for conversions time_t ConvertFileTimeToTime_t(FILETIME *fileTime); bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo); -#ifdef _MSC_VER - #define stat(filename, struct) emu_stat (filename, struct) - #define lstat(filename, struct) emu_stat (filename, struct) - #define fstat(handle, struct) emu_fstat(handle, struct) -#else - inline int stat(const char* filename, struct stat* stat) - { - return emu_stat(filename, stat); - } - inline int lstat(const char* filename, struct stat* stat) - { - return emu_stat(filename, stat); - } - inline int fstat(HANDLE handle, struct stat* stat) - { - return emu_fstat(handle, stat); - } -#endif - -int poll(struct pollfd *ufds, unsigned long nfds, int timeout); -bool EnableBackupRights( void ); +int emu_chdir (const char* pDirName); +int emu_mkdir (const char* pPathName); +int emu_unlink (const char* pFileName); +int emu_fstat (HANDLE file, struct stat* st); +int emu_stat (const char* pName, struct stat* st); +int emu_utimes (const char* pName, const struct timeval[]); +int emu_chmod (const char* pName, mode_t mode); +char* emu_getcwd (char* pBuffer, int BufSize); +int emu_rename (const char* pOldName, const char* pNewName); + +#define chdir(directory) emu_chdir (directory) +#define mkdir(path, mode) emu_mkdir (path) +#define unlink(file) emu_unlink (file) +#define stat(filename, struct) emu_stat (filename, struct) +#define lstat(filename, struct) emu_stat (filename, struct) +#define fstat(handle, struct) emu_fstat (handle, struct) +#define utimes(buffer, times) emu_utimes (buffer, times) +#define chmod(file, mode) emu_chmod (file, mode) +#define getcwd(buffer, size) emu_getcwd (buffer, size) +#define rename(oldname, newname) emu_rename (oldname, newname) -bool ConvertUtf8ToConsole(const char* pString, std::string& rDest); -bool ConvertConsoleToUtf8(const char* pString, std::string& rDest); +int statfs(const char * name, struct statfs * s); -// replacement for _cgetws which requires a relatively recent C runtime lib -int console_read(char* pBuffer, size_t BufferSize); +int poll(struct pollfd *ufds, unsigned long nfds, int timeout); struct iovec { void *iov_base; /* Starting address */ @@ -412,4 +362,43 @@ struct iovec { int readv (int filedes, const struct iovec *vector, size_t count); int writev(int filedes, const struct iovec *vector, size_t count); +// The following functions are not emulations, but utilities for other +// parts of the code where Windows API is used or windows-specific stuff +// is needed, like codepage conversion. + +bool EnableBackupRights( void ); + +bool ConvertEncoding (const std::string& rSource, int sourceCodePage, + std::string& rDest, int destCodePage); +bool ConvertToUtf8 (const std::string& rSource, std::string& rDest, + int sourceCodePage); +bool ConvertFromUtf8 (const std::string& rSource, std::string& rDest, + int destCodePage); +bool ConvertUtf8ToConsole(const char* pString, std::string& rDest); +bool ConvertConsoleToUtf8(const char* pString, std::string& rDest); + +// Utility function which returns a default config file name, +// based on the path of the current executable. +std::string GetDefaultConfigFilePath(const std::string& rName); + +// GetErrorMessage() returns a system error message, like strerror() +// but for Windows error codes. +std::string GetErrorMessage(DWORD errorCode); + +// console_read() is a replacement for _cgetws which requires a +// relatively recent C runtime lib +int console_read(char* pBuffer, size_t BufferSize); + +#ifdef _MSC_VER + /* disable certain compiler warnings to be able to actually see the show-stopper ones */ + #pragma warning(disable:4101) // unreferenced local variable + #pragma warning(disable:4244) // conversion, possible loss of data + #pragma warning(disable:4267) // conversion, possible loss of data + #pragma warning(disable:4311) // pointer truncation + #pragma warning(disable:4700) // uninitialized local variable used (hmmmmm...) + #pragma warning(disable:4805) // unsafe mix of type and type 'bool' in operation + #pragma warning(disable:4800) // forcing value to bool 'true' or 'false' (performance warning) + #pragma warning(disable:4996) // POSIX name for this item is deprecated +#endif // _MSC_VER + #endif // !EMU_INCLUDE && WIN32 diff --git a/lib/win32/getopt_long.cxx b/lib/win32/getopt_long.cxx new file mode 100755 index 00000000..c6800426 --- /dev/null +++ b/lib/win32/getopt_long.cxx @@ -0,0 +1,547 @@ +/* $OpenBSD: getopt_long.c,v 1.20 2005/10/25 15:49:37 jmc Exp $ */
+/* $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $ */
+// Adapted for Box Backup by Chris Wilson <chris+boxbackup@qwirx.com>
+
+/*
+ * Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Sponsored in part by the Defense Advanced Research Projects
+ * Agency (DARPA) and Air Force Research Laboratory, Air Force
+ * Materiel Command, USAF, under agreement number F39502-99-1-0512.
+ */
+/*-
+ * Copyright (c) 2000 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Dieter Baron and Thomas Klausner.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by the NetBSD
+ * Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "Box.h"
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#define REPLACE_GETOPT /* use this getopt as the system getopt(3) */
+#endif
+
+#ifdef REPLACE_GETOPT
+int opterr = 1; /* if error message should be printed */
+int optind = 1; /* index into parent argv vector */
+int optopt = '?'; /* character checked for validity */
+int optreset; /* reset getopt */
+char *optarg; /* argument associated with option */
+#endif
+
+#define PRINT_ERROR ((opterr) && (*options != ':'))
+
+#define FLAG_PERMUTE 0x01 /* permute non-options to the end of argv */
+#define FLAG_ALLARGS 0x02 /* treat non-options as args to option "-1" */
+#define FLAG_LONGONLY 0x04 /* operate as getopt_long_only */
+
+/* return values */
+#define BADCH (int)'?'
+#define BADARG ((*options == ':') ? (int)':' : (int)'?')
+#define INORDER (int)1
+
+#define EMSG ""
+
+static void warnx(const char* fmt, ...)
+{
+ va_list ap;
+ va_start(ap, fmt);
+ vfprintf(stderr, fmt, ap);
+ va_end(ap);
+ fprintf(stderr, "\n");
+}
+
+static int getopt_internal(int, char * const *, const char *,
+ const struct option *, int *, int);
+static int parse_long_options(char * const *, const char *,
+ const struct option *, int *, int);
+static int gcd(int, int);
+static void permute_args(int, int, int, char * const *);
+
+static char *place = EMSG; /* option letter processing */
+
+/* XXX: set optreset to 1 rather than these two */
+static int nonopt_start = -1; /* first non option argument (for permute) */
+static int nonopt_end = -1; /* first option after non options (for permute) */
+
+/* Error messages */
+static const char recargchar[] = "option requires an argument -- %c";
+static const char recargstring[] = "option requires an argument -- %s";
+static const char ambig[] = "ambiguous option -- %.*s";
+static const char noarg[] = "option doesn't take an argument -- %.*s";
+static const char illoptchar[] = "unknown option -- %c";
+static const char illoptstring[] = "unknown option -- %s";
+
+/*
+ * Compute the greatest common divisor of a and b.
+ */
+static int
+gcd(int a, int b)
+{
+ int c;
+
+ c = a % b;
+ while (c != 0) {
+ a = b;
+ b = c;
+ c = a % b;
+ }
+
+ return (b);
+}
+
+/*
+ * Exchange the block from nonopt_start to nonopt_end with the block
+ * from nonopt_end to opt_end (keeping the same order of arguments
+ * in each block).
+ */
+static void
+permute_args(int panonopt_start, int panonopt_end, int opt_end,
+ char * const *nargv)
+{
+ int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos;
+ char *swap;
+
+ /*
+ * compute lengths of blocks and number and size of cycles
+ */
+ nnonopts = panonopt_end - panonopt_start;
+ nopts = opt_end - panonopt_end;
+ ncycle = gcd(nnonopts, nopts);
+ cyclelen = (opt_end - panonopt_start) / ncycle;
+
+ for (i = 0; i < ncycle; i++) {
+ cstart = panonopt_end+i;
+ pos = cstart;
+ for (j = 0; j < cyclelen; j++) {
+ if (pos >= panonopt_end)
+ pos -= nnonopts;
+ else
+ pos += nopts;
+ swap = nargv[pos];
+ /* LINTED const cast */
+ ((char **) nargv)[pos] = nargv[cstart];
+ /* LINTED const cast */
+ ((char **)nargv)[cstart] = swap;
+ }
+ }
+}
+
+/*
+ * parse_long_options --
+ * Parse long options in argc/argv argument vector.
+ * Returns -1 if short_too is set and the option does not match long_options.
+ */
+static int
+parse_long_options(char * const *nargv, const char *options,
+ const struct option *long_options, int *idx, int short_too)
+{
+ char *current_argv, *has_equal;
+ size_t current_argv_len;
+ int i, match;
+
+ current_argv = place;
+ match = -1;
+
+ optind++;
+
+ if ((has_equal = strchr(current_argv, '=')) != NULL) {
+ /* argument found (--option=arg) */
+ current_argv_len = has_equal - current_argv;
+ has_equal++;
+ } else
+ current_argv_len = strlen(current_argv);
+
+ for (i = 0; long_options[i].name; i++) {
+ /* find matching long option */
+ if (strncmp(current_argv, long_options[i].name,
+ current_argv_len))
+ continue;
+
+ if (strlen(long_options[i].name) == current_argv_len) {
+ /* exact match */
+ match = i;
+ break;
+ }
+ /*
+ * If this is a known short option, don't allow
+ * a partial match of a single character.
+ */
+ if (short_too && current_argv_len == 1)
+ continue;
+
+ if (match == -1) /* partial match */
+ match = i;
+ else {
+ /* ambiguous abbreviation */
+ if (PRINT_ERROR)
+ warnx(ambig, (int)current_argv_len,
+ current_argv);
+ optopt = 0;
+ return (BADCH);
+ }
+ }
+ if (match != -1) { /* option found */
+ if (long_options[match].has_arg == no_argument
+ && has_equal) {
+ if (PRINT_ERROR)
+ warnx(noarg, (int)current_argv_len,
+ current_argv);
+ /*
+ * XXX: GNU sets optopt to val regardless of flag
+ */
+ if (long_options[match].flag == NULL)
+ optopt = long_options[match].val;
+ else
+ optopt = 0;
+ return (BADARG);
+ }
+ if (long_options[match].has_arg == required_argument ||
+ long_options[match].has_arg == optional_argument) {
+ if (has_equal)
+ optarg = has_equal;
+ else if (long_options[match].has_arg ==
+ required_argument) {
+ /*
+ * optional argument doesn't use next nargv
+ */
+ optarg = nargv[optind++];
+ }
+ }
+ if ((long_options[match].has_arg == required_argument)
+ && (optarg == NULL)) {
+ /*
+ * Missing argument; leading ':' indicates no error
+ * should be generated.
+ */
+ if (PRINT_ERROR)
+ warnx(recargstring,
+ current_argv);
+ /*
+ * XXX: GNU sets optopt to val regardless of flag
+ */
+ if (long_options[match].flag == NULL)
+ optopt = long_options[match].val;
+ else
+ optopt = 0;
+ --optind;
+ return (BADARG);
+ }
+ } else { /* unknown option */
+ if (short_too) {
+ --optind;
+ return (-1);
+ }
+ if (PRINT_ERROR)
+ warnx(illoptstring, current_argv);
+ optopt = 0;
+ return (BADCH);
+ }
+ if (idx)
+ *idx = match;
+ if (long_options[match].flag) {
+ *long_options[match].flag = long_options[match].val;
+ return (0);
+ } else
+ return (long_options[match].val);
+}
+
+/*
+ * getopt_internal --
+ * Parse argc/argv argument vector. Called by user level routines.
+ */
+static int
+getopt_internal(int nargc, char * const *nargv, const char *options,
+ const struct option *long_options, int *idx, int flags)
+{
+ const char * oli; /* option letter list index */
+ int optchar, short_too;
+ static int posixly_correct = -1;
+
+ if (options == NULL)
+ return (-1);
+
+ /*
+ * Disable GNU extensions if POSIXLY_CORRECT is set or options
+ * string begins with a '+'.
+ */
+ if (posixly_correct == -1)
+ posixly_correct = (getenv("POSIXLY_CORRECT") != NULL);
+ if (posixly_correct || *options == '+')
+ flags &= ~FLAG_PERMUTE;
+ else if (*options == '-')
+ flags |= FLAG_ALLARGS;
+ if (*options == '+' || *options == '-')
+ options++;
+
+ /*
+ * XXX Some GNU programs (like cvs) set optind to 0 instead of
+ * XXX using optreset. Work around this braindamage.
+ */
+ if (optind == 0)
+ optind = optreset = 1;
+
+ optarg = NULL;
+ if (optreset)
+ nonopt_start = nonopt_end = -1;
+start:
+ if (optreset || !*place) { /* update scanning pointer */
+ optreset = 0;
+ if (optind >= nargc) { /* end of argument vector */
+ place = EMSG;
+ if (nonopt_end != -1) {
+ /* do permutation, if we have to */
+ permute_args(nonopt_start, nonopt_end,
+ optind, nargv);
+ optind -= nonopt_end - nonopt_start;
+ }
+ else if (nonopt_start != -1) {
+ /*
+ * If we skipped non-options, set optind
+ * to the first of them.
+ */
+ optind = nonopt_start;
+ }
+ nonopt_start = nonopt_end = -1;
+ return (-1);
+ }
+ if (*(place = nargv[optind]) != '-' ||
+ (place[1] == '\0' && strchr(options, '-') == NULL)) {
+ place = EMSG; /* found non-option */
+ if (flags & FLAG_ALLARGS) {
+ /*
+ * GNU extension:
+ * return non-option as argument to option 1
+ */
+ optarg = nargv[optind++];
+ return (INORDER);
+ }
+ if (!(flags & FLAG_PERMUTE)) {
+ /*
+ * If no permutation wanted, stop parsing
+ * at first non-option.
+ */
+ return (-1);
+ }
+ /* do permutation */
+ if (nonopt_start == -1)
+ nonopt_start = optind;
+ else if (nonopt_end != -1) {
+ permute_args(nonopt_start, nonopt_end,
+ optind, nargv);
+ nonopt_start = optind -
+ (nonopt_end - nonopt_start);
+ nonopt_end = -1;
+ }
+ optind++;
+ /* process next argument */
+ goto start;
+ }
+ if (nonopt_start != -1 && nonopt_end == -1)
+ nonopt_end = optind;
+
+ /*
+ * If we have "-" do nothing, if "--" we are done.
+ */
+ if (place[1] != '\0' && *++place == '-' && place[1] == '\0') {
+ optind++;
+ place = EMSG;
+ /*
+ * We found an option (--), so if we skipped
+ * non-options, we have to permute.
+ */
+ if (nonopt_end != -1) {
+ permute_args(nonopt_start, nonopt_end,
+ optind, nargv);
+ optind -= nonopt_end - nonopt_start;
+ }
+ nonopt_start = nonopt_end = -1;
+ return (-1);
+ }
+ }
+
+ /*
+ * Check long options if:
+ * 1) we were passed some
+ * 2) the arg is not just "-"
+ * 3) either the arg starts with -- we are getopt_long_only()
+ */
+ if (long_options != NULL && place != nargv[optind] &&
+ (*place == '-' || (flags & FLAG_LONGONLY))) {
+ short_too = 0;
+ if (*place == '-')
+ place++; /* --foo long option */
+ else if (*place != ':' && strchr(options, *place) != NULL)
+ short_too = 1; /* could be short option too */
+
+ optchar = parse_long_options(nargv, options, long_options,
+ idx, short_too);
+ if (optchar != -1) {
+ place = EMSG;
+ return (optchar);
+ }
+ }
+
+ if ((optchar = (int)*place++) == (int)':' ||
+ optchar == (int)'-' && *place != '\0' ||
+ (oli = strchr(options, optchar)) == NULL) {
+ /*
+ * If the user specified "-" and '-' isn't listed in
+ * options, return -1 (non-option) as per POSIX.
+ * Otherwise, it is an unknown option character (or ':').
+ */
+ if (optchar == (int)'-' && *place == '\0')
+ return (-1);
+ if (!*place)
+ ++optind;
+ if (PRINT_ERROR)
+ warnx(illoptchar, optchar);
+ optopt = optchar;
+ return (BADCH);
+ }
+ if (long_options != NULL && optchar == 'W' && oli[1] == ';') {
+ /* -W long-option */
+ if (*place) /* no space */
+ /* NOTHING */;
+ else if (++optind >= nargc) { /* no arg */
+ place = EMSG;
+ if (PRINT_ERROR)
+ warnx(recargchar, optchar);
+ optopt = optchar;
+ return (BADARG);
+ } else /* white space */
+ place = nargv[optind];
+ optchar = parse_long_options(nargv, options, long_options,
+ idx, 0);
+ place = EMSG;
+ return (optchar);
+ }
+ if (*++oli != ':') { /* doesn't take argument */
+ if (!*place)
+ ++optind;
+ } else { /* takes (optional) argument */
+ optarg = NULL;
+ if (*place) /* no white space */
+ optarg = place;
+ /* XXX: disable test for :: if PC? (GNU doesn't) */
+ else if (oli[1] != ':') { /* arg not optional */
+ if (++optind >= nargc) { /* no arg */
+ place = EMSG;
+ if (PRINT_ERROR)
+ warnx(recargchar, optchar);
+ optopt = optchar;
+ return (BADARG);
+ } else
+ optarg = nargv[optind];
+ } else if (!(flags & FLAG_PERMUTE)) {
+ /*
+ * If permutation is disabled, we can accept an
+ * optional arg separated by whitespace so long
+ * as it does not start with a dash (-).
+ */
+ if (optind + 1 < nargc && *nargv[optind + 1] != '-')
+ optarg = nargv[++optind];
+ }
+ place = EMSG;
+ ++optind;
+ }
+ /* dump back option letter */
+ return (optchar);
+}
+
+#ifdef REPLACE_GETOPT
+/*
+ * getopt --
+ * Parse argc/argv argument vector.
+ *
+ * [eventually this will replace the BSD getopt]
+ */
+int
+getopt(int nargc, char * const *nargv, const char *options)
+{
+
+ /*
+ * We don't pass FLAG_PERMUTE to getopt_internal() since
+ * the BSD getopt(3) (unlike GNU) has never done this.
+ *
+ * Furthermore, since many privileged programs call getopt()
+ * before dropping privileges it makes sense to keep things
+ * as simple (and bug-free) as possible.
+ */
+ return (getopt_internal(nargc, nargv, options, NULL, NULL, 0));
+}
+#endif /* REPLACE_GETOPT */
+
+/*
+ * getopt_long --
+ * Parse argc/argv argument vector.
+ */
+int
+getopt_long(int nargc, char * const *nargv, const char *options,
+ const struct option *long_options, int *idx)
+{
+
+ return (getopt_internal(nargc, nargv, options, long_options, idx,
+ FLAG_PERMUTE));
+}
+
+/*
+ * getopt_long_only --
+ * Parse argc/argv argument vector.
+ */
+int
+getopt_long_only(int nargc, char * const *nargv, const char *options,
+ const struct option *long_options, int *idx)
+{
+
+ return (getopt_internal(nargc, nargv, options, long_options, idx,
+ FLAG_PERMUTE|FLAG_LONGONLY));
+}
+
diff --git a/modules.txt b/modules.txt index de00f08b..d0d0b76b 100644 --- a/modules.txt +++ b/modules.txt @@ -9,51 +9,37 @@ # Generic support code and modules -OMIT:mingw32 -OMIT:CYGWIN lib/raidfile -END-OMIT - lib/crypto lib/server -lib/win32 lib/server lib/compress -test/common lib/win32 -test/crypto lib/crypto lib/win32 -test/compress lib/compress lib/win32 +lib/intercept -OMIT:mingw32 -test/basicserver lib/server lib/win32 -OMIT:CYGWIN -test/raidfile lib/raidfile -END-OMIT +test/common +test/crypto lib/crypto +test/compress lib/compress +test/raidfile lib/raidfile lib/intercept +test/basicserver lib/server # IF_DISTRIBUTION(boxbackup) # Backup system lib/backupclient lib/server lib/crypto lib/compress - -OMIT:mingw32 -OMIT:CYGWIN lib/backupstore lib/server lib/raidfile lib/backupclient -bin/bbstored lib/raidfile lib/server lib/backupstore lib/backupclient lib/win32 -bin/bbstoreaccounts lib/raidfile lib/backupstore lib/win32 -bin/bbackupobjdump lib/backupclient lib/backupstore lib/win32 -END-OMIT -bin/bbackupd lib/server lib/backupclient lib/win32 -bin/bbackupquery lib/server lib/backupclient lib/win32 -bin/bbackupctl lib/server lib/backupclient lib/win32 +bin/bbackupobjdump lib/backupclient lib/backupstore +bin/bbstored lib/raidfile lib/server lib/backupstore lib/backupclient +bin/bbstoreaccounts lib/raidfile lib/backupstore +bin/bbackupd lib/server lib/backupclient +bin/bbackupquery lib/server lib/backupclient +bin/bbackupctl lib/server lib/backupclient -OMIT:mingw32 -OMIT:CYGWIN test/backupstore bin/bbstored bin/bbstoreaccounts lib/server lib/backupstore lib/backupclient lib/raidfile -test/backupstorefix bin/bbstored bin/bbstoreaccounts lib/backupstore lib/raidfile bin/bbackupquery bin/bbackupd +test/backupstorefix bin/bbstored bin/bbstoreaccounts lib/backupstore lib/raidfile bin/bbackupquery bin/bbackupd bin/bbackupctl test/backupstorepatch bin/bbstored bin/bbstoreaccounts lib/backupstore lib/raidfile test/backupdiff lib/backupclient -test/bbackupd bin/bbackupd bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupctl lib/server lib/backupstore lib/backupclient -END-OMIT +test/bbackupd bin/bbackupd bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupctl lib/server lib/backupstore lib/backupclient lib/intercept # END_IF_DISTRIBUTION diff --git a/parcels.txt b/parcels.txt index 13328e55..5a4ad8f9 100644 --- a/parcels.txt +++ b/parcels.txt @@ -8,24 +8,22 @@ backup-client bin bbackupquery bin bbackupctl script bin/bbackupd/bbackupd-config + script LICENSE.txt -ONLY:mingw32 +ONLY:mingw32,mingw32msvc script bin/bbackupd/win32/installer.iss - script bin/bbackupd/win32/ReadMe.txt script bin/bbackupd/win32/bbackupd.conf +END-ONLY + +ONLY:mingw32 script /bin/mgwz.dll script /bin/mingwm10.dll optional script /bin/pcreposix.dll END-ONLY -OMIT:mingw32 -OMIT:CYGWIN - backup-server bin bbstored bin bbstoreaccounts script bin/bbstored/bbstored-certs script bin/bbstored/bbstored-config script lib/raidfile/raidfile-config - -END-OMIT diff --git a/test/backupdiff/testbackupdiff.cpp b/test/backupdiff/testbackupdiff.cpp index 303861a8..a91d6dfe 100644 --- a/test/backupdiff/testbackupdiff.cpp +++ b/test/backupdiff/testbackupdiff.cpp @@ -375,8 +375,10 @@ int test(int argc, const char *argv[]) { // Want to trace out all the details #ifndef NDEBUG + #ifndef WIN32 BackupStoreFile::TraceDetailsOfDiffProcess = true; #endif + #endif // Create all the test files create_test_files(); @@ -509,8 +511,15 @@ int test(int argc, const char *argv[]) // suck up lots of processor time -- because of lots of matches // found. Check this out! + #ifdef WIN32 + ::fprintf(stdout, "Testing diffing two large streams, " + "may take a while!\n"); + ::fflush(stdout); + #endif + make_file_of_zeros("testfiles/zero.0", 20*1024*1024); make_file_of_zeros("testfiles/zero.1", 200*1024*1024); + // Generate a first encoded file { BackupStoreFilenameClear f0name("zero.0"); @@ -530,7 +539,14 @@ int test(int argc, const char *argv[]) 2000 /* object ID of the file diffing from */, blockindex, IOStream::TimeOutInfinite, 0, 0)); encoded->CopyStreamTo(out); + + printf("Time taken: %d seconds\n", (int)(time(0) - beginTime)); + + #ifdef WIN32 + TEST_THAT(time(0) < (beginTime + 300)); + #else TEST_THAT(time(0) < (beginTime + 40)); + #endif } // Remove zero-files to save disk space remove("testfiles/zero.0"); diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp index 34fc98cd..ca29cdff 100644 --- a/test/backupstore/testbackupstore.cpp +++ b/test/backupstore/testbackupstore.cpp @@ -33,6 +33,7 @@ #include "MemBlockStream.h" #include "BackupClientFileAttributes.h" #include "BackupClientCryptoKeys.h" +#include "ServerControl.h" #include "MemLeakFindOn.h" @@ -1683,7 +1684,10 @@ int test3(int argc, const char *argv[]) "testfiles/clientTrustedCAs.pem"); // First, try logging in without an account having been created... just make sure login fails. - int pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid"); + + int pid = LaunchServer(BBSTORED " testfiles/bbstored.conf", + "testfiles/bbstored.pid"); + TEST_THAT(pid != -1 && pid != 0); if(pid > 0) { @@ -1712,12 +1716,17 @@ int test3(int argc, const char *argv[]) } // Create an account for the test client - TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf create 01234567 0 10000B 20000B") == 0); + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf create 01234567 0 " + "10000B 20000B") == 0); + TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + TEST_THAT(TestDirExists("testfiles/0_0/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_1/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567")); - TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8); // make sure something is written to it + TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8); + // make sure something is written to it TEST_THAT(ServerIsAlive(pid)); @@ -1741,13 +1750,18 @@ int test3(int argc, const char *argv[]) TestRemoteProcessMemLeaks("bbstored.memleaks"); #endif - // Set a new limit on the account -- leave the hard limit high to make sure the target for - // freeing space is the soft limit. - TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf setlimit 01234567 10B 20000B") == 0); + // Set a new limit on the account -- leave the hard limit + // high to make sure the target for freeing space is the + // soft limit. + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf setlimit 01234567 " + "10B 20000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Start things up - pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid"); + pid = LaunchServer(BBSTORED " testfiles/bbstored.conf", + "testfiles/bbstored.pid"); + ::sleep(1); TEST_THAT(ServerIsAlive(pid)); @@ -1763,8 +1777,9 @@ int test3(int argc, const char *argv[]) // Count the objects again recursive_count_objects_results after = {0,0,0}; - recursive_count_objects("localhost", BackupProtocolClientListDirectory::RootDirectory, after); -printf("after.objectsNotDel=%i, deleted=%i, old=%i\n",after.objectsNotDel, after.deleted, after.old); + recursive_count_objects("localhost", + BackupProtocolClientListDirectory::RootDirectory, + after); // If these tests fail then try increasing the timeout above TEST_THAT(after.objectsNotDel == before.objectsNotDel); @@ -1772,7 +1787,9 @@ printf("after.objectsNotDel=%i, deleted=%i, old=%i\n",after.objectsNotDel, after TEST_THAT(after.old == 0); // Set a really small hard limit - TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf setlimit 01234567 10B 20B") == 0); + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf setlimit 01234567 " + "10B 20B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Try to upload a file and create a directory, and check an error is generated @@ -1836,11 +1853,16 @@ int multi_server() printf("Starting server for connection from remote machines...\n"); // Create an account for the test client - TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf create 01234567 0 30000B 40000B") == 0); + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf create 01234567 0 " + "30000B 40000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // First, try logging in without an account having been created... just make sure login fails. - int pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored_multi.conf", "testfiles/bbstored.pid"); + + int pid = LaunchServer(BBSTORED " testfiles/bbstored_multi.conf", + "testfiles/bbstored.pid"); + TEST_THAT(pid != -1 && pid != 0); if(pid > 0) { @@ -1874,12 +1896,6 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName); int test(int argc, const char *argv[]) { #ifdef WIN32 - // Under win32 we must initialise the Winsock library - // before using sockets - - WSADATA info; - TEST_THAT(WSAStartup(0x0101, &info) != SOCKET_ERROR) - // this had better work, or bbstored will die when combining diffs char* file = "foo"; std::string abs = ConvertPathToAbsoluteUnicode(file); @@ -1905,6 +1921,7 @@ int test(int argc, const char *argv[]) CloseHandle(h2); CloseHandle(h1); + delete [] wfile; h1 = openfile("foo", O_CREAT | O_RDWR, 0); TEST_THAT(h1 != INVALID_HANDLE_VALUE); @@ -1932,14 +1949,19 @@ int test(int argc, const char *argv[]) // for seeing what's going on. BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); - // encode in some filenames -- can't do static initialisation because the key won't be set up when these are initialised - for(unsigned int l = 0; l < sizeof(ens_filenames) / sizeof(ens_filenames[0]); ++l) - { - ens[l].fn = BackupStoreFilenameClear(ens_filenames[l]); - } - for(unsigned int l = 0; l < sizeof(uploads_filenames) / sizeof(uploads_filenames[0]); ++l) + // encode in some filenames -- can't do static initialisation + // because the key won't be set up when these are initialised { - uploads[l].name = BackupStoreFilenameClear(uploads_filenames[l]); + MEMLEAKFINDER_NO_LEAKS + + for(unsigned int l = 0; l < sizeof(ens_filenames) / sizeof(ens_filenames[0]); ++l) + { + ens[l].fn = BackupStoreFilenameClear(ens_filenames[l]); + } + for(unsigned int l = 0; l < sizeof(uploads_filenames) / sizeof(uploads_filenames[0]); ++l) + { + uploads[l].name = BackupStoreFilenameClear(uploads_filenames[l]); + } } // Trace errors out diff --git a/test/backupstorefix/testbackupstorefix.cpp b/test/backupstorefix/testbackupstorefix.cpp index 92928cc6..97ff70f3 100644 --- a/test/backupstorefix/testbackupstorefix.cpp +++ b/test/backupstorefix/testbackupstorefix.cpp @@ -29,6 +29,7 @@ #include "RaidFileException.h" #include "StoreStructure.h" #include "BackupStoreFileWire.h" +#include "ServerControl.h" #include "MemLeakFindOn.h" @@ -42,7 +43,7 @@ make some BackupDirectoryStore objects, CheckAndFix(), then verify - all old flags delete store info -add suprious file +add spurious file delete directory (should appear again) change container ID of directory delete a file @@ -65,22 +66,8 @@ std::map<std::string, int32_t> nameToID; std::map<int32_t, bool> objectIsDir; #define RUN_CHECK \ - ::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf check 01234567"); \ - ::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf check 01234567 fix"); - -// Wait a given number of seconds for something to complete -void wait_for_operation(int seconds) -{ - printf("waiting: "); - fflush(stdout); - for(int l = 0; l < seconds; ++l) - { - sleep(1); - printf("."); - fflush(stdout); - } - printf("\n"); -} + ::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf check 01234567"); \ + ::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf check 01234567 fix"); // Get ID of an object given a filename int32_t getID(const char *name) @@ -138,6 +125,7 @@ void CorruptObject(const char *name, int start, const char *rubbish) w.Write(rubbish, rubbish_len); // Copy rest of file r->CopyStreamTo(w); + r->Close(); // Commit w.Commit(true /* convert now */); } @@ -202,9 +190,12 @@ void check_dir_dep(BackupStoreDirectory &dir, checkdepinfoen *ck) void test_dir_fixing() { - fnames[0].SetAsClearFilename("x1"); - fnames[1].SetAsClearFilename("x2"); - fnames[2].SetAsClearFilename("x3"); + { + MEMLEAKFINDER_NO_LEAKS; + fnames[0].SetAsClearFilename("x1"); + fnames[1].SetAsClearFilename("x2"); + fnames[2].SetAsClearFilename("x3"); + } { BackupStoreDirectory dir; @@ -275,7 +266,7 @@ void test_dir_fixing() TEST_THAT(dir.CheckAndFix() == false); check_dir_dep(dir, c1); - // Check that a suprious depends older ref is undone + // Check that a spurious depends older ref is undone e2->SetDependsOlder(1); TEST_THAT(dir.CheckAndFix() == true); TEST_THAT(dir.CheckAndFix() == false); @@ -300,37 +291,52 @@ int test(int argc, const char *argv[]) rcontroller.Initialise("testfiles/raidfile.conf"); // Create an account - TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf create 01234567 0 10000B 20000B") == 0); + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf " + "create 01234567 0 10000B 20000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Start the bbstored server - int pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid"); + int pid = LaunchServer(BBSTORED " testfiles/bbstored.conf", + "testfiles/bbstored.pid"); TEST_THAT(pid != -1 && pid != 0); + if(pid > 0) { ::sleep(1); TEST_THAT(ServerIsAlive(pid)); // Run the perl script to create the initial directories - TEST_THAT_ABORTONFAIL(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl init") == 0); + TEST_THAT_ABORTONFAIL(::system(PERL_EXECUTABLE + " testfiles/testbackupstorefix.pl init") == 0); - int bbackupd_pid = LaunchServer("../../bin/bbackupd/bbackupd testfiles/bbackupd.conf", "testfiles/bbackupd.pid"); + int bbackupd_pid = LaunchServer(BBACKUPD + " testfiles/bbackupd.conf", "testfiles/bbackupd.pid"); TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + if(bbackupd_pid > 0) { ::sleep(1); TEST_THAT(ServerIsAlive(bbackupd_pid)); // Create a nice store directory - wait_for_operation(30); + wait_for_operation(14); // That'll do nicely, stop the server - TEST_THAT(KillServer(bbackupd_pid)); - TestRemoteProcessMemLeaks("bbackupd.memleaks"); + #ifdef WIN32 + terminate_bbackupd(bbackupd_pid); + // implicit check for memory leaks + #else + TEST_THAT(KillServer(bbackupd_pid)); + TestRemoteProcessMemLeaks("bbackupd.memleaks"); + #endif } // Generate a list of all the object IDs - TEST_THAT_ABORTONFAIL(::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf \"list -r\" quit > testfiles/initial-listing.txt") == 0); + TEST_THAT_ABORTONFAIL(::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf \"list -r\" quit " + "> testfiles/initial-listing.txt") == 0); + // And load it in { FILE *f = ::fopen("testfiles/initial-listing.txt", "r"); @@ -341,9 +347,11 @@ int test(int argc, const char *argv[]) char name[256]; while(::fgets(line, sizeof(line), f) != 0) { - TEST_THAT(::sscanf(line, "%x %s %s", &id, flags, name) == 3); + TEST_THAT(::sscanf(line, "%x %s %s", &id, + flags, name) == 3); bool isDir = (::strcmp(flags, "-d---") == 0); //TRACE3("%x,%d,%s\n", id, isDir, name); + MEMLEAKFINDER_NO_LEAKS; nameToID[std::string(name)] = id; objectIsDir[id] = isDir; } @@ -358,19 +366,24 @@ int test(int argc, const char *argv[]) del.Delete(); } { - // Add a suprious file - RaidFileWrite random(discSetNum, storeRoot + "randomfile"); + // Add a spurious file + RaidFileWrite random(discSetNum, + storeRoot + "randomfile"); random.Open(); random.Write("test", 4); random.Commit(true); } + // Fix it RUN_CHECK + // Check everything is as it was - TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 0") == 0); + TEST_THAT(::system(PERL_EXECUTABLE + " testfiles/testbackupstorefix.pl check 0") == 0); // Check the random file doesn't exist { - TEST_THAT(!RaidFileRead::FileExists(discSetNum, storeRoot + "01/randomfile")); + TEST_THAT(!RaidFileRead::FileExists(discSetNum, + storeRoot + "01/randomfile")); } // ------------------------------------------------------------------------------------------------ @@ -410,6 +423,8 @@ int test(int argc, const char *argv[]) file_BlockIndexEntry e[2]; } h; TEST_THAT(file->Read(&h, sizeof(h)) == sizeof(h)); + file->Close(); + // Modify TEST_THAT(box_ntoh64(h.hdr.mOtherFileID) == 0); TEST_THAT(box_ntoh64(h.hdr.mNumBlocks) >= 2); @@ -425,14 +440,16 @@ int test(int argc, const char *argv[]) // Fix it RUN_CHECK // Check - TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 1") == 0); + TEST_THAT(::system(PERL_EXECUTABLE + " testfiles/testbackupstorefix.pl check 1") + == 0); // Check the modified file doesn't exist TEST_THAT(!RaidFileRead::FileExists(discSetNum, fn)); } // ------------------------------------------------------------------------------------------------ - ::printf(" === Delete directory, change container ID of another, duplicate entry in dir, supurious file size, delete file\n"); + ::printf(" === Delete directory, change container ID of another, duplicate entry in dir, spurious file size, delete file\n"); { BackupStoreDirectory dir; LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir); @@ -440,7 +457,7 @@ int test(int argc, const char *argv[]) SaveDirectory("Test1/foreomizes/stemptinevidate/ict", dir); } int64_t duplicatedID = 0; - int64_t notSupriousFileSize = 0; + int64_t notSpuriousFileSize = 0; { BackupStoreDirectory dir; LoadDirectory("Test1/cannes/ict/peep", dir); @@ -458,7 +475,7 @@ int test(int argc, const char *argv[]) BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(BackupStoreDirectory::Entry::Flags_File); TEST_THAT(en != 0); - notSupriousFileSize = en->GetSizeInBlocks(); + notSpuriousFileSize = en->GetSizeInBlocks(); en->SetSizeInBlocks(3473874); TEST_THAT(en->GetSizeInBlocks() == 3473874); } @@ -471,7 +488,8 @@ int test(int argc, const char *argv[]) // Fix it RUN_CHECK // Check everything is as it should be - TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 2") == 0); + TEST_THAT(::system(PERL_EXECUTABLE + " testfiles/testbackupstorefix.pl check 2") == 0); { BackupStoreDirectory dir; LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir); @@ -497,7 +515,7 @@ int test(int argc, const char *argv[]) BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(BackupStoreDirectory::Entry::Flags_File); TEST_THAT(en != 0); - TEST_THAT(en->GetSizeInBlocks() == notSupriousFileSize); + TEST_THAT(en->GetSizeInBlocks() == notSpuriousFileSize); } } @@ -527,7 +545,8 @@ int test(int argc, const char *argv[]) // Fix it RUN_CHECK // Check everything is as it should be - TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 3") == 0); + TEST_THAT(::system(PERL_EXECUTABLE + " testfiles/testbackupstorefix.pl check 3") == 0); { BackupStoreDirectory dir; LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir); @@ -541,18 +560,22 @@ int test(int argc, const char *argv[]) // Fix it RUN_CHECK // Check everything is where it is predicted to be - TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 4") == 0); + TEST_THAT(::system(PERL_EXECUTABLE + " testfiles/testbackupstorefix.pl check 4") == 0); // ------------------------------------------------------------------------------------------------ ::printf(" === Corrupt file and dir\n"); // File - CorruptObject("Test1/foreomizes/stemptinevidate/algoughtnerge", 33, "34i729834298349283479233472983sdfhasgs"); + CorruptObject("Test1/foreomizes/stemptinevidate/algoughtnerge", + 33, "34i729834298349283479233472983sdfhasgs"); // Dir - CorruptObject("Test1/cannes/imulatrougge/foreomizes", 23, "dsf32489sdnadf897fd2hjkesdfmnbsdfcsfoisufio2iofe2hdfkjhsf"); + CorruptObject("Test1/cannes/imulatrougge/foreomizes",23, + "dsf32489sdnadf897fd2hjkesdfmnbsdfcsfoisufio2iofe2hdfkjhsf"); // Fix it RUN_CHECK // Check everything is where it should be - TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 5") == 0); + TEST_THAT(::system(PERL_EXECUTABLE + " testfiles/testbackupstorefix.pl check 5") == 0); // ------------------------------------------------------------------------------------------------ ::printf(" === Overwrite root with a file\n"); @@ -566,13 +589,16 @@ int test(int argc, const char *argv[]) // Fix it RUN_CHECK // Check everything is where it should be - TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl reroot 6") == 0); + TEST_THAT(::system(PERL_EXECUTABLE + " testfiles/testbackupstorefix.pl reroot 6") == 0); // ------------------------------------------------------------------------------------------------ // Stop server TEST_THAT(KillServer(pid)); - TestRemoteProcessMemLeaks("bbstored.memleaks"); + #ifndef WIN32 + TestRemoteProcessMemLeaks("bbstored.memleaks"); + #endif } return 0; diff --git a/test/backupstorepatch/testbackupstorepatch.cpp b/test/backupstorepatch/testbackupstorepatch.cpp index 8e2fa5f6..4fbf296a 100644 --- a/test/backupstorepatch/testbackupstorepatch.cpp +++ b/test/backupstorepatch/testbackupstorepatch.cpp @@ -35,6 +35,7 @@ #include "MemBlockStream.h" #include "BackupClientFileAttributes.h" #include "BackupClientCryptoKeys.h" +#include "ServerControl.h" #include "MemLeakFindOn.h" @@ -283,14 +284,6 @@ void test_depends_in_dirs() int test(int argc, const char *argv[]) { -#ifdef WIN32 - // Under win32 we must initialise the Winsock library - // before using sockets - - WSADATA info; - TEST_THAT(WSAStartup(0x0101, &info) != SOCKET_ERROR) -#endif - // Allocate a buffer buffer = ::malloc(BUFFER_SIZE); TEST_THAT(buffer != 0); @@ -317,9 +310,8 @@ int test(int argc, const char *argv[]) "testfiles/clientTrustedCAs.pem"); // Create an account - TEST_THAT_ABORTONFAIL(RunCommand( - "../../bin/bbstoreaccounts/bbstoreaccounts " - "-c testfiles/bbstored.conf " + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf " "create 01234567 0 30000B 40000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); @@ -330,7 +322,8 @@ int test(int argc, const char *argv[]) test_depends_in_dirs(); // First, try logging in without an account having been created... just make sure login fails. - int pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid"); + int pid = LaunchServer(BBSTORED " testfiles/bbstored.conf", + "testfiles/bbstored.pid"); TEST_THAT(pid != -1 && pid != 0); if(pid > 0) { diff --git a/test/basicserver/TestCommands.cpp b/test/basicserver/TestCommands.cpp index b18a3326..657e79ea 100644 --- a/test/basicserver/TestCommands.cpp +++ b/test/basicserver/TestCommands.cpp @@ -1,7 +1,9 @@ #include "Box.h" +#ifdef HAVE_SYSLOG_H #include <syslog.h> +#endif #include "autogen_TestProtocolServer.h" #include "CollectInBufferStream.h" diff --git a/test/basicserver/testbasicserver.cpp b/test/basicserver/testbasicserver.cpp index 3360cd20..4b33f555 100644 --- a/test/basicserver/testbasicserver.cpp +++ b/test/basicserver/testbasicserver.cpp @@ -28,6 +28,7 @@ #include "TestContext.h" #include "autogen_TestProtocolClient.h" #include "autogen_TestProtocolServer.h" +#include "ServerControl.h" #include "MemLeakFindOn.h" @@ -430,84 +431,139 @@ int test(int argc, const char *argv[]) } } -//printf("SKIPPING TESTS------------------------\n"); -//goto protocolserver; + //printf("SKIPPING TESTS------------------------\n"); + //goto protocolserver; // Launch a basic server { - int pid = LaunchServer("./test srv1 testfiles/srv1.conf", "testfiles/srv1.pid"); + int pid = LaunchServer("./test srv1 testfiles/srv1.conf", + "testfiles/srv1.pid"); + TEST_THAT(pid != -1 && pid != 0); if(pid > 0) { // Check that it's written the expected file - TEST_THAT(TestFileExists("testfiles/srv1.test1")); + TEST_THAT(TestFileExists("testfiles" + DIRECTORY_SEPARATOR "srv1.test1")); TEST_THAT(ServerIsAlive(pid)); + // Move the config file over - TEST_THAT(::rename("testfiles/srv1b.conf", "testfiles/srv1.conf") != -1); - // Get it to reread the config file - TEST_THAT(HUPServer(pid)); - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); - // Check that new file exists - TEST_THAT(TestFileExists("testfiles/srv1.test2")); + #ifdef WIN32 + TEST_THAT(::unlink("testfiles" + DIRECTORY_SEPARATOR "srv1.conf") != -1); + #endif + + TEST_THAT(::rename( + "testfiles" DIRECTORY_SEPARATOR "srv1b.conf", + "testfiles" DIRECTORY_SEPARATOR "srv1.conf") + != -1); + + #ifndef WIN32 + // Get it to reread the config file + TEST_THAT(HUPServer(pid)); + ::sleep(1); + TEST_THAT(ServerIsAlive(pid)); + // Check that new file exists + TEST_THAT(TestFileExists("testfiles" + DIRECTORY_SEPARATOR "srv1.test2")); + #endif // !WIN32 + // Kill it off TEST_THAT(KillServer(pid)); - TestRemoteProcessMemLeaks("generic-daemon.memleaks"); + + #ifndef WIN32 + TestRemoteProcessMemLeaks( + "generic-daemon.memleaks"); + #endif // !WIN32 } } // Launch a test forking server { - int pid = LaunchServer("./test srv2 testfiles/srv2.conf", "testfiles/srv2.pid"); + int pid = LaunchServer("./test srv2 testfiles/srv2.conf", + "testfiles/srv2.pid"); + TEST_THAT(pid != -1 && pid != 0); + if(pid > 0) { // Will it restart? TEST_THAT(ServerIsAlive(pid)); - TEST_THAT(HUPServer(pid)); - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); + + #ifndef WIN32 + TEST_THAT(HUPServer(pid)); + ::sleep(1); + TEST_THAT(ServerIsAlive(pid)); + #endif // !WIN32 + // Make some connections { SocketStream conn1; conn1.Open(Socket::TypeINET, "localhost", 2003); - SocketStream conn2; - conn2.Open(Socket::TypeUNIX, "testfiles/srv2.sock"); - SocketStream conn3; - conn3.Open(Socket::TypeINET, "localhost", 2003); + + #ifndef WIN32 + SocketStream conn2; + conn2.Open(Socket::TypeUNIX, + "testfiles/srv2.sock"); + SocketStream conn3; + conn3.Open(Socket::TypeINET, + "localhost", 2003); + #endif // !WIN32 + // Quick check that reconnections fail - TEST_CHECK_THROWS(conn1.Open(Socket::TypeUNIX, "testfiles/srv2.sock");, ServerException, SocketAlreadyOpen); + TEST_CHECK_THROWS(conn1.Open(Socket::TypeUNIX, + "testfiles/srv2.sock");, + ServerException, SocketAlreadyOpen); + // Stuff some data around std::vector<IOStream *> conns; conns.push_back(&conn1); - conns.push_back(&conn2); - conns.push_back(&conn3); + + #ifndef WIN32 + conns.push_back(&conn2); + conns.push_back(&conn3); + #endif // !WIN32 + Srv2TestConversations(conns); // Implicit close } - // HUP again - TEST_THAT(HUPServer(pid)); - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); + + #ifndef WIN32 + // HUP again + TEST_THAT(HUPServer(pid)); + ::sleep(1); + TEST_THAT(ServerIsAlive(pid)); + #endif // !WIN32 + // Kill it TEST_THAT(KillServer(pid)); ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); - TestRemoteProcessMemLeaks("test-srv2.memleaks"); + + #ifndef WIN32 + TestRemoteProcessMemLeaks("test-srv2.memleaks"); + #endif // !WIN32 } } // Launch a test SSL server { - int pid = LaunchServer("./test srv3 testfiles/srv3.conf", "testfiles/srv3.pid"); + int pid = LaunchServer("./test srv3 testfiles/srv3.conf", + "testfiles/srv3.pid"); + TEST_THAT(pid != -1 && pid != 0); + if(pid > 0) { // Will it restart? TEST_THAT(ServerIsAlive(pid)); - TEST_THAT(HUPServer(pid)); - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); + + #ifndef WIN32 + TEST_THAT(HUPServer(pid)); + ::sleep(1); + TEST_THAT(ServerIsAlive(pid)); + #endif + // Make some connections { // SSL library @@ -522,37 +578,60 @@ int test(int argc, const char *argv[]) SocketStreamTLS conn1; conn1.Open(context, Socket::TypeINET, "localhost", 2003); - SocketStreamTLS conn2; - conn2.Open(context, Socket::TypeUNIX, "testfiles/srv3.sock"); - SocketStreamTLS conn3; - conn3.Open(context, Socket::TypeINET, "localhost", 2003); + #ifndef WIN32 + SocketStreamTLS conn2; + conn2.Open(context, Socket::TypeUNIX, + "testfiles/srv3.sock"); + SocketStreamTLS conn3; + conn3.Open(context, Socket::TypeINET, + "localhost", 2003); + #endif + // Quick check that reconnections fail - TEST_CHECK_THROWS(conn1.Open(context, Socket::TypeUNIX, "testfiles/srv3.sock");, ServerException, SocketAlreadyOpen); + TEST_CHECK_THROWS(conn1.Open(context, + Socket::TypeUNIX, + "testfiles/srv3.sock");, + ServerException, SocketAlreadyOpen); + // Stuff some data around std::vector<IOStream *> conns; conns.push_back(&conn1); - conns.push_back(&conn2); - conns.push_back(&conn3); + + #ifndef WIN32 + conns.push_back(&conn2); + conns.push_back(&conn3); + #endif + Srv2TestConversations(conns); // Implicit close } - // HUP again - TEST_THAT(HUPServer(pid)); - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); + + #ifndef WIN32 + // HUP again + TEST_THAT(HUPServer(pid)); + ::sleep(1); + TEST_THAT(ServerIsAlive(pid)); + #endif + // Kill it TEST_THAT(KillServer(pid)); ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); - TestRemoteProcessMemLeaks("test-srv3.memleaks"); + + #ifndef WIN32 + TestRemoteProcessMemLeaks("test-srv3.memleaks"); + #endif } } //protocolserver: // Launch a test protocol handling server { - int pid = LaunchServer("./test srv4 testfiles/srv4.conf", "testfiles/srv4.pid"); + int pid = LaunchServer("./test srv4 testfiles/srv4.conf", + "testfiles/srv4.pid"); + TEST_THAT(pid != -1 && pid != 0); + if(pid > 0) { ::sleep(1); @@ -560,7 +639,11 @@ int test(int argc, const char *argv[]) // Open a connection to it SocketStream conn; - conn.Open(Socket::TypeUNIX, "testfiles/srv4.sock"); + #ifdef WIN32 + conn.Open(Socket::TypeINET, "localhost", 2003); + #else + conn.Open(Socket::TypeUNIX, "testfiles/srv4.sock"); + #endif // Create a protocol TestProtocolClient protocol(conn); @@ -623,7 +706,10 @@ int test(int argc, const char *argv[]) TEST_THAT(KillServer(pid)); ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); - TestRemoteProcessMemLeaks("test-srv4.memleaks"); + + #ifndef WIN32 + TestRemoteProcessMemLeaks("test-srv4.memleaks"); + #endif } } diff --git a/test/basicserver/testfiles/srv4.conf b/test/basicserver/testfiles/srv4.conf index b4c5627c..f05dff75 100644 --- a/test/basicserver/testfiles/srv4.conf +++ b/test/basicserver/testfiles/srv4.conf @@ -1,6 +1,6 @@ Server { PidFile = testfiles/srv4.pid - ListenAddresses = unix:testfiles/srv4.sock + ListenAddresses = unix:testfiles/srv4.sock,inet:localhost } diff --git a/test/bbackupd/Makefile.extra b/test/bbackupd/Makefile.extra new file mode 100644 index 00000000..1d3f5103 --- /dev/null +++ b/test/bbackupd/Makefile.extra @@ -0,0 +1 @@ +link-extra: ../../bin/bbackupd/autogen_ClientException.o ../../bin/bbackupd/BackupClientContext.o ../../bin/bbackupd/BackupClientDeleteList.o ../../bin/bbackupd/BackupClientDirectoryRecord.o ../../bin/bbackupd/Win32BackupService.o ../../bin/bbackupd/BackupClientInodeToIDMap.o ../../bin/bbackupd/Win32ServiceFunctions.o ../../bin/bbackupd/BackupDaemon.o diff --git a/test/bbackupd/testbbackupd.cpp b/test/bbackupd/testbbackupd.cpp index 0aedb714..49b7f48a 100644 --- a/test/bbackupd/testbbackupd.cpp +++ b/test/bbackupd/testbbackupd.cpp @@ -9,38 +9,65 @@ #include "Box.h" +// do not include MinGW's dirent.h on Win32, +// as we override some of it in lib/win32. + +#ifndef WIN32 + #include <dirent.h> +#endif + #include <stdio.h> #include <sys/types.h> #include <sys/stat.h> #include <limits.h> #include <string.h> -#include <sys/wait.h> #include <unistd.h> + +#ifdef HAVE_SYS_WAIT_H + #include <sys/wait.h> +#endif + #ifdef HAVE_SYS_XATTR_H -#include <cerrno> -#include <sys/xattr.h> + #include <cerrno> + #include <sys/xattr.h> #endif + #include <map> -#include "Test.h" +#ifdef HAVE_SYSCALL + #include <sys/syscall.h> +#endif + +#include "BackupClientCryptoKeys.h" #include "BackupClientFileAttributes.h" -#include "CommonException.h" +#include "BackupClientRestore.h" +#include "BackupDaemon.h" +#include "BackupDaemonConfigVerify.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" #include "BackupStoreException.h" +#include "BoxPortsAndFiles.h" +#include "BoxTime.h" +#include "BoxTimeToUnix.h" +#include "CollectInBufferStream.h" +#include "CommonException.h" +#include "Configuration.h" #include "FileModificationTime.h" -#include "autogen_BackupProtocolClient.h" +#include "FileStream.h" +#include "IOStreamGetLine.h" +#include "LocalProcessStream.h" #include "SSLLib.h" -#include "TLSContext.h" -#include "SocketStreamTLS.h" -#include "BoxPortsAndFiles.h" -#include "BackupStoreConstants.h" +#include "ServerControl.h" #include "Socket.h" -#include "BackupClientRestore.h" -#include "BackupStoreDirectory.h" -#include "BackupClientCryptoKeys.h" -#include "CollectInBufferStream.h" +#include "SocketStreamTLS.h" +#include "TLSContext.h" +#include "Test.h" +#include "Timer.h" #include "Utils.h" -#include "BoxTime.h" -#include "BoxTimeToUnix.h" + +#include "autogen_BackupProtocolClient.h" +#include "intercept.h" +#include "ServerControl.h" #include "MemLeakFindOn.h" @@ -54,18 +81,11 @@ void wait_for_backup_operation(int seconds = TIME_TO_WAIT_FOR_BACKUP_OPERATION) { - printf("waiting: "); - fflush(stdout); - for(int l = 0; l < seconds; ++l) - { - sleep(1); - printf("."); - fflush(stdout); - } - printf("\n"); + wait_for_operation(seconds); } int bbstored_pid = 0; +int bbackupd_pid = 0; #ifdef HAVE_SYS_XATTR_H bool readxattr_into_map(const char *filename, std::map<std::string,std::string> &rOutput) @@ -252,6 +272,9 @@ bool attrmatch(const char *f1, const char *f2) // if link, just make sure other file is a link too, and that the link to names match if((s1.st_mode & S_IFMT) == S_IFLNK) { +#ifdef WIN32 + TEST_FAIL_WITH_MESSAGE("No symlinks on win32!") +#else if((s2.st_mode & S_IFMT) != S_IFLNK) return false; char p1[PATH_MAX], p2[PATH_MAX]; @@ -262,6 +285,7 @@ bool attrmatch(const char *f1, const char *f2) p1[p1l] = '\0'; p2[p2l] = '\0'; return strcmp(p1, p2) == 0; +#endif } // modification times @@ -280,12 +304,15 @@ int test_basics() BackupClientFileAttributes t1; t1.ReadAttributes("testfiles/test1"); TEST_THAT(!t1.IsSymLink()); + +#ifndef WIN32 BackupClientFileAttributes t2; t2.ReadAttributes("testfiles/test2"); TEST_THAT(t2.IsSymLink()); // Check that it's actually been encrypted (search for symlink name encoded in it) void *te = ::memchr(t2.GetBuffer(), 't', t2.GetSize() - 3); TEST_THAT(te == 0 || ::memcmp(te, "test", 4) != 0); +#endif BackupClientFileAttributes t3; TEST_CHECK_THROWS(t3.ReadAttributes("doesn't exist"), CommonException, OSFileError); @@ -298,13 +325,20 @@ int test_basics() // Apply attributes to these new files t1.WriteAttributes("testfiles/test1_n"); +#ifdef WIN32 + t1.WriteAttributes("testfiles/test2_n"); +#else t2.WriteAttributes("testfiles/test2_n"); +#endif + +#ifndef WIN32 TEST_CHECK_THROWS(t1.WriteAttributes("testfiles/test1_nXX"), CommonException, OSFileError); TEST_CHECK_THROWS(t3.WriteAttributes("doesn't exist"), BackupStoreException, AttributesNotLoaded); - // Test that atttributes are vaguely similar + // Test that attributes are vaguely similar TEST_THAT(attrmatch("testfiles/test1", "testfiles/test1_n")); TEST_THAT(attrmatch("testfiles/test2", "testfiles/test2_n")); +#endif // Check encryption, and recovery from encryption // First, check that two attributes taken from the same thing have different encrypted values (think IV) @@ -377,18 +411,22 @@ int test_basics() int test_setupaccount() { - TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf create 01234567 0 1000B 2000B") == 0); + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " + "testfiles/bbstored.conf create 01234567 0 1000B 2000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); return 0; } int test_run_bbstored() { - bbstored_pid = LaunchServer("../../bin/bbstored/bbstored testfiles/bbstored.conf", "testfiles/bbstored.pid"); + std::string cmd = BBSTORED + bbstored_args + " testfiles/bbstored.conf"; + bbstored_pid = LaunchServer(cmd, "testfiles/bbstored.pid"); + TEST_THAT(bbstored_pid != -1 && bbstored_pid != 0); + if(bbstored_pid > 0) { - ::sleep(1); + ::safe_sleep(1); TEST_THAT(ServerIsAlive(bbstored_pid)); return 0; // success } @@ -399,9 +437,13 @@ int test_run_bbstored() int test_kill_bbstored() { TEST_THAT(KillServer(bbstored_pid)); - ::sleep(1); + ::safe_sleep(1); TEST_THAT(!ServerIsAlive(bbstored_pid)); - TestRemoteProcessMemLeaks("bbstored.memleaks"); + + #ifndef WIN32 + TestRemoteProcessMemLeaks("bbstored.memleaks"); + #endif + return 0; } @@ -437,6 +479,7 @@ void terminate_on_alarm(int sigraised) abort(); } +#ifndef WIN32 void do_interrupted_restore(const TLSContext &context, int64_t restoredirid) { int pid = 0; @@ -499,12 +542,216 @@ void do_interrupted_restore(const TLSContext &context, int64_t restoredirid) } } } +#endif // !WIN32 + +void force_sync() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "force-sync") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void wait_for_sync_start() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "wait-for-sync") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void wait_for_sync_end() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "wait-for-end") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void sync_and_wait() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "force-sync") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +#ifdef WIN32 +bool set_file_time(const char* filename, FILETIME creationTime, + FILETIME lastModTime, FILETIME lastAccessTime) +{ + HANDLE handle = openfile(filename, O_RDWR, 0); + TEST_THAT(handle != INVALID_HANDLE_VALUE); + if (handle == INVALID_HANDLE_VALUE) return false; + + BOOL success = SetFileTime(handle, &creationTime, &lastAccessTime, + &lastModTime); + TEST_THAT(success); + + TEST_THAT(CloseHandle(handle)); + return success; +} +#endif + +void intercept_setup_delay(const char *filename, unsigned int delay_after, + int delay_ms, int syscall_to_delay); +bool intercept_triggered(); + +int64_t SearchDir(BackupStoreDirectory& rDir, + const std::string& rChildName) +{ + BackupStoreDirectory::Iterator i(rDir); + BackupStoreFilenameClear child(rChildName.c_str()); + BackupStoreDirectory::Entry *en = i.FindMatchingClearName(child); + if (en == 0) return 0; + int64_t id = en->GetObjectID(); + TEST_THAT(id > 0); + TEST_THAT(id != BackupProtocolClientListDirectory::RootDirectory); + return id; +} + +int start_internal_daemon() +{ + // ensure that no child processes end up running tests! + int own_pid = getpid(); + + BackupDaemon daemon; + int result = daemon.Main("testfiles/bbackupd.conf"); + + TEST_THAT(result == 0); + if (result != 0) + { + printf("Daemon exited with code %d\n", result); + } + + // ensure that no child processes end up running tests! + TEST_THAT(getpid() == own_pid); + if (getpid() != own_pid) + { + // abort! + _exit(1); + } + + TEST_THAT(TestFileExists("testfiles/bbackupd.pid")); + + printf("Waiting for backup daemon to start: "); + int pid = -1; + + for (int i = 0; i < 30; i++) + { + printf("."); + fflush(stdout); + safe_sleep(1); + + if (TestFileExists("testfiles/bbackupd.pid")) + { + pid = ReadPidFile("testfiles/bbackupd.pid"); + } + + if (pid > 0) + { + break; + } + } + + printf(" done.\n"); + fflush(stdout); + + TEST_THAT(pid > 0); + return pid; +} + +void stop_internal_daemon(int pid) +{ + TEST_THAT(KillServer(pid)); + + /* + int status; + TEST_THAT(waitpid(pid, &status, 0) == pid); + TEST_THAT(WIFEXITED(status)); + + if (WIFEXITED(status)) + { + TEST_THAT(WEXITSTATUS(status) == 0); + } + */ +} + +static struct dirent readdir_test_dirent; +static int readdir_test_counter = 0; +static int readdir_stop_time = 0; +static char stat_hook_filename[512]; + +// First test hook, during the directory scanning stage, returns empty. +// This will not match the directory on the store, so a sync will start. +// We set up the next intercept for the same directory by passing NULL. + +struct dirent *readdir_test_hook_2(DIR *dir); + +#ifdef LINUX_WEIRD_LSTAT +int lstat_test_hook(int ver, const char *file_name, struct stat *buf); +#else +int lstat_test_hook(const char *file_name, struct stat *buf); +#endif + +struct dirent *readdir_test_hook_1(DIR *dir) +{ +#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + intercept_setup_readdir_hook(NULL, readdir_test_hook_2); +#endif + return NULL; +} + +// Second test hook, during the directory sync stage, keeps returning +// new filenames until the timer expires, then disables the intercept. + +struct dirent *readdir_test_hook_2(DIR *dir) +{ + if (time(NULL) >= readdir_stop_time) + { +#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + intercept_setup_readdir_hook(NULL, NULL); + intercept_setup_lstat_hook (NULL, NULL); +#endif + // we will not be called again. + } + + // fill in the struct dirent appropriately + memset(&readdir_test_dirent, 0, sizeof(readdir_test_dirent)); + + #ifdef HAVE_STRUCT_DIRENT_D_INO + readdir_test_dirent.d_ino = ++readdir_test_counter; + #endif + + snprintf(readdir_test_dirent.d_name, + sizeof(readdir_test_dirent.d_name), + "test.%d", readdir_test_counter); + + // ensure that when bbackupd stats the file, it gets the + // right answer + snprintf(stat_hook_filename, sizeof(stat_hook_filename), + "testfiles/TestDir1/spacetest/d1/test.%d", + readdir_test_counter); +#ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + intercept_setup_lstat_hook(stat_hook_filename, lstat_test_hook); +#endif + + return &readdir_test_dirent; +} +#ifdef LINUX_WEIRD_LSTAT +int lstat_test_hook(int ver, const char *file_name, struct stat *buf) +#else +int lstat_test_hook(const char *file_name, struct stat *buf) +#endif +{ + // TRACE1("lstat hook triggered for %s", file_name); + memset(buf, 0, sizeof(*buf)); + buf->st_mode = S_IFREG; + return 0; +} int test_bbackupd() { -// // First, wait for a normal period to make sure the last changes attributes are within a normal backup timeframe. -// wait_for_backup_operation(); + // First, wait for a normal period to make sure the last changes + // attributes are within a normal backup timeframe. + // wait_for_backup_operation(); // Connection gubbins TLSContext context; @@ -515,39 +762,349 @@ int test_bbackupd() // unpack the files for the initial test TEST_THAT(::system("rm -rf testfiles/TestDir1") == 0); - TEST_THAT(::system("mkdir testfiles/TestDir1") == 0); - TEST_THAT(::system("gzip -d < testfiles/spacetest1.tgz | ( cd testfiles/TestDir1 && tar xf - )") == 0); + TEST_THAT(::mkdir("testfiles/TestDir1", 0777) == 0); + + #ifdef WIN32 + TEST_THAT(::system("tar xzvf testfiles/spacetest1.tgz " + "-C testfiles/TestDir1") == 0); + #else + TEST_THAT(::system("gzip -d < testfiles/spacetest1.tgz " + "| ( cd testfiles/TestDir1 && tar xf - )") == 0); + #endif + +#ifdef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + printf("\n==== Skipping intercept-based KeepAlive tests " + "on this platform.\n"); +#else + printf("\n==== Testing SSL KeepAlive messages\n"); + + { + #ifdef WIN32 + #error TODO: implement threads on Win32, or this test \ + will not finish properly + #endif + + // bbackupd daemon will try to initialise timers itself + Timers::Cleanup(); + + // something to diff against (empty file doesn't work) + int fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); + TEST_THAT(fd > 0); + char buffer[10000]; + TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_THAT(close(fd) == 0); + + int pid = start_internal_daemon(); + wait_for_backup_operation(); + stop_internal_daemon(pid); + + intercept_setup_delay("testfiles/TestDir1/spacetest/f1", + 0, 2000, SYS_read, 1); + TEST_THAT(unlink("testfiles/bbackupd.log") == 0); + + pid = start_internal_daemon(); + + fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); + TEST_THAT(fd > 0); + // write again, to update the file's timestamp + TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_THAT(close(fd) == 0); + + wait_for_backup_operation(); + // can't test whether intercept was triggered, because + // it's in a different process. + // TEST_THAT(intercept_triggered()); + stop_internal_daemon(pid); + + // check that keepalive was written to logs, and + // diff was not aborted, i.e. upload was a diff + FileStream fs("testfiles/bbackupd.log", O_RDONLY); + IOStreamGetLine reader(fs); + bool found1 = false; + + while (!reader.IsEOF()) + { + std::string line; + TEST_THAT(reader.GetLine(line)); + if (line == "Send GetBlockIndexByName(0x3,\"f1\")") + { + found1 = true; + break; + } + } + + TEST_THAT(found1); + if (found1) + { + std::string line; + TEST_THAT(reader.GetLine(line)); + std::string comp = "Receive Success(0x"; + TEST_THAT(line.substr(0, comp.size()) == comp); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receiving stream, size 124"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Send GetIsAlive()"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receive IsAlive()"); + + TEST_THAT(reader.GetLine(line)); + comp = "Send StoreFile(0x3,"; + TEST_THAT(line.substr(0, comp.size()) == comp); + comp = ",\"f1\")"; + TEST_THAT(line.substr(line.size() - comp.size()) + == comp); + } + + intercept_setup_delay("testfiles/TestDir1/spacetest/f1", + 0, 4000, SYS_read, 1); + pid = start_internal_daemon(); + + fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); + TEST_THAT(fd > 0); + // write again, to update the file's timestamp + TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_THAT(close(fd) == 0); + + wait_for_backup_operation(); + // can't test whether intercept was triggered, because + // it's in a different process. + // TEST_THAT(intercept_triggered()); + stop_internal_daemon(pid); + + // check that the diff was aborted, i.e. upload was not a diff + found1 = false; + + while (!reader.IsEOF()) + { + std::string line; + TEST_THAT(reader.GetLine(line)); + if (line == "Send GetBlockIndexByName(0x3,\"f1\")") + { + found1 = true; + break; + } + } + + TEST_THAT(found1); + if (found1) + { + std::string line; + TEST_THAT(reader.GetLine(line)); + std::string comp = "Receive Success(0x"; + TEST_THAT(line.substr(0, comp.size()) == comp); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receiving stream, size 124"); + + // delaying for 4 seconds in one step means that + // the diff timer and the keepalive timer will + // both expire, and the diff timer is honoured first, + // so there will be no keepalives. + + TEST_THAT(reader.GetLine(line)); + comp = "Send StoreFile(0x3,"; + TEST_THAT(line.substr(0, comp.size()) == comp); + comp = ",0x0,\"f1\")"; + TEST_THAT(line.substr(line.size() - comp.size()) + == comp); + } + + intercept_setup_delay("testfiles/TestDir1/spacetest/f1", + 0, 1000, SYS_read, 3); + pid = start_internal_daemon(); + + fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); + TEST_THAT(fd > 0); + // write again, to update the file's timestamp + TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_THAT(close(fd) == 0); + + wait_for_backup_operation(); + // can't test whether intercept was triggered, because + // it's in a different process. + // TEST_THAT(intercept_triggered()); + stop_internal_daemon(pid); + + // check that the diff was aborted, i.e. upload was not a diff + found1 = false; + + while (!reader.IsEOF()) + { + std::string line; + TEST_THAT(reader.GetLine(line)); + if (line == "Send GetBlockIndexByName(0x3,\"f1\")") + { + found1 = true; + break; + } + } + + TEST_THAT(found1); + if (found1) + { + std::string line; + TEST_THAT(reader.GetLine(line)); + std::string comp = "Receive Success(0x"; + TEST_THAT(line.substr(0, comp.size()) == comp); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receiving stream, size 124"); + + // delaying for 3 seconds in steps of 1 second + // means that the keepalive timer will expire 3 times, + // and on the 3rd time the diff timer will expire too. + // The diff timer is honoured first, so there will be + // only two keepalives. + + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Send GetIsAlive()"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receive IsAlive()"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Send GetIsAlive()"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receive IsAlive()"); + + TEST_THAT(reader.GetLine(line)); + comp = "Send StoreFile(0x3,"; + TEST_THAT(line.substr(0, comp.size()) == comp); + comp = ",0x0,\"f1\")"; + TEST_THAT(line.substr(line.size() - comp.size()) + == comp); + } - int pid = LaunchServer("../../bin/bbackupd/bbackupd testfiles/bbackupd.conf", "testfiles/bbackupd.pid"); - TEST_THAT(pid != -1 && pid != 0); - if(pid > 0) + intercept_setup_readdir_hook("testfiles/TestDir1/spacetest/d1", + readdir_test_hook_1); + + // time for at least two keepalives + readdir_stop_time = time(NULL) + 12 + 2; + + pid = start_internal_daemon(); + + std::string touchfile = + "testfiles/TestDir1/spacetest/d1/touch-me"; + + fd = open(touchfile.c_str(), O_CREAT | O_WRONLY); + TEST_THAT(fd > 0); + // write again, to update the file's timestamp + TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_THAT(close(fd) == 0); + + wait_for_backup_operation(); + // can't test whether intercept was triggered, because + // it's in a different process. + // TEST_THAT(intercept_triggered()); + stop_internal_daemon(pid); + + // check that keepalives were sent during the dir search + found1 = false; + + // skip to next login + while (!reader.IsEOF()) + { + std::string line; + TEST_THAT(reader.GetLine(line)); + if (line == "Send ListDirectory(0x3,0xffffffff,0xc,true)") + { + found1 = true; + break; + } + } + + TEST_THAT(found1); + if (found1) + { + found1 = false; + + while (!reader.IsEOF()) + { + std::string line; + TEST_THAT(reader.GetLine(line)); + if (line == "Send ListDirectory(0x3,0xffffffff,0xc,true)") + { + found1 = true; + break; + } + } + } + + if (found1) + { + std::string line; + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receive Success(0x3)"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receiving stream, size 425"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Send GetIsAlive()"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receive IsAlive()"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Send GetIsAlive()"); + TEST_THAT(reader.GetLine(line)); + TEST_THAT(line == "Receive IsAlive()"); + } + + TEST_THAT(unlink(touchfile.c_str()) == 0); + + // restore timers for rest of tests + Timers::Init(); + } +#endif // PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + + std::string cmd = BBACKUPD + bbackupd_args + " testfiles/bbackupd.conf"; + bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); + + TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + + ::safe_sleep(1); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + if(bbackupd_pid > 0) { - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); + printf("\n==== Testing that backup pauses when store is full\n"); - // First, check storage space handling -- wait for file to be uploaded + // wait for files to be uploaded wait_for_backup_operation(); - //TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf info 01234567") == 0); + // Set limit to something very small - // About 28 blocks will be used at this point. bbackupd will only pause if the size used is - // greater than soft limit + 1/3 of (hard - soft). Set small values for limits accordingly. - TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf setlimit 01234567 10B 40B") == 0); + // About 28 blocks will be used at this point. bbackupd + // will only pause if the size used is greater than + // soft limit + 1/3 of (hard - soft). Set small values + // for limits accordingly. + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " + "testfiles/bbstored.conf setlimit 01234567 9B 10B") + == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Unpack some more files - TEST_THAT(::system("gzip -d < testfiles/spacetest2.tgz | ( cd testfiles/TestDir1 && tar xf - )") == 0); + #ifdef WIN32 + TEST_THAT(::system("tar xzvf testfiles/spacetest2.tgz " + "-C testfiles/TestDir1") == 0); + #else + TEST_THAT(::system("gzip -d < testfiles/spacetest2.tgz " + "| ( cd testfiles/TestDir1 && tar xf - )") == 0); + #endif + // Delete a file and a directory TEST_THAT(::unlink("testfiles/TestDir1/spacetest/d1/f3") == 0); TEST_THAT(::system("rm -rf testfiles/TestDir1/spacetest/d3/d4") == 0); wait_for_backup_operation(); // Make sure there are some differences - int compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query0a.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 2*256); + int compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query0a.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 2); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // Put the limit back - TEST_THAT_ABORTONFAIL(::system("../../bin/bbstoreaccounts/bbstoreaccounts -c testfiles/bbstored.conf setlimit 01234567 1000B 2000B") == 0); + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " + "testfiles/bbstored.conf setlimit 01234567 " + "1000B 2000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Check that the notify script was run @@ -556,185 +1113,1119 @@ int test_bbackupd() TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2")); // unpack the initial files again - TEST_THAT(::system("gzip -d < testfiles/test_base.tgz | ( cd testfiles && tar xf - )") == 0); + #ifdef WIN32 + TEST_THAT(::system("tar xzvf testfiles/test_base.tgz " + "-C testfiles") == 0); + #else + TEST_THAT(::system("gzip -d < testfiles/test_base.tgz " + "| ( cd testfiles && tar xf - )") == 0); + #endif // wait for it to do it's stuff wait_for_backup_operation(); - // Check that the contents of the store are the same as the contents - // of the disc (-a = all, -c = give result in return code) - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query1.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + // Check that the contents of the store are the same + // as the contents of the disc + // (-a = all, -c = give result in return code) + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query1.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + + printf("\n==== Check that read-only directories and " + "their contents can be restored.\n"); + + { + #ifdef WIN32 + TEST_THAT(::system("chmod 0555 testfiles/" + "TestDir1/x1") == 0); + #else + TEST_THAT(chmod("testfiles/TestDir1/x1", + 0555) == 0); + #endif + + wait_for_sync_end(); // too new + wait_for_sync_end(); // should be backed up now + + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-q \"compare -cEQ Test1 testfiles/TestDir1\" " + "quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // check that we can restore it + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-q \"restore Test1 testfiles/restore1\" " + "quit"); + TEST_RETURN(compareReturnValue, 0); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // check that it restored properly + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-q \"compare -cEQ Test1 testfiles/restore1\" " + "quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // put the permissions back to sensible values + #ifdef WIN32 + TEST_THAT(::system("chmod 0755 testfiles/" + "TestDir1/x1") == 0); + #else + TEST_THAT(chmod("testfiles/TestDir1/x1", + 0755) == 0); + #endif + + } + +#ifdef WIN32 + printf("\n==== Check that filenames in UTF-8 " + "can be backed up\n"); + + // We have no guarantee that a random Unicode string can be + // represented in the user's character set, so we go the + // other way, taking three random characters from the + // character set and converting them to Unicode. + // + // We hope that these characters are valid in most + // character sets, but they probably are not in multibyte + // character sets such as Shift-JIS, GB2312, etc. This test + // will probably fail if your system locale is set to + // Chinese, Japanese, etc. where one of these character + // sets is used by default. You can check the character + // set for your system in Control Panel -> Regional + // Options -> General -> Language Settings -> Set Default + // (System Locale). Because bbackupquery converts from + // system locale to UTF-8 via the console code page + // (which you can check from the Command Prompt with "chcp") + // they must also be valid in your code page (850 for + // Western Europe). + // + // In ISO-8859-1 (Danish locale) they are three Danish + // accented characters, which are supported in code page + // 850. Depending on your locale, YYMV (your yak may vomit). + + std::string foreignCharsNative("\x91\x9b\x86"); + std::string foreignCharsUnicode; + TEST_THAT(ConvertConsoleToUtf8(foreignCharsNative.c_str(), + foreignCharsUnicode)); + + std::string basedir("testfiles/TestDir1"); + std::string dirname("test" + foreignCharsUnicode + "testdir"); + std::string dirpath(basedir + "/" + dirname); + TEST_THAT(mkdir(dirpath.c_str(), 0) == 0); + + std::string filename("test" + foreignCharsUnicode + "testfile"); + std::string filepath(dirpath + "/" + filename); + + char cwdbuf[1024]; + TEST_THAT(getcwd(cwdbuf, sizeof(cwdbuf)) == cwdbuf); + std::string cwd = cwdbuf; + + // Test that our emulated chdir() works properly + // with relative and absolute paths + TEST_THAT(::chdir(dirpath.c_str()) == 0); + TEST_THAT(::chdir("../../..") == 0); + TEST_THAT(::chdir(cwd.c_str()) == 0); + + // Check that it can be converted to the system encoding + // (which is what is needed on the command line) + std::string systemDirName; + TEST_THAT(ConvertEncoding(dirname.c_str(), CP_UTF8, + systemDirName, CP_ACP)); + + std::string systemFileName; + TEST_THAT(ConvertEncoding(filename.c_str(), CP_UTF8, + systemFileName, CP_ACP)); + + // Check that it can be converted to the console encoding + // (which is what we will see in the output) + std::string consoleDirName; + TEST_THAT(ConvertUtf8ToConsole(dirname.c_str(), + consoleDirName)); + + std::string consoleFileName; + TEST_THAT(ConvertUtf8ToConsole(filename.c_str(), + consoleFileName)); + + // test that bbackupd will let us lcd into the local + // directory using a relative path + std::string command = BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "\"lcd testfiles/TestDir1/" + systemDirName + "\" " + "quit"; + compareReturnValue = ::system(command.c_str()); + TEST_RETURN(compareReturnValue, 0); + + // and back out again + command = BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "\"lcd testfiles/TestDir1/" + systemDirName + "\" " + "\"lcd ..\" quit"; + compareReturnValue = ::system(command.c_str()); + TEST_RETURN(compareReturnValue, 0); + + // and using an absolute path + command = BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "\"lcd " + cwd + "/testfiles/TestDir1/" + + systemDirName + "\" quit"; + compareReturnValue = ::system(command.c_str()); + TEST_RETURN(compareReturnValue, 0); + + // and back out again + command = BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "\"lcd " + cwd + "/testfiles/TestDir1/" + + systemDirName + "\" " + "\"lcd ..\" quit"; + compareReturnValue = ::system(command.c_str()); + TEST_RETURN(compareReturnValue, 0); + + { + FileStream fs(filepath.c_str(), O_CREAT | O_RDWR); + + std::string data("hello world\n"); + fs.Write(data.c_str(), data.size()); + TEST_THAT(fs.GetPosition() == 12); + fs.Close(); + } + + wait_for_backup_operation(); + // Compare to check that the file was uploaded + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // Check that we can find it in directory listing + { + SocketStreamTLS conn; + conn.Open(context, Socket::TypeINET, "localhost", + BOX_PORT_BBSTORED); + BackupProtocolClient protocol(conn); + protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION); + protocol.QueryLogin(0x01234567, 0); + + int64_t rootDirId = BackupProtocolClientListDirectory + ::RootDirectory; + std::auto_ptr<BackupProtocolClientSuccess> dirreply( + protocol.QueryListDirectory( + rootDirId, false, 0, false)); + std::auto_ptr<IOStream> dirstream( + protocol.ReceiveStream()); + BackupStoreDirectory dir; + dir.ReadFromStream(*dirstream, protocol.GetTimeout()); + + int64_t baseDirId = SearchDir(dir, "Test1"); + TEST_THAT(baseDirId != 0); + dirreply = protocol.QueryListDirectory(baseDirId, + false, 0, false); + dirstream = protocol.ReceiveStream(); + dir.ReadFromStream(*dirstream, protocol.GetTimeout()); + + int64_t testDirId = SearchDir(dir, dirname.c_str()); + TEST_THAT(testDirId != 0); + dirreply = protocol.QueryListDirectory(testDirId, + false, 0, false); + dirstream = protocol.ReceiveStream(); + dir.ReadFromStream(*dirstream, protocol.GetTimeout()); + + TEST_THAT(SearchDir(dir, filename.c_str()) != 0); + // Log out + protocol.QueryFinished(); + } + + + // Check that bbackupquery shows the dir in console encoding + command = BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-q \"list Test1\" quit"; + pid_t bbackupquery_pid; + std::auto_ptr<IOStream> queryout; + queryout = LocalProcessStream(command.c_str(), + bbackupquery_pid); + TEST_THAT(queryout.get() != NULL); + TEST_THAT(bbackupquery_pid != -1); + + IOStreamGetLine reader(*queryout); + std::string line; + bool found = false; + while (!reader.IsEOF()) + { + TEST_THAT(reader.GetLine(line)); + if (line.find(consoleDirName) != std::string::npos) + { + found = true; + } + } + TEST_THAT(!(queryout->StreamDataLeft())); + TEST_THAT(reader.IsEOF()); + TEST_THAT(found); + queryout->Close(); + + // Check that bbackupquery can list the dir when given + // on the command line in system encoding, and shows + // the file in console encoding + command = BBACKUPQUERY " -c testfiles/bbackupd.conf " + "-q \"list Test1/" + systemDirName + "\" quit"; + queryout = LocalProcessStream(command.c_str(), + bbackupquery_pid); + TEST_THAT(queryout.get() != NULL); + TEST_THAT(bbackupquery_pid != -1); + + IOStreamGetLine reader2(*queryout); + found = false; + while (!reader2.IsEOF()) + { + TEST_THAT(reader2.GetLine(line)); + if (line.find(consoleFileName) != std::string::npos) + { + found = true; + } + } + TEST_THAT(!(queryout->StreamDataLeft())); + TEST_THAT(reader2.IsEOF()); + TEST_THAT(found); + queryout->Close(); + + // Check that bbackupquery can compare the dir when given + // on the command line in system encoding. + command = BBACKUPQUERY " -c testfiles/bbackupd.conf " + "-q \"compare -cEQ Test1/" + systemDirName + + " testfiles/TestDir1/" + systemDirName + "\" quit"; + + compareReturnValue = ::system(command.c_str()); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_RETURN(compareReturnValue, 1); + + // Check that bbackupquery can restore the dir when given + // on the command line in system encoding. + command = BBACKUPQUERY " -c testfiles/bbackupd.conf " + "-q \"restore Test1/" + systemDirName + + " testfiles/restore-" + systemDirName + "\" quit"; + + compareReturnValue = ::system(command.c_str()); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_RETURN(compareReturnValue, 0); + + // Compare to make sure it was restored properly. + command = BBACKUPQUERY " -c testfiles/bbackupd.conf " + "-q \"compare -cEQ Test1/" + systemDirName + + " testfiles/restore-" + systemDirName + "\" quit"; + + compareReturnValue = ::system(command.c_str()); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_RETURN(compareReturnValue, 1); + + std::string fileToUnlink = "testfiles/restore-" + + dirname + "/" + filename; + TEST_THAT(::unlink(fileToUnlink.c_str()) == 0); + + // Check that bbackupquery can get the file when given + // on the command line in system encoding. + command = BBACKUPQUERY " -c testfiles/bbackupd.conf " + "-q \"get Test1/" + systemDirName + "/" + + systemFileName + " " + "testfiles/restore-" + + systemDirName + "/" + systemFileName + "\" quit"; + + compareReturnValue = ::system(command.c_str()); + TEST_RETURN(compareReturnValue, 0); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // And after changing directory to a relative path + command = BBACKUPQUERY " -c testfiles/bbackupd.conf -q " + "\"lcd testfiles\" " + "\"cd Test1/" + systemDirName + "\" " + + "\"get " + systemFileName + "\" quit"; + + compareReturnValue = ::system(command.c_str()); + TEST_RETURN(compareReturnValue, 0); + TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks"); + + // cannot overwrite a file that exists, so delete it + std::string tmp = "testfiles/" + filename; + TEST_THAT(::unlink(tmp.c_str()) == 0); + + // And after changing directory to an absolute path + command = BBACKUPQUERY " -c testfiles/bbackupd.conf -q " + "\"lcd " + cwd + "/testfiles\" " + "\"cd Test1/" + systemDirName + "\" " + + "\"get " + systemFileName + "\" quit"; + + compareReturnValue = ::system(command.c_str()); + TEST_RETURN(compareReturnValue, 0); + TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks"); + + // Compare to make sure it was restored properly. + // The Get command does not restore attributes, so + // we must compare without them (-A) to succeed. + command = BBACKUPQUERY " -c testfiles/bbackupd.conf " + "-q \"compare -cAEQ Test1/" + systemDirName + + " testfiles/restore-" + systemDirName + "\" quit"; + + compareReturnValue = ::system(command.c_str()); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_RETURN(compareReturnValue, 1); + + // Compare without attributes. This should fail. + command = BBACKUPQUERY " -c testfiles/bbackupd.conf " + "-q \"compare -cEQ Test1/" + systemDirName + + " testfiles/restore-" + systemDirName + "\" quit"; + + compareReturnValue = ::system(command.c_str()); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_RETURN(compareReturnValue, 2); +#endif // WIN32 + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + printf("\n==== Check that SyncAllowScript is executed and can " + "pause backup\n"); + fflush(stdout); + + { + wait_for_sync_end(); + // we now have 3 seconds before bbackupd + // runs the SyncAllowScript again. + + char* sync_control_file = "testfiles" + DIRECTORY_SEPARATOR "syncallowscript.control"; + int fd = open(sync_control_file, + O_CREAT | O_EXCL | O_WRONLY, 0700); + if (fd <= 0) + { + perror(sync_control_file); + } + TEST_THAT(fd > 0); - printf("Delete file and update another, create symlink.\n"); + char* control_string = "10\n"; + TEST_THAT(write(fd, control_string, + strlen(control_string)) == + (int)strlen(control_string)); + close(fd); + + // this will pause backups, bbackupd will check + // every 10 seconds to see if they are allowed again. + + char* new_test_file = "testfiles" + DIRECTORY_SEPARATOR "TestDir1" + DIRECTORY_SEPARATOR "Added_During_Pause"; + fd = open(new_test_file, + O_CREAT | O_EXCL | O_WRONLY, 0700); + if (fd <= 0) + { + perror(new_test_file); + } + TEST_THAT(fd > 0); + close(fd); + + struct stat st; + + // next poll should happen within the next + // 5 seconds (normally about 3 seconds) + + safe_sleep(1); // 2 seconds before + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + "syncallowscript.notifyran.1", &st) != 0); + safe_sleep(4); // 2 seconds after + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + "syncallowscript.notifyran.1", &st) == 0); + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + "syncallowscript.notifyran.2", &st) != 0); + + // next poll should happen within the next + // 10 seconds (normally about 8 seconds) + + safe_sleep(6); // 2 seconds before + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + "syncallowscript.notifyran.2", &st) != 0); + safe_sleep(4); // 2 seconds after + TEST_THAT(stat("testfiles" DIRECTORY_SEPARATOR + "syncallowscript.notifyran.2", &st) == 0); + + // check that no backup has run (compare fails) + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 2); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + long start_time = time(NULL); + TEST_THAT(unlink(sync_control_file) == 0); + wait_for_sync_start(); + long end_time = time(NULL); + + long wait_time = end_time - start_time + 2; + // should be about 10 seconds + printf("Waited for %ld seconds, should have been %s", + wait_time, control_string); + TEST_THAT(wait_time >= 8); + TEST_THAT(wait_time <= 12); + + wait_for_sync_end(); + // check that backup has run (compare succeeds) + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3a.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + if (failures > 0) + { + // stop early to make debugging easier + return 1; + } + } + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + printf("\n==== Delete file and update another, " + "create symlink.\n"); // Delete a file TEST_THAT(::unlink("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); - // New symlink - TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/symlink-to-dir") == 0); - + + #ifndef WIN32 + // New symlink + TEST_THAT(::symlink("does-not-exist", + "testfiles/TestDir1/symlink-to-dir") == 0); + #endif + // Update a file (will be uploaded as a diff) { - // Check that the file is over the diffing threshold in the bbackupd.conf file - TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") > 1024); + // Check that the file is over the diffing + // threshold in the bbackupd.conf file + TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") + > 1024); // Add a bit to the end FILE *f = ::fopen("testfiles/TestDir1/f45.df", "a"); TEST_THAT(f != 0); ::fprintf(f, "EXTRA STUFF"); ::fclose(f); - TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") > 1024); + TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") + > 1024); } // wait for backup daemon to do it's stuff, and compare again wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query2.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query2.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + // Try a quick compare, just for fun - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query2q.log \"compare -acq\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query2q.log " + "\"compare -acqQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + // Check that store errors are reported neatly + printf("\n==== Create store error\n"); + TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf", + "testfiles/0_0/backup/01234567/info.rf.bak") == 0); + TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf", + "testfiles/0_1/backup/01234567/info.rf.bak") == 0); + TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf", + "testfiles/0_2/backup/01234567/info.rf.bak") == 0); + // Create a file to trigger an upload + { + int fd1 = open("testfiles/TestDir1/force-upload", + O_CREAT | O_EXCL | O_WRONLY, 0700); + TEST_THAT(fd1 > 0); + TEST_THAT(write(fd1, "just do it", 10) == 10); + TEST_THAT(close(fd1) == 0); + wait_for_backup_operation(4); + } + // Wait and test... + wait_for_backup_operation(); + // Check that it was reported correctly + TEST_THAT(TestFileExists("testfiles/notifyran.backup-error.1")); + // Check that the error was only reported once + TEST_THAT(!TestFileExists("testfiles/notifyran.backup-error.2")); + // Fix the store + TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf.bak", + "testfiles/0_0/backup/01234567/info.rf") == 0); + TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf.bak", + "testfiles/0_1/backup/01234567/info.rf") == 0); + TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf.bak", + "testfiles/0_2/backup/01234567/info.rf") == 0); + + // Check that we DO get errors on compare + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3b.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 2); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + // Wait until bbackupd recovers from the exception + wait_for_backup_operation(100); + + // Ensure that the force-upload file gets uploaded, + // meaning that bbackupd recovered + sync_and_wait(); + + // Check that it did get uploaded, and we have no more errors + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3b.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + // Bad case: delete a file/symlink, replace it with a directory - printf("Replace symlink with directory, add new directory\n"); - TEST_THAT(::unlink("testfiles/TestDir1/symlink-to-dir") == 0); - TEST_THAT(::mkdir("testfiles/TestDir1/symlink-to-dir", 0755) == 0); - TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", 0755) == 0); - // NOTE: create a file within the directory to avoid deletion by the housekeeping process later - TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file/contents") == 0); + printf("\n==== Replace symlink with directory, " + "add new directory\n"); + + #ifndef WIN32 + TEST_THAT(::unlink("testfiles/TestDir1/symlink-to-dir") + == 0); + #endif + + TEST_THAT(::mkdir("testfiles/TestDir1/symlink-to-dir", 0755) + == 0); + TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", 0755) + == 0); + + // NOTE: create a file within the directory to + // avoid deletion by the housekeeping process later + + #ifndef WIN32 + TEST_THAT(::symlink("does-not-exist", + "testfiles/TestDir1/x1/dir-to-file/contents") + == 0); + #endif + wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3c.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + // And the inverse, replace a directory with a file/symlink - printf("Replace directory with symlink\n"); - TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file/contents") == 0); + printf("\n==== Replace directory with symlink\n"); + + #ifndef WIN32 + TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file" + "/contents") == 0); + #endif + TEST_THAT(::rmdir("testfiles/TestDir1/x1/dir-to-file") == 0); - TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file") == 0); + + #ifndef WIN32 + TEST_THAT(::symlink("does-not-exist", + "testfiles/TestDir1/x1/dir-to-file") == 0); + #endif + wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3d.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + // And then, put it back to how it was before. - printf("Replace symlink with directory (which was a symlink)\n"); - TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file") == 0); - TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", 0755) == 0); - TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file/contents2") == 0); + printf("\n==== Replace symlink with directory " + "(which was a symlink)\n"); + + #ifndef WIN32 + TEST_THAT(::unlink("testfiles/TestDir1/x1" + "/dir-to-file") == 0); + #endif + + TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", + 0755) == 0); + + #ifndef WIN32 + TEST_THAT(::symlink("does-not-exist", + "testfiles/TestDir1/x1/dir-to-file/contents2") + == 0); + #endif + wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3e.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - // And finally, put it back to how it was before it was put back to how it was before - // This gets lots of nasty things in the store with directories over other old directories. - printf("Put it all back to how it was\n"); - TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file/contents2") == 0); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + // And finally, put it back to how it was before + // it was put back to how it was before + // This gets lots of nasty things in the store with + // directories over other old directories. + printf("\n==== Put it all back to how it was\n"); + + #ifndef WIN32 + TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file" + "/contents2") == 0); + #endif + TEST_THAT(::rmdir("testfiles/TestDir1/x1/dir-to-file") == 0); - TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/x1/dir-to-file") == 0); + + #ifndef WIN32 + TEST_THAT(::symlink("does-not-exist", + "testfiles/TestDir1/x1/dir-to-file") == 0); + #endif + + wait_for_backup_operation(); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3f.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + // rename an untracked file over an + // existing untracked file + printf("\n==== Rename over existing untracked file\n"); + int fd1 = open("testfiles/TestDir1/untracked-1", + O_CREAT | O_EXCL | O_WRONLY, 0700); + int fd2 = open("testfiles/TestDir1/untracked-2", + O_CREAT | O_EXCL | O_WRONLY, 0700); + TEST_THAT(fd1 > 0); + TEST_THAT(fd2 > 0); + TEST_THAT(write(fd1, "hello", 5) == 5); + TEST_THAT(close(fd1) == 0); + safe_sleep(1); + TEST_THAT(write(fd2, "world", 5) == 5); + TEST_THAT(close(fd2) == 0); + TEST_THAT(TestFileExists("testfiles/TestDir1/untracked-1")); + TEST_THAT(TestFileExists("testfiles/TestDir1/untracked-2")); + wait_for_operation(5); + // back up both files wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3g.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - // case which went wrong: rename a tracked file over a deleted file - printf("Rename an existing file over a deleted file\n"); - TEST_THAT(::rename("testfiles/TestDir1/df9834.dsf", "testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); + #ifdef WIN32 + TEST_THAT(::unlink("testfiles/TestDir1/untracked-2") + == 0); + #endif + + TEST_THAT(::rename("testfiles/TestDir1/untracked-1", + "testfiles/TestDir1/untracked-2") == 0); + TEST_THAT(!TestFileExists("testfiles/TestDir1/untracked-1")); + TEST_THAT( TestFileExists("testfiles/TestDir1/untracked-2")); + wait_for_backup_operation(); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3g.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + // case which went wrong: rename a tracked file over an + // existing tracked file + printf("\n==== Rename over existing tracked file\n"); + fd1 = open("testfiles/TestDir1/tracked-1", + O_CREAT | O_EXCL | O_WRONLY, 0700); + fd2 = open("testfiles/TestDir1/tracked-2", + O_CREAT | O_EXCL | O_WRONLY, 0700); + TEST_THAT(fd1 > 0); + TEST_THAT(fd2 > 0); + char buffer[1024]; + TEST_THAT(write(fd1, "hello", 5) == 5); + TEST_THAT(write(fd1, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_THAT(close(fd1) == 0); + safe_sleep(1); + TEST_THAT(write(fd2, "world", 5) == 5); + TEST_THAT(write(fd2, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_THAT(close(fd2) == 0); + TEST_THAT(TestFileExists("testfiles/TestDir1/tracked-1")); + TEST_THAT(TestFileExists("testfiles/TestDir1/tracked-2")); + wait_for_operation(5); + // back up both files + wait_for_backup_operation(); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3h.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + #ifdef WIN32 + TEST_THAT(::unlink("testfiles/TestDir1/tracked-2") + == 0); + #endif + + TEST_THAT(::rename("testfiles/TestDir1/tracked-1", + "testfiles/TestDir1/tracked-2") == 0); + TEST_THAT(!TestFileExists("testfiles/TestDir1/tracked-1")); + TEST_THAT( TestFileExists("testfiles/TestDir1/tracked-2")); wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3s.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3i.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + // case which went wrong: rename a tracked file + // over a deleted file + printf("\n==== Rename an existing file over a deleted file\n"); + TEST_THAT(!TestFileExists("testfiles/TestDir1/x1/dsfdsfs98.fd")); + TEST_THAT(::rename("testfiles/TestDir1/df9834.dsf", + "testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); - printf("Add files with old times, update attributes of one to latest time\n"); + wait_for_backup_operation(); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3j.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + printf("\n==== Add files with old times, update " + "attributes of one to latest time\n"); // Move that file back - TEST_THAT(::rename("testfiles/TestDir1/x1/dsfdsfs98.fd", "testfiles/TestDir1/df9834.dsf") == 0); + TEST_THAT(::rename("testfiles/TestDir1/x1/dsfdsfs98.fd", + "testfiles/TestDir1/df9834.dsf") == 0); // Add some more files - // Because the 'm' option is not used, these files will look very old to the daemon. + // Because the 'm' option is not used, these files will + // look very old to the daemon. // Lucky it'll upload them then! - TEST_THAT(::system("gzip -d < testfiles/test2.tgz | ( cd testfiles && tar xf - )") == 0); - ::chmod("testfiles/TestDir1/sub23/dhsfdss/blf.h", 0415); + #ifdef WIN32 + TEST_THAT(::system("tar xzvf testfiles/test2.tgz " + "-C testfiles") == 0); + #else + TEST_THAT(::system("gzip -d < testfiles/test2.tgz " + "| ( cd testfiles && tar xf - )") == 0); + ::chmod("testfiles/TestDir1/sub23/dhsfdss/blf.h", 0415); + #endif // Wait and test wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3k.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - // Check that modifying files with old timestamps still get added - printf("Modify existing file, but change timestamp to rather old\n"); - // Time critical, so sync - TEST_THAT(::system("../../bin/bbackupctl/bbackupctl -q -c testfiles/bbackupd.conf wait-for-sync") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); - // Then wait a second, to make sure the scan is complete - ::sleep(1); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + // Check that modifying files with old timestamps + // still get added + printf("\n==== Modify existing file, but change timestamp " + "to rather old\n"); + wait_for_sync_end(); + // Then modify an existing file { - chmod("testfiles/TestDir1/sub23/rand.h", 0777); // in the archive, it's read only - FILE *f = fopen("testfiles/TestDir1/sub23/rand.h", "w+"); + // in the archive, it's read only + #ifdef WIN32 + TEST_THAT(::system("chmod 0777 testfiles" + "/TestDir1/sub23/rand.h") == 0); + #else + TEST_THAT(chmod("testfiles/TestDir1/sub23" + "/rand.h", 0777) == 0); + #endif + + FILE *f = fopen("testfiles/TestDir1/sub23/rand.h", + "w+"); + + if (f == 0) + { + perror("Failed to open"); + } + TEST_THAT(f != 0); - fprintf(f, "MODIFIED!\n"); - fclose(f); + + if (f != 0) + { + fprintf(f, "MODIFIED!\n"); + fclose(f); + } + // and then move the time backwards! struct timeval times[2]; - BoxTimeToTimeval(SecondsToBoxTime((time_t)(365*24*60*60)), times[1]); + BoxTimeToTimeval(SecondsToBoxTime( + (time_t)(365*24*60*60)), times[1]); times[0] = times[1]; - TEST_THAT(::utimes("testfiles/TestDir1/sub23/rand.h", times) == 0); + TEST_THAT(::utimes("testfiles/TestDir1/sub23/rand.h", + times) == 0); } + // Wait and test - wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3e.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + wait_for_sync_end(); // files too new + wait_for_sync_end(); // should (not) be backed up this time + + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3l.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + // Add some files and directories which are marked as excluded - printf("Add files and dirs for exclusion test\n"); - TEST_THAT(::system("gzip -d < testfiles/testexclude.tgz | ( cd testfiles && tar xf - )") == 0); + printf("\n==== Add files and dirs for exclusion test\n"); + #ifdef WIN32 + TEST_THAT(::system("tar xzvf testfiles/testexclude.tgz " + "-C testfiles") == 0); + #else + TEST_THAT(::system("gzip -d < " + "testfiles/testexclude.tgz " + "| ( cd testfiles && tar xf - )") == 0); + #endif + // Wait and test - wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3c.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + wait_for_sync_end(); + wait_for_sync_end(); + + // compare with exclusions, should not find differences + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3m.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3d.log \"compare -acE\" quit"); - TEST_THAT(compareReturnValue == 2*256); // should find differences + + // compare without exclusions, should find differences + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3n.log " + "\"compare -acEQ\" quit"); + TEST_RETURN(compareReturnValue, 2); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + // check that the excluded files did not make it + // into the store, and the included files did + printf("\n==== Check that exclude/alwaysinclude commands " + "actually work\n"); + + { + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify( + "testfiles/bbackupd.conf", + &BackupDaemonConfigVerify, errs)); + Configuration& conf(*config); + SSLLib::Initialise(); + TLSContext tlsContext; + std::string certFile(conf.GetKeyValue("CertificateFile")); + std::string keyFile (conf.GetKeyValue("PrivateKeyFile")); + std::string caFile (conf.GetKeyValue("TrustedCAsFile")); + tlsContext.Initialise(false, certFile.c_str(), + keyFile.c_str(), caFile.c_str()); + BackupClientCryptoKeys_Setup( + conf.GetKeyValue("KeysFile").c_str()); + SocketStreamTLS socket; + socket.Open(tlsContext, Socket::TypeINET, + conf.GetKeyValue("StoreHostname").c_str(), + BOX_PORT_BBSTORED); + BackupProtocolClient connection(socket); + connection.Handshake(); + std::auto_ptr<BackupProtocolClientVersion> + serverVersion(connection.QueryVersion( + BACKUP_STORE_SERVER_VERSION)); + if(serverVersion->GetVersion() != + BACKUP_STORE_SERVER_VERSION) + { + THROW_EXCEPTION(BackupStoreException, + WrongServerVersion); + } + connection.QueryLogin( + conf.GetKeyValueInt("AccountNumber"), + BackupProtocolClientLogin::Flags_ReadOnly); + + int64_t rootDirId = BackupProtocolClientListDirectory + ::RootDirectory; + std::auto_ptr<BackupProtocolClientSuccess> dirreply( + connection.QueryListDirectory( + rootDirId, false, 0, false)); + std::auto_ptr<IOStream> dirstream( + connection.ReceiveStream()); + BackupStoreDirectory dir; + dir.ReadFromStream(*dirstream, connection.GetTimeout()); + + int64_t testDirId = SearchDir(dir, "Test1"); + TEST_THAT(testDirId != 0); + dirreply = connection.QueryListDirectory(testDirId, false, 0, false); + dirstream = connection.ReceiveStream(); + dir.ReadFromStream(*dirstream, connection.GetTimeout()); + + TEST_THAT(!SearchDir(dir, "excluded_1")); + TEST_THAT(!SearchDir(dir, "excluded_2")); + TEST_THAT(!SearchDir(dir, "exclude_dir")); + TEST_THAT(!SearchDir(dir, "exclude_dir_2")); + // xx_not_this_dir_22 should not be excluded by + // ExcludeDirsRegex, because it's a file + TEST_THAT(SearchDir (dir, "xx_not_this_dir_22")); + TEST_THAT(!SearchDir(dir, "zEXCLUDEu")); + TEST_THAT(SearchDir (dir, "dont.excludethis")); + TEST_THAT(SearchDir (dir, "xx_not_this_dir_ALWAYSINCLUDE")); + + int64_t sub23id = SearchDir(dir, "sub23"); + TEST_THAT(sub23id != 0); + dirreply = connection.QueryListDirectory(sub23id, false, 0, false); + dirstream = connection.ReceiveStream(); + dir.ReadFromStream(*dirstream, connection.GetTimeout()); + TEST_THAT(!SearchDir(dir, "xx_not_this_dir_22")); + TEST_THAT(!SearchDir(dir, "somefile.excludethis")); + connection.QueryFinished(); + } + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + +#ifndef WIN32 // These tests only work as non-root users. if(::getuid() != 0) { // Check that read errors are reported neatly - printf("Add unreadable files\n"); + printf("\n==== Add unreadable files\n"); + { // Dir and file which can't be read - TEST_THAT(::mkdir("testfiles/TestDir1/sub23/read-fail-test-dir", 0000) == 0); - int fd = ::open("testfiles/TestDir1/read-fail-test-file", O_CREAT | O_WRONLY, 0000); + TEST_THAT(::mkdir("testfiles/TestDir1/sub23" + "/read-fail-test-dir", 0000) == 0); + int fd = ::open("testfiles/TestDir1" + "/read-fail-test-file", + O_CREAT | O_WRONLY, 0000); TEST_THAT(fd != -1); ::close(fd); } + // Wait and test... wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3e.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 2*256); // should find differences + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3o.log " + "\"compare -acQ\" quit"); + + // should find differences + TEST_RETURN(compareReturnValue, 3); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + // Check that it was reported correctly TEST_THAT(TestFileExists("testfiles/notifyran.read-error.1")); + + // Check that the error was only reported once TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.2")); - // Set permissions on file and dir to stop errors in the future - ::chmod("testfiles/TestDir1/sub23/read-fail-test-dir", 0770); - ::chmod("testfiles/TestDir1/read-fail-test-file", 0770); + + // Set permissions on file and dir to stop + // errors in the future + TEST_THAT(::chmod("testfiles/TestDir1/sub23" + "/read-fail-test-dir", 0770) == 0); + TEST_THAT(::chmod("testfiles/TestDir1" + "/read-fail-test-file", 0770) == 0); } +#endif + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; - printf("Continuously update file, check isn't uploaded\n"); + printf("\n==== Continuously update file, " + "check isn't uploaded\n"); - // Make sure everything happens at the same point in the sync cycle: wait until exactly the start of a sync - TEST_THAT(::system("../../bin/bbackupctl/bbackupctl -c testfiles/bbackupd.conf wait-for-sync") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + // Make sure everything happens at the same point in the + // sync cycle: wait until exactly the start of a sync + wait_for_sync_start(); + // Then wait a second, to make sure the scan is complete - ::sleep(1); + ::safe_sleep(1); { // Open a file, then save something to it every second @@ -744,39 +2235,54 @@ int test_bbackupd() TEST_THAT(f != 0); fprintf(f, "Loop iteration %d\n", l); fflush(f); - sleep(1); + fclose(f); + printf("."); fflush(stdout); - ::fclose(f); + safe_sleep(1); } printf("\n"); + fflush(stdout); // Check there's a difference - compareReturnValue = ::system("testfiles/extcheck1.pl"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system("perl testfiles/" + "extcheck1.pl"); + + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - printf("Keep on continuously updating file, check it is uploaded eventually\n"); + printf("\n==== Keep on continuously updating file, " + "check it is uploaded eventually\n"); for(int l = 0; l < 28; ++l) { - FILE *f = ::fopen("testfiles/TestDir1/continousupdate", "w+"); + FILE *f = ::fopen("testfiles/TestDir1/" + "continousupdate", "w+"); TEST_THAT(f != 0); fprintf(f, "Loop 2 iteration %d\n", l); fflush(f); - sleep(1); + fclose(f); + printf("."); fflush(stdout); - ::fclose(f); + safe_sleep(1); } printf("\n"); + fflush(stdout); - compareReturnValue = ::system("testfiles/extcheck2.pl"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system("perl testfiles/" + "extcheck2.pl"); + + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); } - printf("Delete directory, change attributes\n"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + printf("\n==== Delete directory, change attributes\n"); // Delete a directory TEST_THAT(::system("rm -rf testfiles/TestDir1/x1") == 0); @@ -785,109 +2291,288 @@ int test_bbackupd() // Wait and test wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query4.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - printf("Restore files and directories\n"); + printf("\n==== Restore files and directories\n"); int64_t deldirid = 0; int64_t restoredirid = 0; { // connect and log in SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED); + conn.Open(context, Socket::TypeINET, "localhost", + BOX_PORT_BBSTORED); BackupProtocolClient protocol(conn); protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION); - std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, BackupProtocolClientLogin::Flags_ReadOnly)); + std::auto_ptr<BackupProtocolClientLoginConfirmed> + loginConf(protocol.QueryLogin(0x01234567, + BackupProtocolClientLogin::Flags_ReadOnly)); // Find the ID of the Test1 directory - restoredirid = GetDirID(protocol, "Test1", BackupProtocolClientListDirectory::RootDirectory); + restoredirid = GetDirID(protocol, "Test1", + BackupProtocolClientListDirectory::RootDirectory); TEST_THAT(restoredirid != 0); // Test the restoration - TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-Test1", true /* print progress dots */) == Restore_Complete); + TEST_THAT(BackupClientRestore(protocol, restoredirid, + "testfiles/restore-Test1", + true /* print progress dots */) + == Restore_Complete); - // Compare it - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query10.log \"compare -cE Test1 testfiles/restore-Test1\" quit"); - TEST_THAT(compareReturnValue == 1*256); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + // On Win32 we can't open another connection + // to the server, so we'll compare later. // Make sure you can't restore a restored directory - TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-Test1", true /* print progress dots */) == Restore_TargetExists); + TEST_THAT(BackupClientRestore(protocol, restoredirid, + "testfiles/restore-Test1", + true /* print progress dots */) + == Restore_TargetExists); + + // Make sure you can't restore to a nonexistant path + printf("Try to restore to a path that doesn't exist\n"); + TEST_THAT(BackupClientRestore(protocol, restoredirid, + "testfiles/no-such-path/subdir", + true /* print progress dots */) + == Restore_TargetPathNotFound); // Find ID of the deleted directory deldirid = GetDirID(protocol, "x1", restoredirid); TEST_THAT(deldirid != 0); - // Just check it doesn't bomb out -- will check this properly later (when bbackupd is stopped) - TEST_THAT(BackupClientRestore(protocol, deldirid, "testfiles/restore-Test1-x1", true /* print progress dots */, true /* deleted files */) == Restore_Complete); + // Just check it doesn't bomb out -- will check this + // properly later (when bbackupd is stopped) + TEST_THAT(BackupClientRestore(protocol, deldirid, + "testfiles/restore-Test1-x1", + true /* print progress dots */, + true /* deleted files */) + == Restore_Complete); // Log out protocol.QueryFinished(); } - printf("Add files with current time\n"); + // Compare the restored files + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query10.log " + "\"compare -cEQ Test1 testfiles/restore-Test1\" " + "quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + +#ifdef WIN32 + // make one of the files read-only, expect a compare failure + compareReturnValue = ::system("attrib +r " + "testfiles\\restore-Test1\\f1.dat"); + TEST_RETURN(compareReturnValue, 0); + + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query10a.log " + "\"compare -cEQ Test1 testfiles/restore-Test1\" " + "quit"); + TEST_RETURN(compareReturnValue, 2); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // set it back, expect no failures + compareReturnValue = ::system("attrib -r " + "testfiles\\restore-Test1\\f1.dat"); + TEST_RETURN(compareReturnValue, 0); + + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf -l testfiles/query10a.log " + "\"compare -cEQ Test1 testfiles/restore-Test1\" " + "quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // change the timestamp on a file, expect a compare failure + char* testfile = "testfiles\\restore-Test1\\f1.dat"; + HANDLE handle = openfile(testfile, O_RDWR, 0); + TEST_THAT(handle != INVALID_HANDLE_VALUE); + + FILETIME creationTime, lastModTime, lastAccessTime; + TEST_THAT(GetFileTime(handle, &creationTime, &lastAccessTime, + &lastModTime) != 0); + TEST_THAT(CloseHandle(handle)); + + FILETIME dummyTime = lastModTime; + dummyTime.dwHighDateTime -= 100; + + // creation time is backed up, so changing it should cause + // a compare failure + TEST_THAT(set_file_time(testfile, dummyTime, lastModTime, + lastAccessTime)); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query10a.log " + "\"compare -cEQ Test1 testfiles/restore-Test1\" " + "quit"); + TEST_RETURN(compareReturnValue, 2); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // last access time is not backed up, so it cannot be compared + TEST_THAT(set_file_time(testfile, creationTime, lastModTime, + dummyTime)); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query10a.log " + "\"compare -cEQ Test1 testfiles/restore-Test1\" " + "quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // last write time is backed up, so changing it should cause + // a compare failure + TEST_THAT(set_file_time(testfile, creationTime, dummyTime, + lastAccessTime)); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query10a.log " + "\"compare -cEQ Test1 testfiles/restore-Test1\" " + "quit"); + TEST_RETURN(compareReturnValue, 2); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // set back to original values, check that compare succeeds + TEST_THAT(set_file_time(testfile, creationTime, lastModTime, + lastAccessTime)); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query10a.log " + "\"compare -cEQ Test1 testfiles/restore-Test1\" " + "quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); +#endif // WIN32 + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + printf("\n==== Add files with current time\n"); // Add some more files and modify others // Use the m flag this time so they have a recent modification time - TEST_THAT(::system("gzip -d < testfiles/test3.tgz | ( cd testfiles && tar xmf - )") == 0); + #ifdef WIN32 + TEST_THAT(::system("tar xzvmf testfiles/test3.tgz " + "-C testfiles") == 0); + #else + TEST_THAT(::system("gzip -d < testfiles/test3.tgz " + "| ( cd testfiles && tar xmf - )") == 0); + #endif // Wait and test wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query5.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query5.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + // Rename directory - printf("Rename directory\n"); - TEST_THAT(rename("testfiles/TestDir1/sub23/dhsfdss", "testfiles/TestDir1/renamed-dir") == 0); + printf("\n==== Rename directory\n"); + TEST_THAT(rename("testfiles/TestDir1/sub23/dhsfdss", + "testfiles/TestDir1/renamed-dir") == 0); wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query6.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query6.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + // and again, but with quick flag - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query6q.log \"compare -acq\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query6q.log " + "\"compare -acqQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // Rename some files -- one under the threshold, others above - printf("Rename files\n"); - TEST_THAT(rename("testfiles/TestDir1/continousupdate", "testfiles/TestDir1/continousupdate-ren") == 0); - TEST_THAT(rename("testfiles/TestDir1/df324", "testfiles/TestDir1/df324-ren") == 0); - TEST_THAT(rename("testfiles/TestDir1/sub23/find2perl", "testfiles/TestDir1/find2perl-ren") == 0); + printf("\n==== Rename files\n"); + TEST_THAT(rename("testfiles/TestDir1/continousupdate", + "testfiles/TestDir1/continousupdate-ren") == 0); + TEST_THAT(rename("testfiles/TestDir1/df324", + "testfiles/TestDir1/df324-ren") == 0); + TEST_THAT(rename("testfiles/TestDir1/sub23/find2perl", + "testfiles/TestDir1/find2perl-ren") == 0); wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query6.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query6.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - // Check that modifying files with madly in the future timestamps still get added - printf("Create a file with timestamp to way ahead in the future\n"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + // Check that modifying files with madly in the future + // timestamps still get added + printf("\n==== Create a file with timestamp way ahead " + "in the future\n"); + // Time critical, so sync - TEST_THAT(::system("../../bin/bbackupctl/bbackupctl -q -c testfiles/bbackupd.conf wait-for-sync") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + wait_for_sync_start(); + // Then wait a second, to make sure the scan is complete - ::sleep(1); + ::safe_sleep(1); + // Then modify an existing file { - FILE *f = fopen("testfiles/TestDir1/sub23/in-the-future", "w"); + FILE *f = fopen("testfiles/TestDir1/sub23/" + "in-the-future", "w"); TEST_THAT(f != 0); fprintf(f, "Back to the future!\n"); fclose(f); // and then move the time forwards! struct timeval times[2]; - BoxTimeToTimeval(GetCurrentBoxTime() + SecondsToBoxTime((time_t)(365*24*60*60)), times[1]); + BoxTimeToTimeval(GetCurrentBoxTime() + + SecondsToBoxTime((time_t)(365*24*60*60)), + times[1]); times[0] = times[1]; - TEST_THAT(::utimes("testfiles/TestDir1/sub23/in-the-future", times) == 0); + TEST_THAT(::utimes("testfiles/TestDir1/sub23/" + "in-the-future", times) == 0); } + // Wait and test wait_for_backup_operation(); - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query3e.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3e.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - printf("Change client store marker\n"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + printf("\n==== Change client store marker\n"); - // Then... connect to the server, and change the client store marker. See what that does! + // Then... connect to the server, and change the + // client store marker. See what that does! { bool done = false; int tries = 4; @@ -896,7 +2581,8 @@ int test_bbackupd() try { SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED); + conn.Open(context, Socket::TypeINET, + "localhost", BOX_PORT_BBSTORED); BackupProtocolClient protocol(conn); protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION); std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)); // read-write @@ -920,10 +2606,15 @@ int test_bbackupd() TEST_THAT(done); } - printf("Check change of store marker pauses daemon\n"); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + printf("\n==== Check change of store marker pauses daemon\n"); - // Make a change to a file, to detect whether or not it's hanging around - // waiting to retry. + // Make a change to a file, to detect whether or not + // it's hanging around waiting to retry. { FILE *f = ::fopen("testfiles/TestDir1/fileaftermarker", "w"); TEST_THAT(f != 0); @@ -932,56 +2623,114 @@ int test_bbackupd() } // Wait and test that there *are* differences - wait_for_backup_operation((TIME_TO_WAIT_FOR_BACKUP_OPERATION*3) / 2); // little bit longer than usual - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query6.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 2*256); + wait_for_backup_operation((TIME_TO_WAIT_FOR_BACKUP_OPERATION * + 3) / 2); // little bit longer than usual + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query6.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 2); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - - printf("Interrupted restore\n"); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + printf("\n==== Waiting for bbackupd to recover\n"); + // 100 seconds - (12*3/2) + wait_for_operation(82); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + +#ifndef WIN32 + printf("\n==== Interrupted restore\n"); { do_interrupted_restore(context, restoredirid); int64_t resumesize = 0; - TEST_THAT(FileExists("testfiles/restore-interrupt.boxbackupresume", &resumesize)); - TEST_THAT(resumesize > 16); // make sure it has recorded something to resume + TEST_THAT(FileExists("testfiles/" + "restore-interrupt.boxbackupresume", + &resumesize)); + // make sure it has recorded something to resume + TEST_THAT(resumesize > 16); - printf("\nResume restore\n"); + printf("\n==== Resume restore\n"); SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED); + conn.Open(context, Socket::TypeINET, "localhost", + BOX_PORT_BBSTORED); BackupProtocolClient protocol(conn); protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION); - std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)); // read-write + std::auto_ptr<BackupProtocolClientLoginConfirmed> + loginConf(protocol.QueryLogin(0x01234567, + 0 /* read-write */)); - // Check that the restore fn returns resume possible, rather than doing anything - TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-interrupt", true /* print progress dots */) == Restore_ResumePossible); + // Check that the restore fn returns resume possible, + // rather than doing anything + TEST_THAT(BackupClientRestore(protocol, restoredirid, + "testfiles/restore-interrupt", + true /* print progress dots */) + == Restore_ResumePossible); // Then resume it - TEST_THAT(BackupClientRestore(protocol, restoredirid, "testfiles/restore-interrupt", true /* print progress dots */, false /* deleted files */, false /* undelete server */, true /* resume */) == Restore_Complete); + TEST_THAT(BackupClientRestore(protocol, restoredirid, + "testfiles/restore-interrupt", + true /* print progress dots */, + false /* deleted files */, + false /* undelete server */, + true /* resume */) + == Restore_Complete); protocol.QueryFinished(); // Then check it has restored the correct stuff - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query14.log \"compare -cE Test1 testfiles/restore-interrupt\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query14.log " + "\"compare -cEQ Test1 " + "testfiles/restore-interrupt\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); } +#endif // !WIN32 + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + printf("\n==== Check restore deleted files\n"); - printf("Check restore deleted files\n"); { SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED); + conn.Open(context, Socket::TypeINET, "localhost", + BOX_PORT_BBSTORED); BackupProtocolClient protocol(conn); protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION); - std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)); // read-write + std::auto_ptr<BackupProtocolClientLoginConfirmed> + loginConf(protocol.QueryLogin(0x01234567, + 0 /* read-write */)); // Do restore and undelete - TEST_THAT(BackupClientRestore(protocol, deldirid, "testfiles/restore-Test1-x1-2", true /* print progress dots */, true /* deleted files */, true /* undelete on server */) == Restore_Complete); + TEST_THAT(BackupClientRestore(protocol, deldirid, + "testfiles/restore-Test1-x1-2", + true /* print progress dots */, + true /* deleted files */, + true /* undelete on server */) + == Restore_Complete); protocol.QueryFinished(); // Do a compare with the now undeleted files - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query11.log \"compare -cE Test1/x1 testfiles/restore-Test1-x1-2\" quit"); - TEST_THAT(compareReturnValue == 1*256); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query11.log " + "\"compare -cEQ Test1/x1 " + "testfiles/restore-Test1-x1-2\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); } @@ -989,39 +2738,124 @@ int test_bbackupd() TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2")); TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.2")); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + +#ifdef WIN32 + printf("\n==== Testing locked file behaviour:\n"); + + // Test that locked files cannot be backed up, + // and the appropriate error is reported. + // Wait for the sync to finish, so that we have time to work + wait_for_sync_start(); + // Now we have about three seconds to work + + handle = openfile("testfiles/TestDir1/lockedfile", + O_CREAT | O_EXCL | O_LOCK, 0); + TEST_THAT(handle != INVALID_HANDLE_VALUE); + + if (handle != 0) + { + // first sync will ignore the file, it's too new + wait_for_sync_end(); + TEST_THAT(!TestFileExists("testfiles/" + "notifyran.read-error.1")); + + // this sync should try to back up the file, + // and fail, because it's locked + wait_for_sync_end(); + TEST_THAT(TestFileExists("testfiles/" + "notifyran.read-error.1")); + TEST_THAT(!TestFileExists("testfiles/" + "notifyran.read-error.2")); + + // now close the file and check that it is + // backed up on the next run. + CloseHandle(handle); + wait_for_sync_end(); + TEST_THAT(!TestFileExists("testfiles/" + "notifyran.read-error.2")); + + // compare, and check that it works + // reports the correct error message (and finishes) + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query15a.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // open the file again, compare and check that compare + // reports the correct error message (and finishes) + handle = openfile("testfiles/TestDir1/lockedfile", + O_LOCK, 0); + TEST_THAT(handle != INVALID_HANDLE_VALUE); + + compareReturnValue = ::system(BBACKUPQUERY + " -q -c testfiles/bbackupd.conf " + "-l testfiles/query15.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 3); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // close the file again, check that compare + // works again + CloseHandle(handle); + + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query15a.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + } +#endif + // Kill the daemon - TEST_THAT(KillServer(pid)); - ::sleep(1); - TEST_THAT(!ServerIsAlive(pid)); - TestRemoteProcessMemLeaks("bbackupd.memleaks"); + terminate_bbackupd(bbackupd_pid); // Start it again - pid = LaunchServer("../../bin/bbackupd/bbackupd testfiles/bbackupd.conf", "testfiles/bbackupd.pid"); - TEST_THAT(pid != -1 && pid != 0); - if(pid != -1 && pid != 0) + cmd = BBACKUPD + bbackupd_args + " testfiles/bbackupd.conf"; + bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); + + TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + if(bbackupd_pid != -1 && bbackupd_pid != 0) { - // Wait and comapre - wait_for_backup_operation((TIME_TO_WAIT_FOR_BACKUP_OPERATION*3) / 2); // little bit longer than usual - compareReturnValue = ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac\" quit"); - TEST_THAT(compareReturnValue == 1*256); + // Wait and compare (a little bit longer than usual) + wait_for_backup_operation( + (TIME_TO_WAIT_FOR_BACKUP_OPERATION*3) / 2); + compareReturnValue = ::system(BBACKUPQUERY " -q " + "-c testfiles/bbackupd.conf " + "-l testfiles/query4a.log " + "\"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // Kill it again - TEST_THAT(KillServer(pid)); - ::sleep(1); - TEST_THAT(!ServerIsAlive(pid)); - TestRemoteProcessMemLeaks("bbackupd.memleaks"); + terminate_bbackupd(bbackupd_pid); } } // List the files on the server - ::system("../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/queryLIST.log \"list -rotdh\" quit"); + ::system(BBACKUPQUERY " -q -c testfiles/bbackupd.conf " + "-l testfiles/queryLIST.log \"list -rotdh\" quit"); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - - if(::getuid() == 0) - { - ::printf("WARNING: This test was run as root. Some tests have been omitted.\n"); - } + + #ifndef WIN32 + if(::getuid() == 0) + { + ::printf("WARNING: This test was run as root. " + "Some tests have been omitted.\n"); + } + #endif return 0; } @@ -1035,7 +2869,13 @@ int test(int argc, const char *argv[]) BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); // Initial files - TEST_THAT(::system("gzip -d < testfiles/test_base.tgz | ( cd testfiles && tar xf - )") == 0); + #ifdef WIN32 + TEST_THAT(::system("tar xzvf testfiles/test_base.tgz " + "-C testfiles") == 0); + #else + TEST_THAT(::system("gzip -d < testfiles/test_base.tgz " + "| ( cd testfiles && tar xf - )") == 0); + #endif // Do the tests @@ -1049,7 +2889,12 @@ int test(int argc, const char *argv[]) if(r != 0) return r; r = test_bbackupd(); - if(r != 0) return r; + if(r != 0) + { + KillServer(bbackupd_pid); + KillServer(bbstored_pid); + return r; + } test_kill_bbstored(); diff --git a/test/bbackupd/testfiles/bbackupd.conf.in b/test/bbackupd/testfiles/bbackupd.conf.in index 6d381d5b..ce5d5b7c 100644 --- a/test/bbackupd/testfiles/bbackupd.conf.in +++ b/test/bbackupd/testfiles/bbackupd.conf.in @@ -17,13 +17,16 @@ MaxUploadWait = 24 FileTrackingSizeThreshold = 1024 DiffingUploadSizeThreshold = 1024 -MaximumDiffingTime = 8 +MaximumDiffingTime = 3 +KeepAliveTime = 1 -ExtendedLogging = yes +ExtendedLogging = no +ExtendedLogFile = testfiles/bbackupd.log CommandSocket = testfiles/bbackupd.sock -NotifyScript = @PERL@ testfiles/notifyscript.pl +NotifyScript = @TARGET_PERL@ testfiles/notifyscript.pl +SyncAllowScript = @TARGET_PERL@ testfiles/syncallowscript.pl Server { diff --git a/test/bbackupd/testfiles/extcheck1.pl.in b/test/bbackupd/testfiles/extcheck1.pl.in index edbacd0a..2a7c0e9a 100755 --- a/test/bbackupd/testfiles/extcheck1.pl.in +++ b/test/bbackupd/testfiles/extcheck1.pl.in @@ -1,7 +1,9 @@ #!@PERL@ use strict; -unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac\" quit|") +my $flags = $ARGV[0] or ""; + +unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac$flags\" quit 2>&1 |") { print "Couldn't open compare utility\n"; exit 2; @@ -15,14 +17,23 @@ while(<IN>) next unless m/\S/; if(m/continousupdate/) { - $ret = 2 unless m/exists/; + unless (/exists/) + { + print "FAIL: continousupdate line does not match\n"; + $ret = 2; + } $seen = 1; } else { - $ret = 2 unless m/\AWARNING/ || m/\ADifferences/ || /might be reason/ || /probably due to file mod/; + unless (/\AWARNING/ or /\ADifferences/ or /might be reason/ + or /probably due to file mod/) + { + print "FAIL: Summary line does not match\n"; + $ret = 2; + } } - print; + print "READ: $_"; } close IN; diff --git a/test/bbackupd/testfiles/extcheck2.pl.in b/test/bbackupd/testfiles/extcheck2.pl.in index 68baa045..c79bf414 100755 --- a/test/bbackupd/testfiles/extcheck2.pl.in +++ b/test/bbackupd/testfiles/extcheck2.pl.in @@ -1,7 +1,9 @@ #!@PERL@ use strict; -unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac\" quit|") +my $flags = $ARGV[0] or ""; + +unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac$flags\" quit 2>&1 |") { print "Couldn't open compare utility\n"; exit 2; @@ -14,13 +16,23 @@ while(<IN>) next unless m/\S/; if(m/continousupdate/) { - $ret = 2 unless m/contents/ || m/attributes/; + unless (m/contents/ or m/attributes/) + { + print "FAIL: continuousupdate line does not match\n"; + $ret = 2; + } } else { - $ret = 2 unless m/\AWARNING/ || m/\ADifferences/ || /might be reason/ || /probably due to file mod/; + unless (/\AWARNING/ or /\ADifferences/ or /might be reason/ + or /probably due to file mod/) + { + print "FAIL: summary line does not match\n"; + $ret = 2; + } } - print; + + print "READ: $_"; } close IN; diff --git a/test/bbackupd/testfiles/notifyscript.pl.in b/test/bbackupd/testfiles/notifyscript.pl.in index df0e5a2d..89741e46 100755 --- a/test/bbackupd/testfiles/notifyscript.pl.in +++ b/test/bbackupd/testfiles/notifyscript.pl.in @@ -1,4 +1,4 @@ -#!@PERL@ +#!@TARGET_PERL@ my $f = 'testfiles/notifyran.'.$ARGV[0].'.'; diff --git a/test/bbackupd/testfiles/syncallowscript.pl.in b/test/bbackupd/testfiles/syncallowscript.pl.in index af425e90..f2ef1171 100755 --- a/test/bbackupd/testfiles/syncallowscript.pl.in +++ b/test/bbackupd/testfiles/syncallowscript.pl.in @@ -1,4 +1,4 @@ -#!@PERL@ +#!@TARGET_PERL@ use strict; use warnings; diff --git a/test/common/testcommon.cpp b/test/common/testcommon.cpp index ce5cd931..eb057228 100644 --- a/test/common/testcommon.cpp +++ b/test/common/testcommon.cpp @@ -9,13 +9,16 @@ #include "Box.h" +#include <errno.h> #include <stdio.h> +#include <time.h> #include "Test.h" #include "Configuration.h" #include "FdGetLine.h" #include "Guards.h" #include "FileStream.h" +#include "InvisibleTempFileStream.h" #include "IOStreamGetLine.h" #include "NamedLock.h" #include "ReadGatherStream.h" @@ -24,6 +27,12 @@ #include "CommonException.h" #include "Conversion.h" #include "autogen_ConversionException.h" +#include "CollectInBufferStream.h" +#include "Archive.h" +#include "Timer.h" +#include "Logging.h" +#include "ZeroStream.h" +#include "PartialReadStream.h" #include "MemLeakFindOn.h" @@ -130,11 +139,139 @@ ConfigurationVerify verify = 0 }; +class TestLogger : public Logger +{ + private: + bool mTriggered; + Log::Level mTargetLevel; + + public: + TestLogger(Log::Level targetLevel) + : mTriggered(false), mTargetLevel(targetLevel) + { + Logging::Add(this); + } + ~TestLogger() + { + Logging::Remove(this); + } + + bool IsTriggered() { return mTriggered; } + void Reset() { mTriggered = false; } + + virtual bool Log(Log::Level level, const std::string& rFile, + int line, std::string& rMessage) + { + if (level == mTargetLevel) + { + mTriggered = true; + } + return true; + } + + virtual const char* GetType() { return "Test"; } + virtual void SetProgramName(const std::string& rProgramName) { } +}; + int test(int argc, const char *argv[]) { - // Test memory leak detection + // Test PartialReadStream and ReadGatherStream handling of files + // over 2GB (refs #2) + { + char buffer[8]; + + ZeroStream zero(0x80000003); + zero.Seek(0x7ffffffe, IOStream::SeekType_Absolute); + TEST_THAT(zero.GetPosition() == 0x7ffffffe); + TEST_THAT(zero.Read(buffer, 8) == 5); + TEST_THAT(zero.GetPosition() == 0x80000003); + TEST_THAT(zero.Read(buffer, 8) == 0); + zero.Seek(0, IOStream::SeekType_Absolute); + TEST_THAT(zero.GetPosition() == 0); + + char* buffer2 = new char [0x1000000]; + TEST_THAT(buffer2 != NULL); + + PartialReadStream part(zero, 0x80000002); + for (int i = 0; i < 0x80; i++) + { + int read = part.Read(buffer2, 0x1000000); + TEST_THAT(read == 0x1000000); + } + TEST_THAT(part.Read(buffer, 8) == 2); + TEST_THAT(part.Read(buffer, 8) == 0); + + delete [] buffer2; + + ReadGatherStream gather(false); + zero.Seek(0, IOStream::SeekType_Absolute); + int component = gather.AddComponent(&zero); + gather.AddBlock(component, 0x80000002); + TEST_THAT(gather.Read(buffer, 8) == 8); + } + + // Test self-deleting temporary file streams + { + std::string tempfile("testfiles/tempfile"); + TEST_CHECK_THROWS(InvisibleTempFileStream fs(tempfile.c_str()), + CommonException, OSFileOpenError); + InvisibleTempFileStream fs(tempfile.c_str(), O_CREAT); + + #ifdef WIN32 + // file is still visible under Windows + TEST_THAT(TestFileExists(tempfile.c_str())); + + // opening it again should work + InvisibleTempFileStream fs2(tempfile.c_str()); + TEST_THAT(TestFileExists(tempfile.c_str())); + + // opening it to create should work + InvisibleTempFileStream fs3(tempfile.c_str(), O_CREAT); + TEST_THAT(TestFileExists(tempfile.c_str())); + + // opening it to create exclusively should fail + TEST_CHECK_THROWS(InvisibleTempFileStream fs4(tempfile.c_str(), + O_CREAT | O_EXCL), CommonException, OSFileOpenError); + + fs2.Close(); + #else + // file is not visible under Unix + TEST_THAT(!TestFileExists(tempfile.c_str())); + + // opening it again should fail + TEST_CHECK_THROWS(InvisibleTempFileStream fs2(tempfile.c_str()), + CommonException, OSFileOpenError); + + // opening it to create should work + InvisibleTempFileStream fs3(tempfile.c_str(), O_CREAT); + TEST_THAT(!TestFileExists(tempfile.c_str())); + + // opening it to create exclusively should work + InvisibleTempFileStream fs4(tempfile.c_str(), O_CREAT | O_EXCL); + TEST_THAT(!TestFileExists(tempfile.c_str())); + + fs4.Close(); + #endif + + fs.Close(); + fs3.Close(); + + // now that it's closed, it should be invisible on all platforms + TEST_THAT(!TestFileExists(tempfile.c_str())); + } + + // Test that memory leak detection doesn't crash + { + char *test = new char[1024]; + delete [] test; + MemBlockStream *s = new MemBlockStream(test,12); + delete s; + } + #ifdef BOX_MEMORY_LEAK_TESTING { + Timers::Cleanup(); + TEST_THAT(memleakfinder_numleaks() == 0); void *block = ::malloc(12); TEST_THAT(memleakfinder_numleaks() == 1); @@ -150,9 +287,85 @@ int test(int argc, const char *argv[]) TEST_THAT(memleakfinder_numleaks() == 1); delete [] test; TEST_THAT(memleakfinder_numleaks() == 0); + + Timers::Init(); } #endif // BOX_MEMORY_LEAK_TESTING + + // test main() initialises timers for us, so uninitialise them + Timers::Cleanup(); + + // Check that using timer methods without initialisation + // throws an assertion failure. Can only do this in debug mode + #ifndef NDEBUG + TEST_CHECK_THROWS(Timers::Add(*(Timer*)NULL), + CommonException, AssertFailed); + TEST_CHECK_THROWS(Timers::Remove(*(Timer*)NULL), + CommonException, AssertFailed); + #endif + + // TEST_CHECK_THROWS(Timers::Signal(), CommonException, AssertFailed); + #ifndef NDEBUG + TEST_CHECK_THROWS(Timers::Cleanup(), CommonException, + AssertFailed); + #endif + // Check that we can initialise the timers + Timers::Init(); + + // Check that double initialisation throws an exception + #ifndef NDEBUG + TEST_CHECK_THROWS(Timers::Init(), CommonException, + AssertFailed); + #endif + + // Check that we can clean up the timers + Timers::Cleanup(); + + // Check that double cleanup throws an exception + #ifndef NDEBUG + TEST_CHECK_THROWS(Timers::Cleanup(), CommonException, + AssertFailed); + #endif + + Timers::Init(); + + Timer t0(0); // should never expire + Timer t1(1); + Timer t2(2); + Timer t3(3); + + TEST_THAT(!t0.HasExpired()); + TEST_THAT(!t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + + t1 = Timer(1); + t2 = Timer(2); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(!t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(t3.HasExpired()); + + // Leave timers initialised for rest of test. + // Test main() will cleanup after test finishes. static char *testfilelines[] = { @@ -537,7 +750,7 @@ int test(int argc, const char *argv[]) TEST_THAT(elist.SizeOfDefiniteList() == 4); // Add regex entries - #ifdef HAVE_REGEX_H + #ifdef HAVE_REGEX_SUPPORT elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")); elist.AddRegexEntries(std::string("")); TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[:not_valid")), CommonException, BadRegularExpression); @@ -569,7 +782,7 @@ int test(int argc, const char *argv[]) TEST_THAT(elist.IsExcluded("thingdefthree") == !CASE_SENSITIVE); - #ifdef HAVE_REGEX_H + #ifdef HAVE_REGEX_SUPPORT TEST_THAT(elist.IsExcluded(std::string("b.reg")) == true); TEST_THAT(elist.IsExcluded(std::string("B.reg")) == !CASE_SENSITIVE); TEST_THAT(elist.IsExcluded(std::string("b.Reg")) == !CASE_SENSITIVE); @@ -583,9 +796,86 @@ int test(int argc, const char *argv[]) #endif #undef CASE_SENSITIVE + + TestLogger logger(Log::WARNING); + TEST_THAT(!logger.IsTriggered()); + elist.AddDefiniteEntries(std::string("/foo")); + TEST_THAT(!logger.IsTriggered()); + elist.AddDefiniteEntries(std::string("/foo/")); + TEST_THAT(logger.IsTriggered()); + logger.Reset(); + elist.AddDefiniteEntries(std::string("/foo" + DIRECTORY_SEPARATOR)); + TEST_THAT(logger.IsTriggered()); + logger.Reset(); + elist.AddDefiniteEntries(std::string("/foo" + DIRECTORY_SEPARATOR "bar\x01/foo")); + TEST_THAT(!logger.IsTriggered()); + elist.AddDefiniteEntries(std::string("/foo" + DIRECTORY_SEPARATOR "bar\x01/foo" + DIRECTORY_SEPARATOR)); + TEST_THAT(logger.IsTriggered()); } test_conversions(); + // test that we can use Archive and CollectInBufferStream + // to read and write arbitrary types to a memory buffer + + { + CollectInBufferStream buffer; + ASSERT(buffer.GetPosition() == 0); + + { + Archive archive(buffer, 0); + ASSERT(buffer.GetPosition() == 0); + + archive.Write((bool) true); + archive.Write((bool) false); + archive.Write((int) 0x12345678); + archive.Write((int) 0x87654321); + archive.Write((int64_t) 0x0badfeedcafebabeLL); + archive.Write((uint64_t) 0xfeedfacedeadf00dLL); + archive.Write((uint8_t) 0x01); + archive.Write((uint8_t) 0xfe); + archive.Write(std::string("hello world!")); + archive.Write(std::string("goodbye cruel world!")); + } + + CollectInBufferStream buf2; + buf2.Write(buffer.GetBuffer(), buffer.GetSize()); + TEST_THAT(buf2.GetPosition() == buffer.GetSize()); + + buf2.SetForReading(); + TEST_THAT(buf2.GetPosition() == 0); + + { + Archive archive(buf2, 0); + TEST_THAT(buf2.GetPosition() == 0); + + bool b; + archive.Read(b); TEST_THAT(b == true); + archive.Read(b); TEST_THAT(b == false); + + int i; + archive.Read(i); TEST_THAT(i == 0x12345678); + archive.Read(i); TEST_THAT((unsigned int)i == 0x87654321); + + uint64_t i64; + archive.Read(i64); TEST_THAT(i64 == 0x0badfeedcafebabeLL); + archive.Read(i64); TEST_THAT(i64 == 0xfeedfacedeadf00dLL); + + uint8_t i8; + archive.Read(i8); TEST_THAT(i8 == 0x01); + archive.Read(i8); TEST_THAT(i8 == 0xfe); + + std::string s; + archive.Read(s); TEST_THAT(s == "hello world!"); + archive.Read(s); TEST_THAT(s == "goodbye cruel world!"); + + TEST_THAT(!buf2.StreamDataLeft()); + } + } + return 0; } diff --git a/test/raidfile/testraidfile.cpp b/test/raidfile/testraidfile.cpp index 1e510486..40703de5 100644 --- a/test/raidfile/testraidfile.cpp +++ b/test/raidfile/testraidfile.cpp @@ -12,7 +12,10 @@ #include <fcntl.h> #include <unistd.h> #include <errno.h> + +#ifdef HAVE_SYSCALL #include <sys/syscall.h> +#endif #include <string.h> @@ -207,16 +210,24 @@ void testReadingFileContents(int set, const char *filename, void *data, int data bytesread += r; } TEST_THAT(!readstream4.StreamDataLeft()); // check IOStream interface is correct + pread.reset(); // Be nasty, and create some errors for the RAID stuff to recover from... if(TestRAIDProperties) { char stripe1fn[256], stripe1fnRename[256]; - sprintf(stripe1fn, "testfiles/%d_%d/%s.rf", set, startDisc, filename); - sprintf(stripe1fnRename, "testfiles/%d_%d/%s.rf-REMOVED", set, startDisc, filename); + sprintf(stripe1fn, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR "%s.rf", set, startDisc, filename); + sprintf(stripe1fnRename, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR "%s.rf-REMOVED", set, startDisc, + filename); char stripe2fn[256], stripe2fnRename[256]; - sprintf(stripe2fn, "testfiles/%d_%d/%s.rf", set, (startDisc + 1) % RAID_NUMBER_DISCS, filename); - sprintf(stripe2fnRename, "testfiles/%d_%d/%s.rf-REMOVED", set, (startDisc + 1) % RAID_NUMBER_DISCS, filename); + sprintf(stripe2fn, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR "%s.rf", set, + (startDisc + 1) % RAID_NUMBER_DISCS, filename); + sprintf(stripe2fnRename, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR "%s.rf-REMOVED", set, + (startDisc + 1) % RAID_NUMBER_DISCS, filename); // Read with stripe1 + parity TEST_THAT(::rename(stripe2fn, stripe2fnRename) == 0); @@ -252,9 +263,15 @@ void testReadingFileContents(int set, const char *filename, void *data, int data } mungefilename[m++] = '\0'; char stripe1munge[256]; - sprintf(stripe1munge, "testfiles/%d_%d/.raidfile-unreadable/%s", set, startDisc, mungefilename); + sprintf(stripe1munge, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR ".raidfile-unreadable" + DIRECTORY_SEPARATOR "%s", set, startDisc, + mungefilename); char stripe2munge[256]; - sprintf(stripe2munge, "testfiles/%d_%d/.raidfile-unreadable/%s", set, (startDisc + 1) % RAID_NUMBER_DISCS, mungefilename); + sprintf(stripe2munge, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR ".raidfile-unreadable" + DIRECTORY_SEPARATOR "%s", set, + (startDisc + 1) % RAID_NUMBER_DISCS, mungefilename); #ifdef TRF_CAN_INTERCEPT @@ -359,10 +376,12 @@ void testReadWriteFileDo(int set, const char *filename, void *data, int datasize write4.Write(data, datasize); // This time, don't discard and transform it to a RAID File char writefnPre[256]; - sprintf(writefnPre, "testfiles/%d_%d/%s.rfwX", set, startDisc, filename); + sprintf(writefnPre, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR "%s.rfwX", set, startDisc, filename); TEST_THAT(TestFileExists(writefnPre)); char writefn[256]; - sprintf(writefn, "testfiles/%d_%d/%s.rfw", set, startDisc, filename); + sprintf(writefn, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR "%s.rfw", set, startDisc, filename); int usageInBlocks = write4.GetDiscUsageInBlocks(); write4.Commit(DoTransform); // Check that files are nicely done... @@ -390,14 +409,19 @@ void testReadWriteFileDo(int set, const char *filename, void *data, int datasize fs1 = ((fullblocks / 2)+1) * RAID_BLOCK_SIZE; } char stripe1fn[256]; - sprintf(stripe1fn, "testfiles/%d_%d/%s.rf", set, startDisc, filename); + sprintf(stripe1fn, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR "%s.rf", set, startDisc, filename); TEST_THAT(TestGetFileSize(stripe1fn) == fs1); char stripe2fn[256]; - sprintf(stripe2fn, "testfiles/%d_%d/%s.rf", set, (startDisc + 1) % RAID_NUMBER_DISCS, filename); + sprintf(stripe2fn, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR "%s.rf", set, + (startDisc + 1) % RAID_NUMBER_DISCS, filename); TEST_THAT(TestGetFileSize(stripe2fn) == (int)(datasize - fs1)); // Parity file size char parityfn[256]; - sprintf(parityfn, "testfiles/%d_%d/%s.rf", set, (startDisc + 2) % RAID_NUMBER_DISCS, filename); + sprintf(parityfn, "testfiles" DIRECTORY_SEPARATOR "%d_%d" + DIRECTORY_SEPARATOR "%s.rf", set, + (startDisc + 2) % RAID_NUMBER_DISCS, filename); // Mildly complex calculation unsigned int blocks = datasize / RAID_BLOCK_SIZE; unsigned int bytesOver = datasize % RAID_BLOCK_SIZE; @@ -436,14 +460,16 @@ void testReadWriteFileDo(int set, const char *filename, void *data, int datasize if(datasize > (3*1024)) { int f; - TEST_THAT((f = ::open(stripe1fn, O_RDONLY, 0)) != -1); + TEST_THAT((f = ::open(stripe1fn, O_RDONLY | O_BINARY, + 0)) != -1); TEST_THAT(sizeof(testblock) == ::read(f, testblock, sizeof(testblock))); for(unsigned int q = 0; q < sizeof(testblock); ++q) { TEST_THAT(testblock[q] == ((char*)data)[q]); } ::close(f); - TEST_THAT((f = ::open(stripe2fn, O_RDONLY, 0)) != -1); + TEST_THAT((f = ::open(stripe2fn, O_RDONLY | O_BINARY, + 0)) != -1); TEST_THAT(sizeof(testblock) == ::read(f, testblock, sizeof(testblock))); for(unsigned int q = 0; q < sizeof(testblock); ++q) { @@ -536,7 +562,9 @@ void test_overwrites() // Generate a random pre-existing write file (and ensure that it doesn't exist already) int f; - TEST_THAT((f = ::open("testfiles/0_2/overwrite_B.rfwX", O_WRONLY | O_CREAT | O_EXCL, 0755)) != -1); + TEST_THAT((f = ::open("testfiles" DIRECTORY_SEPARATOR "0_2" + DIRECTORY_SEPARATOR "overwrite_B.rfwX", + O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755)) != -1); TEST_THAT(::write(f, "TESTTEST", 8) == 8); ::close(f); @@ -557,7 +585,7 @@ int test(int argc, const char *argv[]) // Initialise the controller RaidFileController &rcontroller = RaidFileController::GetController(); - rcontroller.Initialise("testfiles/raidfile.conf"); + rcontroller.Initialise("testfiles" DIRECTORY_SEPARATOR "raidfile.conf"); // some data char data[TEST_DATA_SIZE]; @@ -574,9 +602,12 @@ int test(int argc, const char *argv[]) // Try creating a directory RaidFileWrite::CreateDirectory(0, "test-dir"); - TEST_THAT(TestDirExists("testfiles/0_0/test-dir")); - TEST_THAT(TestDirExists("testfiles/0_1/test-dir")); - TEST_THAT(TestDirExists("testfiles/0_2/test-dir")); + TEST_THAT(TestDirExists("testfiles" DIRECTORY_SEPARATOR "0_0" + DIRECTORY_SEPARATOR "test-dir")); + TEST_THAT(TestDirExists("testfiles" DIRECTORY_SEPARATOR "0_1" + DIRECTORY_SEPARATOR "test-dir")); + TEST_THAT(TestDirExists("testfiles" DIRECTORY_SEPARATOR "0_2" + DIRECTORY_SEPARATOR "test-dir")); TEST_THAT(RaidFileRead::DirectoryExists(0, "test-dir")); TEST_THAT(!RaidFileRead::DirectoryExists(0, "test-dir-not")); @@ -608,9 +639,12 @@ int test(int argc, const char *argv[]) // Before it's deleted, check to see the contents are as expected int f; - TEST_THAT((f = ::open("testfiles/0_2/test1.rfwX", O_RDONLY, 0)) >= 0); + TEST_THAT((f = ::open("testfiles" DIRECTORY_SEPARATOR "0_2" + DIRECTORY_SEPARATOR "test1.rfwX", O_RDONLY | O_BINARY, 0)) + >= 0); char buffer[sizeof(data)]; - TEST_THAT(::read(f, buffer, sizeof(buffer)) == sizeof(buffer)); + int bytes_read = ::read(f, buffer, sizeof(buffer)); + TEST_THAT(bytes_read == sizeof(buffer)); for(unsigned int l = 0; l < 1024; ++l) { TEST_THAT(buffer[l] == data[l]); @@ -624,7 +658,8 @@ int test(int argc, const char *argv[]) TEST_THAT(buffer[l+2048+sizeof(data2)] == data2[l]); } TEST_THAT(::lseek(f, sizeof(data), SEEK_SET) == sizeof(buffer)); - TEST_THAT(::read(f, buffer, sizeof(buffer)) == sizeof(buffer)); + bytes_read = ::read(f, buffer, sizeof(buffer)); + TEST_THAT(bytes_read == sizeof(buffer)); for(unsigned int l = 0; l < 1024; ++l) { TEST_THAT(buffer[l] == data[l]); @@ -635,7 +670,9 @@ int test(int argc, const char *argv[]) // Commit the data write1.Commit(); - TEST_THAT((f = ::open("testfiles/0_2/test1.rfw", O_RDONLY, 0)) >= 0); + TEST_THAT((f = ::open("testfiles" DIRECTORY_SEPARATOR "0_2" + DIRECTORY_SEPARATOR "test1.rfw", O_RDONLY | O_BINARY, 0)) + >= 0); ::close(f); // Now try and read it @@ -687,7 +724,9 @@ int test(int argc, const char *argv[]) write2.Write(data, sizeof(data)); // This time, discard it write2.Discard(); - TEST_THAT((f = ::open("testfiles/0_2/test1.rfw", O_RDONLY, 0)) == -1); + TEST_THAT((f = ::open("testfiles" DIRECTORY_SEPARATOR "0_2" + DIRECTORY_SEPARATOR "test1.rfw", O_RDONLY | O_BINARY, 0)) + == -1); // And leaving it there... RaidFileWrite writeLeave(0, "test1"); @@ -695,7 +734,9 @@ int test(int argc, const char *argv[]) writeLeave.Write(data, sizeof(data)); // This time, commit it writeLeave.Commit(); - TEST_THAT((f = ::open("testfiles/0_2/test1.rfw", O_RDONLY, 0)) != -1); + TEST_THAT((f = ::open("testfiles" DIRECTORY_SEPARATOR "0_2" + DIRECTORY_SEPARATOR "test1.rfw", O_RDONLY | O_BINARY, 0)) + != -1); ::close(f); // Then check that the thing will refuse to open it again. @@ -712,7 +753,8 @@ int test(int argc, const char *argv[]) write3b.Write(data + 3, sizeof(data) - 3); write3b.Commit(); // Test it - testReadingFileContents(0, "test1", data+3, sizeof(data) - 3, false /*TestRAIDProperties*/); + testReadingFileContents(0, "test1", data+3, sizeof(data) - 3, false + /* TestRAIDProperties */); // And once again, but this time making it a raid file RaidFileWrite write3c(0, "test1"); @@ -721,7 +763,8 @@ int test(int argc, const char *argv[]) write3c.Write(data + 7, sizeof(data) - 7); write3c.Commit(true); // make RAID // Test it - testReadingFileContents(0, "test1", data+7, sizeof(data) - 7, false /*TestRAIDProperties*/); + testReadingFileContents(0, "test1", data+7, sizeof(data) - 7, false + /*TestRAIDProperties*/); // Test opening a file which doesn't exist TEST_CHECK_THROWS( @@ -736,20 +779,23 @@ int test(int argc, const char *argv[]) w.Commit(true); // Try removing the parity file - TEST_THAT(::rename("testfiles/0_0/damage.rf", "testfiles/0_0/damage.rf-NT") == 0); + TEST_THAT(::rename("testfiles" DIRECTORY_SEPARATOR "0_0" + DIRECTORY_SEPARATOR "damage.rf", + "testfiles" DIRECTORY_SEPARATOR "0_0" + DIRECTORY_SEPARATOR "damage.rf-NT") == 0); { std::auto_ptr<RaidFileRead> pr0 = RaidFileRead::Open(0, "damage"); pr0->Read(buffer, sizeof(data)); } - TEST_THAT(::rename("testfiles/0_0/damage.rf-NT", "testfiles/0_0/damage.rf") == 0); - + TEST_THAT(::rename("testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "damage.rf-NT", "testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "damage.rf") == 0); + // Delete one of the files - TEST_THAT(::unlink("testfiles/0_1/damage.rf") == 0); // stripe 1 + TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_1" DIRECTORY_SEPARATOR "damage.rf") == 0); // stripe 1 #ifdef TRF_CAN_INTERCEPT // Open it and read... { - intercept_setup_error("testfiles/0_2/damage.rf", 0, EIO, SYS_read); // stripe 2 + intercept_setup_error("testfiles" DIRECTORY_SEPARATOR "0_2" DIRECTORY_SEPARATOR "damage.rf", 0, EIO, SYS_read); // stripe 2 std::auto_ptr<RaidFileRead> pr1 = RaidFileRead::Open(0, "damage"); TEST_CHECK_THROWS( pr1->Read(buffer, sizeof(data)), @@ -761,7 +807,7 @@ int test(int argc, const char *argv[]) #endif //TRF_CAN_INTERCEPT // Delete another - TEST_THAT(::unlink("testfiles/0_0/damage.rf") == 0); // parity + TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "damage.rf") == 0); // parity TEST_CHECK_THROWS( std::auto_ptr<RaidFileRead> pread2 = RaidFileRead::Open(0, "damage"), @@ -772,22 +818,22 @@ int test(int argc, const char *argv[]) { RaidFileWrite::CreateDirectory(0, "dirread"); // Make some contents - RaidFileWrite::CreateDirectory(0, "dirread/dfsdf1"); - RaidFileWrite::CreateDirectory(0, "dirread/ponwq2"); + RaidFileWrite::CreateDirectory(0, "dirread" DIRECTORY_SEPARATOR "dfsdf1"); + RaidFileWrite::CreateDirectory(0, "dirread" DIRECTORY_SEPARATOR "ponwq2"); { - RaidFileWrite w(0, "dirread/sdf9873241"); + RaidFileWrite w(0, "dirread" DIRECTORY_SEPARATOR "sdf9873241"); w.Open(); w.Write(data, sizeof(data)); w.Commit(true); } { - RaidFileWrite w(0, "dirread/fsdcxjni3242"); + RaidFileWrite w(0, "dirread" DIRECTORY_SEPARATOR "fsdcxjni3242"); w.Open(); w.Write(data, sizeof(data)); w.Commit(true); } { - RaidFileWrite w(0, "dirread/cskjnds3"); + RaidFileWrite w(0, "dirread" DIRECTORY_SEPARATOR "cskjnds3"); w.Open(); w.Write(data, sizeof(data)); w.Commit(false); @@ -803,15 +849,15 @@ int test(int argc, const char *argv[]) TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_DirsOnly, names)); TEST_THAT(list_matches(names, dir_list1)); // Delete things - TEST_THAT(::unlink("testfiles/0_0/dirread/sdf9873241.rf") == 0); + TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_0" DIRECTORY_SEPARATOR "dirread" DIRECTORY_SEPARATOR "sdf9873241.rf") == 0); TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names)); TEST_THAT(list_matches(names, file_list1)); // Delete something else so that it's not recoverable - TEST_THAT(::unlink("testfiles/0_1/dirread/sdf9873241.rf") == 0); + TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_1" DIRECTORY_SEPARATOR "dirread" DIRECTORY_SEPARATOR "sdf9873241.rf") == 0); TEST_THAT(false == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names)); TEST_THAT(list_matches(names, file_list1)); // And finally... - TEST_THAT(::unlink("testfiles/0_2/dirread/sdf9873241.rf") == 0); + TEST_THAT(::unlink("testfiles" DIRECTORY_SEPARATOR "0_2" DIRECTORY_SEPARATOR "dirread" DIRECTORY_SEPARATOR "sdf9873241.rf") == 0); TEST_THAT(true == RaidFileRead::ReadDirectoryContents(0, std::string("dirread"), RaidFileRead::DirReadType_FilesOnly, names)); TEST_THAT(list_matches(names, file_list2)); } diff --git a/test/win32/testlibwin32.cpp b/test/win32/testlibwin32.cpp index ca2989d8..fb735c56 100644 --- a/test/win32/testlibwin32.cpp +++ b/test/win32/testlibwin32.cpp @@ -6,12 +6,214 @@ #ifdef WIN32 +#include <assert.h> +#include <AccCtrl.h> +#include <Aclapi.h> + #include "../../bin/bbackupd/BackupDaemon.h" #include "BoxPortsAndFiles.h" #include "emu.h" int main(int argc, char* argv[]) { + // ACL tests + char* exename = getenv("WINDIR"); + + PSID psidOwner; + PSID psidGroup; + PACL pDacl; + PSECURITY_DESCRIPTOR pSecurityDesc; + + DWORD result = GetNamedSecurityInfo( + exename, // pObjectName + SE_FILE_OBJECT, // ObjectType + DACL_SECURITY_INFORMATION | // SecurityInfo + GROUP_SECURITY_INFORMATION | + OWNER_SECURITY_INFORMATION, + &psidOwner, // ppsidOwner, + &psidGroup, // ppsidGroup, + &pDacl, // ppDacl, + NULL, // ppSacl, + &pSecurityDesc // ppSecurityDescriptor + ); + if (result != ERROR_SUCCESS) + { + printf("Error getting security info for '%s': error %d", + exename, result); + } + assert(result == ERROR_SUCCESS); + + char namebuf[1024]; + char domainbuf[1024]; + SID_NAME_USE nametype; + DWORD namelen = sizeof(namebuf); + DWORD domainlen = sizeof(domainbuf); + + assert(LookupAccountSid(NULL, psidOwner, namebuf, &namelen, + domainbuf, &domainlen, &nametype)); + + printf("Owner:\n"); + printf("User name: %s\n", namebuf); + printf("Domain name: %s\n", domainbuf); + printf("Name type: %d\n", nametype); + printf("\n"); + + namelen = sizeof(namebuf); + domainlen = sizeof(domainbuf); + + assert(LookupAccountSid(NULL, psidGroup, namebuf, &namelen, + domainbuf, &domainlen, &nametype)); + + printf("Group:\n"); + printf("User name: %s\n", namebuf); + printf("Domain name: %s\n", domainbuf); + printf("Name type: %d\n", nametype); + printf("\n"); + + ULONG numEntries; + PEXPLICIT_ACCESS pEntries; + result = GetExplicitEntriesFromAcl + ( + pDacl, // pAcl + &numEntries, // pcCountOfExplicitEntries, + &pEntries // pListOfExplicitEntries + ); + assert(result == ERROR_SUCCESS); + + printf("Found %lu explicit DACL entries for '%s'\n\n", + (unsigned long)numEntries, exename); + + for (ULONG i = 0; i < numEntries; i++) + { + EXPLICIT_ACCESS* pEntry = &(pEntries[i]); + printf("DACL entry %lu:\n", (unsigned long)i); + + DWORD perms = pEntry->grfAccessPermissions; + printf(" Access permissions: ", perms); + + #define PRINT_PERM(name) \ + if (perms & name) \ + { \ + printf(#name " "); \ + perms &= ~name; \ + } + + PRINT_PERM(FILE_ADD_FILE); + PRINT_PERM(FILE_ADD_SUBDIRECTORY); + PRINT_PERM(FILE_ALL_ACCESS); + PRINT_PERM(FILE_APPEND_DATA); + PRINT_PERM(FILE_CREATE_PIPE_INSTANCE); + PRINT_PERM(FILE_DELETE_CHILD); + PRINT_PERM(FILE_EXECUTE); + PRINT_PERM(FILE_LIST_DIRECTORY); + PRINT_PERM(FILE_READ_ATTRIBUTES); + PRINT_PERM(FILE_READ_DATA); + PRINT_PERM(FILE_READ_EA); + PRINT_PERM(FILE_TRAVERSE); + PRINT_PERM(FILE_WRITE_ATTRIBUTES); + PRINT_PERM(FILE_WRITE_DATA); + PRINT_PERM(FILE_WRITE_EA); + PRINT_PERM(STANDARD_RIGHTS_READ); + PRINT_PERM(STANDARD_RIGHTS_WRITE); + PRINT_PERM(SYNCHRONIZE); + PRINT_PERM(DELETE); + PRINT_PERM(READ_CONTROL); + PRINT_PERM(WRITE_DAC); + PRINT_PERM(WRITE_OWNER); + PRINT_PERM(MAXIMUM_ALLOWED); + PRINT_PERM(GENERIC_ALL); + PRINT_PERM(GENERIC_EXECUTE); + PRINT_PERM(GENERIC_WRITE); + PRINT_PERM(GENERIC_READ); + printf("\n"); + + if (perms) + { + printf(" Bits left over: %08x\n", perms); + } + assert(!perms); + + printf(" Access mode: "); + switch(pEntry->grfAccessMode) + { + case NOT_USED_ACCESS: + printf("NOT_USED_ACCESS\n"); break; + case GRANT_ACCESS: + printf("GRANT_ACCESS\n"); break; + case DENY_ACCESS: + printf("DENY_ACCESS\n"); break; + case REVOKE_ACCESS: + printf("REVOKE_ACCESS\n"); break; + case SET_AUDIT_SUCCESS: + printf("SET_AUDIT_SUCCESS\n"); break; + case SET_AUDIT_FAILURE: + printf("SET_AUDIT_FAILURE\n"); break; + default: + printf("Unknown (%08x)\n", pEntry->grfAccessMode); + } + + printf(" Trustee: "); + assert(pEntry->Trustee.pMultipleTrustee == NULL); + assert(pEntry->Trustee.MultipleTrusteeOperation == NO_MULTIPLE_TRUSTEE); + switch(pEntry->Trustee.TrusteeForm) + { + case TRUSTEE_IS_SID: + { + PSID trusteeSid = (PSID)(pEntry->Trustee.ptstrName); + + namelen = sizeof(namebuf); + domainlen = sizeof(domainbuf); + + assert(LookupAccountSid(NULL, trusteeSid, namebuf, &namelen, + domainbuf, &domainlen, &nametype)); + + printf("SID of %s\\%s (%d)\n", domainbuf, namebuf, nametype); + } + break; + case TRUSTEE_IS_NAME: + printf("Name\n"); break; + case TRUSTEE_BAD_FORM: + printf("Bad form\n"); assert(0); + case TRUSTEE_IS_OBJECTS_AND_SID: + printf("Objects and SID\n"); break; + case TRUSTEE_IS_OBJECTS_AND_NAME: + printf("Objects and name\n"); break; + default: + printf("Unknown form\n"); assert(0); + } + + printf(" Trustee type: "); + switch(pEntry->Trustee.TrusteeType) + { + case TRUSTEE_IS_UNKNOWN: + printf("Unknown type.\n"); break; + case TRUSTEE_IS_USER: + printf("User\n"); break; + case TRUSTEE_IS_GROUP: + printf("Group\n"); break; + case TRUSTEE_IS_DOMAIN: + printf("Domain\n"); break; + case TRUSTEE_IS_ALIAS: + printf("Alias\n"); break; + case TRUSTEE_IS_WELL_KNOWN_GROUP: + printf("Well-known group\n"); break; + case TRUSTEE_IS_DELETED: + printf("Deleted account\n"); break; + case TRUSTEE_IS_INVALID: + printf("Invalid trustee type\n"); break; + case TRUSTEE_IS_COMPUTER: + printf("Computer\n"); break; + default: + printf("Unknown type %d\n", pEntry->Trustee.TrusteeType); + assert(0); + } + + printf("\n"); + } + + assert(LocalFree((HLOCAL)pEntries) == 0); + assert(LocalFree((HLOCAL)pSecurityDesc) == 0); + chdir("c:\\tmp"); openfile("test", O_CREAT, 0); struct stat ourfs; @@ -25,12 +227,11 @@ int main(int argc, char* argv[]) do { info = readdir(ourDir); - if ( info ) printf("File/Dir name is : %s\r\n", info->d_name); - - }while ( info != NULL ); + if (info) printf("File/Dir name is : %s\r\n", info->d_name); + } + while (info != NULL); closedir(ourDir); - } std::string diry("C:\\Projects\\boxbuild\\testfiles\\"); @@ -41,12 +242,12 @@ int main(int argc, char* argv[]) do { info = readdir(ourDir); - if ( info == NULL ) break; + if (info == NULL) break; std::string file(diry + info->d_name); stat(file.c_str(), &ourfs); - if ( info ) printf("File/Dir name is : %s\r\n", info->d_name); - - }while ( info != NULL ); + if (info) printf("File/Dir name is : %s\r\n", info->d_name); + } + while ( info != NULL ); closedir(ourDir); @@ -54,47 +255,70 @@ int main(int argc, char* argv[]) stat("c:\\windows", &ourfs); stat("c:\\autoexec.bat", &ourfs); - printf("Finished dir read"); -#if 0 - //remove - sleepycat include a version of getopt - mine never REALLY worked ! - //test our getopt function - std::string commline("-q -c fgfgfg -f -l hello"); + printf("Finished dir read\n"); - int c; - while((c = getopt(commline.size(), (char * const *)commline.c_str(), "qwc:l:")) != -1) + //test our getopt function + char * test_argv[] = { - printf("switch = %c, param is %s\r\n", c, optarg); - } -#endif + "foobar.exe", + "-qwc", + "-", + "-c", + "fgfgfg", + "-f", + "-l", + "hello", + "-", + "force-sync", + NULL + }; + int test_argc; + for (test_argc = 0; test_argv[test_argc]; test_argc++) { } + const char* opts = "qwc:l:"; + + assert(getopt(test_argc, test_argv, opts) == 'q'); + assert(getopt(test_argc, test_argv, opts) == 'w'); + assert(getopt(test_argc, test_argv, opts) == 'c'); + assert(strcmp(optarg, "-") == 0); + assert(getopt(test_argc, test_argv, opts) == 'c'); + assert(strcmp(optarg, "fgfgfg") == 0); + assert(getopt(test_argc, test_argv, opts) == '?'); + assert(optopt == 'f'); + assert(getopt(test_argc, test_argv, opts) == 'l'); + assert(strcmp(optarg, "hello") == 0); + assert(getopt(test_argc, test_argv, opts) == -1); + // assert(optopt == 0); // no more options + assert(strcmp(test_argv[optind], "-") == 0); + assert(strcmp(test_argv[optind+1], "force-sync") == 0); //end of getopt test //now test our statfs funct stat("c:\\cert.cer", &ourfs); - - char *timee; timee = ctime(&ourfs.st_mtime); - if ( S_ISREG(ourfs.st_mode)) + if (S_ISREG(ourfs.st_mode)) { - printf("is a normal file"); + printf("is a normal file\n"); } else { - printf("is a directory?"); + printf("is a directory?\n"); + exit(1); } - lstat("c:\\windows", &ourfs); + lstat(getenv("WINDIR"), &ourfs); if ( S_ISDIR(ourfs.st_mode)) { - printf("is a directory"); + printf("is a directory\n"); } else { - printf("is a file?"); + printf("is a file?\n"); + exit(1); } //test the syslog functions @@ -105,6 +329,7 @@ int main(int argc, char* argv[]) closelog(); + /* //first off get the path name for the default char buf[MAX_PATH]; @@ -112,7 +337,7 @@ int main(int argc, char* argv[]) std::string buffer(buf); std::string conf("-c " + buffer.substr(0,(buffer.find("win32test.exe"))) + "bbackupd.conf"); //std::string conf( "-c " + buffer.substr(0,(buffer.find("bbackupd.exe"))) + "bbackupd.conf"); - + */ return 0; } @@ -3,6 +3,8 @@ echo quick and dirty to get up and running by generating the required files
echo using Cygwin and Perl
+copy .\infrastructure\BoxPlatform.pm.in .\infrastructure\BoxPlatform.pm
+
cd .\bin\bbackupquery\ & perl ./../../bin/bbackupquery/makedocumentation.pl.in
cd ..\..\
@@ -16,6 +18,10 @@ cd .\lib\common & perl ./../../lib/common/makeexception.pl.in CommonException.tx cd ..\..\
+cd .\bin\bbackupd & perl ./../../lib/common/makeexception.pl.in ClientException.txt
+
+cd ..\..\
+
cd .\lib\crypto & perl ./../../lib/common/makeexception.pl.in CipherException.txt
cd ..\..\
@@ -24,6 +30,4 @@ echo server parts - which appears as though some of the clients rely on cd .\lib\server & perl ./../../lib/common/makeexception.pl.in ServerException.txt & perl ./../../lib/common/makeexception.pl.in ConnectionException.txt
cd ..\..\
-copy lib\win32\config.h.win32 lib\common\BoxConfig.h
-
perl -pe 's/@PERL@/perl/' ./test/bbackupd/testfiles/bbackupd.conf.in > .\test\bbackupd\testfiles\bbackupd.conf
|