diff options
| -rw-r--r-- | BUGS.txt | 5 | ||||
| -rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.cpp | 176 | ||||
| -rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.h | 4 | ||||
| -rw-r--r-- | bin/bbackupd/BackupDaemon.cpp | 458 | ||||
| -rw-r--r-- | bin/bbackupd/BackupDaemon.h | 10 | ||||
| -rwxr-xr-x | bin/bbackupd/bbackupd-config | 8 | ||||
| -rw-r--r-- | configure.ac | 7 | ||||
| -rw-r--r-- | lib/backupclient/BackupDaemonConfigVerify.cpp | 2 | ||||
| -rw-r--r-- | lib/common/Archive.h | 161 | ||||
| -rw-r--r-- | lib/common/CommonException.txt | 1 | ||||
| -rw-r--r-- | lib/common/ExcludeList.cpp | 181 | ||||
| -rw-r--r-- | lib/common/ExcludeList.h | 6 | ||||
| -rw-r--r-- | lib/server/Daemon.cpp | 88 | ||||
| -rw-r--r-- | lib/server/Daemon.h | 13 |
14 files changed, 1098 insertions, 22 deletions
@@ -8,3 +8,8 @@ Bugs * if bbackupd gets an error then a signal, it may not wait it's full 100 seconds before retrying. And then won't stop the cycle... * bbackupquery restore, if not root, then won't do file ownership properly, but won't alert the user to this fact * empty (real) directories in the store aren't deleted when they're empty (and will never be used again) -- uses up disc space unnecessarily +* need unit tests for SSL keepalives and state saving (serialisation) +* make Archive derive from Protocol +* more automated tests for win32 +* change off_t to box_off_t in preparation for win32 large file support +* support large files on win32 by using native *i64 functions instead of posix diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index 3e8ceeb2..49193ccf 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -25,6 +25,7 @@ #include "FileModificationTime.h" #include "BackupDaemon.h" #include "BackupStoreException.h" +#include "Archive.h" #include "MemLeakFindOn.h" @@ -1226,3 +1227,178 @@ BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, Backu BackupClientDirectoryRecord::SyncParams::~SyncParams() { } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +{ + // Make deletion recursive + DeleteSubDirectories(); + + // Delete maps + if(mpPendingEntries != 0) + { + delete mpPendingEntries; + mpPendingEntries = 0; + } + + // + // + // + rArchive.Read(mObjectID); + rArchive.Read(mSubDirName); + rArchive.Read(mInitialSyncDone); + rArchive.Read(mSyncDone); + + // + // + // + int64_t iCount = 0; + rArchive.Read(iCount); + + if (iCount != sizeof(mStateChecksum)/sizeof(mStateChecksum[0])) + { + // we have some kind of internal system representation change: throw for now + THROW_EXCEPTION(CommonException, Internal) + } + + for (int v = 0; v < iCount; v++) + { + // Load each checksum entry + rArchive.Read(mStateChecksum[v]); + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + // load each pending entry + mpPendingEntries = new std::map<std::string, box_time_t>; + if (!mpPendingEntries) + { + throw std::bad_alloc(); + } + + for (int v = 0; v < iCount; v++) + { + std::string strItem; + box_time_t btItem; + + rArchive.Read(strItem); + rArchive.Read(btItem); + (*mpPendingEntries)[strItem] = btItem; + } + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + for (int v = 0; v < iCount; v++) + { + std::string strItem; + rArchive.Read(strItem); + + BackupClientDirectoryRecord* pSubDirRecord = + new BackupClientDirectoryRecord(0, ""); + // will be deserialized anyway, give it id 0 for now + + if (!pSubDirRecord) + { + throw std::bad_alloc(); + } + + /***** RECURSE *****/ + pSubDirRecord->Deserialize(rArchive); + mSubDirectories[strItem] = pSubDirRecord; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mObjectID); + rArchive.Write(mSubDirName); + rArchive.Write(mInitialSyncDone); + rArchive.Write(mSyncDone); + + // + // + // + int64_t iCount = 0; + + // when reading back the archive, we will + // need to know how many items there are. + iCount = sizeof(mStateChecksum) / sizeof(mStateChecksum[0]); + rArchive.Write(iCount); + + for (int v = 0; v < iCount; v++) + { + rArchive.Write(mStateChecksum[v]); + } + + // + // + // + if (!mpPendingEntries) + { + iCount = 0; + rArchive.Write(iCount); + } + else + { + iCount = mpPendingEntries->size(); + rArchive.Write(iCount); + + for (std::map<std::string, box_time_t>::const_iterator + i = mpPendingEntries->begin(); + i != mpPendingEntries->end(); i++) + { + rArchive.Write(i->first); + rArchive.Write(i->second); + } + } + // + // + // + iCount = mSubDirectories.size(); + rArchive.Write(iCount); + + for (std::map<std::string, BackupClientDirectoryRecord*>::const_iterator + i = mSubDirectories.begin(); + i != mSubDirectories.end(); i++) + { + const BackupClientDirectoryRecord* pSubItem = i->second; + ASSERT(pSubItem); + + rArchive.Write(i->first); + pSubItem->Serialize(rArchive); + } +} diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h index 99354bc8..b7b84984 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.h +++ b/bin/bbackupd/BackupClientDirectoryRecord.h @@ -18,6 +18,7 @@ #include "BackupStoreDirectory.h" #include "MD5Digest.h" +class Archive; class BackupClientContext; class BackupDaemon; @@ -34,6 +35,9 @@ class BackupClientDirectoryRecord public: BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName); ~BackupClientDirectoryRecord(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; private: BackupClientDirectoryRecord(const BackupClientDirectoryRecord &); public: diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp index 07a8db4d..a6d25f0f 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/bin/bbackupd/BackupDaemon.cpp @@ -10,12 +10,19 @@ #include "Box.h" #include <stdio.h> +#include <string.h> #include <unistd.h> -#ifndef WIN32 +#ifdef HAVE_SIGNAL_H #include <signal.h> +#endif +#ifdef HAVE_SYSLOG_H #include <syslog.h> +#endif +#ifdef HAVE_SYS_PARAM_H #include <sys/param.h> +#endif +#ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> #endif #ifdef HAVE_SYS_MOUNT_H @@ -61,6 +68,7 @@ #include "LocalProcessStream.h" #include "IOStreamGetLine.h" #include "Conversion.h" +#include "Archive.h" #include "MemLeakFindOn.h" @@ -467,10 +475,18 @@ void BackupDaemon::Run2() // When the last sync started (only updated if the store was not full when the sync ended) box_time_t lastSyncTime = 0; + // -------------------------------------------------------------------------------------------- + + // And what's the current client store marker? + int64_t clientStoreMarker = + BackupClientContext::ClientStoreMarker_NotKnown; + // haven't contacted the store yet + + DeserializeStoreObjectInfo(clientStoreMarker, lastSyncTime, + nextSyncTime); + // -------------------------------------------------------------------------------------------- - // And what's the current client store marker? - int64_t clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // haven't contacted the store yet // Set state SetState(State_Idle); @@ -674,6 +690,13 @@ void BackupDaemon::Run2() // Log ::syslog(LOG_INFO, "Finished scan of local files"); + + // -------------------------------------------------------------------------------------------- + + // We had a successful backup, save the store info + SerializeStoreObjectInfo(clientStoreMarker, lastSyncTime, nextSyncTime); + + // -------------------------------------------------------------------------------------------- } catch(BoxException &e) { @@ -1831,6 +1854,18 @@ void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) mUnusedRootDirEntries.clear(); } +// -------------------------------------------------------------------------- + +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mNumEntries; + int64_t mObjectID; // this object ID + int64_t mContainerID; // ID of container + uint64_t mAttributesModTime; + int32_t mOptionsPresent; // bit mask of optional sections / features present + +} loc_StreamFormat; // -------------------------------------------------------------------------- // @@ -1870,6 +1905,177 @@ BackupDaemon::Location::~Location() } } +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Location::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Location::Deserialize(Archive &rArchive) +{ + // + // + // + mpDirectoryRecord.reset(NULL); + if (mpExcludeFiles) + { + delete mpExcludeFiles; + mpExcludeFiles = NULL; + } + if (mpExcludeDirs) + { + delete mpExcludeDirs; + mpExcludeDirs = NULL; + } + + // + // + // + rArchive.Read(mName); + rArchive.Read(mPath); + rArchive.Read(mIDMapIndex); + + // + // + // + int64_t aMagicMarker = 0; + rArchive.Read(aMagicMarker); + + if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); + if (!pSubRecord) + throw std::bad_alloc(); + + mpDirectoryRecord.reset(pSubRecord); + mpDirectoryRecord->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(CommonException, Internal) + } + + // + // + // + rArchive.Read(aMagicMarker); + + if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpExcludeFiles = new ExcludeList; + if (!mpExcludeFiles) + throw std::bad_alloc(); + + mpExcludeFiles->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(CommonException, Internal) + } + + // + // + // + rArchive.Read(aMagicMarker); + + if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpExcludeDirs = new ExcludeList; + if (!mpExcludeDirs) + throw std::bad_alloc(); + + mpExcludeDirs->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(CommonException, Internal) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Location::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Location::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mName); + rArchive.Write(mPath); + rArchive.Write(mIDMapIndex); + + // + // + // + if (mpDirectoryRecord.get() == NULL) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpDirectoryRecord->Serialize(rArchive); + } + + // + // + // + if (!mpExcludeFiles) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpExcludeFiles->Serialize(rArchive); + } + + // + // + // + if (!mpExcludeDirs) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpExcludeDirs->Serialize(rArchive); + } +} // -------------------------------------------------------------------------- // @@ -1901,3 +2107,249 @@ BackupDaemon::CommandSocketInfo::~CommandSocketInfo() mpGetLine = 0; } } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) +// Purpose: Serializes remote directory and file information into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- + +static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; +static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; +static const int STOREOBJECTINFO_VERSION = 1; + +void BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const +{ + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if (StoreObjectInfoFile.size() <= 0) + { + return; + } + + try + { + FileStream aFile(StoreObjectInfoFile.c_str(), + O_WRONLY | O_CREAT | O_TRUNC); + Archive anArchive(aFile, 0); + + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE); + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); + anArchive.Write(STOREOBJECTINFO_VERSION); + anArchive.Write(GetLoadedConfigModifiedTime()); + anArchive.Write(aClientStoreMarker); + anArchive.Write(theLastSyncTime); + anArchive.Write(theNextSyncTime); + + // + // + // + int64_t iCount = mLocations.size(); + anArchive.Write(iCount); + + for (int v = 0; v < iCount; v++) + { + ASSERT(mLocations[v]); + mLocations[v]->Serialize(anArchive); + } + + // + // + // + iCount = mIDMapMounts.size(); + anArchive.Write(iCount); + + for (int v = 0; v < iCount; v++) + anArchive.Write(mIDMapMounts[v]); + + // + // + // + aFile.Close(); + ::syslog(LOG_INFO, "Saved store object info file '%s'", + StoreObjectInfoFile.c_str()); + } + catch (...) + { + ::syslog(LOG_WARNING, "Requested store object info file '%s' " + "not accessible or could not be created", + StoreObjectInfoFile.c_str()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime) +// Purpose: Deserializes remote directory and file information from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime) +{ + // + // + // + DeleteAllLocations(); + + // + // + // + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if (StoreObjectInfoFile.size() <= 0) + { + return; + } + + try + { + FileStream aFile(StoreObjectInfoFile.c_str(), O_RDONLY); + Archive anArchive(aFile, 0); + + // + // see if the content looks like a valid serialised archive + // + int iMagicValue = 0; + anArchive.Read(iMagicValue); + + if (iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) + { + ::syslog(LOG_WARNING, "Store object info file '%s' " + "is not a valid or compatible serialised " + "archive. Will re-cache from store.", + StoreObjectInfoFile.c_str()); + return; + } + + // + // get a bit optimistic and read in a string identifier + // + std::string strMagicValue; + anArchive.Read(strMagicValue); + + if (strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) + { + ::syslog(LOG_WARNING, "Store object info file '%s' " + "is not a valid or compatible serialised " + "archive. Will re-cache from store.", + StoreObjectInfoFile.c_str()); + return; + } + + // + // check if we are loading some future format + // version by mistake + // + int iVersion = 0; + anArchive.Read(iVersion); + + if (iVersion != STOREOBJECTINFO_VERSION) + { + ::syslog(LOG_WARNING, "Store object info file '%s' " + "version [%d] unsupported. " + "Will re-cache from store.", + StoreObjectInfoFile.c_str(), + iVersion); + return; + } + + // + // check if this state file is even valid + // for the loaded bbackupd.conf file + // + box_time_t lastKnownConfigModTime; + anArchive.Read(lastKnownConfigModTime); + + if (lastKnownConfigModTime != GetLoadedConfigModifiedTime()) + { + ::syslog(LOG_WARNING, "Store object info file '%s' " + "out of date. Will re-cache from store", + StoreObjectInfoFile.c_str()); + return; + } + + // + // this is it, go at it + // + anArchive.Read(aClientStoreMarker); + anArchive.Read(theLastSyncTime); + anArchive.Read(theNextSyncTime); + + // + // + // + int64_t iCount = 0; + anArchive.Read(iCount); + + for (int v = 0; v < iCount; v++) + { + Location* pLocation = new Location; + if (!pLocation) + throw std::bad_alloc(); + + pLocation->Deserialize(anArchive); + mLocations.push_back(pLocation); + } + + // + // + // + iCount = 0; + anArchive.Read(iCount); + + for (int v = 0; v < iCount; v++) + { + std::string strItem; + anArchive.Read(strItem); + + mIDMapMounts.push_back(strItem); + } + + // + // + // + aFile.Close(); + ::syslog(LOG_INFO, "Loaded store object info file '%s', " + "version [%d]", StoreObjectInfoFile.c_str(), + iVersion); + + if (::unlink(StoreObjectInfoFile.c_str()) != 0) + { + ::syslog(LOG_ERR, "Failed to delete the old " + "store object info file '%s': %s", + StoreObjectInfoFile.c_str(), strerror(errno)); + } + } + catch (...) + { + DeleteAllLocations(); + + aClientStoreMarker = + BackupClientContext::ClientStoreMarker_NotKnown; + theLastSyncTime = 0; + theNextSyncTime = 0; + + ::syslog(LOG_WARNING, "Requested store object info file '%s' " + "does not exist, not accessible, or inconsistent. " + "Will re-cache from store.", + StoreObjectInfoFile.c_str()); + } +} diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h index e6798798..cef09630 100644 --- a/bin/bbackupd/BackupDaemon.h +++ b/bin/bbackupd/BackupDaemon.h @@ -14,8 +14,8 @@ #include <string> #include <memory> -#include "Daemon.h" #include "BoxTime.h" +#include "Daemon.h" #include "Socket.h" #include "SocketListen.h" #include "SocketStream.h" @@ -27,6 +27,7 @@ class Configuration; class BackupClientInodeToIDMap; class ExcludeList; class IOStreamGetLine; +class Archive; // -------------------------------------------------------------------------- // @@ -41,6 +42,10 @@ class BackupDaemon : public Daemon public: BackupDaemon(); ~BackupDaemon(); + + // methods below do partial (specialized) serialization of client state only + void SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const; + void DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime); private: BackupDaemon(const BackupDaemon &); public: @@ -117,6 +122,9 @@ private: public: Location(); ~Location(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; private: Location(const Location &); // copy not allowed Location &operator=(const Location &); diff --git a/bin/bbackupd/bbackupd-config b/bin/bbackupd/bbackupd-config index c5e52282..0d8dd4d1 100755 --- a/bin/bbackupd/bbackupd-config +++ b/bin/bbackupd/bbackupd-config @@ -382,6 +382,14 @@ MaximumDiffingTime = 20 CommandSocket = /var/run/bbackupd.sock +# Uncomment the StoreObjectInfoFile to enable the experimental archiving +# of the daemon's state (including client store marker and configuration) +# between backup runs. This saves time and increases efficiency when +# bbackupd is frequently stopped and started, since it removes the need +# to rescan all directories on the remote server. However, it is new and +# not yet heavily tested, so use with caution. + +# StoreObjectInfoFile = $working_dir/bbackupd.state Server { diff --git a/configure.ac b/configure.ac index 2ed3812a..8a4175ce 100644 --- a/configure.ac +++ b/configure.ac @@ -74,9 +74,10 @@ Upgrade or read the documentation for alternatives]]) AC_HEADER_DIRENT AC_HEADER_STDC AC_HEADER_SYS_WAIT -AC_CHECK_HEADERS([execinfo.h netinet/in.h regex.h sys/types.h sys/xattr.h]) -AC_CHECK_HEADERS([sys/endian.h asm/byteorder.h syslog.h signal.h sys/time.h]) -AC_CHECK_HEADERS([time.h]) +AC_CHECK_HEADERS([execinfo.h regex.h signal.h syslog.h time.h]) +AC_CHECK_HEADERS([asm/byteorder.h]) +AC_CHECK_HEADERS([netinet/in.h]) +AC_CHECK_HEADERS([sys/endian.h sys/param.h sys/types.h sys/wait.h sys/xattr.h sys/time.h]) ### Checks for typedefs, structures, and compiler characteristics. diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index f1d5bd16..89ad4d54 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -85,6 +85,7 @@ static const ConfigurationVerifyKey verifyrootkeys[] = {"CommandSocket", 0, 0, 0}, // not compulsory to have this {"KeepAliveTime", 0, ConfigTest_IsInt, 0}, // optional + {"StoreObjectInfoFile", 0, 0, 0}, // optional {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full @@ -103,4 +104,3 @@ const ConfigurationVerify BackupDaemonConfigVerify = ConfigTest_Exists | ConfigTest_LastEntry, 0 }; - diff --git a/lib/common/Archive.h b/lib/common/Archive.h new file mode 100644 index 00000000..b70f12c4 --- /dev/null +++ b/lib/common/Archive.h @@ -0,0 +1,161 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Archive.h +// Purpose: Backup daemon state archive +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- + +#ifndef ARCHIVE__H +#define ARCHIVE__H + +#include <vector> +#include <string> +#include <memory> + +#include "IOStream.h" +#include "Guards.h" + +#define ARCHIVE_GET_SIZE(hdr) (( ((uint8_t)((hdr)[0])) | ( ((uint8_t)((hdr)[1])) << 8)) >> 2) + +#define ARCHIVE_MAGIC_VALUE_RECURSE 0x4449525F +#define ARCHIVE_MAGIC_VALUE_NOOP 0x5449525F + +class Archive +{ +public: + Archive(IOStream &Stream, int Timeout) + : mrStream(Stream) + { + mTimeout = Timeout; + } +private: + // no copying + Archive(const Archive &); + Archive & operator=(const Archive &); +public: + ~Archive() + { + } + // + // + // + void Write(bool Item) + { + Write((int) Item); + } + void Write(int Item) + { + int32_t privItem = htonl(Item); + mrStream.Write(&privItem, sizeof(privItem)); + } + void Write(int64_t Item) + { + int64_t privItem = box_hton64(Item); + mrStream.Write(&privItem, sizeof(privItem)); + } + void Write(uint64_t Item) + { + uint64_t privItem = box_hton64(Item); + mrStream.Write(&privItem, sizeof(privItem)); + } + void Write(uint8_t Item) + { + int privItem = Item; + Write(privItem); + } + void Write(const std::string &Item) + { + int size = Item.size(); + Write(size); + mrStream.Write(Item.c_str(), size); + } + // + // + // + void Read(bool &rItemOut) + { + int privItem; + Read(privItem); + + if (privItem) + { + rItemOut = true; + } + else + { + rItemOut = false; + } + } + void Read(int &rItemOut) + { + int32_t privItem; + if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + rItemOut = ntohl(privItem); + } + void Read(int64_t &rItemOut) + { + int64_t privItem; + if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + rItemOut = box_ntoh64(privItem); + } + void Read(uint64_t &rItemOut) + { + uint64_t privItem; + if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + rItemOut = box_ntoh64(privItem); + } + void Read(uint8_t &rItemOut) + { + int privItem; + Read(privItem); + rItemOut = privItem; + } + void Read(std::string &rItemOut) + { + int size; + Read(size); + + // Assume most strings are relatively small + char buf[256]; + if(size < (int) sizeof(buf)) + { + // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us + if(!mrStream.ReadFullBuffer(buf, size, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + // assign to this string, storing the header and the extra payload + rItemOut.assign(buf, size); + } + else + { + // Block of memory to hold it + MemoryBlockGuard<char*> dataB(size); + char *ppayload = dataB; + + // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us + if(!mrStream.ReadFullBuffer(ppayload, size, 0 /* not interested in bytes read if this fails */, mTimeout)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + } + // assign to this string, storing the header and the extra pPayload + rItemOut.assign(ppayload, size); + } + } +private: + IOStream &mrStream; + int mTimeout; +}; + +#endif // ARCHIVE__H diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt index f852b7d7..5fa443d0 100644 --- a/lib/common/CommonException.txt +++ b/lib/common/CommonException.txt @@ -43,3 +43,4 @@ KEventErrorRemove 35 KQueueNotSupportedOnThisPlatform 36 IOStreamGetLineNotEnoughDataToIgnore 37 Bad value passed to IOStreamGetLine::IgnoreBufferedData() TempDirPathTooLong 38 Your temporary directory path is too long. Check the TMP and TEMP environment variables. +ArchiveBlockIncompleteRead 39 The Store Object Info File is too short or corrupted, and will be rewritten automatically when the next backup completes. diff --git a/lib/common/ExcludeList.cpp b/lib/common/ExcludeList.cpp index 556a2079..9b2e3acb 100644 --- a/lib/common/ExcludeList.cpp +++ b/lib/common/ExcludeList.cpp @@ -17,6 +17,7 @@ #include "ExcludeList.h" #include "Utils.h" #include "Configuration.h" +#include "Archive.h" #include "MemLeakFindOn.h" @@ -130,6 +131,8 @@ void ExcludeList::AddRegexEntries(const std::string &rEntries) // Store in list of regular expressions mRegex.push_back(pregex); + // Store in list of regular expression string for Serialize + mRegexStr.push_back(i->c_str()); } catch(...) { @@ -213,7 +216,183 @@ void ExcludeList::SetAlwaysIncludeList(ExcludeList *pAlwaysInclude) mpAlwaysInclude = pAlwaysInclude; } +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void ExcludeList::Deserialize(Archive & rArchive) +{ + // + // + // + mDefinite.clear(); - +#ifndef PLATFORM_REGEX_NOT_SUPPORTED + // free regex memory + while(mRegex.size() > 0) + { + regex_t *pregex = mRegex.back(); + mRegex.pop_back(); + // Free regex storage, and the structure itself + ::regfree(pregex); + delete pregex; + } + mRegexStr.clear(); +#endif + // Clean up exceptions list + if(mpAlwaysInclude != 0) + { + delete mpAlwaysInclude; + mpAlwaysInclude = 0; + } + + // + // + // + int64_t iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + for (int v = 0; v < iCount; v++) + { + // load each one + std::string strItem; + rArchive.Read(strItem); + mDefinite.insert(strItem); + } + } + + // + // + // +#ifndef PLATFORM_REGEX_NOT_SUPPORTED + rArchive.Read(iCount); + + if (iCount > 0) + { + for (int v = 0; v < iCount; v++) + { + std::string strItem; + rArchive.Read(strItem); + + // Allocate memory + regex_t* pregex = new regex_t; + + try + { + // Compile + if(::regcomp(pregex, strItem.c_str(), + REG_EXTENDED | REG_NOSUB) != 0) + { + THROW_EXCEPTION(CommonException, + BadRegularExpression) + } + + // Store in list of regular expressions + mRegex.push_back(pregex); + + // Store in list of regular expression strings + // for Serialize + mRegexStr.push_back(strItem); + } + catch(...) + { + delete pregex; + throw; + } + } + } +#endif // PLATFORM_REGEX_NOT_SUPPORTED + + // + // + // + int64_t aMagicMarker = 0; + rArchive.Read(aMagicMarker); + + if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpAlwaysInclude = new ExcludeList; + if (!mpAlwaysInclude) + { + throw std::bad_alloc(); + } + + mpAlwaysInclude->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(CommonException, Internal) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: ExcludeList::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void ExcludeList::Serialize(Archive & rArchive) const +{ + // + // + // + int64_t iCount = mDefinite.size(); + rArchive.Write(iCount); + + for (std::set<std::string>::const_iterator i = mDefinite.begin(); + i != mDefinite.end(); i++) + { + rArchive.Write(*i); + } + + // + // + // +#ifndef PLATFORM_REGEX_NOT_SUPPORTED + // don't even try to save compiled regular expressions, + // use string copies instead. + ASSERT(mRegex.size() == mRegexStr.size()); + + iCount = mRegexStr.size(); + rArchive.Write(iCount); + + for (std::vector<std::string>::const_iterator i = mRegexStr.begin(); + i != mRegexStr.end(); i++) + { + rArchive.Write(*i); + } +#endif // PLATFORM_REGEX_NOT_SUPPORTED + + // + // + // + if (!mpAlwaysInclude) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpAlwaysInclude->Serialize(rArchive); + } +} diff --git a/lib/common/ExcludeList.h b/lib/common/ExcludeList.h index 5324d226..720b6788 100644 --- a/lib/common/ExcludeList.h +++ b/lib/common/ExcludeList.h @@ -19,6 +19,8 @@ typedef int regex_t; #endif +class Archive; + // -------------------------------------------------------------------------- // // Class @@ -33,6 +35,9 @@ public: ExcludeList(); ~ExcludeList(); + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; + void AddDefiniteEntries(const std::string &rEntries); void AddRegexEntries(const std::string &rEntries); @@ -55,6 +60,7 @@ private: std::set<std::string> mDefinite; #ifdef HAVE_REGEX_H std::vector<regex_t *> mRegex; + std::vector<std::string> mRegexStr; // save original regular expression string-based source for Serialize #endif // For exceptions to the excludes diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp index a4dfdaec..9c820b22 100644 --- a/lib/server/Daemon.cpp +++ b/lib/server/Daemon.cpp @@ -9,14 +9,15 @@ #include "Box.h" +#include <errno.h> #include <stdio.h> #include <unistd.h> #include <signal.h> #include <string.h> #include <stdarg.h> -#ifndef WIN32 -#include <syslog.h> +#ifdef HAVE_SYSLOG_H + #include <syslog.h> #endif #include "Daemon.h" @@ -24,6 +25,7 @@ #include "ServerException.h" #include "Guards.h" #include "UnixUser.h" +#include "FileModificationTime.h" #include "MemLeakFindOn.h" @@ -92,22 +94,21 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) } std::string pidFileName; - const char *configfile = 0; try { // Find filename of config file - configfile = DefaultConfigFile; + mConfigFileName = DefaultConfigFile; if(argc >= 2) { // First argument is config file, or it's -c and the next arg is the config file if(::strcmp(argv[1], "-c") == 0 && argc >= 3) { - configfile = argv[2]; + mConfigFileName = argv[2]; } else { - configfile = argv[1]; + mConfigFileName = argv[1]; } } @@ -123,19 +124,25 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) // Load the configuration file. std::string errors; - std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(configfile, GetConfigVerify(), errors); + std::auto_ptr<Configuration> pconfig = + Configuration::LoadAndVerify( + mConfigFileName.c_str(), + GetConfigVerify(), errors); // Got errors? if(pconfig.get() == 0 || !errors.empty()) { // Tell user about errors - fprintf(stderr, "%s: Errors in config file %s:\n%s", DaemonName(), configfile, errors.c_str()); + fprintf(stderr, "%s: Errors in config file %s:\n%s", + DaemonName(), mConfigFileName.c_str(), + errors.c_str()); // And give up return 1; } // Store configuration mpConfiguration = pconfig.release(); + mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); // Server configuration const Configuration &serverConfig(mpConfiguration->GetSubConfiguration("Server")); @@ -228,7 +235,8 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) // open the log ::openlog(DaemonName(), LOG_PID, LOG_LOCAL6); // Log the start message - ::syslog(LOG_INFO, "Starting daemon (config: %s) (version " BOX_VERSION ")", configfile); + ::syslog(LOG_INFO, "Starting daemon (config: %s) (version " + BOX_VERSION ")", mConfigFileName.c_str()); #ifndef WIN32 // Write PID to file @@ -306,15 +314,23 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) if(mReloadConfigWanted && !mTerminateWanted) { // Need to reload that config file... - ::syslog(LOG_INFO, "Reloading configuration (config: %s)", configfile); + ::syslog(LOG_INFO, "Reloading configuration " + "(config: %s)", + mConfigFileName.c_str()); std::string errors; - std::auto_ptr<Configuration> pconfig = Configuration::LoadAndVerify(configfile, GetConfigVerify(), errors); + std::auto_ptr<Configuration> pconfig = + Configuration::LoadAndVerify( + mConfigFileName.c_str(), + GetConfigVerify(), errors); // Got errors? if(pconfig.get() == 0 || !errors.empty()) { // Tell user about errors - ::syslog(LOG_ERR, "Errors in config file %s:\n%s", configfile, errors.c_str()); + ::syslog(LOG_ERR, "Errors in config " + "file %s:\n%s", + mConfigFileName.c_str(), + errors.c_str()); // And give up return 1; } @@ -325,6 +341,8 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) // Store configuration mpConfiguration = pconfig.release(); + mLoadedConfigModifiedTime = + GetConfigFileModifiedTime(); // Stop being marked for loading config again mReloadConfigWanted = false; @@ -547,3 +565,49 @@ void Daemon::SetProcessTitle(const char *format, ...) #endif // HAVE_SETPROCTITLE } + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::GetConfigFileModifiedTime() +// Purpose: Returns the timestamp when the configuration file +// was last modified +// +// Created: 2006/01/29 +// +// -------------------------------------------------------------------------- + +box_time_t Daemon::GetConfigFileModifiedTime() const +{ + struct stat st; + + if(::stat(GetConfigFileName().c_str(), &st) != 0) + { + if (errno == ENOENT) + { + return 0; + } + THROW_EXCEPTION(CommonException, OSFileError) + } + + return FileModificationTime(st); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::GetLoadedConfigModifiedTime() +// Purpose: Returns the timestamp when the configuration file +// had been last modified, at the time when it was +// loaded +// +// Created: 2006/01/29 +// +// -------------------------------------------------------------------------- + +box_time_t Daemon::GetLoadedConfigModifiedTime() const +{ + return mLoadedConfigModifiedTime; +} + diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h index a7b9488b..dbc83e98 100644 --- a/lib/server/Daemon.h +++ b/lib/server/Daemon.h @@ -16,6 +16,10 @@ #ifndef DAEMON__H #define DAEMON__H +#include <string> + +#include "BoxTime.h" + class Configuration; class ConfigurationVerify; @@ -40,7 +44,8 @@ public: virtual void Run(); const Configuration &GetConfiguration() const; - + const std::string &GetConfigFileName() const {return mConfigFileName;} + virtual const char *DaemonName() const; virtual const char *DaemonBanner() const; virtual const ConfigurationVerify *GetConfigVerify() const; @@ -57,12 +62,18 @@ public: virtual void EnterChild(); static void SetProcessTitle(const char *format, ...); + +protected: + box_time_t GetLoadedConfigModifiedTime() const; private: static void SignalHandler(int sigraised); + box_time_t GetConfigFileModifiedTime() const; private: + std::string mConfigFileName; Configuration *mpConfiguration; + box_time_t mLoadedConfigModifiedTime; bool mReloadConfigWanted; bool mTerminateWanted; static Daemon *spDaemon; |