diff options
Diffstat (limited to 'bin')
-rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.cpp | 176 | ||||
-rw-r--r-- | bin/bbackupd/BackupClientDirectoryRecord.h | 4 | ||||
-rw-r--r-- | bin/bbackupd/BackupDaemon.cpp | 458 | ||||
-rw-r--r-- | bin/bbackupd/BackupDaemon.h | 10 | ||||
-rwxr-xr-x | bin/bbackupd/bbackupd-config | 8 |
5 files changed, 652 insertions, 4 deletions
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index 3e8ceeb2..49193ccf 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -25,6 +25,7 @@ #include "FileModificationTime.h" #include "BackupDaemon.h" #include "BackupStoreException.h" +#include "Archive.h" #include "MemLeakFindOn.h" @@ -1226,3 +1227,178 @@ BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, Backu BackupClientDirectoryRecord::SyncParams::~SyncParams() { } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Deserialize(Archive & rArchive) +{ + // Make deletion recursive + DeleteSubDirectories(); + + // Delete maps + if(mpPendingEntries != 0) + { + delete mpPendingEntries; + mpPendingEntries = 0; + } + + // + // + // + rArchive.Read(mObjectID); + rArchive.Read(mSubDirName); + rArchive.Read(mInitialSyncDone); + rArchive.Read(mSyncDone); + + // + // + // + int64_t iCount = 0; + rArchive.Read(iCount); + + if (iCount != sizeof(mStateChecksum)/sizeof(mStateChecksum[0])) + { + // we have some kind of internal system representation change: throw for now + THROW_EXCEPTION(CommonException, Internal) + } + + for (int v = 0; v < iCount; v++) + { + // Load each checksum entry + rArchive.Read(mStateChecksum[v]); + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + // load each pending entry + mpPendingEntries = new std::map<std::string, box_time_t>; + if (!mpPendingEntries) + { + throw std::bad_alloc(); + } + + for (int v = 0; v < iCount; v++) + { + std::string strItem; + box_time_t btItem; + + rArchive.Read(strItem); + rArchive.Read(btItem); + (*mpPendingEntries)[strItem] = btItem; + } + } + + // + // + // + iCount = 0; + rArchive.Read(iCount); + + if (iCount > 0) + { + for (int v = 0; v < iCount; v++) + { + std::string strItem; + rArchive.Read(strItem); + + BackupClientDirectoryRecord* pSubDirRecord = + new BackupClientDirectoryRecord(0, ""); + // will be deserialized anyway, give it id 0 for now + + if (!pSubDirRecord) + { + throw std::bad_alloc(); + } + + /***** RECURSE *****/ + pSubDirRecord->Deserialize(rArchive); + mSubDirectories[strItem] = pSubDirRecord; + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mObjectID); + rArchive.Write(mSubDirName); + rArchive.Write(mInitialSyncDone); + rArchive.Write(mSyncDone); + + // + // + // + int64_t iCount = 0; + + // when reading back the archive, we will + // need to know how many items there are. + iCount = sizeof(mStateChecksum) / sizeof(mStateChecksum[0]); + rArchive.Write(iCount); + + for (int v = 0; v < iCount; v++) + { + rArchive.Write(mStateChecksum[v]); + } + + // + // + // + if (!mpPendingEntries) + { + iCount = 0; + rArchive.Write(iCount); + } + else + { + iCount = mpPendingEntries->size(); + rArchive.Write(iCount); + + for (std::map<std::string, box_time_t>::const_iterator + i = mpPendingEntries->begin(); + i != mpPendingEntries->end(); i++) + { + rArchive.Write(i->first); + rArchive.Write(i->second); + } + } + // + // + // + iCount = mSubDirectories.size(); + rArchive.Write(iCount); + + for (std::map<std::string, BackupClientDirectoryRecord*>::const_iterator + i = mSubDirectories.begin(); + i != mSubDirectories.end(); i++) + { + const BackupClientDirectoryRecord* pSubItem = i->second; + ASSERT(pSubItem); + + rArchive.Write(i->first); + pSubItem->Serialize(rArchive); + } +} diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h index 99354bc8..b7b84984 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.h +++ b/bin/bbackupd/BackupClientDirectoryRecord.h @@ -18,6 +18,7 @@ #include "BackupStoreDirectory.h" #include "MD5Digest.h" +class Archive; class BackupClientContext; class BackupDaemon; @@ -34,6 +35,9 @@ class BackupClientDirectoryRecord public: BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName); ~BackupClientDirectoryRecord(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; private: BackupClientDirectoryRecord(const BackupClientDirectoryRecord &); public: diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp index 07a8db4d..a6d25f0f 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/bin/bbackupd/BackupDaemon.cpp @@ -10,12 +10,19 @@ #include "Box.h" #include <stdio.h> +#include <string.h> #include <unistd.h> -#ifndef WIN32 +#ifdef HAVE_SIGNAL_H #include <signal.h> +#endif +#ifdef HAVE_SYSLOG_H #include <syslog.h> +#endif +#ifdef HAVE_SYS_PARAM_H #include <sys/param.h> +#endif +#ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> #endif #ifdef HAVE_SYS_MOUNT_H @@ -61,6 +68,7 @@ #include "LocalProcessStream.h" #include "IOStreamGetLine.h" #include "Conversion.h" +#include "Archive.h" #include "MemLeakFindOn.h" @@ -467,10 +475,18 @@ void BackupDaemon::Run2() // When the last sync started (only updated if the store was not full when the sync ended) box_time_t lastSyncTime = 0; + // -------------------------------------------------------------------------------------------- + + // And what's the current client store marker? + int64_t clientStoreMarker = + BackupClientContext::ClientStoreMarker_NotKnown; + // haven't contacted the store yet + + DeserializeStoreObjectInfo(clientStoreMarker, lastSyncTime, + nextSyncTime); + // -------------------------------------------------------------------------------------------- - // And what's the current client store marker? - int64_t clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // haven't contacted the store yet // Set state SetState(State_Idle); @@ -674,6 +690,13 @@ void BackupDaemon::Run2() // Log ::syslog(LOG_INFO, "Finished scan of local files"); + + // -------------------------------------------------------------------------------------------- + + // We had a successful backup, save the store info + SerializeStoreObjectInfo(clientStoreMarker, lastSyncTime, nextSyncTime); + + // -------------------------------------------------------------------------------------------- } catch(BoxException &e) { @@ -1831,6 +1854,18 @@ void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) mUnusedRootDirEntries.clear(); } +// -------------------------------------------------------------------------- + +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mNumEntries; + int64_t mObjectID; // this object ID + int64_t mContainerID; // ID of container + uint64_t mAttributesModTime; + int32_t mOptionsPresent; // bit mask of optional sections / features present + +} loc_StreamFormat; // -------------------------------------------------------------------------- // @@ -1870,6 +1905,177 @@ BackupDaemon::Location::~Location() } } +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Location::Deserialize(Archive & rArchive) +// Purpose: Deserializes this object instance from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Location::Deserialize(Archive &rArchive) +{ + // + // + // + mpDirectoryRecord.reset(NULL); + if (mpExcludeFiles) + { + delete mpExcludeFiles; + mpExcludeFiles = NULL; + } + if (mpExcludeDirs) + { + delete mpExcludeDirs; + mpExcludeDirs = NULL; + } + + // + // + // + rArchive.Read(mName); + rArchive.Read(mPath); + rArchive.Read(mIDMapIndex); + + // + // + // + int64_t aMagicMarker = 0; + rArchive.Read(aMagicMarker); + + if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + BackupClientDirectoryRecord *pSubRecord = new BackupClientDirectoryRecord(0, ""); + if (!pSubRecord) + throw std::bad_alloc(); + + mpDirectoryRecord.reset(pSubRecord); + mpDirectoryRecord->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(CommonException, Internal) + } + + // + // + // + rArchive.Read(aMagicMarker); + + if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpExcludeFiles = new ExcludeList; + if (!mpExcludeFiles) + throw std::bad_alloc(); + + mpExcludeFiles->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(CommonException, Internal) + } + + // + // + // + rArchive.Read(aMagicMarker); + + if (aMagicMarker == ARCHIVE_MAGIC_VALUE_NOOP) + { + // NOOP + } + else if (aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) + { + mpExcludeDirs = new ExcludeList; + if (!mpExcludeDirs) + throw std::bad_alloc(); + + mpExcludeDirs->Deserialize(rArchive); + } + else + { + // there is something going on here + THROW_EXCEPTION(CommonException, Internal) + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Location::Serialize(Archive & rArchive) +// Purpose: Serializes this object instance into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Location::Serialize(Archive & rArchive) const +{ + // + // + // + rArchive.Write(mName); + rArchive.Write(mPath); + rArchive.Write(mIDMapIndex); + + // + // + // + if (mpDirectoryRecord.get() == NULL) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpDirectoryRecord->Serialize(rArchive); + } + + // + // + // + if (!mpExcludeFiles) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpExcludeFiles->Serialize(rArchive); + } + + // + // + // + if (!mpExcludeDirs) + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; + rArchive.Write(aMagicMarker); + } + else + { + int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows + rArchive.Write(aMagicMarker); + + mpExcludeDirs->Serialize(rArchive); + } +} // -------------------------------------------------------------------------- // @@ -1901,3 +2107,249 @@ BackupDaemon::CommandSocketInfo::~CommandSocketInfo() mpGetLine = 0; } } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) +// Purpose: Serializes remote directory and file information into a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- + +static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; +static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; +static const int STOREOBJECTINFO_VERSION = 1; + +void BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const +{ + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if (StoreObjectInfoFile.size() <= 0) + { + return; + } + + try + { + FileStream aFile(StoreObjectInfoFile.c_str(), + O_WRONLY | O_CREAT | O_TRUNC); + Archive anArchive(aFile, 0); + + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_VALUE); + anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); + anArchive.Write(STOREOBJECTINFO_VERSION); + anArchive.Write(GetLoadedConfigModifiedTime()); + anArchive.Write(aClientStoreMarker); + anArchive.Write(theLastSyncTime); + anArchive.Write(theNextSyncTime); + + // + // + // + int64_t iCount = mLocations.size(); + anArchive.Write(iCount); + + for (int v = 0; v < iCount; v++) + { + ASSERT(mLocations[v]); + mLocations[v]->Serialize(anArchive); + } + + // + // + // + iCount = mIDMapMounts.size(); + anArchive.Write(iCount); + + for (int v = 0; v < iCount; v++) + anArchive.Write(mIDMapMounts[v]); + + // + // + // + aFile.Close(); + ::syslog(LOG_INFO, "Saved store object info file '%s'", + StoreObjectInfoFile.c_str()); + } + catch (...) + { + ::syslog(LOG_WARNING, "Requested store object info file '%s' " + "not accessible or could not be created", + StoreObjectInfoFile.c_str()); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime) +// Purpose: Deserializes remote directory and file information from a stream of bytes, using an Archive abstraction. +// +// Created: 2005/04/11 +// +// -------------------------------------------------------------------------- +void BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime) +{ + // + // + // + DeleteAllLocations(); + + // + // + // + if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) + { + return; + } + + std::string StoreObjectInfoFile = + GetConfiguration().GetKeyValue("StoreObjectInfoFile"); + + if (StoreObjectInfoFile.size() <= 0) + { + return; + } + + try + { + FileStream aFile(StoreObjectInfoFile.c_str(), O_RDONLY); + Archive anArchive(aFile, 0); + + // + // see if the content looks like a valid serialised archive + // + int iMagicValue = 0; + anArchive.Read(iMagicValue); + + if (iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) + { + ::syslog(LOG_WARNING, "Store object info file '%s' " + "is not a valid or compatible serialised " + "archive. Will re-cache from store.", + StoreObjectInfoFile.c_str()); + return; + } + + // + // get a bit optimistic and read in a string identifier + // + std::string strMagicValue; + anArchive.Read(strMagicValue); + + if (strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) + { + ::syslog(LOG_WARNING, "Store object info file '%s' " + "is not a valid or compatible serialised " + "archive. Will re-cache from store.", + StoreObjectInfoFile.c_str()); + return; + } + + // + // check if we are loading some future format + // version by mistake + // + int iVersion = 0; + anArchive.Read(iVersion); + + if (iVersion != STOREOBJECTINFO_VERSION) + { + ::syslog(LOG_WARNING, "Store object info file '%s' " + "version [%d] unsupported. " + "Will re-cache from store.", + StoreObjectInfoFile.c_str(), + iVersion); + return; + } + + // + // check if this state file is even valid + // for the loaded bbackupd.conf file + // + box_time_t lastKnownConfigModTime; + anArchive.Read(lastKnownConfigModTime); + + if (lastKnownConfigModTime != GetLoadedConfigModifiedTime()) + { + ::syslog(LOG_WARNING, "Store object info file '%s' " + "out of date. Will re-cache from store", + StoreObjectInfoFile.c_str()); + return; + } + + // + // this is it, go at it + // + anArchive.Read(aClientStoreMarker); + anArchive.Read(theLastSyncTime); + anArchive.Read(theNextSyncTime); + + // + // + // + int64_t iCount = 0; + anArchive.Read(iCount); + + for (int v = 0; v < iCount; v++) + { + Location* pLocation = new Location; + if (!pLocation) + throw std::bad_alloc(); + + pLocation->Deserialize(anArchive); + mLocations.push_back(pLocation); + } + + // + // + // + iCount = 0; + anArchive.Read(iCount); + + for (int v = 0; v < iCount; v++) + { + std::string strItem; + anArchive.Read(strItem); + + mIDMapMounts.push_back(strItem); + } + + // + // + // + aFile.Close(); + ::syslog(LOG_INFO, "Loaded store object info file '%s', " + "version [%d]", StoreObjectInfoFile.c_str(), + iVersion); + + if (::unlink(StoreObjectInfoFile.c_str()) != 0) + { + ::syslog(LOG_ERR, "Failed to delete the old " + "store object info file '%s': %s", + StoreObjectInfoFile.c_str(), strerror(errno)); + } + } + catch (...) + { + DeleteAllLocations(); + + aClientStoreMarker = + BackupClientContext::ClientStoreMarker_NotKnown; + theLastSyncTime = 0; + theNextSyncTime = 0; + + ::syslog(LOG_WARNING, "Requested store object info file '%s' " + "does not exist, not accessible, or inconsistent. " + "Will re-cache from store.", + StoreObjectInfoFile.c_str()); + } +} diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h index e6798798..cef09630 100644 --- a/bin/bbackupd/BackupDaemon.h +++ b/bin/bbackupd/BackupDaemon.h @@ -14,8 +14,8 @@ #include <string> #include <memory> -#include "Daemon.h" #include "BoxTime.h" +#include "Daemon.h" #include "Socket.h" #include "SocketListen.h" #include "SocketStream.h" @@ -27,6 +27,7 @@ class Configuration; class BackupClientInodeToIDMap; class ExcludeList; class IOStreamGetLine; +class Archive; // -------------------------------------------------------------------------- // @@ -41,6 +42,10 @@ class BackupDaemon : public Daemon public: BackupDaemon(); ~BackupDaemon(); + + // methods below do partial (specialized) serialization of client state only + void SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const; + void DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime); private: BackupDaemon(const BackupDaemon &); public: @@ -117,6 +122,9 @@ private: public: Location(); ~Location(); + + void Deserialize(Archive & rArchive); + void Serialize(Archive & rArchive) const; private: Location(const Location &); // copy not allowed Location &operator=(const Location &); diff --git a/bin/bbackupd/bbackupd-config b/bin/bbackupd/bbackupd-config index c5e52282..0d8dd4d1 100755 --- a/bin/bbackupd/bbackupd-config +++ b/bin/bbackupd/bbackupd-config @@ -382,6 +382,14 @@ MaximumDiffingTime = 20 CommandSocket = /var/run/bbackupd.sock +# Uncomment the StoreObjectInfoFile to enable the experimental archiving +# of the daemon's state (including client store marker and configuration) +# between backup runs. This saves time and increases efficiency when +# bbackupd is frequently stopped and started, since it removes the need +# to rescan all directories on the remote server. However, it is new and +# not yet heavily tested, so use with caution. + +# StoreObjectInfoFile = $working_dir/bbackupd.state Server { |