From 99f8ce096bc5569adbfea1911dbcda24c28d8d8b Mon Sep 17 00:00:00 2001 From: Ben Summers Date: Fri, 14 Oct 2005 08:50:54 +0000 Subject: Box Backup 0.09 with a few tweeks --- lib/backupstore/BackupStoreAccountDatabase.cpp | 370 +++++++++++ lib/backupstore/BackupStoreAccountDatabase.h | 75 +++ lib/backupstore/BackupStoreAccounts.cpp | 162 +++++ lib/backupstore/BackupStoreAccounts.h | 47 ++ lib/backupstore/BackupStoreCheck.cpp | 745 ++++++++++++++++++++++ lib/backupstore/BackupStoreCheck.h | 199 ++++++ lib/backupstore/BackupStoreCheck2.cpp | 841 +++++++++++++++++++++++++ lib/backupstore/BackupStoreCheckData.cpp | 205 ++++++ lib/backupstore/BackupStoreConfigVerify.cpp | 48 ++ lib/backupstore/BackupStoreConfigVerify.h | 18 + lib/backupstore/BackupStoreInfo.cpp | 592 +++++++++++++++++ lib/backupstore/BackupStoreInfo.h | 111 ++++ lib/backupstore/StoreStructure.cpp | 95 +++ lib/backupstore/StoreStructure.h | 32 + 14 files changed, 3540 insertions(+) create mode 100755 lib/backupstore/BackupStoreAccountDatabase.cpp create mode 100755 lib/backupstore/BackupStoreAccountDatabase.h create mode 100755 lib/backupstore/BackupStoreAccounts.cpp create mode 100755 lib/backupstore/BackupStoreAccounts.h create mode 100644 lib/backupstore/BackupStoreCheck.cpp create mode 100644 lib/backupstore/BackupStoreCheck.h create mode 100644 lib/backupstore/BackupStoreCheck2.cpp create mode 100644 lib/backupstore/BackupStoreCheckData.cpp create mode 100755 lib/backupstore/BackupStoreConfigVerify.cpp create mode 100755 lib/backupstore/BackupStoreConfigVerify.h create mode 100755 lib/backupstore/BackupStoreInfo.cpp create mode 100755 lib/backupstore/BackupStoreInfo.h create mode 100755 lib/backupstore/StoreStructure.cpp create mode 100755 lib/backupstore/StoreStructure.h (limited to 'lib/backupstore') diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp new file mode 100755 index 00000000..91a6b758 --- /dev/null +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -0,0 +1,370 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccountDatabase.cpp +// Purpose: Database of accounts for the backup store +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include +#include +#include +#include + +#include "BackupStoreAccountDatabase.h" +#include "Guards.h" +#include "FdGetLine.h" +#include "BackupStoreException.h" +#include "CommonException.h" +#include "FileModificationTime.h" + +#include "MemLeakFindOn.h" + +class _BackupStoreAccountDatabase +{ +public: + std::string mFilename; + std::map mDatabase; + box_time_t mModificationTime; +}; + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *) +// Purpose: Constructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename) + : pImpl(new _BackupStoreAccountDatabase) +{ + pImpl->mFilename = Filename; + pImpl->mModificationTime = 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::~BackupStoreAccountDatabase() +// Purpose: Destructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::~BackupStoreAccountDatabase() +{ + delete pImpl; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::Entry() +// Purpose: Default constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::Entry() + : mID(-1), + mDiscSet(-1) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::Entry(int32_t, int) +// Purpose: Constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::Entry(int32_t ID, int DiscSet) + : mID(ID), + mDiscSet(DiscSet) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::Entry(const Entry &) +// Purpose: Copy constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::Entry(const Entry &rEntry) + : mID(rEntry.mID), + mDiscSet(rEntry.mDiscSet) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Entry::~Entry() +// Purpose: Destructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccountDatabase::Entry::~Entry() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Read(const char *) +// Purpose: Read in a database from disc +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupStoreAccountDatabase::Read(const char *Filename) +{ + // Database object to use + std::auto_ptr db(new BackupStoreAccountDatabase(Filename)); + + // Read in the file + db->ReadFile(); + + // Return to called + return db; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::ReadFile() +// Purpose: Read the file off disc +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::ReadFile() const +{ + // Open file + FileHandleGuard<> file(pImpl->mFilename.c_str()); + + // Clear existing entries + pImpl->mDatabase.clear(); + + // Read in lines + FdGetLine getLine(file); + + while(!getLine.IsEOF()) + { + // Read and split up line + std::string l(getLine.GetLine(true)); + + if(!l.empty()) + { + // Check... + int32_t id; + int discSet; + if(::sscanf(l.c_str(), "%x:%d", &id, &discSet) != 2) + { + THROW_EXCEPTION(BackupStoreException, BadAccountDatabaseFile) + } + + // Make a new entry + pImpl->mDatabase[id] = Entry(id, discSet); + } + } + + // Store the modification time of the file + pImpl->mModificationTime = GetDBFileModificationTime(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::CheckUpToDate() +// Purpose: Private. Ensure that the in memory database matches the one on disc +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::CheckUpToDate() const +{ + if(pImpl->mModificationTime != GetDBFileModificationTime()) + { + // File has changed -- load it in again + ReadFile(); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::GetDBFileModificationTime() +// Purpose: Get the current modification time of the database +// Created: 21/1/04 +// +// -------------------------------------------------------------------------- +box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const +{ + struct stat st; + if(::stat(pImpl->mFilename.c_str(), &st) == -1) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + return FileModificationTime(st); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::Write() +// Purpose: Write the database back to disc after modifying it +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::Write() +{ + // Open file for writing + // Would use this... + // FileHandleGuard file(pImpl->mFilename.c_str()); + // but gcc fails randomly on it on some platforms. Weird. + + int file = ::open(pImpl->mFilename.c_str(), O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + if(file == -1) + { + THROW_EXCEPTION(CommonException, OSFileOpenError) + } + + try + { + // Then write each entry + for(std::map::const_iterator i(pImpl->mDatabase.begin()); + i != pImpl->mDatabase.end(); ++i) + { + // Write out the entry + char line[256]; // more than enough for a couple of integers in string form + int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet()); + if(::write(file, line, s) != s) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + } + + ::close(file); + } + catch(...) + { + ::close(file); + throw; + } + + // Done. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::EntryExists(int32_t) +// Purpose: Does an entry exist in the database? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreAccountDatabase::EntryExists(int32_t ID) const +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + return pImpl->mDatabase.find(ID) != pImpl->mDatabase.end(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::GetEntry(int32_t) +// Purpose: Retrieve an entry +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(int32_t ID) const +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + std::map::const_iterator i(pImpl->mDatabase.find(ID)); + if(i == pImpl->mDatabase.end()) + { + THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry) + } + + return i->second; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::AddEntry(int32_t, int) +// Purpose: Add a new entry to the database +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::AddEntry(int32_t ID, int DiscSet) +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + pImpl->mDatabase[ID] = Entry(ID, DiscSet); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::DeleteEntry(int32_t) +// Purpose: Delete an entry from the database +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::DeleteEntry(int32_t ID) +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + std::map::iterator i(pImpl->mDatabase.find(ID)); + if(i == pImpl->mDatabase.end()) + { + THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry) + } + + pImpl->mDatabase.erase(i); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccountDatabase::GetAllAccountIDs(std::vector) +// Purpose: +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreAccountDatabase::GetAllAccountIDs(std::vector &rIDsOut) +{ + // Check that we're using the latest version of the database + CheckUpToDate(); + + // Delete everything in the output list + rIDsOut.clear(); + + std::map::iterator i(pImpl->mDatabase.begin()); + for(; i != pImpl->mDatabase.end(); ++i) + { + rIDsOut.push_back(i->first); + } +} + + + + diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h new file mode 100755 index 00000000..8d6e7ad8 --- /dev/null +++ b/lib/backupstore/BackupStoreAccountDatabase.h @@ -0,0 +1,75 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccountDatabase.h +// Purpose: Database of accounts for the backup store +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREACCOUNTDATABASE__H +#define BACKUPSTOREACCOUNTDATABASE__H + +#include +#include + +#include "BoxTime.h" + +class _BackupStoreAccountDatabase; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreAccountDatabase +// Purpose: Database of accounts for the backup store +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +class BackupStoreAccountDatabase +{ +public: + friend class _BackupStoreAccountDatabase; // to stop compiler warnings + ~BackupStoreAccountDatabase(); +private: + BackupStoreAccountDatabase(const char *Filename); + BackupStoreAccountDatabase(const BackupStoreAccountDatabase &); +public: + + static std::auto_ptr Read(const char *Filename); + void Write(); + + class Entry + { + public: + Entry(); + Entry(int32_t ID, int DiscSet); + Entry(const Entry &rEntry); + ~Entry(); + + int32_t GetID() const {return mID;} + int GetDiscSet() const {return mDiscSet;} + + private: + int32_t mID; + int mDiscSet; + }; + + bool EntryExists(int32_t ID) const; + const Entry &GetEntry(int32_t ID) const; + void AddEntry(int32_t ID, int DiscSet); + void DeleteEntry(int32_t ID); + + // This interface should change in the future. But for now it'll do. + void GetAllAccountIDs(std::vector &rIDsOut); + +private: + void ReadFile() const; // const in concept only + void CheckUpToDate() const; // const in concept only + box_time_t GetDBFileModificationTime() const; + +private: + mutable _BackupStoreAccountDatabase *pImpl; +}; + +#endif // BACKUPSTOREACCOUNTDATABASE__H + diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp new file mode 100755 index 00000000..36d9cad3 --- /dev/null +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -0,0 +1,162 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccounts.cpp +// Purpose: Account management for backup store server +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "BoxPortsAndFiles.h" +#include "BackupStoreAccounts.h" +#include "BackupStoreAccountDatabase.h" +#include "RaidFileWrite.h" +#include "BackupStoreInfo.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" +#include "UnixUser.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &) +// Purpose: Constructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase) + : mrDatabase(rDatabase) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::~BackupStoreAccounts() +// Purpose: Destructor +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +BackupStoreAccounts::~BackupStoreAccounts() +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &) +// Purpose: Create a new account on the specified disc set. +// If rAsUsername is not empty, then the account information will be written under the +// username specified. +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername) +{ + { + // Become the user specified in the config file? + std::auto_ptr user; + if(!rAsUsername.empty()) + { + // Username specified, change... + user.reset(new UnixUser(rAsUsername.c_str())); + user->ChangeProcessUser(true /* temporary */); + // Change will be undone at the end of this function + } + + // Get directory name + std::string dirName(MakeAccountRootDir(ID, DiscSet)); + + // Create a directory on disc + RaidFileWrite::CreateDirectory(DiscSet, dirName, true /* recursive */); + + // Create an info file + BackupStoreInfo::CreateNew(ID, dirName, DiscSet, SizeSoftLimit, SizeHardLimit); + + // And an empty directory + BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + int64_t rootDirSize = 0; + // Write it, knowing the directory scheme + { + RaidFileWrite rf(DiscSet, dirName + "o01"); + rf.Open(); + rootDir.WriteToStream(rf); + rootDirSize = rf.GetDiscUsageInBlocks(); + rf.Commit(true); + } + + // Update the store info to reflect the size of the root directory + std::auto_ptr info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */)); + info->ChangeBlocksUsed(rootDirSize); + info->ChangeBlocksInDirectories(rootDirSize); + + // Save it back + info->Save(); + } + + // As the original user... + + // Create the entry in the database + mrDatabase.AddEntry(ID, DiscSet); + + // Write the database back + mrDatabase.Write(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::GetAccountRoot(int32_t, std::string &, int &) +// Purpose: Gets the root of an account, returning the info via references +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const +{ + // Find the account + const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID)); + + rRootDirOut = MakeAccountRootDir(ID, en.GetDiscSet()); + rDiscSetOut = en.GetDiscSet(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::MakeAccountRootDir(int32_t, int) +// Purpose: Private. Generates a root directory name for the account +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const +{ + char accid[64]; // big enough! + ::sprintf(accid, "%08x/", ID); + return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED DIRECTORY_SEPARATOR) + accid); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreAccounts::AccountExists(int32_t) +// Purpose: Does an account exist? +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreAccounts::AccountExists(int32_t ID) +{ + return mrDatabase.EntryExists(ID); +} + + diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h new file mode 100755 index 00000000..0c3dd103 --- /dev/null +++ b/lib/backupstore/BackupStoreAccounts.h @@ -0,0 +1,47 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreAccounts.h +// Purpose: Account management for backup store server +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREACCOUNTS__H +#define BACKUPSTOREACCOUNTS__H + +#include + +class BackupStoreAccountDatabase; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreAccounts +// Purpose: Account management for backup store server +// Created: 2003/08/21 +// +// -------------------------------------------------------------------------- +class BackupStoreAccounts +{ +public: + BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase); + ~BackupStoreAccounts(); +private: + BackupStoreAccounts(const BackupStoreAccounts &rToCopy); + +public: + void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername); + + bool AccountExists(int32_t ID); + void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const; + +private: + std::string MakeAccountRootDir(int32_t ID, int DiscSet) const; + +private: + BackupStoreAccountDatabase &mrDatabase; +}; + +#endif // BACKUPSTOREACCOUNTS__H + diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp new file mode 100644 index 00000000..fb48c3da --- /dev/null +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -0,0 +1,745 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheck.cpp +// Purpose: Check a store for consistency +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include +#include + +#include "BackupStoreCheck.h" +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreFile.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::BackupStoreCheck(const std::string &, int, int32_t, bool, bool) +// Purpose: Constructor +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet) + : mStoreRoot(rStoreRoot), + mDiscSetNumber(DiscSetNumber), + mAccountID(AccountID), + mFixErrors(FixErrors), + mQuiet(Quiet), + mNumberErrorsFound(0), + mLastIDInInfo(0), + mpInfoLastBlock(0), + mInfoLastBlockEntries(0), + mLostDirNameSerial(0), + mLostAndFoundDirectoryID(0), + mBlocksUsed(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::~BackupStoreCheck() +// Purpose: Destructor +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +BackupStoreCheck::~BackupStoreCheck() +{ + // Clean up + FreeInfo(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::Check() +// Purpose: Perform the check on the given account +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::Check() +{ + // Lock the account + { + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + + bool gotLock = false; + int triesLeft = 8; + do + { + gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */); + + if(!gotLock) + { + --triesLeft; + ::sleep(1); + } + } while(!gotLock && triesLeft > 0); + + if(!gotLock) + { + // Couldn't lock the account -- just stop now + if(!mQuiet) + { + ::printf("Couldn't lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server.\n"); + } + THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount) + } + } + + if(!mQuiet && mFixErrors) + { + ::printf("NOTE: Will fix errors encountered during checking.\n"); + } + + // Phase 1, check objects + if(!mQuiet) + { + ::printf("Check store account ID %08x\nPhase 1, check objects...\n", mAccountID); + } + CheckObjects(); + + // Phase 2, check directories + if(!mQuiet) + { + ::printf("Phase 2, check directories...\n"); + } + CheckDirectories(); + + // Phase 3, check root + if(!mQuiet) + { + ::printf("Phase 3, check root...\n"); + } + CheckRoot(); + + // Phase 4, check unattached objects + if(!mQuiet) + { + ::printf("Phase 4, fix unattached objects...\n"); + } + CheckUnattachedObjects(); + + // Phase 5, fix bad info + if(!mQuiet) + { + ::printf("Phase 5, fix unrecovered inconsistencies...\n"); + } + FixDirsWithWrongContainerID(); + FixDirsWithLostDirs(); + + // Phase 6, regenerate store info + if(!mQuiet) + { + ::printf("Phase 6, regenerate store info...\n"); + } + WriteNewStoreInfo(); + +// DUMP_OBJECT_INFO + + if(mNumberErrorsFound > 0) + { + ::printf("%lld errors found\n", mNumberErrorsFound); + if(!mFixErrors) + { + ::printf("NOTE: No changes to the store account have been made.\n"); + } + if(!mFixErrors && mNumberErrorsFound > 0) + { + ::printf("Run again with fix option to fix these errors\n"); + } + if(mNumberErrorsFound > 0) + { + ::printf("You should now use bbackupquery on the client machine to examine the store.\n"); + if(mLostAndFoundDirectoryID != 0) + { + ::printf("A lost+found directory was created in the account root.\n"\ + "This contains files and directories which could not be matched to existing directories.\n"\ + "bbackupd will delete this directory in a few days time.\n"); + } + } + } + else + { + ::printf("Store account checked, no errors found.\n"); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: static TwoDigitHexToInt(const char *, int &) +// Purpose: Convert a two digit hex string to an int, returning whether it's valid or not +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut) +{ + int n = 0; + // Char 0 + if(String[0] >= '0' && String[0] <= '9') + { + n = (String[0] - '0') << 4; + } + else if(String[0] >= 'a' && String[0] <= 'f') + { + n = ((String[0] - 'a') + 0xa) << 4; + } + else + { + return false; + } + // Char 1 + if(String[1] >= '0' && String[1] <= '9') + { + n |= String[1] - '0'; + } + else if(String[1] >= 'a' && String[1] <= 'f') + { + n |= (String[1] - 'a') + 0xa; + } + else + { + return false; + } + + // Return a valid number + rNumberOut = n; + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckObjects() +// Purpose: Read in the contents of the directory, recurse to other levels, +// checking objects for sanity and readability +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckObjects() +{ + // Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything + int64_t maxDir = 0; + + // Find the maximum directory starting ID + { + // Make sure the starting root dir doesn't end with '/'. + std::string start(mStoreRoot); + if(start.size() > 0 && start[start.size() - 1] == '/') + { + start.resize(start.size() - 1); + } + + maxDir = CheckObjectsScanDir(0, 1, mStoreRoot); + TRACE1("Max dir starting ID is %llx\n", maxDir); + } + + // Then go through and scan all the objects within those directories + for(int64_t d = 0; d <= maxDir; d += (1< dirs; + RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName, + RaidFileRead::DirReadType_DirsOnly, dirs); + + for(std::vector::const_iterator i(dirs.begin()); i != dirs.end(); ++i) + { + // Check to see if it's the right name + int n = 0; + if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n) + && n < (1< maxID) + { + maxID = mi; + } + } + else + { + ::printf("Spurious or invalid directory %s/%s found%s -- delete manually\n", rDirName.c_str(), (*i).c_str(), mFixErrors?", deleting":""); + ++mNumberErrorsFound; + } + } + } + + return maxID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckObjectsDir(int64_t) +// Purpose: Check all the files within this directory which has the given starting ID. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckObjectsDir(int64_t StartID) +{ + // Make directory name -- first generate the filename of an entry in it + std::string dirName; + StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */); + // Check expectations + ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == '/'); + // Remove the filename from it + dirName.resize(dirName.size() - 4); // four chars for "/o00" + + // Check directory exists + if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName)) + { + TRACE1("RaidFile dir %s does not exist\n", dirName.c_str()); + return; + } + + // Read directory contents + std::vector files; + RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName, + RaidFileRead::DirReadType_FilesOnly, files); + + // Array of things present + bool idsPresent[(1<::const_iterator i(files.begin()); i != files.end(); ++i) + { + bool fileOK = true; + int n = 0; + if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n) + && n < (1< file(RaidFileRead::Open(mDiscSetNumber, rFilename)); + size = file->GetDiscUsageInBlocks(); + + // Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile + uint32_t signature; + if(file->Read(&signature, sizeof(signature)) != sizeof(signature)) + { + // Too short, can't read signature from it + return false; + } + // Seek back to beginning + file->Seek(0, IOStream::SeekType_Absolute); + + // Then... check depending on the type + switch(ntohl(signature)) + { + case OBJECTMAGIC_FILE_MAGIC_VALUE_V1: +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + case OBJECTMAGIC_FILE_MAGIC_VALUE_V0: +#endif + // File... check + containerID = CheckFile(ObjectID, *file); + break; + + case OBJECTMAGIC_DIR_MAGIC_VALUE: + isFile = false; + containerID = CheckDirInitial(ObjectID, *file); + break; + + default: + // Unknown signature. Bad file. Very bad file. + return false; + break; + } + + // Add to usage counts + int64_t s = file->GetDiscUsageInBlocks(); + mBlocksUsed += s; + if(!isFile) + { + mBlocksInDirectories += s; + } + } + catch(...) + { + // Error caught, not a good file then, let it be deleted + return false; + } + + // Got a container ID? (ie check was successful) + if(containerID == -1) + { + return false; + } + + // Add to list of IDs known about + AddID(ObjectID, containerID, size, isFile); + + // Report success + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckFile(int64_t, IOStream &) +// Purpose: Do check on file, return original container ID if OK, or -1 on error +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) +{ + // Check that it's not the root directory ID. Having a file as the root directory would be bad. + if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) + { + // Get that dodgy thing deleted! + ::printf("Have file as root directory. This is bad.\n"); + return -1; + } + + // Check the format of the file, and obtain the container ID + int64_t originalContainerID = -1; + if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */, + &originalContainerID)) + { + // Didn't verify + return -1; + } + + return originalContainerID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &) +// Purpose: Do initial check on directory, return container ID if OK, or -1 on error +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream) +{ + // Simply attempt to read in the directory + BackupStoreDirectory dir; + dir.ReadFromStream(rStream, IOStream::TimeOutInfinite); + + // Check object ID + if(dir.GetObjectID() != ObjectID) + { + // Wrong object ID + return -1; + } + + // Return container ID + return dir.GetContainerID(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckDirectories() +// Purpose: Check the directories +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckDirectories() +{ + // Phase 1 did this: + // Checked that all the directories are readable + // Built a list of all directories and files which exist on the store + // + // This phase will check all the files in the directories, make + // a note of all directories which are missing, and do initial fixing. + + // Scan all objects + for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + IDBlock *pblock = i->second; + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + + for(int e = 0; e < bentries; ++e) + { + uint8_t flags = GetFlags(pblock, e); + if(flags & Flags_IsDir) + { + // Found a directory. Read it in. + std::string filename; + StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */); + BackupStoreDirectory dir; + { + std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Flag for modifications + bool isModified = false; + + // Check for validity + if(dir.CheckAndFix()) + { + // Wasn't quite right, and has been modified + ::printf("Directory ID %llx has bad structure\n", pblock->mID[e]); + ++mNumberErrorsFound; + isModified = true; + } + + // Go through, and check that everything in that directory exists and is valid + std::vector toDelete; + + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + // Lookup the item + int32_t iIndex; + IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); + bool badEntry = false; + if(piBlock != 0) + { + // Found. Get flags + uint8_t iflags = GetFlags(piBlock, iIndex); + + // Is the type the same? + if(((iflags & Flags_IsDir) == Flags_IsDir) + != ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir)) + { + // Entry is of wrong type + ::printf("Directory ID %llx references object %llx which has a different type than expected.\n", pblock->mID[e], en->GetObjectID()); + badEntry = true; + } + else + { + // Check that the entry is not already contained. + if(iflags & Flags_IsContained) + { + badEntry = true; + ::printf("Directory ID %llx references object %llx which is already contained.\n", pblock->mID[e], en->GetObjectID()); + } + else + { + // Not already contained -- mark as contained + SetFlags(piBlock, iIndex, iflags | Flags_IsContained); + + // Check that the container ID of the object is correct + if(piBlock->mContainer[iIndex] != pblock->mID[e]) + { + // Needs fixing... + if(iflags & Flags_IsDir) + { + // Add to will fix later list + ::printf("Directory ID %llx has wrong container ID.\n", en->GetObjectID()); + mDirsWithWrongContainerID.push_back(en->GetObjectID()); + } + else + { + // This is OK for files, they might move + ::printf("File ID %llx has different container ID, probably moved\n", en->GetObjectID()); + } + + // Fix entry for now + piBlock->mContainer[iIndex] = pblock->mID[e]; + } + } + } + + // Check the object size, if it's OK and a file + if(!badEntry && !((iflags & Flags_IsDir) == Flags_IsDir)) + { + if(en->GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[iIndex]) + { + // Correct + en->SetSizeInBlocks(piBlock->mObjectSizeInBlocks[iIndex]); + // Mark as changed + isModified = true; + // Tell user + ::printf("Directory ID %llx has wrong size for object %llx\n", pblock->mID[e], en->GetObjectID()); + } + } + } + else + { + // Item can't be found. Is it a directory? + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) + { + // Store the directory for later attention + mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e]; + } + else + { + // Just remove the entry + badEntry = true; + ::printf("Directory ID %llx references object %llx which does not exist.\n", pblock->mID[e], en->GetObjectID()); + } + } + + // Is this entry worth keeping? + if(badEntry) + { + toDelete.push_back(en->GetObjectID()); + } + else + { + // Add to sizes? + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) + { + mBlocksInOldFiles += en->GetSizeInBlocks(); + } + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) + { + mBlocksInDeletedFiles += en->GetSizeInBlocks(); + } + } + } + + if(toDelete.size() > 0) + { + // Delete entries from directory + for(std::vector::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) + { + dir.DeleteEntry(*d); + } + + // Mark as modified + isModified = true; + + // Check the directory again, now that entries have been removed + dir.CheckAndFix(); + + // Errors found + ++mNumberErrorsFound; + } + + if(isModified && mFixErrors) + { + ::printf("Fixing directory ID %llx\n", pblock->mID[e]); + + // Save back to disc + RaidFileWrite fixed(mDiscSetNumber, filename); + fixed.Open(true /* allow overwriting */); + dir.WriteToStream(fixed); + // Commit it + fixed.Commit(true /* convert to raid representation now */); + } + } + } + } + +} + + diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h new file mode 100644 index 00000000..3f48312a --- /dev/null +++ b/lib/backupstore/BackupStoreCheck.h @@ -0,0 +1,199 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheck.h +// Purpose: Check a store for consistency +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECHECK__H +#define BACKUPSTORECHECK__H + +#include +#include +#include +#include + +#include "NamedLock.h" +class IOStream; +class BackupStoreFilename; + +/* + +The following problems can be fixed: + + * Spurious files deleted + * Corrupted files deleted + * Root ID as file, deleted + * Dirs with wrong object id inside, deleted + * Direcetory entries pointing to non-existant files, deleted + * Doubly references files have second reference deleted + * Wrong directory container IDs fixed + * Missing root recreated + * Reattach files which exist, but aren't referenced + - files go into directory original directory, if it still exists + - missing directories are inferred, and recreated + - or if all else fails, go into lost+found + - file dir entries take the original name and mod time + - directories go into lost+found + * Container IDs on directories corrected + * Inside directories, + - only one object per name has old version clear + - IDs aren't duplicated + * Bad store info files regenerated + * Bad sizes of files in directories fixed + +*/ + + +// Size of blocks in the list of IDs +#ifdef NDEBUG + #define BACKUPSTORECHECK_BLOCK_SIZE (64*1024) +#else + #define BACKUPSTORECHECK_BLOCK_SIZE 8 +#endif + +// The object ID type -- can redefine to uint32_t to produce a lower memory version for smaller stores +typedef int64_t BackupStoreCheck_ID_t; +// Can redefine the size type for lower memory usage too +typedef int64_t BackupStoreCheck_Size_t; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreCheck +// Purpose: Check a store for consistency +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +class BackupStoreCheck +{ +public: + BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet); + ~BackupStoreCheck(); +private: + // no copying + BackupStoreCheck(const BackupStoreCheck &); + BackupStoreCheck &operator=(const BackupStoreCheck &); +public: + + // Do the exciting things + void Check(); + + bool ErrorsFound() {return mNumberErrorsFound > 0;} + +private: + enum + { + // Bit mask + Flags_IsDir = 1, + Flags_IsContained = 2, + // Mask + Flags__MASK = 3, + // Number of bits + Flags__NumFlags = 2, + // Items per uint8_t + Flags__NumItemsPerEntry = 4 // ie 8 / 2 + }; + + typedef struct + { + // Note use arrays within the block, rather than the more obvious array of + // objects, to be more memory efficient -- think alignment of the byte values. + uint8_t mFlags[BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry]; + BackupStoreCheck_ID_t mID[BACKUPSTORECHECK_BLOCK_SIZE]; + BackupStoreCheck_ID_t mContainer[BACKUPSTORECHECK_BLOCK_SIZE]; + BackupStoreCheck_Size_t mObjectSizeInBlocks[BACKUPSTORECHECK_BLOCK_SIZE]; + } IDBlock; + + // Phases of the check + void CheckObjects(); + void CheckDirectories(); + void CheckRoot(); + void CheckUnattachedObjects(); + void FixDirsWithWrongContainerID(); + void FixDirsWithLostDirs(); + void WriteNewStoreInfo(); + + // Checking functions + int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName); + void CheckObjectsDir(int64_t StartID); + bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename); + int64_t CheckFile(int64_t ObjectID, IOStream &rStream); + int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); + + // Fixing functions + bool TryToRecreateDirectory(int64_t MissingDirectoryID); + void InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory); + int64_t GetLostAndFoundDirID(); + void CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID); + + // Data handling + void FreeInfo(); + void AddID(BackupStoreCheck_ID_t ID, BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile); + IDBlock *LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut); + inline void SetFlags(IDBlock *pBlock, int32_t Index, uint8_t Flags) + { + ASSERT(pBlock != 0); + ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE); + ASSERT(Flags < (1 << Flags__NumFlags)); + + pBlock->mFlags[Index / Flags__NumItemsPerEntry] + |= (Flags << ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)); + } + inline uint8_t GetFlags(IDBlock *pBlock, int32_t Index) + { + ASSERT(pBlock != 0); + ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE); + + return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK; + } + +#ifndef NDEBUG + void DumpObjectInfo(); + #define DUMP_OBJECT_INFO DumpObjectInfo(); +#else + #define DUMP_OBJECT_INFO +#endif + +private: + std::string mStoreRoot; + int mDiscSetNumber; + int32_t mAccountID; + bool mFixErrors; + bool mQuiet; + + int64_t mNumberErrorsFound; + + // Lock for the store account + NamedLock mAccountLock; + + // Storage for ID data + typedef std::map Info_t; + Info_t mInfo; + BackupStoreCheck_ID_t mLastIDInInfo; + IDBlock *mpInfoLastBlock; + int32_t mInfoLastBlockEntries; + + // List of stuff to fix + std::vector mDirsWithWrongContainerID; + // This is a map of lost dir ID -> existing dir ID + std::map mDirsWhichContainLostDirs; + + // Set of extra directories added + std::set mDirsAdded; + + // Misc stuff + int32_t mLostDirNameSerial; + int64_t mLostAndFoundDirectoryID; + + // Usage + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; +}; + +#endif // BACKUPSTORECHECK__H + diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp new file mode 100644 index 00000000..fe91d00d --- /dev/null +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -0,0 +1,841 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheck2.cpp +// Purpose: More backup store checking +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include + +#include "BackupStoreCheck.h" +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" +#include "BackupStoreInfo.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckRoot() +// Purpose: Check the root directory exists. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckRoot() +{ + int32_t index = 0; + IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index); + + if(pblock != 0) + { + // Found it. Which is lucky. Mark it as contained. + SetFlags(pblock, index, Flags_IsContained); + } + else + { + ::printf("Root directory doesn't exist\n"); + + ++mNumberErrorsFound; + + if(mFixErrors) + { + // Create a new root directory + CreateBlankDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CreateBlankDirectory(int64_t, int64_t) +// Purpose: Creates a blank directory +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID) +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + BackupStoreDirectory dir(DirectoryID, ContainingDirID); + + // Serialise to disc + std::string filename; + StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); + RaidFileWrite obj(mDiscSetNumber, filename); + obj.Open(false /* don't allow overwriting */); + dir.WriteToStream(obj); + int64_t size = obj.GetDiscUsageInBlocks(); + obj.Commit(true /* convert to raid now */); + + // Record the fact we've done this + mDirsAdded.insert(DirectoryID); + + // Add to sizes + mBlocksUsed += size; + mBlocksInDirectories += size; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::CheckUnattachedObjects() +// Purpose: Check for objects which aren't attached to anything +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::CheckUnattachedObjects() +{ + // Scan all objects, finding ones which have no container + for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + IDBlock *pblock = i->second; + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + + for(int e = 0; e < bentries; ++e) + { + uint8_t flags = GetFlags(pblock, e); + if((flags & Flags_IsContained) == 0) + { + // Unattached object... + ::printf("Object %llx is unattached.\n", pblock->mID[e]); + ++mNumberErrorsFound; + + // What's to be done? + int64_t putIntoDirectoryID = 0; + + if((flags & Flags_IsDir) == Flags_IsDir) + { + // Directory. Just put into lost and found. + putIntoDirectoryID = GetLostAndFoundDirID(); + } + else + { + // File. Only attempt to attach it somewhere if it isn't a patch + { + int64_t diffFromObjectID = 0; + std::string filename; + StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */); + // The easiest way to do this is to verify it again. Not such a bad penalty, because + // this really shouldn't be done very often. + { + std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); + BackupStoreFile::VerifyEncodedFileFormat(*file, &diffFromObjectID); + } + + // If not zero, then it depends on another file, which may or may not be available. + // Just delete it to be safe. + if(diffFromObjectID != 0) + { + ::printf("Object %llx is unattached, and is a patch. Deleting, cannot reliably recover.\n", pblock->mID[e]); + + // Delete this object instead + if(mFixErrors) + { + RaidFileWrite del(mDiscSetNumber, filename); + del.Delete(); + } + + // Move on to next item + continue; + } + } + + // Files contain their original filename, so perhaps the orginal directory still exists, + // or we can infer the existance of a directory? + // Look for a matching entry in the mDirsWhichContainLostDirs map. + // Can't do this with a directory, because the name just wouldn't be known, which is + // pretty useless as bbackupd would just delete it. So better to put it in lost+found + // where the admin can do something about it. + int32_t dirindex; + IDBlock *pdirblock = LookupID(pblock->mContainer[e], dirindex); + if(pdirblock != 0) + { + // Something with that ID has been found. Is it a directory? + if(GetFlags(pdirblock, dirindex) & Flags_IsDir) + { + // Directory exists, add to that one + putIntoDirectoryID = pblock->mContainer[e]; + } + else + { + // Not a directory. Use lost and found dir + putIntoDirectoryID = GetLostAndFoundDirID(); + } + } + else if(mDirsAdded.find(pblock->mContainer[e]) != mDirsAdded.end() + || TryToRecreateDirectory(pblock->mContainer[e])) + { + // The directory reappeared, or was created somehow elsewhere + putIntoDirectoryID = pblock->mContainer[e]; + } + else + { + putIntoDirectoryID = GetLostAndFoundDirID(); + } + } + ASSERT(putIntoDirectoryID != 0); + + // Add it to the directory + InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID, + ((flags & Flags_IsDir) == Flags_IsDir)); + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::TryToRecreateDirectory(int64_t) +// Purpose: Recreate a missing directory +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) +{ + // During the directory checking phase, a map of "missing directory" to + // containing directory was built. If we can find it here, then it's + // something which can be recreated! + std::map::iterator missing( + mDirsWhichContainLostDirs.find(MissingDirectoryID)); + if(missing == mDirsWhichContainLostDirs.end()) + { + // Not a missing directory, can't recreate. + return false; + } + + // Can recreate this! Wooo! + if(!mFixErrors) + { + ::printf("Missing directory %llx could be recreated\n", MissingDirectoryID); + mDirsAdded.insert(MissingDirectoryID); + return true; + } + ::printf("Recreating missing directory %llx\n", MissingDirectoryID); + + // Create a blank directory + BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */); + // Note that this directory already contains a directory entry pointing to + // this dir, so it doesn't have to be added. + + // Serialise to disc + std::string filename; + StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(false /* don't allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + + // Record the fact we've done this + mDirsAdded.insert(MissingDirectoryID); + + // Remove the entry from the map, so this doesn't happen again + mDirsWhichContainLostDirs.erase(missing); + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::GetLostAndFoundDirID() +// Purpose: Returns the ID of the lost and found directory, creating it if necessary +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreCheck::GetLostAndFoundDirID() +{ + // Already allocated it? + if(mLostAndFoundDirectoryID != 0) + { + return mLostAndFoundDirectoryID; + } + + if(!mFixErrors) + { + // The result will never be used anyway if errors aren't being fixed + return 1; + } + + // Load up the root directory + BackupStoreDirectory dir; + std::string filename; + StoreStructure::MakeObjectFilename(BACKUPSTORE_ROOT_DIRECTORY_ID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + { + std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Find a suitable name + BackupStoreFilename lostAndFound; + int n = 0; + while(true) + { + char name[32]; + ::sprintf(name, "lost+found%d", n++); + lostAndFound.SetAsClearFilename(name); + if(!dir.NameInUse(lostAndFound)) + { + // Found a name which can be used + ::printf("Lost and found dir has name %s\n", name); + break; + } + } + + // Allocate an ID + int64_t id = mLastIDInInfo + 1; + + // Create a blank directory + CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID); + + // Add an entry for it + dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0); + + // Write out root dir + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + + // Store + mLostAndFoundDirectoryID = id; + + // Tell caller + return mLostAndFoundDirectoryID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool) +// Purpose: +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory) +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + // Data for the object + BackupStoreFilename objectStoreFilename; + int64_t modTime = 100; // something which isn't zero or a special time + int32_t sizeInBlocks = 0; // suitable for directories + + if(IsDirectory) + { + // Directory -- simply generate a name for it. + char name[32]; + ::sprintf(name, "dir%08x", mLostDirNameSerial++); + objectStoreFilename.SetAsClearFilename(name); + } + else + { + // Files require a little more work... + // Open file + std::string fileFilename; + StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */); + std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, fileFilename)); + // Fill in size information + sizeInBlocks = file->GetDiscUsageInBlocks(); + // Read in header + file_StreamFormat hdr; + if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + )) + { + // This should never happen, everything has been checked before. + THROW_EXCEPTION(BackupStoreException, Internal) + } + // This tells us nice things + modTime = ntoh64(hdr.mModificationTime); + // And the filename comes next + objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Directory object + BackupStoreDirectory dir; + + // Generate filename + std::string filename; + StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + + // Read it in + { + std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Add a new entry in an appropraite place + dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks, + IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); + + // Fix any flags which have been broken, which there's a good change of going + dir.CheckAndFix(); + + // Write it out + if(mFixErrors) + { + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::FixDirsWithWrongContainerID() +// Purpose: Rewrites container IDs where required +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::FixDirsWithWrongContainerID() +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + // Run through things which need fixing + for(std::vector::iterator i(mDirsWithWrongContainerID.begin()); + i != mDirsWithWrongContainerID.end(); ++i) + { + int32_t index = 0; + IDBlock *pblock = LookupID(*i, index); + if(pblock == 0) continue; + + // Load in + BackupStoreDirectory dir; + std::string filename; + StoreStructure::MakeObjectFilename(*i, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + { + std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Adjust container ID + dir.SetContainerID(pblock->mContainer[index]); + + // Write it out + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::FixDirsWithLostDirs() +// Purpose: Fix directories +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::FixDirsWithLostDirs() +{ + if(!mFixErrors) + { + // Don't do anything if we're not supposed to fix errors + return; + } + + // Run through things which need fixing + for(std::map::iterator i(mDirsWhichContainLostDirs.begin()); + i != mDirsWhichContainLostDirs.end(); ++i) + { + int32_t index = 0; + IDBlock *pblock = LookupID(i->second, index); + if(pblock == 0) continue; + + // Load in + BackupStoreDirectory dir; + std::string filename; + StoreStructure::MakeObjectFilename(i->second, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); + { + std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); + dir.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Delete the dodgy entry + dir.DeleteEntry(i->first); + + // Fix it up + dir.CheckAndFix(); + + // Write it out + RaidFileWrite root(mDiscSetNumber, filename); + root.Open(true /* allow overwriting */); + dir.WriteToStream(root); + root.Commit(true /* convert to raid now */); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::WriteNewStoreInfo() +// Purpose: Regenerate store info +// Created: 23/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::WriteNewStoreInfo() +{ + // Attempt to load the existing store info file + std::auto_ptr poldInfo; + try + { + poldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release()); + } + catch(...) + { + ::printf("Load of existing store info failed, regenerating.\n"); + ++mNumberErrorsFound; + } + + // Minimum soft and hard limits + int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024; + int64_t minHard = ((minSoft * 11) / 10) + 1024; + + // Need to do anything? + if(poldInfo.get() != 0 && mNumberErrorsFound == 0 && poldInfo->GetAccountID() == mAccountID) + { + // Leave the store info as it is, no need to alter it because nothing really changed, + // and the only essential thing was that the account ID was correct, which is was. + return; + } + + // NOTE: We will always build a new store info, so the client store marker gets changed. + + // Work out the new limits + int64_t softLimit = minSoft; + int64_t hardLimit = minHard; + if(poldInfo.get() != 0 && poldInfo->GetBlocksSoftLimit() > minSoft) + { + softLimit = poldInfo->GetBlocksSoftLimit(); + } + else + { + ::printf("NOTE: Soft limit for account changed to ensure housekeeping doesn't delete files on next run\n"); + } + if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard) + { + hardLimit = poldInfo->GetBlocksHardLimit(); + } + else + { + ::printf("NOTE: Hard limit for account changed to ensure housekeeping doesn't delete files on next run\n"); + } + + // Object ID + int64_t lastObjID = mLastIDInInfo; + if(mLostAndFoundDirectoryID != 0) + { + mLastIDInInfo++; + } + + // Build a new store info + std::auto_ptr info(BackupStoreInfo::CreateForRegeneration( + mAccountID, + mStoreRoot, + mDiscSetNumber, + lastObjID, + mBlocksUsed, + mBlocksInOldFiles, + mBlocksInDeletedFiles, + mBlocksInDirectories, + softLimit, + hardLimit)); + + // Save to disc? + if(mFixErrors) + { + info->Save(); + ::printf("New store info file written successfully.\n"); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::CheckAndFix() +// Purpose: Check the directory for obvious logical problems, and fix them. +// Return true if the directory was changed. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreDirectory::CheckAndFix() +{ + bool changed = false; + + // Check that if a file depends on a new version, that version is in this directory + { + std::vector::iterator i(mEntries.begin()); + for(; i != mEntries.end(); ++i) + { + int64_t dependsNewer = (*i)->GetDependsNewer(); + if(dependsNewer != 0) + { + BackupStoreDirectory::Entry *newerEn = FindEntryByID(dependsNewer); + if(newerEn == 0) + { + // Depends on something, but it isn't there. + TRACE2("Entry id %llx removed because depends on newer version %llx which doesn't exist\n", (*i)->GetObjectID(), dependsNewer); + + // Remove + delete *i; + mEntries.erase(i); + + // Start again at the beginning of the vector, the iterator is now invalid + i = mEntries.begin(); + + // Mark as changed + changed = true; + } + else + { + // Check that newerEn has it marked + if(newerEn->GetDependsOlder() != (*i)->GetObjectID()) + { + // Wrong entry + TRACE3("Entry id %llx, correcting DependsOlder to %llx, was %llx\n", dependsNewer, (*i)->GetObjectID(), newerEn->GetDependsOlder()); + newerEn->SetDependsOlder((*i)->GetObjectID()); + // Mark as changed + changed = true; + } + } + } + } + } + + // Check that if a file has a dependency marked, it exists, and remove it if it doesn't + { + std::vector::iterator i(mEntries.begin()); + for(; i != mEntries.end(); ++i) + { + int64_t dependsOlder = (*i)->GetDependsOlder(); + if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0) + { + // Has an older version marked, but this doesn't exist. Remove this mark + TRACE2("Entry id %llx was marked that %llx depended on it, which doesn't exist, dependency info cleaered\n", (*i)->GetObjectID(), dependsOlder); + + (*i)->SetDependsOlder(0); + + // Mark as changed + changed = true; + } + } + } + + bool ch = false; + do + { + // Reset change marker + ch = false; + + // Search backwards -- so see newer versions first + std::vector::iterator i(mEntries.end()); + if(i == mEntries.begin()) + { + // Directory is empty, stop now + return changed; // changed flag + } + + // Records of things seen + std::set idsEncountered; + std::set filenamesEncountered; + + do + { + // Look at previous + --i; + + bool removeEntry = false; + if((*i) == 0) + { + TRACE0("Remove because null pointer found\n"); + removeEntry = true; + } + else + { + bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir); + + // Check mutually exclusive flags + if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File)) + { + // Bad! Unset the file flag + TRACE1("Entry %llx: File flag set when dir flag set\n", (*i)->GetObjectID()); + (*i)->RemoveFlags(Entry::Flags_File); + changed = true; + } + + // Check... + if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end()) + { + // ID already seen, or type doesn't match + TRACE1("Entry %llx: Remove because ID already seen\n", (*i)->GetObjectID()); + removeEntry = true; + } + else + { + // Haven't already seen this ID, remember it + idsEncountered.insert((*i)->GetObjectID()); + + // Check to see if the name has already been encountered -- if not, then it + // needs to have the old version flag set + if(filenamesEncountered.find((*i)->GetName()) != filenamesEncountered.end()) + { + // Seen before -- check old version flag set + if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion + && ((*i)->GetFlags() & Entry::Flags_Deleted) == 0) + { + // Not set, set it + TRACE1("Entry %llx: Set old flag\n", (*i)->GetObjectID()); + (*i)->AddFlags(Entry::Flags_OldVersion); + changed = true; + } + } + else + { + // Check old version flag NOT set + if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion) + { + // Set, unset it + TRACE1("Entry %llx: Old flag unset\n", (*i)->GetObjectID()); + (*i)->RemoveFlags(Entry::Flags_OldVersion); + changed = true; + } + + // Remember filename + filenamesEncountered.insert((*i)->GetName()); + } + } + } + + if(removeEntry) + { + // Mark something as changed, in loop + ch = true; + + // Mark something as globally changed + changed = true; + + // erase the thing from the list + Entry *pentry = (*i); + mEntries.erase(i); + + // And delete the entry object + delete pentry; + + // Stop going around this loop, as the iterator is now invalid + break; + } + } while(i != mEntries.begin()); + + } while(ch != false); + + return changed; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::AddUnattactedObject(...) +// Purpose: Adds an object which is currently unattached. Assume that CheckAndFix() will be called afterwards. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreDirectory::AddUnattactedObject(const BackupStoreFilename &rName, + box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags) +{ + Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, + ModificationTime /* use as attr mod time too */); + try + { + // Want to order this just before the first object which has a higher ID, + // which is the place it's most likely to be correct. + std::vector::iterator i(mEntries.begin()); + for(; i != mEntries.end(); ++i) + { + if((*i)->GetObjectID() > ObjectID) + { + // Found a good place to insert it + break; + } + } + if(i == mEntries.end()) + { + mEntries.push_back(pnew); + } + else + { + mEntries.insert(i, 1 /* just the one copy */, pnew); + } + } + catch(...) + { + delete pnew; + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreDirectory::NameInUse(const BackupStoreFilename &) +// Purpose: Returns true if the name is currently in use in the directory +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreDirectory::NameInUse(const BackupStoreFilename &rName) +{ + for(std::vector::iterator i(mEntries.begin()); i != mEntries.end(); ++i) + { + if((*i)->GetName() == rName) + { + return true; + } + } + + return false; +} + + diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp new file mode 100644 index 00000000..f22c8339 --- /dev/null +++ b/lib/backupstore/BackupStoreCheckData.cpp @@ -0,0 +1,205 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreCheckData.cpp +// Purpose: Data handling for store checking +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include + +#include "BackupStoreCheck.h" +#include "autogen_BackupStoreException.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::FreeInfo() +// Purpose: Free all the data stored +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::FreeInfo() +{ + // Free all the blocks + for(Info_t::iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + ::free(i->second); + } + + // Clear the contents of the map + mInfo.clear(); + + // Reset the last ID, just in case + mpInfoLastBlock = 0; + mInfoLastBlockEntries = 0; + mLastIDInInfo = 0; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::AddID(BackupStoreCheck_ID_t, BackupStoreCheck_ID_t, bool) +// Purpose: Add an ID to the list +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::AddID(BackupStoreCheck_ID_t ID, + BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile) +{ + // Check ID is OK. + if(ID <= mLastIDInInfo) + { + THROW_EXCEPTION(BackupStoreException, InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing) + } + + // Can this go in the current block? + if(mpInfoLastBlock == 0 || mInfoLastBlockEntries >= BACKUPSTORECHECK_BLOCK_SIZE) + { + // No. Allocate a new one + IDBlock *pblk = (IDBlock*)::malloc(sizeof(IDBlock)); + if(pblk == 0) + { + throw std::bad_alloc(); + } + // Zero all the flags entries + for(int z = 0; z < (BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry); ++z) + { + pblk->mFlags[z] = 0; + } + // Store in map + mInfo[ID] = pblk; + // Allocated and stored OK, setup for use + mpInfoLastBlock = pblk; + mInfoLastBlockEntries = 0; + } + ASSERT(mpInfoLastBlock != 0 && mInfoLastBlockEntries < BACKUPSTORECHECK_BLOCK_SIZE); + + // Add to block + mpInfoLastBlock->mID[mInfoLastBlockEntries] = ID; + mpInfoLastBlock->mContainer[mInfoLastBlockEntries] = Container; + mpInfoLastBlock->mObjectSizeInBlocks[mInfoLastBlockEntries] = ObjectSize; + SetFlags(mpInfoLastBlock, mInfoLastBlockEntries, IsFile?(0):(Flags_IsDir)); + + // Increment size + ++mInfoLastBlockEntries; + + // Store last ID + mLastIDInInfo = ID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::LookupID(BackupStoreCheck_ID_t, int32_t +// Purpose: Look up an ID. Return the block it's in, or zero if not found, and the +// index within that block if the thing is found. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut) +{ + IDBlock *pblock = 0; + + // Find the lower matching block who's first entry is not less than ID + Info_t::const_iterator ib(mInfo.lower_bound(ID)); + + // Was there a block + if(ib == mInfo.end()) + { + // Block wasn't found... could be in last block + pblock = mpInfoLastBlock; + } + else + { + // Found it as first entry? + if(ib->first == ID) + { + rIndexOut = 0; + return ib->second; + } + + // Go back one block as it's not the first entry in this one + if(ib == mInfo.begin()) + { + // Was first block, can't go back + return 0; + } + // Go back... + --ib; + + // So, the ID will be in this block, if it's in anything + pblock = ib->second; + } + + ASSERT(pblock != 0); + if(pblock == 0) return 0; + + // How many entries are there in the block + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + + // Do binary search within block + int high = bentries; + int low = -1; + while(high - low > 1) + { + int i = (high + low) / 2; + if(ID <= pblock->mID[i]) + { + high = i; + } + else + { + low = i; + } + } + if(ID == pblock->mID[high]) + { + // Found + rIndexOut = high; + return pblock; + } + + // Not found + return 0; +} + + +#ifndef NDEBUG +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreCheck::DumpObjectInfo() +// Purpose: Debug only. Trace out all object info. +// Created: 22/4/04 +// +// -------------------------------------------------------------------------- +void BackupStoreCheck::DumpObjectInfo() +{ + for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) + { + IDBlock *pblock = i->second; + int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; + TRACE2("BLOCK @ 0x%08x, %d entries\n", pblock, bentries); + + for(int e = 0; e < bentries; ++e) + { + uint8_t flags = GetFlags(pblock, e); + TRACE4("id %llx, c %llx, %s, %s\n", + pblock->mID[e], pblock->mContainer[e], + (flags & Flags_IsDir)?"dir":"file", + (flags & Flags_IsContained)?"contained":"unattached"); + } + } +} +#endif + diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp new file mode 100755 index 00000000..6fa05d06 --- /dev/null +++ b/lib/backupstore/BackupStoreConfigVerify.cpp @@ -0,0 +1,48 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreConfigVerify.h +// Purpose: Configuration definition for the backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" +#include "BackupStoreConfigVerify.h" +#include "ServerTLS.h" +#include "BoxPortsAndFiles.h" + +#include "MemLeakFindOn.h" + +static const ConfigurationVerifyKey verifyserverkeys[] = +{ + SERVERTLS_VERIFY_SERVER_KEYS(0) // no default listen addresses +}; + +static const ConfigurationVerify verifyserver[] = +{ + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } +}; + +static const ConfigurationVerifyKey verifyrootkeys[] = +{ + {"AccountDatabase", 0, ConfigTest_Exists, 0}, + {"TimeBetweenHousekeeping", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, + {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file + {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0} +}; + +const ConfigurationVerify BackupConfigFileVerify = +{ + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 +}; diff --git a/lib/backupstore/BackupStoreConfigVerify.h b/lib/backupstore/BackupStoreConfigVerify.h new file mode 100755 index 00000000..815cfaed --- /dev/null +++ b/lib/backupstore/BackupStoreConfigVerify.h @@ -0,0 +1,18 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreConfigVerify.h +// Purpose: Configuration definition for the backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTORECONFIGVERIFY__H +#define BACKUPSTORECONFIGVERIFY__H + +#include "Configuration.h" + +extern const ConfigurationVerify BackupConfigFileVerify; + +#endif // BACKUPSTORECONFIGVERIFY__H + diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp new file mode 100755 index 00000000..a9effe00 --- /dev/null +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -0,0 +1,592 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreInfo.cpp +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "BackupStoreInfo.h" +#include "BackupStoreException.h" +#include "RaidFileWrite.h" +#include "RaidFileRead.h" + +#include "MemLeakFindOn.h" + +// set packing to one byte +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +// ****************** +// make sure the defaults in CreateNew are modified! +// ****************** +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mAccountID; + int64_t mClientStoreMarker; + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mBlocksSoftLimit; + int64_t mBlocksHardLimit; + uint32_t mCurrentMarkNumber; + uint32_t mOptionsPresent; // bit mask of optional elements present + int64_t mNumberDeletedDirectories; + // Then loads of int64_t IDs for the deleted directories +} info_StreamFormat; + +#define INFO_MAGIC_VALUE 0x34832476 + +// Use default packing +#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +#ifdef NDEBUG + #define NUM_DELETED_DIRS_BLOCK 256 +#else + #define NUM_DELETED_DIRS_BLOCK 2 +#endif + +#define INFO_FILENAME "info" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::BackupStoreInfo() +// Purpose: Default constructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreInfo::BackupStoreInfo() + : mAccountID(-1), + mDiscSet(-1), + mReadOnly(true), + mIsModified(false), + mClientStoreMarker(0), + mLastObjectIDUsed(-1), + mBlocksUsed(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::~BackupStoreInfo +// Purpose: Destructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreInfo::~BackupStoreInfo() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::CreateNew(int32_t, const std::string &, int) +// Purpose: Create a new info file on disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + // Initial header (is entire file) + info_StreamFormat hdr = { + htonl(INFO_MAGIC_VALUE), // mMagicValue + htonl(AccountID), // mAccountID + 0, // mClientStoreMarker + hton64(1), // mLastObjectIDUsed (which is the root directory) + 0, // mBlocksUsed + 0, // mBlocksInOldFiles + 0, // mBlocksInDeletedFiles + 0, // mBlocksInDirectories + hton64(BlockSoftLimit), // mBlocksSoftLimit + hton64(BlockHardLimit), // mBlocksHardLimit + 0, // mCurrentMarkNumber + 0, // mOptionsPresent + 0 // mNumberDeletedDirectories + }; + + // Generate the filename + ASSERT(rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + std::string fn(rRootDir + INFO_FILENAME); + + // Open the file for writing + RaidFileWrite rf(DiscSet, fn); + rf.Open(false); // no overwriting, as this is a new file + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Done. +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::Load(int32_t, const std::string &, int, bool) +// Purpose: Loads the info from disc, given the root information. Can be marked as read only. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupStoreInfo::Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID) +{ + // Generate the filename + std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); + + // Open the file for reading (passing on optional request for revision ID) + std::auto_ptr rf(RaidFileRead::Open(DiscSet, fn, pRevisionID)); + + // Read in a header + info_StreamFormat hdr; + if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Check it + if(ntohl(hdr.mMagicValue) != INFO_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != AccountID) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // Make new object + std::auto_ptr info(new BackupStoreInfo); + + // Put in basic location info + info->mAccountID = AccountID; + info->mDiscSet = DiscSet; + info->mFilename = fn; + info->mReadOnly = ReadOnly; + + // Insert info from file + info->mClientStoreMarker = ntoh64(hdr.mClientStoreMarker); + info->mLastObjectIDUsed = ntoh64(hdr.mLastObjectIDUsed); + info->mBlocksUsed = ntoh64(hdr.mBlocksUsed); + info->mBlocksInOldFiles = ntoh64(hdr.mBlocksInOldFiles); + info->mBlocksInDeletedFiles = ntoh64(hdr.mBlocksInDeletedFiles); + info->mBlocksInDirectories = ntoh64(hdr.mBlocksInDirectories); + info->mBlocksSoftLimit = ntoh64(hdr.mBlocksSoftLimit); + info->mBlocksHardLimit = ntoh64(hdr.mBlocksHardLimit); + + // Load up array of deleted objects + int64_t numDelObj = ntoh64(hdr.mNumberDeletedDirectories); + + // Then load them in + if(numDelObj > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int64_t toload = numDelObj; + while(toload > 0) + { + // How many in this one? + int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload)); + + if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Add them + for(int t = 0; t < b; ++t) + { + info->mDeletedDirectories.push_back(ntoh64(objs[t])); + } + + // Number loaded + toload -= b; + } + } + + // Final check + if(static_cast(info->mDeletedDirectories.size()) != numDelObj) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // return it to caller + return info; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::CreateForRegeneration(...) +// Purpose: Return an object which can be used to save for regeneration. +// Created: 23/4/04 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupStoreInfo::CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, + int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, + int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + // Generate the filename + std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); + + // Make new object + std::auto_ptr info(new BackupStoreInfo); + + // Put in basic info + info->mAccountID = AccountID; + info->mDiscSet = DiscSet; + info->mFilename = fn; + info->mReadOnly = false; + + // Insert info starting info + info->mClientStoreMarker = 0; + info->mLastObjectIDUsed = LastObjectID; + info->mBlocksUsed = BlocksUsed; + info->mBlocksInOldFiles = BlocksInOldFiles; + info->mBlocksInDeletedFiles = BlocksInDeletedFiles; + info->mBlocksInDirectories = BlocksInDirectories; + info->mBlocksSoftLimit = BlockSoftLimit; + info->mBlocksHardLimit = BlockHardLimit; + + // return it to caller + return info; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::Save() +// Purpose: Save modified info back to disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::Save() +{ + // Make sure we're initialised (although should never come to this) + if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Can we do this? + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Then... open a write file + RaidFileWrite rf(mDiscSet, mFilename); + rf.Open(true); // allow overwriting + + // Make header + info_StreamFormat hdr; + hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); + hdr.mAccountID = htonl(mAccountID); + hdr.mClientStoreMarker = hton64(mClientStoreMarker); + hdr.mLastObjectIDUsed = hton64(mLastObjectIDUsed); + hdr.mBlocksUsed = hton64(mBlocksUsed); + hdr.mBlocksInOldFiles = hton64(mBlocksInOldFiles); + hdr.mBlocksInDeletedFiles = hton64(mBlocksInDeletedFiles); + hdr.mBlocksInDirectories = hton64(mBlocksInDirectories); + hdr.mBlocksSoftLimit = hton64(mBlocksSoftLimit); + hdr.mBlocksHardLimit = hton64(mBlocksHardLimit); + hdr.mCurrentMarkNumber = 0; + hdr.mOptionsPresent = 0; + hdr.mNumberDeletedDirectories = hton64(mDeletedDirectories.size()); + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Write the deleted object list + if(mDeletedDirectories.size() > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int tosave = mDeletedDirectories.size(); + std::vector::iterator i(mDeletedDirectories.begin()); + while(tosave > 0) + { + // How many in this one? + int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); + + // Add them + for(int t = 0; t < b; ++t) + { + ASSERT(i != mDeletedDirectories.end()); + objs[t] = hton64((*i)); + i++; + } + + // Write + rf.Write(objs, b * sizeof(int64_t)); + + // Number saved + tosave -= b; + } + } + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Mark is as not modified + mIsModified = false; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksUsed(int32_t) +// Purpose: Change number of blocks used, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksUsed + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksUsed += Delta; + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksInOldFiles(int32_t) +// Purpose: Change number of blocks in old files, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInOldFiles + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInOldFiles += Delta; + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksInDeletedFiles(int32_t) +// Purpose: Change number of blocks in deleted files, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInDeletedFiles + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInDeletedFiles += Delta; + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeBlocksInDirectories(int32_t) +// Purpose: Change number of blocks in directories, by a delta amount +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInDirectories + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInDirectories += Delta; + + mIsModified = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::CorrectAllUsedValues(int64_t, int64_t, int64_t, int64_t) +// Purpose: Set all the usage counts to specific values -- use for correcting in housekeeping +// if something when wrong during the backup connection, and the store info wasn't +// saved back to disc. +// Created: 15/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Set the values + mBlocksUsed = Used; + mBlocksInOldFiles = InOldFiles; + mBlocksInDeletedFiles = InDeletedFiles; + mBlocksInDirectories = InDirectories; + + mIsModified = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::AddDeletedDirectory(int64_t) +// Purpose: Add a directory ID to the deleted list +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::AddDeletedDirectory(int64_t DirID) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + mDeletedDirectories.push_back(DirID); + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::RemovedDeletedDirectory(int64_t) +// Purpose: Remove a directory from the deleted list +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + std::vector::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID)); + if(i == mDeletedDirectories.end()) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList) + } + mDeletedDirectories.erase(i); + + mIsModified = true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::ChangeLimits(int64_t, int64_t) +// Purpose: Change the soft and hard limits +// Created: 15/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + mBlocksSoftLimit = BlockSoftLimit; + mBlocksHardLimit = BlockHardLimit; + + mIsModified = true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::AllocateObjectID() +// Purpose: Allocate an ID for a new object in the store. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreInfo::AllocateObjectID() +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if(mLastObjectIDUsed < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised) + } + + // Return the next object ID + return ++mLastObjectIDUsed; + + mIsModified = true; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreInfo::SetClientStoreMarker(int64_t) +// Purpose: Sets the client store marker +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + mClientStoreMarker = ClientStoreMarker; + + mIsModified = true; +} + + + diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h new file mode 100755 index 00000000..1aba5edb --- /dev/null +++ b/lib/backupstore/BackupStoreInfo.h @@ -0,0 +1,111 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreInfo.h +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREINFO__H +#define BACKUPSTOREINFO__H + +#include +#include +#include + +class BackupStoreCheck; + +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreInfo +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +class BackupStoreInfo +{ + friend class BackupStoreCheck; +public: + ~BackupStoreInfo(); +private: + // Creation through static functions only + BackupStoreInfo(); + // No copying allowed + BackupStoreInfo(const BackupStoreInfo &); + +public: + // Create a New account, saving a blank info object to the disc + static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit); + + // Load it from the store + static std::auto_ptr Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0); + + // Has info been modified? + bool IsModified() const {return mIsModified;} + + // Save modified infomation back to store + void Save(); + + // Data access functions + int32_t GetAccountID() const {return mAccountID;} + int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;} + int64_t GetBlocksUsed() const {return mBlocksUsed;} + int64_t GetBlocksInOldFiles() const {return mBlocksInOldFiles;} + int64_t GetBlocksInDeletedFiles() const {return mBlocksInDeletedFiles;} + int64_t GetBlocksInDirectories() const {return mBlocksInDirectories;} + const std::vector &GetDeletedDirectories() const {return mDeletedDirectories;} + int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;} + int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;} + bool IsReadOnly() const {return mReadOnly;} + int GetDiscSetNumber() const {return mDiscSet;} + + // Data modification functions + void ChangeBlocksUsed(int64_t Delta); + void ChangeBlocksInOldFiles(int64_t Delta); + void ChangeBlocksInDeletedFiles(int64_t Delta); + void ChangeBlocksInDirectories(int64_t Delta); + void CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories); + void AddDeletedDirectory(int64_t DirID); + void RemovedDeletedDirectory(int64_t DirID); + void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit); + + // Object IDs + int64_t AllocateObjectID(); + + // Client marker set and get + int64_t GetClientStoreMarker() {return mClientStoreMarker;} + void SetClientStoreMarker(int64_t ClientStoreMarker); + +private: + static std::auto_ptr CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, + int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, + int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit); + +private: + // Location information + int32_t mAccountID; + int mDiscSet; + std::string mFilename; + bool mReadOnly; + bool mIsModified; + + // Client infomation + int64_t mClientStoreMarker; + + // Account information + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mBlocksSoftLimit; + int64_t mBlocksHardLimit; + std::vector mDeletedDirectories; +}; + + +#endif // BACKUPSTOREINFO__H + + diff --git a/lib/backupstore/StoreStructure.cpp b/lib/backupstore/StoreStructure.cpp new file mode 100755 index 00000000..45a1ce91 --- /dev/null +++ b/lib/backupstore/StoreStructure.cpp @@ -0,0 +1,95 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreStructure.cpp +// Purpose: +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "RaidFileController.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StoreStructure::MakeObjectFilename(int64_t, const std::string &, int, std::string &, bool) +// Purpose: Builds the object filename for a given object, given a root. Optionally ensure that the +// directory exists. +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- +void StoreStructure::MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists) +{ + const static char *hex = "0123456789abcdef"; + + // Set output to root string + rFilenameOut = rStoreRoot; + + // get the id value from the stored object ID so we can do + // bitwise operations on it. + uint64_t id = (uint64_t)ObjectID; + + // get leafname, shift the bits which make up the leafname off + unsigned int leafname(id & STORE_ID_SEGMENT_MASK); + id >>= STORE_ID_SEGMENT_LENGTH; + + // build pathname + while(id != 0) + { + // assumes that the segments are no bigger than 8 bits + int v = id & STORE_ID_SEGMENT_MASK; + rFilenameOut += hex[(v & 0xf0) >> 4]; + rFilenameOut += hex[v & 0xf]; + rFilenameOut += DIRECTORY_SEPARATOR_ASCHAR; + + // shift the bits we used off the pathname + id >>= STORE_ID_SEGMENT_LENGTH; + } + + // Want to make sure this exists? + if(EnsureDirectoryExists) + { + if(!RaidFileRead::DirectoryExists(DiscSet, rFilenameOut)) + { + // Create it + RaidFileWrite::CreateDirectory(DiscSet, rFilenameOut, true /* recusive */); + } + } + + // append the filename + rFilenameOut += 'o'; + rFilenameOut += hex[(leafname & 0xf0) >> 4]; + rFilenameOut += hex[leafname & 0xf]; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: StoreStructure::MakeWriteLockFilename(const std::string &, int, std::string &) +// Purpose: Generate the on disc filename of the write lock file +// Created: 15/12/03 +// +// -------------------------------------------------------------------------- +void StoreStructure::MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut) +{ + // Find the disc set + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(DiscSet)); + + // Make the filename + std::string writeLockFile(rdiscSet[0] + DIRECTORY_SEPARATOR + rStoreRoot + "write.lock"); + + // Return it to the caller + rFilenameOut = writeLockFile; +} + + diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h new file mode 100755 index 00000000..094c0deb --- /dev/null +++ b/lib/backupstore/StoreStructure.h @@ -0,0 +1,32 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreStructure.h +// Purpose: Functions for placing files in the store +// Created: 11/12/03 +// +// -------------------------------------------------------------------------- + +#ifndef STORESTRUCTURE__H +#define STORESTRUCTURE__H + +#include + +#ifdef NDEBUG + #define STORE_ID_SEGMENT_LENGTH 8 + #define STORE_ID_SEGMENT_MASK 0xff +#else + // Debug we'll use lots and lots of directories to stress things + #define STORE_ID_SEGMENT_LENGTH 2 + #define STORE_ID_SEGMENT_MASK 0x03 +#endif + + +namespace StoreStructure +{ + void MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists); + void MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut); +}; + +#endif // STORESTRUCTURE__H + -- cgit v1.2.3