summaryrefslogtreecommitdiff
path: root/lib/backupstore
diff options
context:
space:
mode:
authorBen Summers <ben@fluffy.co.uk>2005-10-14 08:50:54 +0000
committerBen Summers <ben@fluffy.co.uk>2005-10-14 08:50:54 +0000
commit99f8ce096bc5569adbfea1911dbcda24c28d8d8b (patch)
tree049c302161fea1f2f6223e1e8f3c40d9e8aadc8b /lib/backupstore
Box Backup 0.09 with a few tweeks
Diffstat (limited to 'lib/backupstore')
-rwxr-xr-xlib/backupstore/BackupStoreAccountDatabase.cpp370
-rwxr-xr-xlib/backupstore/BackupStoreAccountDatabase.h75
-rwxr-xr-xlib/backupstore/BackupStoreAccounts.cpp162
-rwxr-xr-xlib/backupstore/BackupStoreAccounts.h47
-rw-r--r--lib/backupstore/BackupStoreCheck.cpp745
-rw-r--r--lib/backupstore/BackupStoreCheck.h199
-rw-r--r--lib/backupstore/BackupStoreCheck2.cpp841
-rw-r--r--lib/backupstore/BackupStoreCheckData.cpp205
-rwxr-xr-xlib/backupstore/BackupStoreConfigVerify.cpp48
-rwxr-xr-xlib/backupstore/BackupStoreConfigVerify.h18
-rwxr-xr-xlib/backupstore/BackupStoreInfo.cpp592
-rwxr-xr-xlib/backupstore/BackupStoreInfo.h111
-rwxr-xr-xlib/backupstore/StoreStructure.cpp95
-rwxr-xr-xlib/backupstore/StoreStructure.h32
14 files changed, 3540 insertions, 0 deletions
diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp
new file mode 100755
index 00000000..91a6b758
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccountDatabase.cpp
@@ -0,0 +1,370 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccountDatabase.cpp
+// Purpose: Database of accounts for the backup store
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <string>
+#include <map>
+#include <stdio.h>
+#include <sys/stat.h>
+
+#include "BackupStoreAccountDatabase.h"
+#include "Guards.h"
+#include "FdGetLine.h"
+#include "BackupStoreException.h"
+#include "CommonException.h"
+#include "FileModificationTime.h"
+
+#include "MemLeakFindOn.h"
+
+class _BackupStoreAccountDatabase
+{
+public:
+ std::string mFilename;
+ std::map<int32_t, BackupStoreAccountDatabase::Entry> mDatabase;
+ box_time_t mModificationTime;
+};
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *)
+// Purpose: Constructor
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename)
+ : pImpl(new _BackupStoreAccountDatabase)
+{
+ pImpl->mFilename = Filename;
+ pImpl->mModificationTime = 0;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::~BackupStoreAccountDatabase()
+// Purpose: Destructor
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::~BackupStoreAccountDatabase()
+{
+ delete pImpl;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::Entry()
+// Purpose: Default constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry()
+ : mID(-1),
+ mDiscSet(-1)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::Entry(int32_t, int)
+// Purpose: Constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry(int32_t ID, int DiscSet)
+ : mID(ID),
+ mDiscSet(DiscSet)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::Entry(const Entry &)
+// Purpose: Copy constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::Entry(const Entry &rEntry)
+ : mID(rEntry.mID),
+ mDiscSet(rEntry.mDiscSet)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Entry::~Entry()
+// Purpose: Destructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccountDatabase::Entry::~Entry()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Read(const char *)
+// Purpose: Read in a database from disc
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const char *Filename)
+{
+ // Database object to use
+ std::auto_ptr<BackupStoreAccountDatabase> db(new BackupStoreAccountDatabase(Filename));
+
+ // Read in the file
+ db->ReadFile();
+
+ // Return to called
+ return db;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::ReadFile()
+// Purpose: Read the file off disc
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::ReadFile() const
+{
+ // Open file
+ FileHandleGuard<> file(pImpl->mFilename.c_str());
+
+ // Clear existing entries
+ pImpl->mDatabase.clear();
+
+ // Read in lines
+ FdGetLine getLine(file);
+
+ while(!getLine.IsEOF())
+ {
+ // Read and split up line
+ std::string l(getLine.GetLine(true));
+
+ if(!l.empty())
+ {
+ // Check...
+ int32_t id;
+ int discSet;
+ if(::sscanf(l.c_str(), "%x:%d", &id, &discSet) != 2)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadAccountDatabaseFile)
+ }
+
+ // Make a new entry
+ pImpl->mDatabase[id] = Entry(id, discSet);
+ }
+ }
+
+ // Store the modification time of the file
+ pImpl->mModificationTime = GetDBFileModificationTime();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::CheckUpToDate()
+// Purpose: Private. Ensure that the in memory database matches the one on disc
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::CheckUpToDate() const
+{
+ if(pImpl->mModificationTime != GetDBFileModificationTime())
+ {
+ // File has changed -- load it in again
+ ReadFile();
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::GetDBFileModificationTime()
+// Purpose: Get the current modification time of the database
+// Created: 21/1/04
+//
+// --------------------------------------------------------------------------
+box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const
+{
+ struct stat st;
+ if(::stat(pImpl->mFilename.c_str(), &st) == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+
+ return FileModificationTime(st);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::Write()
+// Purpose: Write the database back to disc after modifying it
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::Write()
+{
+ // Open file for writing
+ // Would use this...
+ // FileHandleGuard<O_WRONLY | O_TRUNC> file(pImpl->mFilename.c_str());
+ // but gcc fails randomly on it on some platforms. Weird.
+
+ int file = ::open(pImpl->mFilename.c_str(), O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+ if(file == -1)
+ {
+ THROW_EXCEPTION(CommonException, OSFileOpenError)
+ }
+
+ try
+ {
+ // Then write each entry
+ for(std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.begin());
+ i != pImpl->mDatabase.end(); ++i)
+ {
+ // Write out the entry
+ char line[256]; // more than enough for a couple of integers in string form
+ int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet());
+ if(::write(file, line, s) != s)
+ {
+ THROW_EXCEPTION(CommonException, OSFileError)
+ }
+ }
+
+ ::close(file);
+ }
+ catch(...)
+ {
+ ::close(file);
+ throw;
+ }
+
+ // Done.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::EntryExists(int32_t)
+// Purpose: Does an entry exist in the database?
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreAccountDatabase::EntryExists(int32_t ID) const
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ return pImpl->mDatabase.find(ID) != pImpl->mDatabase.end();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::GetEntry(int32_t)
+// Purpose: Retrieve an entry
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(int32_t ID) const
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ std::map<int32_t, BackupStoreAccountDatabase::Entry>::const_iterator i(pImpl->mDatabase.find(ID));
+ if(i == pImpl->mDatabase.end())
+ {
+ THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry)
+ }
+
+ return i->second;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::AddEntry(int32_t, int)
+// Purpose: Add a new entry to the database
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::AddEntry(int32_t ID, int DiscSet)
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ pImpl->mDatabase[ID] = Entry(ID, DiscSet);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::DeleteEntry(int32_t)
+// Purpose: Delete an entry from the database
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::DeleteEntry(int32_t ID)
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.find(ID));
+ if(i == pImpl->mDatabase.end())
+ {
+ THROW_EXCEPTION(BackupStoreException, AccountDatabaseNoSuchEntry)
+ }
+
+ pImpl->mDatabase.erase(i);
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t>)
+// Purpose:
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccountDatabase::GetAllAccountIDs(std::vector<int32_t> &rIDsOut)
+{
+ // Check that we're using the latest version of the database
+ CheckUpToDate();
+
+ // Delete everything in the output list
+ rIDsOut.clear();
+
+ std::map<int32_t, BackupStoreAccountDatabase::Entry>::iterator i(pImpl->mDatabase.begin());
+ for(; i != pImpl->mDatabase.end(); ++i)
+ {
+ rIDsOut.push_back(i->first);
+ }
+}
+
+
+
+
diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h
new file mode 100755
index 00000000..8d6e7ad8
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccountDatabase.h
@@ -0,0 +1,75 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccountDatabase.h
+// Purpose: Database of accounts for the backup store
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREACCOUNTDATABASE__H
+#define BACKUPSTOREACCOUNTDATABASE__H
+
+#include <memory>
+#include <vector>
+
+#include "BoxTime.h"
+
+class _BackupStoreAccountDatabase;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreAccountDatabase
+// Purpose: Database of accounts for the backup store
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupStoreAccountDatabase
+{
+public:
+ friend class _BackupStoreAccountDatabase; // to stop compiler warnings
+ ~BackupStoreAccountDatabase();
+private:
+ BackupStoreAccountDatabase(const char *Filename);
+ BackupStoreAccountDatabase(const BackupStoreAccountDatabase &);
+public:
+
+ static std::auto_ptr<BackupStoreAccountDatabase> Read(const char *Filename);
+ void Write();
+
+ class Entry
+ {
+ public:
+ Entry();
+ Entry(int32_t ID, int DiscSet);
+ Entry(const Entry &rEntry);
+ ~Entry();
+
+ int32_t GetID() const {return mID;}
+ int GetDiscSet() const {return mDiscSet;}
+
+ private:
+ int32_t mID;
+ int mDiscSet;
+ };
+
+ bool EntryExists(int32_t ID) const;
+ const Entry &GetEntry(int32_t ID) const;
+ void AddEntry(int32_t ID, int DiscSet);
+ void DeleteEntry(int32_t ID);
+
+ // This interface should change in the future. But for now it'll do.
+ void GetAllAccountIDs(std::vector<int32_t> &rIDsOut);
+
+private:
+ void ReadFile() const; // const in concept only
+ void CheckUpToDate() const; // const in concept only
+ box_time_t GetDBFileModificationTime() const;
+
+private:
+ mutable _BackupStoreAccountDatabase *pImpl;
+};
+
+#endif // BACKUPSTOREACCOUNTDATABASE__H
+
diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp
new file mode 100755
index 00000000..36d9cad3
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccounts.cpp
@@ -0,0 +1,162 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccounts.cpp
+// Purpose: Account management for backup store server
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "BoxPortsAndFiles.h"
+#include "BackupStoreAccounts.h"
+#include "BackupStoreAccountDatabase.h"
+#include "RaidFileWrite.h"
+#include "BackupStoreInfo.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+#include "UnixUser.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &)
+// Purpose: Constructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccounts::BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase)
+ : mrDatabase(rDatabase)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::~BackupStoreAccounts()
+// Purpose: Destructor
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+BackupStoreAccounts::~BackupStoreAccounts()
+{
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &)
+// Purpose: Create a new account on the specified disc set.
+// If rAsUsername is not empty, then the account information will be written under the
+// username specified.
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername)
+{
+ {
+ // Become the user specified in the config file?
+ std::auto_ptr<UnixUser> user;
+ if(!rAsUsername.empty())
+ {
+ // Username specified, change...
+ user.reset(new UnixUser(rAsUsername.c_str()));
+ user->ChangeProcessUser(true /* temporary */);
+ // Change will be undone at the end of this function
+ }
+
+ // Get directory name
+ std::string dirName(MakeAccountRootDir(ID, DiscSet));
+
+ // Create a directory on disc
+ RaidFileWrite::CreateDirectory(DiscSet, dirName, true /* recursive */);
+
+ // Create an info file
+ BackupStoreInfo::CreateNew(ID, dirName, DiscSet, SizeSoftLimit, SizeHardLimit);
+
+ // And an empty directory
+ BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID);
+ int64_t rootDirSize = 0;
+ // Write it, knowing the directory scheme
+ {
+ RaidFileWrite rf(DiscSet, dirName + "o01");
+ rf.Open();
+ rootDir.WriteToStream(rf);
+ rootDirSize = rf.GetDiscUsageInBlocks();
+ rf.Commit(true);
+ }
+
+ // Update the store info to reflect the size of the root directory
+ std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */));
+ info->ChangeBlocksUsed(rootDirSize);
+ info->ChangeBlocksInDirectories(rootDirSize);
+
+ // Save it back
+ info->Save();
+ }
+
+ // As the original user...
+
+ // Create the entry in the database
+ mrDatabase.AddEntry(ID, DiscSet);
+
+ // Write the database back
+ mrDatabase.Write();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::GetAccountRoot(int32_t, std::string &, int &)
+// Purpose: Gets the root of an account, returning the info via references
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const
+{
+ // Find the account
+ const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID));
+
+ rRootDirOut = MakeAccountRootDir(ID, en.GetDiscSet());
+ rDiscSetOut = en.GetDiscSet();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::MakeAccountRootDir(int32_t, int)
+// Purpose: Private. Generates a root directory name for the account
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const
+{
+ char accid[64]; // big enough!
+ ::sprintf(accid, "%08x/", ID);
+ return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED DIRECTORY_SEPARATOR) + accid);
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreAccounts::AccountExists(int32_t)
+// Purpose: Does an account exist?
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+bool BackupStoreAccounts::AccountExists(int32_t ID)
+{
+ return mrDatabase.EntryExists(ID);
+}
+
+
diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h
new file mode 100755
index 00000000..0c3dd103
--- /dev/null
+++ b/lib/backupstore/BackupStoreAccounts.h
@@ -0,0 +1,47 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreAccounts.h
+// Purpose: Account management for backup store server
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREACCOUNTS__H
+#define BACKUPSTOREACCOUNTS__H
+
+#include <string>
+
+class BackupStoreAccountDatabase;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreAccounts
+// Purpose: Account management for backup store server
+// Created: 2003/08/21
+//
+// --------------------------------------------------------------------------
+class BackupStoreAccounts
+{
+public:
+ BackupStoreAccounts(BackupStoreAccountDatabase &rDatabase);
+ ~BackupStoreAccounts();
+private:
+ BackupStoreAccounts(const BackupStoreAccounts &rToCopy);
+
+public:
+ void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername);
+
+ bool AccountExists(int32_t ID);
+ void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const;
+
+private:
+ std::string MakeAccountRootDir(int32_t ID, int DiscSet) const;
+
+private:
+ BackupStoreAccountDatabase &mrDatabase;
+};
+
+#endif // BACKUPSTOREACCOUNTS__H
+
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
new file mode 100644
index 00000000..fb48c3da
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -0,0 +1,745 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheck.cpp
+// Purpose: Check a store for consistency
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "BackupStoreCheck.h"
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::BackupStoreCheck(const std::string &, int, int32_t, bool, bool)
+// Purpose: Constructor
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet)
+ : mStoreRoot(rStoreRoot),
+ mDiscSetNumber(DiscSetNumber),
+ mAccountID(AccountID),
+ mFixErrors(FixErrors),
+ mQuiet(Quiet),
+ mNumberErrorsFound(0),
+ mLastIDInInfo(0),
+ mpInfoLastBlock(0),
+ mInfoLastBlockEntries(0),
+ mLostDirNameSerial(0),
+ mLostAndFoundDirectoryID(0),
+ mBlocksUsed(0),
+ mBlocksInOldFiles(0),
+ mBlocksInDeletedFiles(0),
+ mBlocksInDirectories(0)
+{
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::~BackupStoreCheck()
+// Purpose: Destructor
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::~BackupStoreCheck()
+{
+ // Clean up
+ FreeInfo();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::Check()
+// Purpose: Perform the check on the given account
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::Check()
+{
+ // Lock the account
+ {
+ std::string writeLockFilename;
+ StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename);
+
+ bool gotLock = false;
+ int triesLeft = 8;
+ do
+ {
+ gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */);
+
+ if(!gotLock)
+ {
+ --triesLeft;
+ ::sleep(1);
+ }
+ } while(!gotLock && triesLeft > 0);
+
+ if(!gotLock)
+ {
+ // Couldn't lock the account -- just stop now
+ if(!mQuiet)
+ {
+ ::printf("Couldn't lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server.\n");
+ }
+ THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount)
+ }
+ }
+
+ if(!mQuiet && mFixErrors)
+ {
+ ::printf("NOTE: Will fix errors encountered during checking.\n");
+ }
+
+ // Phase 1, check objects
+ if(!mQuiet)
+ {
+ ::printf("Check store account ID %08x\nPhase 1, check objects...\n", mAccountID);
+ }
+ CheckObjects();
+
+ // Phase 2, check directories
+ if(!mQuiet)
+ {
+ ::printf("Phase 2, check directories...\n");
+ }
+ CheckDirectories();
+
+ // Phase 3, check root
+ if(!mQuiet)
+ {
+ ::printf("Phase 3, check root...\n");
+ }
+ CheckRoot();
+
+ // Phase 4, check unattached objects
+ if(!mQuiet)
+ {
+ ::printf("Phase 4, fix unattached objects...\n");
+ }
+ CheckUnattachedObjects();
+
+ // Phase 5, fix bad info
+ if(!mQuiet)
+ {
+ ::printf("Phase 5, fix unrecovered inconsistencies...\n");
+ }
+ FixDirsWithWrongContainerID();
+ FixDirsWithLostDirs();
+
+ // Phase 6, regenerate store info
+ if(!mQuiet)
+ {
+ ::printf("Phase 6, regenerate store info...\n");
+ }
+ WriteNewStoreInfo();
+
+// DUMP_OBJECT_INFO
+
+ if(mNumberErrorsFound > 0)
+ {
+ ::printf("%lld errors found\n", mNumberErrorsFound);
+ if(!mFixErrors)
+ {
+ ::printf("NOTE: No changes to the store account have been made.\n");
+ }
+ if(!mFixErrors && mNumberErrorsFound > 0)
+ {
+ ::printf("Run again with fix option to fix these errors\n");
+ }
+ if(mNumberErrorsFound > 0)
+ {
+ ::printf("You should now use bbackupquery on the client machine to examine the store.\n");
+ if(mLostAndFoundDirectoryID != 0)
+ {
+ ::printf("A lost+found directory was created in the account root.\n"\
+ "This contains files and directories which could not be matched to existing directories.\n"\
+ "bbackupd will delete this directory in a few days time.\n");
+ }
+ }
+ }
+ else
+ {
+ ::printf("Store account checked, no errors found.\n");
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: static TwoDigitHexToInt(const char *, int &)
+// Purpose: Convert a two digit hex string to an int, returning whether it's valid or not
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut)
+{
+ int n = 0;
+ // Char 0
+ if(String[0] >= '0' && String[0] <= '9')
+ {
+ n = (String[0] - '0') << 4;
+ }
+ else if(String[0] >= 'a' && String[0] <= 'f')
+ {
+ n = ((String[0] - 'a') + 0xa) << 4;
+ }
+ else
+ {
+ return false;
+ }
+ // Char 1
+ if(String[1] >= '0' && String[1] <= '9')
+ {
+ n |= String[1] - '0';
+ }
+ else if(String[1] >= 'a' && String[1] <= 'f')
+ {
+ n |= (String[1] - 'a') + 0xa;
+ }
+ else
+ {
+ return false;
+ }
+
+ // Return a valid number
+ rNumberOut = n;
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckObjects()
+// Purpose: Read in the contents of the directory, recurse to other levels,
+// checking objects for sanity and readability
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckObjects()
+{
+ // Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything
+ int64_t maxDir = 0;
+
+ // Find the maximum directory starting ID
+ {
+ // Make sure the starting root dir doesn't end with '/'.
+ std::string start(mStoreRoot);
+ if(start.size() > 0 && start[start.size() - 1] == '/')
+ {
+ start.resize(start.size() - 1);
+ }
+
+ maxDir = CheckObjectsScanDir(0, 1, mStoreRoot);
+ TRACE1("Max dir starting ID is %llx\n", maxDir);
+ }
+
+ // Then go through and scan all the objects within those directories
+ for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH))
+ {
+ CheckObjectsDir(d);
+ }
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckObjectsScanDir(int64_t, int, int, const std::string &)
+// Purpose: Read in the contents of the directory, recurse to other levels,
+// return the maximum starting ID of any directory found.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName)
+{
+ //TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID);
+
+ int64_t maxID = StartID;
+
+ // Read in all the directories, and recurse downwards
+ {
+ std::vector<std::string> dirs;
+ RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName,
+ RaidFileRead::DirReadType_DirsOnly, dirs);
+
+ for(std::vector<std::string>::const_iterator i(dirs.begin()); i != dirs.end(); ++i)
+ {
+ // Check to see if it's the right name
+ int n = 0;
+ if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n)
+ && n < (1<<STORE_ID_SEGMENT_LENGTH))
+ {
+ // Next level down
+ int64_t mi = CheckObjectsScanDir(StartID | (n << (Level * STORE_ID_SEGMENT_LENGTH)), Level + 1,
+ rDirName + DIRECTORY_SEPARATOR + *i);
+ // Found a greater starting ID?
+ if(mi > maxID)
+ {
+ maxID = mi;
+ }
+ }
+ else
+ {
+ ::printf("Spurious or invalid directory %s/%s found%s -- delete manually\n", rDirName.c_str(), (*i).c_str(), mFixErrors?", deleting":"");
+ ++mNumberErrorsFound;
+ }
+ }
+ }
+
+ return maxID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckObjectsDir(int64_t)
+// Purpose: Check all the files within this directory which has the given starting ID.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
+{
+ // Make directory name -- first generate the filename of an entry in it
+ std::string dirName;
+ StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */);
+ // Check expectations
+ ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == '/');
+ // Remove the filename from it
+ dirName.resize(dirName.size() - 4); // four chars for "/o00"
+
+ // Check directory exists
+ if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName))
+ {
+ TRACE1("RaidFile dir %s does not exist\n", dirName.c_str());
+ return;
+ }
+
+ // Read directory contents
+ std::vector<std::string> files;
+ RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName,
+ RaidFileRead::DirReadType_FilesOnly, files);
+
+ // Array of things present
+ bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)];
+ for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l)
+ {
+ idsPresent[l] = false;
+ }
+
+ // Parse each entry, building up a list of object IDs which are present in the dir.
+ // This is done so that whatever order is retured from the directory, objects are scanned
+ // in order.
+ // Filename must begin with a 'o' and be three characters long, otherwise it gets deleted.
+ for(std::vector<std::string>::const_iterator i(files.begin()); i != files.end(); ++i)
+ {
+ bool fileOK = true;
+ int n = 0;
+ if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n)
+ && n < (1<<STORE_ID_SEGMENT_LENGTH))
+ {
+ // Filename is valid, mark as existing
+ idsPresent[n] = true;
+ }
+ else
+ {
+ // info file in root dir is OK!
+ if(StartID != 0 || ::strcmp("info", (*i).c_str()) != 0)
+ {
+ fileOK = false;
+ }
+ }
+
+ if(!fileOK)
+ {
+ // Unexpected or bad file, delete it
+ ::printf("Spurious file %s/%s found%s\n", dirName.c_str(), (*i).c_str(), mFixErrors?", deleting":"");
+ ++mNumberErrorsFound;
+ if(mFixErrors)
+ {
+ RaidFileWrite del(mDiscSetNumber, dirName + DIRECTORY_SEPARATOR + *i);
+ del.Delete();
+ }
+ }
+ }
+
+ // Check all the objects found in this directory
+ for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i)
+ {
+ if(idsPresent[i])
+ {
+ // Check the object is OK, and add entry
+ char leaf[8];
+ ::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i);
+ if(!CheckAndAddObject(StartID | i, dirName + leaf))
+ {
+ // File was bad, delete it
+ ::printf("Corrupted file %s%s found%s\n", dirName.c_str(), leaf, mFixErrors?", deleting":"");
+ ++mNumberErrorsFound;
+ if(mFixErrors)
+ {
+ RaidFileWrite del(mDiscSetNumber, dirName + leaf);
+ del.Delete();
+ }
+ }
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &)
+// Purpose: Check a specific object and add it to the list if it's OK -- if
+// there are any errors with the reading, return false and it'll be deleted.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename)
+{
+ // Info on object...
+ bool isFile = true;
+ int64_t containerID = -1;
+ int64_t size = -1;
+
+ try
+ {
+ // Open file
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename));
+ size = file->GetDiscUsageInBlocks();
+
+ // Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile
+ uint32_t signature;
+ if(file->Read(&signature, sizeof(signature)) != sizeof(signature))
+ {
+ // Too short, can't read signature from it
+ return false;
+ }
+ // Seek back to beginning
+ file->Seek(0, IOStream::SeekType_Absolute);
+
+ // Then... check depending on the type
+ switch(ntohl(signature))
+ {
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
+#endif
+ // File... check
+ containerID = CheckFile(ObjectID, *file);
+ break;
+
+ case OBJECTMAGIC_DIR_MAGIC_VALUE:
+ isFile = false;
+ containerID = CheckDirInitial(ObjectID, *file);
+ break;
+
+ default:
+ // Unknown signature. Bad file. Very bad file.
+ return false;
+ break;
+ }
+
+ // Add to usage counts
+ int64_t s = file->GetDiscUsageInBlocks();
+ mBlocksUsed += s;
+ if(!isFile)
+ {
+ mBlocksInDirectories += s;
+ }
+ }
+ catch(...)
+ {
+ // Error caught, not a good file then, let it be deleted
+ return false;
+ }
+
+ // Got a container ID? (ie check was successful)
+ if(containerID == -1)
+ {
+ return false;
+ }
+
+ // Add to list of IDs known about
+ AddID(ObjectID, containerID, size, isFile);
+
+ // Report success
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckFile(int64_t, IOStream &)
+// Purpose: Do check on file, return original container ID if OK, or -1 on error
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream)
+{
+ // Check that it's not the root directory ID. Having a file as the root directory would be bad.
+ if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
+ {
+ // Get that dodgy thing deleted!
+ ::printf("Have file as root directory. This is bad.\n");
+ return -1;
+ }
+
+ // Check the format of the file, and obtain the container ID
+ int64_t originalContainerID = -1;
+ if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */,
+ &originalContainerID))
+ {
+ // Didn't verify
+ return -1;
+ }
+
+ return originalContainerID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &)
+// Purpose: Do initial check on directory, return container ID if OK, or -1 on error
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream)
+{
+ // Simply attempt to read in the directory
+ BackupStoreDirectory dir;
+ dir.ReadFromStream(rStream, IOStream::TimeOutInfinite);
+
+ // Check object ID
+ if(dir.GetObjectID() != ObjectID)
+ {
+ // Wrong object ID
+ return -1;
+ }
+
+ // Return container ID
+ return dir.GetContainerID();
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckDirectories()
+// Purpose: Check the directories
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckDirectories()
+{
+ // Phase 1 did this:
+ // Checked that all the directories are readable
+ // Built a list of all directories and files which exist on the store
+ //
+ // This phase will check all the files in the directories, make
+ // a note of all directories which are missing, and do initial fixing.
+
+ // Scan all objects
+ for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ IDBlock *pblock = i->second;
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+
+ for(int e = 0; e < bentries; ++e)
+ {
+ uint8_t flags = GetFlags(pblock, e);
+ if(flags & Flags_IsDir)
+ {
+ // Found a directory. Read it in.
+ std::string filename;
+ StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */);
+ BackupStoreDirectory dir;
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Flag for modifications
+ bool isModified = false;
+
+ // Check for validity
+ if(dir.CheckAndFix())
+ {
+ // Wasn't quite right, and has been modified
+ ::printf("Directory ID %llx has bad structure\n", pblock->mID[e]);
+ ++mNumberErrorsFound;
+ isModified = true;
+ }
+
+ // Go through, and check that everything in that directory exists and is valid
+ std::vector<int64_t> toDelete;
+
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next()) != 0)
+ {
+ // Lookup the item
+ int32_t iIndex;
+ IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
+ bool badEntry = false;
+ if(piBlock != 0)
+ {
+ // Found. Get flags
+ uint8_t iflags = GetFlags(piBlock, iIndex);
+
+ // Is the type the same?
+ if(((iflags & Flags_IsDir) == Flags_IsDir)
+ != ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir))
+ {
+ // Entry is of wrong type
+ ::printf("Directory ID %llx references object %llx which has a different type than expected.\n", pblock->mID[e], en->GetObjectID());
+ badEntry = true;
+ }
+ else
+ {
+ // Check that the entry is not already contained.
+ if(iflags & Flags_IsContained)
+ {
+ badEntry = true;
+ ::printf("Directory ID %llx references object %llx which is already contained.\n", pblock->mID[e], en->GetObjectID());
+ }
+ else
+ {
+ // Not already contained -- mark as contained
+ SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
+
+ // Check that the container ID of the object is correct
+ if(piBlock->mContainer[iIndex] != pblock->mID[e])
+ {
+ // Needs fixing...
+ if(iflags & Flags_IsDir)
+ {
+ // Add to will fix later list
+ ::printf("Directory ID %llx has wrong container ID.\n", en->GetObjectID());
+ mDirsWithWrongContainerID.push_back(en->GetObjectID());
+ }
+ else
+ {
+ // This is OK for files, they might move
+ ::printf("File ID %llx has different container ID, probably moved\n", en->GetObjectID());
+ }
+
+ // Fix entry for now
+ piBlock->mContainer[iIndex] = pblock->mID[e];
+ }
+ }
+ }
+
+ // Check the object size, if it's OK and a file
+ if(!badEntry && !((iflags & Flags_IsDir) == Flags_IsDir))
+ {
+ if(en->GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[iIndex])
+ {
+ // Correct
+ en->SetSizeInBlocks(piBlock->mObjectSizeInBlocks[iIndex]);
+ // Mark as changed
+ isModified = true;
+ // Tell user
+ ::printf("Directory ID %llx has wrong size for object %llx\n", pblock->mID[e], en->GetObjectID());
+ }
+ }
+ }
+ else
+ {
+ // Item can't be found. Is it a directory?
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir)
+ {
+ // Store the directory for later attention
+ mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e];
+ }
+ else
+ {
+ // Just remove the entry
+ badEntry = true;
+ ::printf("Directory ID %llx references object %llx which does not exist.\n", pblock->mID[e], en->GetObjectID());
+ }
+ }
+
+ // Is this entry worth keeping?
+ if(badEntry)
+ {
+ toDelete.push_back(en->GetObjectID());
+ }
+ else
+ {
+ // Add to sizes?
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion)
+ {
+ mBlocksInOldFiles += en->GetSizeInBlocks();
+ }
+ if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted)
+ {
+ mBlocksInDeletedFiles += en->GetSizeInBlocks();
+ }
+ }
+ }
+
+ if(toDelete.size() > 0)
+ {
+ // Delete entries from directory
+ for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
+ {
+ dir.DeleteEntry(*d);
+ }
+
+ // Mark as modified
+ isModified = true;
+
+ // Check the directory again, now that entries have been removed
+ dir.CheckAndFix();
+
+ // Errors found
+ ++mNumberErrorsFound;
+ }
+
+ if(isModified && mFixErrors)
+ {
+ ::printf("Fixing directory ID %llx\n", pblock->mID[e]);
+
+ // Save back to disc
+ RaidFileWrite fixed(mDiscSetNumber, filename);
+ fixed.Open(true /* allow overwriting */);
+ dir.WriteToStream(fixed);
+ // Commit it
+ fixed.Commit(true /* convert to raid representation now */);
+ }
+ }
+ }
+ }
+
+}
+
+
diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h
new file mode 100644
index 00000000..3f48312a
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck.h
@@ -0,0 +1,199 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheck.h
+// Purpose: Check a store for consistency
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECHECK__H
+#define BACKUPSTORECHECK__H
+
+#include <string>
+#include <map>
+#include <vector>
+#include <set>
+
+#include "NamedLock.h"
+class IOStream;
+class BackupStoreFilename;
+
+/*
+
+The following problems can be fixed:
+
+ * Spurious files deleted
+ * Corrupted files deleted
+ * Root ID as file, deleted
+ * Dirs with wrong object id inside, deleted
+ * Direcetory entries pointing to non-existant files, deleted
+ * Doubly references files have second reference deleted
+ * Wrong directory container IDs fixed
+ * Missing root recreated
+ * Reattach files which exist, but aren't referenced
+ - files go into directory original directory, if it still exists
+ - missing directories are inferred, and recreated
+ - or if all else fails, go into lost+found
+ - file dir entries take the original name and mod time
+ - directories go into lost+found
+ * Container IDs on directories corrected
+ * Inside directories,
+ - only one object per name has old version clear
+ - IDs aren't duplicated
+ * Bad store info files regenerated
+ * Bad sizes of files in directories fixed
+
+*/
+
+
+// Size of blocks in the list of IDs
+#ifdef NDEBUG
+ #define BACKUPSTORECHECK_BLOCK_SIZE (64*1024)
+#else
+ #define BACKUPSTORECHECK_BLOCK_SIZE 8
+#endif
+
+// The object ID type -- can redefine to uint32_t to produce a lower memory version for smaller stores
+typedef int64_t BackupStoreCheck_ID_t;
+// Can redefine the size type for lower memory usage too
+typedef int64_t BackupStoreCheck_Size_t;
+
+// --------------------------------------------------------------------------
+//
+// Class
+// Name: BackupStoreCheck
+// Purpose: Check a store for consistency
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+class BackupStoreCheck
+{
+public:
+ BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet);
+ ~BackupStoreCheck();
+private:
+ // no copying
+ BackupStoreCheck(const BackupStoreCheck &);
+ BackupStoreCheck &operator=(const BackupStoreCheck &);
+public:
+
+ // Do the exciting things
+ void Check();
+
+ bool ErrorsFound() {return mNumberErrorsFound > 0;}
+
+private:
+ enum
+ {
+ // Bit mask
+ Flags_IsDir = 1,
+ Flags_IsContained = 2,
+ // Mask
+ Flags__MASK = 3,
+ // Number of bits
+ Flags__NumFlags = 2,
+ // Items per uint8_t
+ Flags__NumItemsPerEntry = 4 // ie 8 / 2
+ };
+
+ typedef struct
+ {
+ // Note use arrays within the block, rather than the more obvious array of
+ // objects, to be more memory efficient -- think alignment of the byte values.
+ uint8_t mFlags[BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry];
+ BackupStoreCheck_ID_t mID[BACKUPSTORECHECK_BLOCK_SIZE];
+ BackupStoreCheck_ID_t mContainer[BACKUPSTORECHECK_BLOCK_SIZE];
+ BackupStoreCheck_Size_t mObjectSizeInBlocks[BACKUPSTORECHECK_BLOCK_SIZE];
+ } IDBlock;
+
+ // Phases of the check
+ void CheckObjects();
+ void CheckDirectories();
+ void CheckRoot();
+ void CheckUnattachedObjects();
+ void FixDirsWithWrongContainerID();
+ void FixDirsWithLostDirs();
+ void WriteNewStoreInfo();
+
+ // Checking functions
+ int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName);
+ void CheckObjectsDir(int64_t StartID);
+ bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename);
+ int64_t CheckFile(int64_t ObjectID, IOStream &rStream);
+ int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream);
+
+ // Fixing functions
+ bool TryToRecreateDirectory(int64_t MissingDirectoryID);
+ void InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory);
+ int64_t GetLostAndFoundDirID();
+ void CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID);
+
+ // Data handling
+ void FreeInfo();
+ void AddID(BackupStoreCheck_ID_t ID, BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile);
+ IDBlock *LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut);
+ inline void SetFlags(IDBlock *pBlock, int32_t Index, uint8_t Flags)
+ {
+ ASSERT(pBlock != 0);
+ ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE);
+ ASSERT(Flags < (1 << Flags__NumFlags));
+
+ pBlock->mFlags[Index / Flags__NumItemsPerEntry]
+ |= (Flags << ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags));
+ }
+ inline uint8_t GetFlags(IDBlock *pBlock, int32_t Index)
+ {
+ ASSERT(pBlock != 0);
+ ASSERT(Index < BACKUPSTORECHECK_BLOCK_SIZE);
+
+ return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK;
+ }
+
+#ifndef NDEBUG
+ void DumpObjectInfo();
+ #define DUMP_OBJECT_INFO DumpObjectInfo();
+#else
+ #define DUMP_OBJECT_INFO
+#endif
+
+private:
+ std::string mStoreRoot;
+ int mDiscSetNumber;
+ int32_t mAccountID;
+ bool mFixErrors;
+ bool mQuiet;
+
+ int64_t mNumberErrorsFound;
+
+ // Lock for the store account
+ NamedLock mAccountLock;
+
+ // Storage for ID data
+ typedef std::map<BackupStoreCheck_ID_t, IDBlock*> Info_t;
+ Info_t mInfo;
+ BackupStoreCheck_ID_t mLastIDInInfo;
+ IDBlock *mpInfoLastBlock;
+ int32_t mInfoLastBlockEntries;
+
+ // List of stuff to fix
+ std::vector<BackupStoreCheck_ID_t> mDirsWithWrongContainerID;
+ // This is a map of lost dir ID -> existing dir ID
+ std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs;
+
+ // Set of extra directories added
+ std::set<BackupStoreCheck_ID_t> mDirsAdded;
+
+ // Misc stuff
+ int32_t mLostDirNameSerial;
+ int64_t mLostAndFoundDirectoryID;
+
+ // Usage
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+};
+
+#endif // BACKUPSTORECHECK__H
+
diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp
new file mode 100644
index 00000000..fe91d00d
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheck2.cpp
@@ -0,0 +1,841 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheck2.cpp
+// Purpose: More backup store checking
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "BackupStoreCheck.h"
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreObjectMagic.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreFileWire.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreInfo.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckRoot()
+// Purpose: Check the root directory exists.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckRoot()
+{
+ int32_t index = 0;
+ IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index);
+
+ if(pblock != 0)
+ {
+ // Found it. Which is lucky. Mark it as contained.
+ SetFlags(pblock, index, Flags_IsContained);
+ }
+ else
+ {
+ ::printf("Root directory doesn't exist\n");
+
+ ++mNumberErrorsFound;
+
+ if(mFixErrors)
+ {
+ // Create a new root directory
+ CreateBlankDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID);
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CreateBlankDirectory(int64_t, int64_t)
+// Purpose: Creates a blank directory
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t ContainingDirID)
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ BackupStoreDirectory dir(DirectoryID, ContainingDirID);
+
+ // Serialise to disc
+ std::string filename;
+ StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */);
+ RaidFileWrite obj(mDiscSetNumber, filename);
+ obj.Open(false /* don't allow overwriting */);
+ dir.WriteToStream(obj);
+ int64_t size = obj.GetDiscUsageInBlocks();
+ obj.Commit(true /* convert to raid now */);
+
+ // Record the fact we've done this
+ mDirsAdded.insert(DirectoryID);
+
+ // Add to sizes
+ mBlocksUsed += size;
+ mBlocksInDirectories += size;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::CheckUnattachedObjects()
+// Purpose: Check for objects which aren't attached to anything
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::CheckUnattachedObjects()
+{
+ // Scan all objects, finding ones which have no container
+ for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ IDBlock *pblock = i->second;
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+
+ for(int e = 0; e < bentries; ++e)
+ {
+ uint8_t flags = GetFlags(pblock, e);
+ if((flags & Flags_IsContained) == 0)
+ {
+ // Unattached object...
+ ::printf("Object %llx is unattached.\n", pblock->mID[e]);
+ ++mNumberErrorsFound;
+
+ // What's to be done?
+ int64_t putIntoDirectoryID = 0;
+
+ if((flags & Flags_IsDir) == Flags_IsDir)
+ {
+ // Directory. Just put into lost and found.
+ putIntoDirectoryID = GetLostAndFoundDirID();
+ }
+ else
+ {
+ // File. Only attempt to attach it somewhere if it isn't a patch
+ {
+ int64_t diffFromObjectID = 0;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */);
+ // The easiest way to do this is to verify it again. Not such a bad penalty, because
+ // this really shouldn't be done very often.
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ BackupStoreFile::VerifyEncodedFileFormat(*file, &diffFromObjectID);
+ }
+
+ // If not zero, then it depends on another file, which may or may not be available.
+ // Just delete it to be safe.
+ if(diffFromObjectID != 0)
+ {
+ ::printf("Object %llx is unattached, and is a patch. Deleting, cannot reliably recover.\n", pblock->mID[e]);
+
+ // Delete this object instead
+ if(mFixErrors)
+ {
+ RaidFileWrite del(mDiscSetNumber, filename);
+ del.Delete();
+ }
+
+ // Move on to next item
+ continue;
+ }
+ }
+
+ // Files contain their original filename, so perhaps the orginal directory still exists,
+ // or we can infer the existance of a directory?
+ // Look for a matching entry in the mDirsWhichContainLostDirs map.
+ // Can't do this with a directory, because the name just wouldn't be known, which is
+ // pretty useless as bbackupd would just delete it. So better to put it in lost+found
+ // where the admin can do something about it.
+ int32_t dirindex;
+ IDBlock *pdirblock = LookupID(pblock->mContainer[e], dirindex);
+ if(pdirblock != 0)
+ {
+ // Something with that ID has been found. Is it a directory?
+ if(GetFlags(pdirblock, dirindex) & Flags_IsDir)
+ {
+ // Directory exists, add to that one
+ putIntoDirectoryID = pblock->mContainer[e];
+ }
+ else
+ {
+ // Not a directory. Use lost and found dir
+ putIntoDirectoryID = GetLostAndFoundDirID();
+ }
+ }
+ else if(mDirsAdded.find(pblock->mContainer[e]) != mDirsAdded.end()
+ || TryToRecreateDirectory(pblock->mContainer[e]))
+ {
+ // The directory reappeared, or was created somehow elsewhere
+ putIntoDirectoryID = pblock->mContainer[e];
+ }
+ else
+ {
+ putIntoDirectoryID = GetLostAndFoundDirID();
+ }
+ }
+ ASSERT(putIntoDirectoryID != 0);
+
+ // Add it to the directory
+ InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID,
+ ((flags & Flags_IsDir) == Flags_IsDir));
+ }
+ }
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::TryToRecreateDirectory(int64_t)
+// Purpose: Recreate a missing directory
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID)
+{
+ // During the directory checking phase, a map of "missing directory" to
+ // containing directory was built. If we can find it here, then it's
+ // something which can be recreated!
+ std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator missing(
+ mDirsWhichContainLostDirs.find(MissingDirectoryID));
+ if(missing == mDirsWhichContainLostDirs.end())
+ {
+ // Not a missing directory, can't recreate.
+ return false;
+ }
+
+ // Can recreate this! Wooo!
+ if(!mFixErrors)
+ {
+ ::printf("Missing directory %llx could be recreated\n", MissingDirectoryID);
+ mDirsAdded.insert(MissingDirectoryID);
+ return true;
+ }
+ ::printf("Recreating missing directory %llx\n", MissingDirectoryID);
+
+ // Create a blank directory
+ BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */);
+ // Note that this directory already contains a directory entry pointing to
+ // this dir, so it doesn't have to be added.
+
+ // Serialise to disc
+ std::string filename;
+ StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */);
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(false /* don't allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+
+ // Record the fact we've done this
+ mDirsAdded.insert(MissingDirectoryID);
+
+ // Remove the entry from the map, so this doesn't happen again
+ mDirsWhichContainLostDirs.erase(missing);
+
+ return true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::GetLostAndFoundDirID()
+// Purpose: Returns the ID of the lost and found directory, creating it if necessary
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreCheck::GetLostAndFoundDirID()
+{
+ // Already allocated it?
+ if(mLostAndFoundDirectoryID != 0)
+ {
+ return mLostAndFoundDirectoryID;
+ }
+
+ if(!mFixErrors)
+ {
+ // The result will never be used anyway if errors aren't being fixed
+ return 1;
+ }
+
+ // Load up the root directory
+ BackupStoreDirectory dir;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(BACKUPSTORE_ROOT_DIRECTORY_ID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Find a suitable name
+ BackupStoreFilename lostAndFound;
+ int n = 0;
+ while(true)
+ {
+ char name[32];
+ ::sprintf(name, "lost+found%d", n++);
+ lostAndFound.SetAsClearFilename(name);
+ if(!dir.NameInUse(lostAndFound))
+ {
+ // Found a name which can be used
+ ::printf("Lost and found dir has name %s\n", name);
+ break;
+ }
+ }
+
+ // Allocate an ID
+ int64_t id = mLastIDInInfo + 1;
+
+ // Create a blank directory
+ CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID);
+
+ // Add an entry for it
+ dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0);
+
+ // Write out root dir
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+
+ // Store
+ mLostAndFoundDirectoryID = id;
+
+ // Tell caller
+ return mLostAndFoundDirectoryID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool)
+// Purpose:
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory)
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ // Data for the object
+ BackupStoreFilename objectStoreFilename;
+ int64_t modTime = 100; // something which isn't zero or a special time
+ int32_t sizeInBlocks = 0; // suitable for directories
+
+ if(IsDirectory)
+ {
+ // Directory -- simply generate a name for it.
+ char name[32];
+ ::sprintf(name, "dir%08x", mLostDirNameSerial++);
+ objectStoreFilename.SetAsClearFilename(name);
+ }
+ else
+ {
+ // Files require a little more work...
+ // Open file
+ std::string fileFilename;
+ StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */);
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, fileFilename));
+ // Fill in size information
+ sizeInBlocks = file->GetDiscUsageInBlocks();
+ // Read in header
+ file_StreamFormat hdr;
+ if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
+#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
+ && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
+#endif
+ ))
+ {
+ // This should never happen, everything has been checked before.
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+ // This tells us nice things
+ modTime = ntoh64(hdr.mModificationTime);
+ // And the filename comes next
+ objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Directory object
+ BackupStoreDirectory dir;
+
+ // Generate filename
+ std::string filename;
+ StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+
+ // Read it in
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Add a new entry in an appropraite place
+ dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks,
+ IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File));
+
+ // Fix any flags which have been broken, which there's a good change of going
+ dir.CheckAndFix();
+
+ // Write it out
+ if(mFixErrors)
+ {
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::FixDirsWithWrongContainerID()
+// Purpose: Rewrites container IDs where required
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FixDirsWithWrongContainerID()
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ // Run through things which need fixing
+ for(std::vector<BackupStoreCheck_ID_t>::iterator i(mDirsWithWrongContainerID.begin());
+ i != mDirsWithWrongContainerID.end(); ++i)
+ {
+ int32_t index = 0;
+ IDBlock *pblock = LookupID(*i, index);
+ if(pblock == 0) continue;
+
+ // Load in
+ BackupStoreDirectory dir;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(*i, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Adjust container ID
+ dir.SetContainerID(pblock->mContainer[index]);
+
+ // Write it out
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::FixDirsWithLostDirs()
+// Purpose: Fix directories
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FixDirsWithLostDirs()
+{
+ if(!mFixErrors)
+ {
+ // Don't do anything if we're not supposed to fix errors
+ return;
+ }
+
+ // Run through things which need fixing
+ for(std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t>::iterator i(mDirsWhichContainLostDirs.begin());
+ i != mDirsWhichContainLostDirs.end(); ++i)
+ {
+ int32_t index = 0;
+ IDBlock *pblock = LookupID(i->second, index);
+ if(pblock == 0) continue;
+
+ // Load in
+ BackupStoreDirectory dir;
+ std::string filename;
+ StoreStructure::MakeObjectFilename(i->second, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */);
+ {
+ std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
+ dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
+ }
+
+ // Delete the dodgy entry
+ dir.DeleteEntry(i->first);
+
+ // Fix it up
+ dir.CheckAndFix();
+
+ // Write it out
+ RaidFileWrite root(mDiscSetNumber, filename);
+ root.Open(true /* allow overwriting */);
+ dir.WriteToStream(root);
+ root.Commit(true /* convert to raid now */);
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::WriteNewStoreInfo()
+// Purpose: Regenerate store info
+// Created: 23/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::WriteNewStoreInfo()
+{
+ // Attempt to load the existing store info file
+ std::auto_ptr<BackupStoreInfo> poldInfo;
+ try
+ {
+ poldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release());
+ }
+ catch(...)
+ {
+ ::printf("Load of existing store info failed, regenerating.\n");
+ ++mNumberErrorsFound;
+ }
+
+ // Minimum soft and hard limits
+ int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024;
+ int64_t minHard = ((minSoft * 11) / 10) + 1024;
+
+ // Need to do anything?
+ if(poldInfo.get() != 0 && mNumberErrorsFound == 0 && poldInfo->GetAccountID() == mAccountID)
+ {
+ // Leave the store info as it is, no need to alter it because nothing really changed,
+ // and the only essential thing was that the account ID was correct, which is was.
+ return;
+ }
+
+ // NOTE: We will always build a new store info, so the client store marker gets changed.
+
+ // Work out the new limits
+ int64_t softLimit = minSoft;
+ int64_t hardLimit = minHard;
+ if(poldInfo.get() != 0 && poldInfo->GetBlocksSoftLimit() > minSoft)
+ {
+ softLimit = poldInfo->GetBlocksSoftLimit();
+ }
+ else
+ {
+ ::printf("NOTE: Soft limit for account changed to ensure housekeeping doesn't delete files on next run\n");
+ }
+ if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard)
+ {
+ hardLimit = poldInfo->GetBlocksHardLimit();
+ }
+ else
+ {
+ ::printf("NOTE: Hard limit for account changed to ensure housekeeping doesn't delete files on next run\n");
+ }
+
+ // Object ID
+ int64_t lastObjID = mLastIDInInfo;
+ if(mLostAndFoundDirectoryID != 0)
+ {
+ mLastIDInInfo++;
+ }
+
+ // Build a new store info
+ std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration(
+ mAccountID,
+ mStoreRoot,
+ mDiscSetNumber,
+ lastObjID,
+ mBlocksUsed,
+ mBlocksInOldFiles,
+ mBlocksInDeletedFiles,
+ mBlocksInDirectories,
+ softLimit,
+ hardLimit));
+
+ // Save to disc?
+ if(mFixErrors)
+ {
+ info->Save();
+ ::printf("New store info file written successfully.\n");
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::CheckAndFix()
+// Purpose: Check the directory for obvious logical problems, and fix them.
+// Return true if the directory was changed.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDirectory::CheckAndFix()
+{
+ bool changed = false;
+
+ // Check that if a file depends on a new version, that version is in this directory
+ {
+ std::vector<Entry*>::iterator i(mEntries.begin());
+ for(; i != mEntries.end(); ++i)
+ {
+ int64_t dependsNewer = (*i)->GetDependsNewer();
+ if(dependsNewer != 0)
+ {
+ BackupStoreDirectory::Entry *newerEn = FindEntryByID(dependsNewer);
+ if(newerEn == 0)
+ {
+ // Depends on something, but it isn't there.
+ TRACE2("Entry id %llx removed because depends on newer version %llx which doesn't exist\n", (*i)->GetObjectID(), dependsNewer);
+
+ // Remove
+ delete *i;
+ mEntries.erase(i);
+
+ // Start again at the beginning of the vector, the iterator is now invalid
+ i = mEntries.begin();
+
+ // Mark as changed
+ changed = true;
+ }
+ else
+ {
+ // Check that newerEn has it marked
+ if(newerEn->GetDependsOlder() != (*i)->GetObjectID())
+ {
+ // Wrong entry
+ TRACE3("Entry id %llx, correcting DependsOlder to %llx, was %llx\n", dependsNewer, (*i)->GetObjectID(), newerEn->GetDependsOlder());
+ newerEn->SetDependsOlder((*i)->GetObjectID());
+ // Mark as changed
+ changed = true;
+ }
+ }
+ }
+ }
+ }
+
+ // Check that if a file has a dependency marked, it exists, and remove it if it doesn't
+ {
+ std::vector<Entry*>::iterator i(mEntries.begin());
+ for(; i != mEntries.end(); ++i)
+ {
+ int64_t dependsOlder = (*i)->GetDependsOlder();
+ if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0)
+ {
+ // Has an older version marked, but this doesn't exist. Remove this mark
+ TRACE2("Entry id %llx was marked that %llx depended on it, which doesn't exist, dependency info cleaered\n", (*i)->GetObjectID(), dependsOlder);
+
+ (*i)->SetDependsOlder(0);
+
+ // Mark as changed
+ changed = true;
+ }
+ }
+ }
+
+ bool ch = false;
+ do
+ {
+ // Reset change marker
+ ch = false;
+
+ // Search backwards -- so see newer versions first
+ std::vector<Entry*>::iterator i(mEntries.end());
+ if(i == mEntries.begin())
+ {
+ // Directory is empty, stop now
+ return changed; // changed flag
+ }
+
+ // Records of things seen
+ std::set<int64_t> idsEncountered;
+ std::set<BackupStoreFilename> filenamesEncountered;
+
+ do
+ {
+ // Look at previous
+ --i;
+
+ bool removeEntry = false;
+ if((*i) == 0)
+ {
+ TRACE0("Remove because null pointer found\n");
+ removeEntry = true;
+ }
+ else
+ {
+ bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir);
+
+ // Check mutually exclusive flags
+ if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File))
+ {
+ // Bad! Unset the file flag
+ TRACE1("Entry %llx: File flag set when dir flag set\n", (*i)->GetObjectID());
+ (*i)->RemoveFlags(Entry::Flags_File);
+ changed = true;
+ }
+
+ // Check...
+ if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end())
+ {
+ // ID already seen, or type doesn't match
+ TRACE1("Entry %llx: Remove because ID already seen\n", (*i)->GetObjectID());
+ removeEntry = true;
+ }
+ else
+ {
+ // Haven't already seen this ID, remember it
+ idsEncountered.insert((*i)->GetObjectID());
+
+ // Check to see if the name has already been encountered -- if not, then it
+ // needs to have the old version flag set
+ if(filenamesEncountered.find((*i)->GetName()) != filenamesEncountered.end())
+ {
+ // Seen before -- check old version flag set
+ if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion
+ && ((*i)->GetFlags() & Entry::Flags_Deleted) == 0)
+ {
+ // Not set, set it
+ TRACE1("Entry %llx: Set old flag\n", (*i)->GetObjectID());
+ (*i)->AddFlags(Entry::Flags_OldVersion);
+ changed = true;
+ }
+ }
+ else
+ {
+ // Check old version flag NOT set
+ if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion)
+ {
+ // Set, unset it
+ TRACE1("Entry %llx: Old flag unset\n", (*i)->GetObjectID());
+ (*i)->RemoveFlags(Entry::Flags_OldVersion);
+ changed = true;
+ }
+
+ // Remember filename
+ filenamesEncountered.insert((*i)->GetName());
+ }
+ }
+ }
+
+ if(removeEntry)
+ {
+ // Mark something as changed, in loop
+ ch = true;
+
+ // Mark something as globally changed
+ changed = true;
+
+ // erase the thing from the list
+ Entry *pentry = (*i);
+ mEntries.erase(i);
+
+ // And delete the entry object
+ delete pentry;
+
+ // Stop going around this loop, as the iterator is now invalid
+ break;
+ }
+ } while(i != mEntries.begin());
+
+ } while(ch != false);
+
+ return changed;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::AddUnattactedObject(...)
+// Purpose: Adds an object which is currently unattached. Assume that CheckAndFix() will be called afterwards.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreDirectory::AddUnattactedObject(const BackupStoreFilename &rName,
+ box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags)
+{
+ Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags,
+ ModificationTime /* use as attr mod time too */);
+ try
+ {
+ // Want to order this just before the first object which has a higher ID,
+ // which is the place it's most likely to be correct.
+ std::vector<Entry*>::iterator i(mEntries.begin());
+ for(; i != mEntries.end(); ++i)
+ {
+ if((*i)->GetObjectID() > ObjectID)
+ {
+ // Found a good place to insert it
+ break;
+ }
+ }
+ if(i == mEntries.end())
+ {
+ mEntries.push_back(pnew);
+ }
+ else
+ {
+ mEntries.insert(i, 1 /* just the one copy */, pnew);
+ }
+ }
+ catch(...)
+ {
+ delete pnew;
+ throw;
+ }
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreDirectory::NameInUse(const BackupStoreFilename &)
+// Purpose: Returns true if the name is currently in use in the directory
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDirectory::NameInUse(const BackupStoreFilename &rName)
+{
+ for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i)
+ {
+ if((*i)->GetName() == rName)
+ {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+
diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp
new file mode 100644
index 00000000..f22c8339
--- /dev/null
+++ b/lib/backupstore/BackupStoreCheckData.cpp
@@ -0,0 +1,205 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreCheckData.cpp
+// Purpose: Data handling for store checking
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <memory>
+
+#include "BackupStoreCheck.h"
+#include "autogen_BackupStoreException.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::FreeInfo()
+// Purpose: Free all the data stored
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::FreeInfo()
+{
+ // Free all the blocks
+ for(Info_t::iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ ::free(i->second);
+ }
+
+ // Clear the contents of the map
+ mInfo.clear();
+
+ // Reset the last ID, just in case
+ mpInfoLastBlock = 0;
+ mInfoLastBlockEntries = 0;
+ mLastIDInInfo = 0;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::AddID(BackupStoreCheck_ID_t, BackupStoreCheck_ID_t, bool)
+// Purpose: Add an ID to the list
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::AddID(BackupStoreCheck_ID_t ID,
+ BackupStoreCheck_ID_t Container, BackupStoreCheck_Size_t ObjectSize, bool IsFile)
+{
+ // Check ID is OK.
+ if(ID <= mLastIDInInfo)
+ {
+ THROW_EXCEPTION(BackupStoreException, InternalAlgorithmErrorCheckIDNotMonotonicallyIncreasing)
+ }
+
+ // Can this go in the current block?
+ if(mpInfoLastBlock == 0 || mInfoLastBlockEntries >= BACKUPSTORECHECK_BLOCK_SIZE)
+ {
+ // No. Allocate a new one
+ IDBlock *pblk = (IDBlock*)::malloc(sizeof(IDBlock));
+ if(pblk == 0)
+ {
+ throw std::bad_alloc();
+ }
+ // Zero all the flags entries
+ for(int z = 0; z < (BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry); ++z)
+ {
+ pblk->mFlags[z] = 0;
+ }
+ // Store in map
+ mInfo[ID] = pblk;
+ // Allocated and stored OK, setup for use
+ mpInfoLastBlock = pblk;
+ mInfoLastBlockEntries = 0;
+ }
+ ASSERT(mpInfoLastBlock != 0 && mInfoLastBlockEntries < BACKUPSTORECHECK_BLOCK_SIZE);
+
+ // Add to block
+ mpInfoLastBlock->mID[mInfoLastBlockEntries] = ID;
+ mpInfoLastBlock->mContainer[mInfoLastBlockEntries] = Container;
+ mpInfoLastBlock->mObjectSizeInBlocks[mInfoLastBlockEntries] = ObjectSize;
+ SetFlags(mpInfoLastBlock, mInfoLastBlockEntries, IsFile?(0):(Flags_IsDir));
+
+ // Increment size
+ ++mInfoLastBlockEntries;
+
+ // Store last ID
+ mLastIDInInfo = ID;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::LookupID(BackupStoreCheck_ID_t, int32_t
+// Purpose: Look up an ID. Return the block it's in, or zero if not found, and the
+// index within that block if the thing is found.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, int32_t &rIndexOut)
+{
+ IDBlock *pblock = 0;
+
+ // Find the lower matching block who's first entry is not less than ID
+ Info_t::const_iterator ib(mInfo.lower_bound(ID));
+
+ // Was there a block
+ if(ib == mInfo.end())
+ {
+ // Block wasn't found... could be in last block
+ pblock = mpInfoLastBlock;
+ }
+ else
+ {
+ // Found it as first entry?
+ if(ib->first == ID)
+ {
+ rIndexOut = 0;
+ return ib->second;
+ }
+
+ // Go back one block as it's not the first entry in this one
+ if(ib == mInfo.begin())
+ {
+ // Was first block, can't go back
+ return 0;
+ }
+ // Go back...
+ --ib;
+
+ // So, the ID will be in this block, if it's in anything
+ pblock = ib->second;
+ }
+
+ ASSERT(pblock != 0);
+ if(pblock == 0) return 0;
+
+ // How many entries are there in the block
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+
+ // Do binary search within block
+ int high = bentries;
+ int low = -1;
+ while(high - low > 1)
+ {
+ int i = (high + low) / 2;
+ if(ID <= pblock->mID[i])
+ {
+ high = i;
+ }
+ else
+ {
+ low = i;
+ }
+ }
+ if(ID == pblock->mID[high])
+ {
+ // Found
+ rIndexOut = high;
+ return pblock;
+ }
+
+ // Not found
+ return 0;
+}
+
+
+#ifndef NDEBUG
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreCheck::DumpObjectInfo()
+// Purpose: Debug only. Trace out all object info.
+// Created: 22/4/04
+//
+// --------------------------------------------------------------------------
+void BackupStoreCheck::DumpObjectInfo()
+{
+ for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
+ {
+ IDBlock *pblock = i->second;
+ int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
+ TRACE2("BLOCK @ 0x%08x, %d entries\n", pblock, bentries);
+
+ for(int e = 0; e < bentries; ++e)
+ {
+ uint8_t flags = GetFlags(pblock, e);
+ TRACE4("id %llx, c %llx, %s, %s\n",
+ pblock->mID[e], pblock->mContainer[e],
+ (flags & Flags_IsDir)?"dir":"file",
+ (flags & Flags_IsContained)?"contained":"unattached");
+ }
+ }
+}
+#endif
+
diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp
new file mode 100755
index 00000000..6fa05d06
--- /dev/null
+++ b/lib/backupstore/BackupStoreConfigVerify.cpp
@@ -0,0 +1,48 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreConfigVerify.h
+// Purpose: Configuration definition for the backup store server
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "BackupStoreConfigVerify.h"
+#include "ServerTLS.h"
+#include "BoxPortsAndFiles.h"
+
+#include "MemLeakFindOn.h"
+
+static const ConfigurationVerifyKey verifyserverkeys[] =
+{
+ SERVERTLS_VERIFY_SERVER_KEYS(0) // no default listen addresses
+};
+
+static const ConfigurationVerify verifyserver[] =
+{
+ {
+ "Server",
+ 0,
+ verifyserverkeys,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+ }
+};
+
+static const ConfigurationVerifyKey verifyrootkeys[] =
+{
+ {"AccountDatabase", 0, ConfigTest_Exists, 0},
+ {"TimeBetweenHousekeeping", 0, ConfigTest_Exists | ConfigTest_IsInt, 0},
+ {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file
+ {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0}
+};
+
+const ConfigurationVerify BackupConfigFileVerify =
+{
+ "root",
+ verifyserver,
+ verifyrootkeys,
+ ConfigTest_Exists | ConfigTest_LastEntry,
+ 0
+};
diff --git a/lib/backupstore/BackupStoreConfigVerify.h b/lib/backupstore/BackupStoreConfigVerify.h
new file mode 100755
index 00000000..815cfaed
--- /dev/null
+++ b/lib/backupstore/BackupStoreConfigVerify.h
@@ -0,0 +1,18 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreConfigVerify.h
+// Purpose: Configuration definition for the backup store server
+// Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTORECONFIGVERIFY__H
+#define BACKUPSTORECONFIGVERIFY__H
+
+#include "Configuration.h"
+
+extern const ConfigurationVerify BackupConfigFileVerify;
+
+#endif // BACKUPSTORECONFIGVERIFY__H
+
diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp
new file mode 100755
index 00000000..a9effe00
--- /dev/null
+++ b/lib/backupstore/BackupStoreInfo.cpp
@@ -0,0 +1,592 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreInfo.cpp
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <algorithm>
+
+#include "BackupStoreInfo.h"
+#include "BackupStoreException.h"
+#include "RaidFileWrite.h"
+#include "RaidFileRead.h"
+
+#include "MemLeakFindOn.h"
+
+// set packing to one byte
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "BeginStructPackForWire.h"
+#else
+BEGIN_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+// ******************
+// make sure the defaults in CreateNew are modified!
+// ******************
+typedef struct
+{
+ int32_t mMagicValue; // also the version number
+ int32_t mAccountID;
+ int64_t mClientStoreMarker;
+ int64_t mLastObjectIDUsed;
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+ int64_t mBlocksSoftLimit;
+ int64_t mBlocksHardLimit;
+ uint32_t mCurrentMarkNumber;
+ uint32_t mOptionsPresent; // bit mask of optional elements present
+ int64_t mNumberDeletedDirectories;
+ // Then loads of int64_t IDs for the deleted directories
+} info_StreamFormat;
+
+#define INFO_MAGIC_VALUE 0x34832476
+
+// Use default packing
+#ifdef STRUCTURE_PATCKING_FOR_WIRE_USE_HEADERS
+#include "EndStructPackForWire.h"
+#else
+END_STRUCTURE_PACKING_FOR_WIRE
+#endif
+
+#ifdef NDEBUG
+ #define NUM_DELETED_DIRS_BLOCK 256
+#else
+ #define NUM_DELETED_DIRS_BLOCK 2
+#endif
+
+#define INFO_FILENAME "info"
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::BackupStoreInfo()
+// Purpose: Default constructor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreInfo::BackupStoreInfo()
+ : mAccountID(-1),
+ mDiscSet(-1),
+ mReadOnly(true),
+ mIsModified(false),
+ mClientStoreMarker(0),
+ mLastObjectIDUsed(-1),
+ mBlocksUsed(0),
+ mBlocksInOldFiles(0),
+ mBlocksInDeletedFiles(0)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::~BackupStoreInfo
+// Purpose: Destructor
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+BackupStoreInfo::~BackupStoreInfo()
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::CreateNew(int32_t, const std::string &, int)
+// Purpose: Create a new info file on disc
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+ // Initial header (is entire file)
+ info_StreamFormat hdr = {
+ htonl(INFO_MAGIC_VALUE), // mMagicValue
+ htonl(AccountID), // mAccountID
+ 0, // mClientStoreMarker
+ hton64(1), // mLastObjectIDUsed (which is the root directory)
+ 0, // mBlocksUsed
+ 0, // mBlocksInOldFiles
+ 0, // mBlocksInDeletedFiles
+ 0, // mBlocksInDirectories
+ hton64(BlockSoftLimit), // mBlocksSoftLimit
+ hton64(BlockHardLimit), // mBlocksHardLimit
+ 0, // mCurrentMarkNumber
+ 0, // mOptionsPresent
+ 0 // mNumberDeletedDirectories
+ };
+
+ // Generate the filename
+ ASSERT(rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR);
+ std::string fn(rRootDir + INFO_FILENAME);
+
+ // Open the file for writing
+ RaidFileWrite rf(DiscSet, fn);
+ rf.Open(false); // no overwriting, as this is a new file
+
+ // Write header
+ rf.Write(&hdr, sizeof(hdr));
+
+ // Commit it to disc, converting it to RAID now
+ rf.Commit(true);
+
+ // Done.
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::Load(int32_t, const std::string &, int, bool)
+// Purpose: Loads the info from disc, given the root information. Can be marked as read only.
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID)
+{
+ // Generate the filename
+ std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME);
+
+ // Open the file for reading (passing on optional request for revision ID)
+ std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID));
+
+ // Read in a header
+ info_StreamFormat hdr;
+ if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+ }
+
+ // Check it
+ if(ntohl(hdr.mMagicValue) != INFO_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != AccountID)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+ }
+
+ // Make new object
+ std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
+
+ // Put in basic location info
+ info->mAccountID = AccountID;
+ info->mDiscSet = DiscSet;
+ info->mFilename = fn;
+ info->mReadOnly = ReadOnly;
+
+ // Insert info from file
+ info->mClientStoreMarker = ntoh64(hdr.mClientStoreMarker);
+ info->mLastObjectIDUsed = ntoh64(hdr.mLastObjectIDUsed);
+ info->mBlocksUsed = ntoh64(hdr.mBlocksUsed);
+ info->mBlocksInOldFiles = ntoh64(hdr.mBlocksInOldFiles);
+ info->mBlocksInDeletedFiles = ntoh64(hdr.mBlocksInDeletedFiles);
+ info->mBlocksInDirectories = ntoh64(hdr.mBlocksInDirectories);
+ info->mBlocksSoftLimit = ntoh64(hdr.mBlocksSoftLimit);
+ info->mBlocksHardLimit = ntoh64(hdr.mBlocksHardLimit);
+
+ // Load up array of deleted objects
+ int64_t numDelObj = ntoh64(hdr.mNumberDeletedDirectories);
+
+ // Then load them in
+ if(numDelObj > 0)
+ {
+ int64_t objs[NUM_DELETED_DIRS_BLOCK];
+
+ int64_t toload = numDelObj;
+ while(toload > 0)
+ {
+ // How many in this one?
+ int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload));
+
+ if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */))
+ {
+ THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo)
+ }
+
+ // Add them
+ for(int t = 0; t < b; ++t)
+ {
+ info->mDeletedDirectories.push_back(ntoh64(objs[t]));
+ }
+
+ // Number loaded
+ toload -= b;
+ }
+ }
+
+ // Final check
+ if(static_cast<int64_t>(info->mDeletedDirectories.size()) != numDelObj)
+ {
+ THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad)
+ }
+
+ // return it to caller
+ return info;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::CreateForRegeneration(...)
+// Purpose: Return an object which can be used to save for regeneration.
+// Created: 23/4/04
+//
+// --------------------------------------------------------------------------
+std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration(int32_t AccountID, const std::string &rRootDir,
+ int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles,
+ int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+ // Generate the filename
+ std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME);
+
+ // Make new object
+ std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo);
+
+ // Put in basic info
+ info->mAccountID = AccountID;
+ info->mDiscSet = DiscSet;
+ info->mFilename = fn;
+ info->mReadOnly = false;
+
+ // Insert info starting info
+ info->mClientStoreMarker = 0;
+ info->mLastObjectIDUsed = LastObjectID;
+ info->mBlocksUsed = BlocksUsed;
+ info->mBlocksInOldFiles = BlocksInOldFiles;
+ info->mBlocksInDeletedFiles = BlocksInDeletedFiles;
+ info->mBlocksInDirectories = BlocksInDirectories;
+ info->mBlocksSoftLimit = BlockSoftLimit;
+ info->mBlocksHardLimit = BlockHardLimit;
+
+ // return it to caller
+ return info;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::Save()
+// Purpose: Save modified info back to disc
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::Save()
+{
+ // Make sure we're initialised (although should never come to this)
+ if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1)
+ {
+ THROW_EXCEPTION(BackupStoreException, Internal)
+ }
+
+ // Can we do this?
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ // Then... open a write file
+ RaidFileWrite rf(mDiscSet, mFilename);
+ rf.Open(true); // allow overwriting
+
+ // Make header
+ info_StreamFormat hdr;
+ hdr.mMagicValue = htonl(INFO_MAGIC_VALUE);
+ hdr.mAccountID = htonl(mAccountID);
+ hdr.mClientStoreMarker = hton64(mClientStoreMarker);
+ hdr.mLastObjectIDUsed = hton64(mLastObjectIDUsed);
+ hdr.mBlocksUsed = hton64(mBlocksUsed);
+ hdr.mBlocksInOldFiles = hton64(mBlocksInOldFiles);
+ hdr.mBlocksInDeletedFiles = hton64(mBlocksInDeletedFiles);
+ hdr.mBlocksInDirectories = hton64(mBlocksInDirectories);
+ hdr.mBlocksSoftLimit = hton64(mBlocksSoftLimit);
+ hdr.mBlocksHardLimit = hton64(mBlocksHardLimit);
+ hdr.mCurrentMarkNumber = 0;
+ hdr.mOptionsPresent = 0;
+ hdr.mNumberDeletedDirectories = hton64(mDeletedDirectories.size());
+
+ // Write header
+ rf.Write(&hdr, sizeof(hdr));
+
+ // Write the deleted object list
+ if(mDeletedDirectories.size() > 0)
+ {
+ int64_t objs[NUM_DELETED_DIRS_BLOCK];
+
+ int tosave = mDeletedDirectories.size();
+ std::vector<int64_t>::iterator i(mDeletedDirectories.begin());
+ while(tosave > 0)
+ {
+ // How many in this one?
+ int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave));
+
+ // Add them
+ for(int t = 0; t < b; ++t)
+ {
+ ASSERT(i != mDeletedDirectories.end());
+ objs[t] = hton64((*i));
+ i++;
+ }
+
+ // Write
+ rf.Write(objs, b * sizeof(int64_t));
+
+ // Number saved
+ tosave -= b;
+ }
+ }
+
+ // Commit it to disc, converting it to RAID now
+ rf.Commit(true);
+
+ // Mark is as not modified
+ mIsModified = false;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksUsed(int32_t)
+// Purpose: Change number of blocks used, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksUsed + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksUsed += Delta;
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksInOldFiles(int32_t)
+// Purpose: Change number of blocks in old files, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksInOldFiles + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksInOldFiles += Delta;
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksInDeletedFiles(int32_t)
+// Purpose: Change number of blocks in deleted files, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksInDeletedFiles + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksInDeletedFiles += Delta;
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeBlocksInDirectories(int32_t)
+// Purpose: Change number of blocks in directories, by a delta amount
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if((mBlocksInDirectories + Delta) < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative)
+ }
+
+ mBlocksInDirectories += Delta;
+
+ mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::CorrectAllUsedValues(int64_t, int64_t, int64_t, int64_t)
+// Purpose: Set all the usage counts to specific values -- use for correcting in housekeeping
+// if something when wrong during the backup connection, and the store info wasn't
+// saved back to disc.
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ // Set the values
+ mBlocksUsed = Used;
+ mBlocksInOldFiles = InOldFiles;
+ mBlocksInDeletedFiles = InDeletedFiles;
+ mBlocksInDirectories = InDirectories;
+
+ mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::AddDeletedDirectory(int64_t)
+// Purpose: Add a directory ID to the deleted list
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::AddDeletedDirectory(int64_t DirID)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ mDeletedDirectories.push_back(DirID);
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::RemovedDeletedDirectory(int64_t)
+// Purpose: Remove a directory from the deleted list
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ std::vector<int64_t>::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID));
+ if(i == mDeletedDirectories.end())
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList)
+ }
+ mDeletedDirectories.erase(i);
+
+ mIsModified = true;
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::ChangeLimits(int64_t, int64_t)
+// Purpose: Change the soft and hard limits
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ mBlocksSoftLimit = BlockSoftLimit;
+ mBlocksHardLimit = BlockHardLimit;
+
+ mIsModified = true;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::AllocateObjectID()
+// Purpose: Allocate an ID for a new object in the store.
+// Created: 2003/09/03
+//
+// --------------------------------------------------------------------------
+int64_t BackupStoreInfo::AllocateObjectID()
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+ if(mLastObjectIDUsed < 0)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised)
+ }
+
+ // Return the next object ID
+ return ++mLastObjectIDUsed;
+
+ mIsModified = true;
+}
+
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: BackupStoreInfo::SetClientStoreMarker(int64_t)
+// Purpose: Sets the client store marker
+// Created: 2003/10/29
+//
+// --------------------------------------------------------------------------
+void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker)
+{
+ if(mReadOnly)
+ {
+ THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly)
+ }
+
+ mClientStoreMarker = ClientStoreMarker;
+
+ mIsModified = true;
+}
+
+
+
diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h
new file mode 100755
index 00000000..1aba5edb
--- /dev/null
+++ b/lib/backupstore/BackupStoreInfo.h
@@ -0,0 +1,111 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreInfo.h
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREINFO__H
+#define BACKUPSTOREINFO__H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+class BackupStoreCheck;
+
+// --------------------------------------------------------------------------
+//
+// File
+// Name: BackupStoreInfo
+// Purpose: Main backup store information storage
+// Created: 2003/08/28
+//
+// --------------------------------------------------------------------------
+class BackupStoreInfo
+{
+ friend class BackupStoreCheck;
+public:
+ ~BackupStoreInfo();
+private:
+ // Creation through static functions only
+ BackupStoreInfo();
+ // No copying allowed
+ BackupStoreInfo(const BackupStoreInfo &);
+
+public:
+ // Create a New account, saving a blank info object to the disc
+ static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+ // Load it from the store
+ static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0);
+
+ // Has info been modified?
+ bool IsModified() const {return mIsModified;}
+
+ // Save modified infomation back to store
+ void Save();
+
+ // Data access functions
+ int32_t GetAccountID() const {return mAccountID;}
+ int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;}
+ int64_t GetBlocksUsed() const {return mBlocksUsed;}
+ int64_t GetBlocksInOldFiles() const {return mBlocksInOldFiles;}
+ int64_t GetBlocksInDeletedFiles() const {return mBlocksInDeletedFiles;}
+ int64_t GetBlocksInDirectories() const {return mBlocksInDirectories;}
+ const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;}
+ int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;}
+ int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;}
+ bool IsReadOnly() const {return mReadOnly;}
+ int GetDiscSetNumber() const {return mDiscSet;}
+
+ // Data modification functions
+ void ChangeBlocksUsed(int64_t Delta);
+ void ChangeBlocksInOldFiles(int64_t Delta);
+ void ChangeBlocksInDeletedFiles(int64_t Delta);
+ void ChangeBlocksInDirectories(int64_t Delta);
+ void CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int64_t InDeletedFiles, int64_t InDirectories);
+ void AddDeletedDirectory(int64_t DirID);
+ void RemovedDeletedDirectory(int64_t DirID);
+ void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+ // Object IDs
+ int64_t AllocateObjectID();
+
+ // Client marker set and get
+ int64_t GetClientStoreMarker() {return mClientStoreMarker;}
+ void SetClientStoreMarker(int64_t ClientStoreMarker);
+
+private:
+ static std::auto_ptr<BackupStoreInfo> CreateForRegeneration(int32_t AccountID, const std::string &rRootDir,
+ int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles,
+ int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit);
+
+private:
+ // Location information
+ int32_t mAccountID;
+ int mDiscSet;
+ std::string mFilename;
+ bool mReadOnly;
+ bool mIsModified;
+
+ // Client infomation
+ int64_t mClientStoreMarker;
+
+ // Account information
+ int64_t mLastObjectIDUsed;
+ int64_t mBlocksUsed;
+ int64_t mBlocksInOldFiles;
+ int64_t mBlocksInDeletedFiles;
+ int64_t mBlocksInDirectories;
+ int64_t mBlocksSoftLimit;
+ int64_t mBlocksHardLimit;
+ std::vector<int64_t> mDeletedDirectories;
+};
+
+
+#endif // BACKUPSTOREINFO__H
+
+
diff --git a/lib/backupstore/StoreStructure.cpp b/lib/backupstore/StoreStructure.cpp
new file mode 100755
index 00000000..45a1ce91
--- /dev/null
+++ b/lib/backupstore/StoreStructure.cpp
@@ -0,0 +1,95 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: StoreStructure.cpp
+// Purpose:
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include "StoreStructure.h"
+#include "RaidFileRead.h"
+#include "RaidFileWrite.h"
+#include "RaidFileController.h"
+
+#include "MemLeakFindOn.h"
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: StoreStructure::MakeObjectFilename(int64_t, const std::string &, int, std::string &, bool)
+// Purpose: Builds the object filename for a given object, given a root. Optionally ensure that the
+// directory exists.
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void StoreStructure::MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists)
+{
+ const static char *hex = "0123456789abcdef";
+
+ // Set output to root string
+ rFilenameOut = rStoreRoot;
+
+ // get the id value from the stored object ID so we can do
+ // bitwise operations on it.
+ uint64_t id = (uint64_t)ObjectID;
+
+ // get leafname, shift the bits which make up the leafname off
+ unsigned int leafname(id & STORE_ID_SEGMENT_MASK);
+ id >>= STORE_ID_SEGMENT_LENGTH;
+
+ // build pathname
+ while(id != 0)
+ {
+ // assumes that the segments are no bigger than 8 bits
+ int v = id & STORE_ID_SEGMENT_MASK;
+ rFilenameOut += hex[(v & 0xf0) >> 4];
+ rFilenameOut += hex[v & 0xf];
+ rFilenameOut += DIRECTORY_SEPARATOR_ASCHAR;
+
+ // shift the bits we used off the pathname
+ id >>= STORE_ID_SEGMENT_LENGTH;
+ }
+
+ // Want to make sure this exists?
+ if(EnsureDirectoryExists)
+ {
+ if(!RaidFileRead::DirectoryExists(DiscSet, rFilenameOut))
+ {
+ // Create it
+ RaidFileWrite::CreateDirectory(DiscSet, rFilenameOut, true /* recusive */);
+ }
+ }
+
+ // append the filename
+ rFilenameOut += 'o';
+ rFilenameOut += hex[(leafname & 0xf0) >> 4];
+ rFilenameOut += hex[leafname & 0xf];
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+// Name: StoreStructure::MakeWriteLockFilename(const std::string &, int, std::string &)
+// Purpose: Generate the on disc filename of the write lock file
+// Created: 15/12/03
+//
+// --------------------------------------------------------------------------
+void StoreStructure::MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut)
+{
+ // Find the disc set
+ RaidFileController &rcontroller(RaidFileController::GetController());
+ RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(DiscSet));
+
+ // Make the filename
+ std::string writeLockFile(rdiscSet[0] + DIRECTORY_SEPARATOR + rStoreRoot + "write.lock");
+
+ // Return it to the caller
+ rFilenameOut = writeLockFile;
+}
+
+
diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h
new file mode 100755
index 00000000..094c0deb
--- /dev/null
+++ b/lib/backupstore/StoreStructure.h
@@ -0,0 +1,32 @@
+// --------------------------------------------------------------------------
+//
+// File
+// Name: StoreStructure.h
+// Purpose: Functions for placing files in the store
+// Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#ifndef STORESTRUCTURE__H
+#define STORESTRUCTURE__H
+
+#include <string>
+
+#ifdef NDEBUG
+ #define STORE_ID_SEGMENT_LENGTH 8
+ #define STORE_ID_SEGMENT_MASK 0xff
+#else
+ // Debug we'll use lots and lots of directories to stress things
+ #define STORE_ID_SEGMENT_LENGTH 2
+ #define STORE_ID_SEGMENT_MASK 0x03
+#endif
+
+
+namespace StoreStructure
+{
+ void MakeObjectFilename(int64_t ObjectID, const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut, bool EnsureDirectoryExists);
+ void MakeWriteLockFilename(const std::string &rStoreRoot, int DiscSet, std::string &rFilenameOut);
+};
+
+#endif // STORESTRUCTURE__H
+