diff options
Diffstat (limited to 'lib/backupstore')
| -rw-r--r-- | lib/backupstore/BackupStoreAccountDatabase.cpp | 8 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreAccountDatabase.h | 8 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreAccounts.cpp | 23 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreAccounts.h | 10 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreCheck.cpp | 304 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreCheck.h | 36 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreCheck2.cpp | 104 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreCheckData.cpp | 17 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreConfigVerify.cpp | 10 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreInfo.cpp | 267 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreInfo.h | 47 | ||||
| -rw-r--r-- | lib/backupstore/BackupStoreRefCountDatabase.cpp | 117 |
12 files changed, 951 insertions, 0 deletions
diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp index 201491a3..ec44de0e 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.cpp +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -40,7 +40,11 @@ public: // Created: 2003/08/20 // // -------------------------------------------------------------------------- +<<<<<<< HEAD +BackupStoreAccountDatabase::BackupStoreAccountDatabase(const char *Filename) +======= BackupStoreAccountDatabase::BackupStoreAccountDatabase(const std::string& Filename) +>>>>>>> 0.12 : pImpl(new _BackupStoreAccountDatabase) { pImpl->mFilename = Filename; @@ -123,7 +127,11 @@ BackupStoreAccountDatabase::Entry::~Entry() // Created: 2003/08/21 // // -------------------------------------------------------------------------- +<<<<<<< HEAD +std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const char *Filename) +======= std::auto_ptr<BackupStoreAccountDatabase> BackupStoreAccountDatabase::Read(const std::string& Filename) +>>>>>>> 0.12 { // Database object to use std::auto_ptr<BackupStoreAccountDatabase> db(new BackupStoreAccountDatabase(Filename)); diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h index f9665c7d..cb19b01b 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.h +++ b/lib/backupstore/BackupStoreAccountDatabase.h @@ -31,11 +31,19 @@ public: friend class _BackupStoreAccountDatabase; // to stop compiler warnings ~BackupStoreAccountDatabase(); private: +<<<<<<< HEAD + BackupStoreAccountDatabase(const char *Filename); + BackupStoreAccountDatabase(const BackupStoreAccountDatabase &); +public: + + static std::auto_ptr<BackupStoreAccountDatabase> Read(const char *Filename); +======= BackupStoreAccountDatabase(const std::string& Filename); BackupStoreAccountDatabase(const BackupStoreAccountDatabase &); public: static std::auto_ptr<BackupStoreAccountDatabase> Read(const std::string& Filename); +>>>>>>> 0.12 void Write(); class Entry diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp index 18500fc1..8cb23a6a 100644 --- a/lib/backupstore/BackupStoreAccounts.cpp +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -11,6 +11,16 @@ #include <stdio.h> +<<<<<<< HEAD +#include "BoxPortsAndFiles.h" +#include "BackupStoreAccounts.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreRefCountDatabase.h" +#include "RaidFileWrite.h" +#include "BackupStoreInfo.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" +======= #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreConstants.h" @@ -21,6 +31,7 @@ #include "BoxPortsAndFiles.h" #include "RaidFileWrite.h" #include "StoreStructure.h" +>>>>>>> 0.12 #include "UnixUser.h" #include "MemLeakFindOn.h" @@ -57,8 +68,13 @@ BackupStoreAccounts::~BackupStoreAccounts() // Function // Name: BackupStoreAccounts::Create(int32_t, int, int64_t, int64_t, const std::string &) // Purpose: Create a new account on the specified disc set. +<<<<<<< HEAD +// If rAsUsername is not empty, then the account information will be written under the +// username specified. +======= // If rAsUsername is not empty, then the account information will be written under the // username specified. +>>>>>>> 0.12 // Created: 2003/08/21 // // -------------------------------------------------------------------------- @@ -104,7 +120,10 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, dirName, DiscSet, false /* ReadWrite */)); info->ChangeBlocksUsed(rootDirSize); info->ChangeBlocksInDirectories(rootDirSize); +<<<<<<< HEAD +======= info->AdjustNumDirectories(1); +>>>>>>> 0.12 // Save it back info->Save(); @@ -170,6 +189,9 @@ bool BackupStoreAccounts::AccountExists(int32_t ID) return mrDatabase.EntryExists(ID); } +<<<<<<< HEAD + +======= void BackupStoreAccounts::LockAccount(int32_t ID, NamedLock& rNamedLock) { const BackupStoreAccountDatabase::Entry &en(mrDatabase.GetEntry(ID)); @@ -201,3 +223,4 @@ void BackupStoreAccounts::LockAccount(int32_t ID, NamedLock& rNamedLock) "lock on account " << ID); } } +>>>>>>> 0.12 diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h index 3163f15c..e0e420bb 100644 --- a/lib/backupstore/BackupStoreAccounts.h +++ b/lib/backupstore/BackupStoreAccounts.h @@ -13,7 +13,10 @@ #include <string> #include "BackupStoreAccountDatabase.h" +<<<<<<< HEAD +======= #include "NamedLock.h" +>>>>>>> 0.12 // -------------------------------------------------------------------------- // @@ -32,8 +35,12 @@ private: BackupStoreAccounts(const BackupStoreAccounts &rToCopy); public: +<<<<<<< HEAD + void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername); +======= void Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername); +>>>>>>> 0.12 bool AccountExists(int32_t ID); void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const; @@ -42,7 +49,10 @@ public: { return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet()); } +<<<<<<< HEAD +======= void LockAccount(int32_t ID, NamedLock& rNamedLock); +>>>>>>> 0.12 private: static std::string MakeAccountRootDir(int32_t ID, int DiscSet); diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index f2302337..79a61a77 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -11,6 +11,19 @@ #include <stdio.h> #include <string.h> +<<<<<<< HEAD +#include <unistd.h> + +#include "BackupStoreCheck.h" +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreFile.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" +======= #ifdef HAVE_UNISTD_H # include <unistd.h> @@ -29,6 +42,7 @@ #include "RaidFileWrite.h" #include "StoreStructure.h" #include "Utils.h" +>>>>>>> 0.12 #include "MemLeakFindOn.h" @@ -54,6 +68,11 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum mLostDirNameSerial(0), mLostAndFoundDirectoryID(0), mBlocksUsed(0), +<<<<<<< HEAD + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0) +======= mBlocksInCurrentFiles(0), mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), @@ -62,6 +81,7 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum mNumOldFiles(0), mNumDeletedFiles(0), mNumDirectories(0) +>>>>>>> 0.12 { } @@ -85,16 +105,51 @@ BackupStoreCheck::~BackupStoreCheck() // // Function // Name: BackupStoreCheck::Check() +<<<<<<< HEAD +// Purpose: Perform the check on the given account +======= // Purpose: Perform the check on the given account. You need to // hold a lock on the account before calling this! +>>>>>>> 0.12 // Created: 21/4/04 // // -------------------------------------------------------------------------- void BackupStoreCheck::Check() { +<<<<<<< HEAD + // Lock the account + { + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + + bool gotLock = false; + int triesLeft = 8; + do + { + gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */); + + if(!gotLock) + { + --triesLeft; + ::sleep(1); + } + } while(!gotLock && triesLeft > 0); + + if(!gotLock) + { + // Couldn't lock the account -- just stop now + if(!mQuiet) + { + BOX_ERROR("Failed to lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server."); + } + THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount) + } + } +======= std::string writeLockFilename; StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); ASSERT(FileExists(writeLockFilename)); +>>>>>>> 0.12 if(!mQuiet && mFixErrors) { @@ -284,6 +339,8 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const // Read in all the directories, and recurse downwards { +<<<<<<< HEAD +======= // If any of the directories is missing, create it. RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); @@ -311,6 +368,7 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const } } +>>>>>>> 0.12 std::vector<std::string> dirs; RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName, RaidFileRead::DirReadType_DirsOnly, dirs); @@ -333,7 +391,11 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const } else { +<<<<<<< HEAD + BOX_WARNING("Spurious or invalid directory " << +======= BOX_ERROR("Spurious or invalid directory " << +>>>>>>> 0.12 rDirName << DIRECTORY_SEPARATOR << (*i) << " found, " << (mFixErrors?"deleting":"delete manually")); @@ -350,8 +412,12 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const // // Function // Name: BackupStoreCheck::CheckObjectsDir(int64_t) +<<<<<<< HEAD +// Purpose: Check all the files within this directory which has the given starting ID. +======= // Purpose: Check all the files within this directory which has // the given starting ID. +>>>>>>> 0.12 // Created: 22/4/04 // // -------------------------------------------------------------------------- @@ -399,6 +465,15 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) // Filename is valid, mark as existing idsPresent[n] = true; } +<<<<<<< HEAD + else + { + // info file in root dir is OK! + if(StartID != 0 || ::strcmp("info", (*i).c_str()) != 0) + { + fileOK = false; + } +======= // No other files should be present in subdirectories else if(StartID != 0) { @@ -412,12 +487,17 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) else { fileOK = false; +>>>>>>> 0.12 } if(!fileOK) { // Unexpected or bad file, delete it +<<<<<<< HEAD + BOX_WARNING("Spurious file " << dirName << +======= BOX_ERROR("Spurious file " << dirName << +>>>>>>> 0.12 DIRECTORY_SEPARATOR << (*i) << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; @@ -440,7 +520,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) if(!CheckAndAddObject(StartID | i, dirName + leaf)) { // File was bad, delete it +<<<<<<< HEAD + BOX_WARNING("Corrupted file " << dirName << +======= BOX_ERROR("Corrupted file " << dirName << +>>>>>>> 0.12 leaf << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; @@ -458,6 +542,15 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) // -------------------------------------------------------------------------- // // Function +<<<<<<< HEAD +// Name: BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &) +// Purpose: Check a specific object and add it to the list if it's OK -- if +// there are any errors with the reading, return false and it'll be deleted. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename) +======= // Name: BackupStoreCheck::CheckAndAddObject(int64_t, // const std::string &) // Purpose: Check a specific object and add it to the list @@ -468,6 +561,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) // -------------------------------------------------------------------------- bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename) +>>>>>>> 0.12 { // Info on object... bool isFile = true; @@ -477,12 +571,19 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, try { // Open file +<<<<<<< HEAD + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename)); + size = file->GetDiscUsageInBlocks(); + + // Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile +======= std::auto_ptr<RaidFileRead> file( RaidFileRead::Open(mDiscSetNumber, rFilename)); size = file->GetDiscUsageInBlocks(); // Read in first four bytes -- don't have to worry about // retrying if not all bytes read as is RaidFile +>>>>>>> 0.12 uint32_t signature; if(file->Read(&signature, sizeof(signature)) != sizeof(signature)) { @@ -513,6 +614,16 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, return false; break; } +<<<<<<< HEAD + + // Add to usage counts + mBlocksUsed += size; + if(!isFile) + { + mBlocksInDirectories += size; + } +======= +>>>>>>> 0.12 } catch(...) { @@ -525,6 +636,12 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, { return false; } +<<<<<<< HEAD + + // Add to list of IDs known about + AddID(ObjectID, containerID, size, isFile); + +======= // Add to list of IDs known about AddID(ObjectID, containerID, size, isFile); @@ -576,6 +693,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, } } +>>>>>>> 0.12 // Report success return true; } @@ -585,15 +703,23 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, // // Function // Name: BackupStoreCheck::CheckFile(int64_t, IOStream &) +<<<<<<< HEAD +// Purpose: Do check on file, return original container ID if OK, or -1 on error +======= // Purpose: Do check on file, return original container ID // if OK, or -1 on error +>>>>>>> 0.12 // Created: 22/4/04 // // -------------------------------------------------------------------------- int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) { +<<<<<<< HEAD + // Check that it's not the root directory ID. Having a file as the root directory would be bad. +======= // Check that it's not the root directory ID. Having a file as // the root directory would be bad. +>>>>>>> 0.12 if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) { // Get that dodgy thing deleted! @@ -603,8 +729,12 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) // Check the format of the file, and obtain the container ID int64_t originalContainerID = -1; +<<<<<<< HEAD + if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */, +======= if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */, +>>>>>>> 0.12 &originalContainerID)) { // Didn't verify @@ -619,8 +749,12 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) // // Function // Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &) +<<<<<<< HEAD +// Purpose: Do initial check on directory, return container ID if OK, or -1 on error +======= // Purpose: Do initial check on directory, return container ID // if OK, or -1 on error +>>>>>>> 0.12 // Created: 22/4/04 // // -------------------------------------------------------------------------- @@ -659,12 +793,16 @@ void BackupStoreCheck::CheckDirectories() // This phase will check all the files in the directories, make // a note of all directories which are missing, and do initial fixing. +<<<<<<< HEAD + // Scan all objects +======= // The root directory is not contained inside another directory, so // it has no directory entry to scan, but we have to count it // somewhere, so we'll count it here. mNumDirectories++; // Scan all objects. +>>>>>>> 0.12 for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) { IDBlock *pblock = i->second; @@ -685,6 +823,24 @@ void BackupStoreCheck::CheckDirectories() } // Flag for modifications +<<<<<<< HEAD + bool isModified = false; + + // Check for validity + if(dir.CheckAndFix()) + { + // Wasn't quite right, and has been modified + BOX_WARNING("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " has bad structure"); + ++mNumberErrorsFound; + isModified = true; + } + + // Go through, and check that everything in that directory exists and is valid + std::vector<int64_t> toDelete; + +======= bool isModified = CheckDirectory(dir); // Check the directory again, now that entries have been removed @@ -715,10 +871,157 @@ void BackupStoreCheck::CheckDirectories() } // Count valid entries +>>>>>>> 0.12 BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { +<<<<<<< HEAD + // Lookup the item + int32_t iIndex; + IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); + bool badEntry = false; + if(piBlock != 0) + { + // Found. Get flags + uint8_t iflags = GetFlags(piBlock, iIndex); + + // Is the type the same? + if(((iflags & Flags_IsDir) == Flags_IsDir) + != ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir)) + { + // Entry is of wrong type + BOX_WARNING("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " references object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " which has a different type than expected."); + badEntry = true; + } + else + { + // Check that the entry is not already contained. + if(iflags & Flags_IsContained) + { + BOX_WARNING("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " references object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " which is already contained."); + badEntry = true; + } + else + { + // Not already contained -- mark as contained + SetFlags(piBlock, iIndex, iflags | Flags_IsContained); + + // Check that the container ID of the object is correct + if(piBlock->mContainer[iIndex] != pblock->mID[e]) + { + // Needs fixing... + if(iflags & Flags_IsDir) + { + // Add to will fix later list + BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " has wrong container ID."); + mDirsWithWrongContainerID.push_back(en->GetObjectID()); + } + else + { + // This is OK for files, they might move + BOX_WARNING("File ID " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " has different container ID, probably moved"); + } + + // Fix entry for now + piBlock->mContainer[iIndex] = pblock->mID[e]; + } + } + } + + // Check the object size, if it's OK and a file + if(!badEntry && !((iflags & Flags_IsDir) == Flags_IsDir)) + { + if(en->GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[iIndex]) + { + // Correct + en->SetSizeInBlocks(piBlock->mObjectSizeInBlocks[iIndex]); + // Mark as changed + isModified = true; + // Tell user + BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " has wrong size for object " << BOX_FORMAT_OBJECTID(en->GetObjectID())); + } + } + } + else + { + // Item can't be found. Is it a directory? + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) + { + // Store the directory for later attention + mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e]; + } + else + { + // Just remove the entry + badEntry = true; + BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist."); + } + } + + // Is this entry worth keeping? + if(badEntry) + { + toDelete.push_back(en->GetObjectID()); + } + else + { + // Add to sizes? + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) + { + mBlocksInOldFiles += en->GetSizeInBlocks(); + } + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) + { + mBlocksInDeletedFiles += en->GetSizeInBlocks(); + } + } + } + + if(toDelete.size() > 0) + { + // Delete entries from directory + for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) + { + dir.DeleteEntry(*d); + } + + // Mark as modified + isModified = true; + + // Check the directory again, now that entries have been removed + dir.CheckAndFix(); + + // Errors found + ++mNumberErrorsFound; + } + + if(isModified && mFixErrors) + { + BOX_WARNING("Fixing directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e])); + + // Save back to disc + RaidFileWrite fixed(mDiscSetNumber, filename); + fixed.Open(true /* allow overwriting */); + dir.WriteToStream(fixed); + // Commit it + fixed.Commit(true /* convert to raid representation now */); + } + } + } + } + +} + +======= int32_t iIndex; IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); @@ -951,4 +1254,5 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, return !badEntry; } +>>>>>>> 0.12 diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h index 178a873a..cddd8a4d 100644 --- a/lib/backupstore/BackupStoreCheck.h +++ b/lib/backupstore/BackupStoreCheck.h @@ -16,8 +16,11 @@ #include <set> #include "NamedLock.h" +<<<<<<< HEAD +======= #include "BackupStoreDirectory.h" +>>>>>>> 0.12 class IOStream; class BackupStoreFilename; @@ -28,8 +31,14 @@ The following problems can be fixed: * Spurious files deleted * Corrupted files deleted * Root ID as file, deleted +<<<<<<< HEAD + * Dirs with wrong object id inside, deleted + * Direcetory entries pointing to non-existant files, deleted + * Doubly references files have second reference deleted +======= * Dirs with wrong object id in header, deleted * Doubly referenced files have second reference deleted +>>>>>>> 0.12 * Wrong directory container IDs fixed * Missing root recreated * Reattach files which exist, but aren't referenced @@ -42,9 +51,13 @@ The following problems can be fixed: * Inside directories, - only one object per name has old version clear - IDs aren't duplicated +<<<<<<< HEAD + * Bad store info files regenerated +======= - entries pointing to non-existant files are deleted - patches depending on non-existent objects are deleted * Bad store info and refcount files regenerated +>>>>>>> 0.12 * Bad sizes of files in directories fixed */ @@ -85,10 +98,13 @@ public: void Check(); bool ErrorsFound() {return mNumberErrorsFound > 0;} +<<<<<<< HEAD +======= inline int64_t GetNumErrorsFound() { return mNumberErrorsFound; } +>>>>>>> 0.12 private: enum @@ -127,9 +143,12 @@ private: int64_t CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName); void CheckObjectsDir(int64_t StartID); bool CheckAndAddObject(int64_t ObjectID, const std::string &rFilename); +<<<<<<< HEAD +======= bool CheckDirectory(BackupStoreDirectory& dir); bool CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified); +>>>>>>> 0.12 int64_t CheckFile(int64_t ObjectID, IOStream &rStream); int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); @@ -166,12 +185,19 @@ private: #else #define DUMP_OBJECT_INFO #endif +<<<<<<< HEAD + +======= +>>>>>>> 0.12 private: std::string mStoreRoot; int mDiscSetNumber; int32_t mAccountID; +<<<<<<< HEAD +======= std::string mAccountName; +>>>>>>> 0.12 bool mFixErrors; bool mQuiet; @@ -190,8 +216,12 @@ private: // List of stuff to fix std::vector<BackupStoreCheck_ID_t> mDirsWithWrongContainerID; // This is a map of lost dir ID -> existing dir ID +<<<<<<< HEAD + std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs; +======= std::map<BackupStoreCheck_ID_t, BackupStoreCheck_ID_t> mDirsWhichContainLostDirs; +>>>>>>> 0.12 // Set of extra directories added std::set<BackupStoreCheck_ID_t> mDirsAdded; @@ -202,6 +232,11 @@ private: // Usage int64_t mBlocksUsed; +<<<<<<< HEAD + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; +======= int64_t mBlocksInCurrentFiles; int64_t mBlocksInOldFiles; int64_t mBlocksInDeletedFiles; @@ -210,6 +245,7 @@ private: int64_t mNumOldFiles; int64_t mNumDeletedFiles; int64_t mNumDirectories; +>>>>>>> 0.12 }; #endif // BACKUPSTORECHECK__H diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 90e21e7f..341ac524 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -12,6 +12,19 @@ #include <stdio.h> #include <string.h> +<<<<<<< HEAD +#include "BackupStoreCheck.h" +#include "StoreStructure.h" +#include "RaidFileRead.h" +#include "RaidFileWrite.h" +#include "autogen_BackupStoreException.h" +#include "BackupStoreObjectMagic.h" +#include "BackupStoreFile.h" +#include "BackupStoreFileWire.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreConstants.h" +#include "BackupStoreInfo.h" +======= #include "autogen_BackupStoreException.h" #include "BackupStoreCheck.h" #include "BackupStoreConstants.h" @@ -24,6 +37,7 @@ #include "RaidFileRead.h" #include "RaidFileWrite.h" #include "StoreStructure.h" +>>>>>>> 0.12 #include "MemLeakFindOn.h" @@ -138,7 +152,11 @@ void BackupStoreCheck::CheckUnattachedObjects() if((flags & Flags_IsContained) == 0) { // Unattached object... +<<<<<<< HEAD + BOX_WARNING("Object " << +======= BOX_ERROR("Object " << +>>>>>>> 0.12 BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached."); ++mNumberErrorsFound; @@ -158,7 +176,10 @@ void BackupStoreCheck::CheckUnattachedObjects() int64_t diffFromObjectID = 0; std::string filename; StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */); +<<<<<<< HEAD +======= +>>>>>>> 0.12 // The easiest way to do this is to verify it again. Not such a bad penalty, because // this really shouldn't be done very often. { @@ -384,7 +405,11 @@ void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, } // Add a new entry in an appropriate place +<<<<<<< HEAD + mDirectory.AddUnattactedObject(objectStoreFilename, modTime, +======= mDirectory.AddUnattachedObject(objectStoreFilename, modTime, +>>>>>>> 0.12 ObjectID, sizeInBlocks, IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); } @@ -575,6 +600,19 @@ void BackupStoreCheck::FixDirsWithLostDirs() void BackupStoreCheck::WriteNewStoreInfo() { // Attempt to load the existing store info file +<<<<<<< HEAD + std::auto_ptr<BackupStoreInfo> poldInfo; + try + { + poldInfo.reset(BackupStoreInfo::Load(mAccountID, mStoreRoot, mDiscSetNumber, true /* read only */).release()); + } + catch(...) + { + BOX_WARNING("Load of existing store info failed, regenerating."); + ++mNumberErrorsFound; + } + +======= std::auto_ptr<BackupStoreInfo> pOldInfo; try { @@ -592,14 +630,19 @@ void BackupStoreCheck::WriteNewStoreInfo() "deleted files: " << mNumDeletedFiles << "), " "directories: " << mNumDirectories); +>>>>>>> 0.12 // Minimum soft and hard limits int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024; int64_t minHard = ((minSoft * 11) / 10) + 1024; // Need to do anything? +<<<<<<< HEAD + if(poldInfo.get() != 0 && mNumberErrorsFound == 0 && poldInfo->GetAccountID() == mAccountID) +======= if(pOldInfo.get() != 0 && mNumberErrorsFound == 0 && pOldInfo->GetAccountID() == mAccountID) +>>>>>>> 0.12 { // Leave the store info as it is, no need to alter it because nothing really changed, // and the only essential thing was that the account ID was correct, which is was. @@ -611,6 +654,23 @@ void BackupStoreCheck::WriteNewStoreInfo() // Work out the new limits int64_t softLimit = minSoft; int64_t hardLimit = minHard; +<<<<<<< HEAD + if(poldInfo.get() != 0 && poldInfo->GetBlocksSoftLimit() > minSoft) + { + softLimit = poldInfo->GetBlocksSoftLimit(); + } + else + { + BOX_WARNING("Soft limit for account changed to ensure housekeeping doesn't delete files on next run."); + } + if(poldInfo.get() != 0 && poldInfo->GetBlocksHardLimit() > minHard) + { + hardLimit = poldInfo->GetBlocksHardLimit(); + } + else + { + BOX_WARNING("Hard limit for account changed to ensure housekeeping doesn't delete files on next run."); +======= if(pOldInfo.get() != 0 && pOldInfo->GetBlocksSoftLimit() > minSoft) { softLimit = pOldInfo->GetBlocksSoftLimit(); @@ -628,6 +688,7 @@ void BackupStoreCheck::WriteNewStoreInfo() { BOX_WARNING("Hard limit for account changed to ensure " "housekeeping doesn't delete files on next run."); +>>>>>>> 0.12 } // Object ID @@ -638,6 +699,10 @@ void BackupStoreCheck::WriteNewStoreInfo() } // Build a new store info +<<<<<<< HEAD + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration( + mAccountID, +======= std::auto_ptr<MemBlockStream> extra_data; if(pOldInfo.get()) { @@ -650,15 +715,22 @@ void BackupStoreCheck::WriteNewStoreInfo() std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::CreateForRegeneration( mAccountID, mAccountName, +>>>>>>> 0.12 mStoreRoot, mDiscSetNumber, lastObjID, mBlocksUsed, +<<<<<<< HEAD +======= mBlocksInCurrentFiles, +>>>>>>> 0.12 mBlocksInOldFiles, mBlocksInDeletedFiles, mBlocksInDirectories, softLimit, +<<<<<<< HEAD + hardLimit)); +======= hardLimit, (pOldInfo.get() ? pOldInfo->IsAccountEnabled() : true), *extra_data)); @@ -671,6 +743,7 @@ void BackupStoreCheck::WriteNewStoreInfo() { mNumberErrorsFound += info->ReportChangesTo(*pOldInfo); } +>>>>>>> 0.12 // Save to disc? if(mFixErrors) @@ -697,12 +770,16 @@ bool BackupStoreDirectory::CheckAndFix() bool changed = false; // Check that if a file depends on a new version, that version is in this directory +<<<<<<< HEAD + { +======= bool restart; do { restart = false; +>>>>>>> 0.12 std::vector<Entry*>::iterator i(mEntries.begin()); for(; i != mEntries.end(); ++i) { @@ -713,7 +790,11 @@ bool BackupStoreDirectory::CheckAndFix() if(newerEn == 0) { // Depends on something, but it isn't there. +<<<<<<< HEAD + BOX_TRACE("Entry id " << FMT_i << +======= BOX_WARNING("Entry id " << FMT_i << +>>>>>>> 0.12 " removed because depends " "on newer version " << FMT_OID(dependsNewer) << @@ -723,12 +804,20 @@ bool BackupStoreDirectory::CheckAndFix() delete *i; mEntries.erase(i); +<<<<<<< HEAD + // Start again at the beginning of the vector, the iterator is now invalid + i = mEntries.begin(); + + // Mark as changed + changed = true; +======= // Mark as changed changed = true; // Start again at the beginning of the vector, the iterator is now invalid restart = true; break; +>>>>>>> 0.12 } else { @@ -750,7 +839,10 @@ bool BackupStoreDirectory::CheckAndFix() } } } +<<<<<<< HEAD +======= while(restart); +>>>>>>> 0.12 // Check that if a file has a dependency marked, it exists, and remove it if it doesn't { @@ -875,7 +967,11 @@ bool BackupStoreDirectory::CheckAndFix() // erase the thing from the list Entry *pentry = (*i); mEntries.erase(i); +<<<<<<< HEAD + +======= +>>>>>>> 0.12 // And delete the entry object delete pentry; @@ -893,12 +989,20 @@ bool BackupStoreDirectory::CheckAndFix() // -------------------------------------------------------------------------- // // Function +<<<<<<< HEAD +// Name: BackupStoreDirectory::AddUnattactedObject(...) +======= // Name: BackupStoreDirectory::AddUnattachedObject(...) +>>>>>>> 0.12 // Purpose: Adds an object which is currently unattached. Assume that CheckAndFix() will be called afterwards. // Created: 22/4/04 // // -------------------------------------------------------------------------- +<<<<<<< HEAD +void BackupStoreDirectory::AddUnattactedObject(const BackupStoreFilename &rName, +======= void BackupStoreDirectory::AddUnattachedObject(const BackupStoreFilename &rName, +>>>>>>> 0.12 box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags) { Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp index ec606d52..c89b5082 100644 --- a/lib/backupstore/BackupStoreCheckData.cpp +++ b/lib/backupstore/BackupStoreCheckData.cpp @@ -65,11 +65,23 @@ void BackupStoreCheck::AddID(BackupStoreCheck_ID_t ID, if(mpInfoLastBlock == 0 || mInfoLastBlockEntries >= BACKUPSTORECHECK_BLOCK_SIZE) { // No. Allocate a new one +<<<<<<< HEAD + IDBlock *pblk = (IDBlock*)::malloc(sizeof(IDBlock)); +======= IDBlock *pblk = (IDBlock*)calloc(1, sizeof(IDBlock)); +>>>>>>> 0.12 if(pblk == 0) { throw std::bad_alloc(); } +<<<<<<< HEAD + // Zero all the flags entries + for(int z = 0; z < (BACKUPSTORECHECK_BLOCK_SIZE * Flags__NumFlags / Flags__NumItemsPerEntry); ++z) + { + pblk->mFlags[z] = 0; + } +======= +>>>>>>> 0.12 // Store in map mInfo[ID] = pblk; // Allocated and stored OK, setup for use @@ -136,8 +148,13 @@ BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, pblock = ib->second; } +<<<<<<< HEAD + ASSERT(pblock != 0); + if(pblock == 0) return 0; +======= if(pblock == 0) return 0; ASSERT(pblock != 0); +>>>>>>> 0.12 // How many entries are there in the block int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp index 921adfa4..c2344634 100644 --- a/lib/backupstore/BackupStoreConfigVerify.cpp +++ b/lib/backupstore/BackupStoreConfigVerify.cpp @@ -38,7 +38,17 @@ static const ConfigurationVerifyKey verifyrootkeys[] = ConfigTest_Exists | ConfigTest_IsInt), ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), // make value "yes" to enable in config file +<<<<<<< HEAD + + #ifdef WIN32 + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry) + #else + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry, + BOX_FILE_RAIDFILE_DEFAULT_CONFIG) + #endif +======= ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry) +>>>>>>> 0.12 }; const ConfigurationVerify BackupConfigFileVerify = diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index b6714709..c58ae99a 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -11,7 +11,10 @@ #include <algorithm> +<<<<<<< HEAD +======= #include "Archive.h" +>>>>>>> 0.12 #include "BackupStoreInfo.h" #include "BackupStoreException.h" #include "RaidFileWrite.h" @@ -19,12 +22,57 @@ #include "MemLeakFindOn.h" +<<<<<<< HEAD +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +// ****************** +// make sure the defaults in CreateNew are modified! +// ****************** +typedef struct +{ + int32_t mMagicValue; // also the version number + int32_t mAccountID; + int64_t mClientStoreMarker; + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mBlocksSoftLimit; + int64_t mBlocksHardLimit; + uint32_t mCurrentMarkNumber; + uint32_t mOptionsPresent; // bit mask of optional elements present + int64_t mNumberDeletedDirectories; + // Then loads of int64_t IDs for the deleted directories +} info_StreamFormat; + +#define INFO_MAGIC_VALUE 0x34832476 + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +======= +>>>>>>> 0.12 #ifdef BOX_RELEASE_BUILD #define NUM_DELETED_DIRS_BLOCK 256 #else #define NUM_DELETED_DIRS_BLOCK 2 #endif +<<<<<<< HEAD +#define INFO_FILENAME "info" + +======= +>>>>>>> 0.12 // -------------------------------------------------------------------------- // // Function @@ -41,6 +89,10 @@ BackupStoreInfo::BackupStoreInfo() mClientStoreMarker(0), mLastObjectIDUsed(-1), mBlocksUsed(0), +<<<<<<< HEAD + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0) +======= mBlocksInCurrentFiles(0), mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), @@ -50,6 +102,7 @@ BackupStoreInfo::BackupStoreInfo() mNumDeletedFiles(0), mNumDirectories(0), mAccountEnabled(true) +>>>>>>> 0.12 { } @@ -75,6 +128,41 @@ BackupStoreInfo::~BackupStoreInfo() // -------------------------------------------------------------------------- void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit) { +<<<<<<< HEAD + // Initial header (is entire file) + info_StreamFormat hdr = { + htonl(INFO_MAGIC_VALUE), // mMagicValue + htonl(AccountID), // mAccountID + 0, // mClientStoreMarker + box_hton64(1), // mLastObjectIDUsed (which is the root directory) + 0, // mBlocksUsed + 0, // mBlocksInOldFiles + 0, // mBlocksInDeletedFiles + 0, // mBlocksInDirectories + box_hton64(BlockSoftLimit), // mBlocksSoftLimit + box_hton64(BlockHardLimit), // mBlocksHardLimit + 0, // mCurrentMarkNumber + 0, // mOptionsPresent + 0 // mNumberDeletedDirectories + }; + + // Generate the filename + ASSERT(rRootDir[rRootDir.size() - 1] == '/' || + rRootDir[rRootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + std::string fn(rRootDir + INFO_FILENAME); + + // Open the file for writing + RaidFileWrite rf(DiscSet, fn); + rf.Open(false); // no overwriting, as this is a new file + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Done. +======= BackupStoreInfo info; info.mAccountID = AccountID; info.mDiscSet = DiscSet; @@ -90,11 +178,40 @@ void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, info.mExtraData.SetForReading(); // extra data is empty in this case info.Save(false); +>>>>>>> 0.12 } // -------------------------------------------------------------------------- // // Function +<<<<<<< HEAD +// Name: BackupStoreInfo::Load(int32_t, const std::string &, int, bool) +// Purpose: Loads the info from disc, given the root information. Can be marked as read only. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID) +{ + // Generate the filename + std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); + + // Open the file for reading (passing on optional request for revision ID) + std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID)); + + // Read in a header + info_StreamFormat hdr; + if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Check it + if(ntohl(hdr.mMagicValue) != INFO_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != AccountID) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + +======= // Name: BackupStoreInfo::Load(int32_t, const std::string &, // int, bool) // Purpose: Loads the info from disc, given the root @@ -138,6 +255,7 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, fn, BackupStoreException, BadStoreInfoOnLoad); } +>>>>>>> 0.12 // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); @@ -146,6 +264,23 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, info->mDiscSet = DiscSet; info->mFilename = fn; info->mReadOnly = ReadOnly; +<<<<<<< HEAD + + // Insert info from file + info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker); + info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed); + info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); + info->mBlocksInOldFiles = box_ntoh64(hdr.mBlocksInOldFiles); + info->mBlocksInDeletedFiles = box_ntoh64(hdr.mBlocksInDeletedFiles); + info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories); + info->mBlocksSoftLimit = box_ntoh64(hdr.mBlocksSoftLimit); + info->mBlocksHardLimit = box_ntoh64(hdr.mBlocksHardLimit); + + // Load up array of deleted objects + int64_t numDelObj = box_ntoh64(hdr.mNumberDeletedDirectories); + + // Then load them in +======= int64_t numDelObj = 0; if (v1) @@ -213,6 +348,7 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, } // Then load the list of deleted directories +>>>>>>> 0.12 if(numDelObj > 0) { int64_t objs[NUM_DELETED_DIRS_BLOCK]; @@ -244,6 +380,8 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) } +<<<<<<< HEAD +======= if(v2) { @@ -264,6 +402,7 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, rf->CopyStreamTo(info->mExtraData); } info->mExtraData.SetForReading(); +>>>>>>> 0.12 // return it to caller return info; @@ -278,6 +417,14 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, // Created: 23/4/04 // // -------------------------------------------------------------------------- +<<<<<<< HEAD +std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, + int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, + int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit) +{ + // Generate the filename + std::string fn(rRootDir + DIRECTORY_SEPARATOR INFO_FILENAME); +======= std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( int32_t AccountID, const std::string& rAccountName, const std::string &rRootDir, int DiscSet, @@ -289,12 +436,23 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( { // Generate the filename std::string fn(rRootDir + INFO_FILENAME); +>>>>>>> 0.12 // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); // Put in basic info info->mAccountID = AccountID; +<<<<<<< HEAD + info->mDiscSet = DiscSet; + info->mFilename = fn; + info->mReadOnly = false; + + // Insert info starting info + info->mClientStoreMarker = 0; + info->mLastObjectIDUsed = LastObjectID; + info->mBlocksUsed = BlocksUsed; +======= info->mAccountName = rAccountName; info->mDiscSet = DiscSet; info->mFilename = fn; @@ -305,16 +463,21 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( info->mLastObjectIDUsed = LastObjectID; info->mBlocksUsed = BlocksUsed; info->mBlocksInCurrentFiles = BlocksInCurrentFiles; +>>>>>>> 0.12 info->mBlocksInOldFiles = BlocksInOldFiles; info->mBlocksInDeletedFiles = BlocksInDeletedFiles; info->mBlocksInDirectories = BlocksInDirectories; info->mBlocksSoftLimit = BlockSoftLimit; info->mBlocksHardLimit = BlockHardLimit; +<<<<<<< HEAD + +======= info->mAccountEnabled = AccountEnabled; ExtraData.CopyStreamTo(info->mExtraData); info->mExtraData.SetForReading(); +>>>>>>> 0.12 // return it to caller return info; } @@ -323,12 +486,20 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( // -------------------------------------------------------------------------- // // Function +<<<<<<< HEAD +// Name: BackupStoreInfo::Save() +======= // Name: BackupStoreInfo::Save(bool allowOverwrite) +>>>>>>> 0.12 // Purpose: Save modified info back to disc // Created: 2003/08/28 // // -------------------------------------------------------------------------- +<<<<<<< HEAD +void BackupStoreInfo::Save() +======= void BackupStoreInfo::Save(bool allowOverwrite) +>>>>>>> 0.12 { // Make sure we're initialised (although should never come to this) if(mFilename.empty() || mAccountID == -1 || mDiscSet == -1) @@ -344,6 +515,29 @@ void BackupStoreInfo::Save(bool allowOverwrite) // Then... open a write file RaidFileWrite rf(mDiscSet, mFilename); +<<<<<<< HEAD + rf.Open(true); // allow overwriting + + // Make header + info_StreamFormat hdr; + hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); + hdr.mAccountID = htonl(mAccountID); + hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); + hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); + hdr.mBlocksUsed = box_hton64(mBlocksUsed); + hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); + hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); + hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); + hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); + hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); + hdr.mCurrentMarkNumber = 0; + hdr.mOptionsPresent = 0; + hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); + + // Write header + rf.Write(&hdr, sizeof(hdr)); + +======= rf.Open(allowOverwrite); // Make header @@ -370,6 +564,7 @@ void BackupStoreInfo::Save(bool allowOverwrite) int64_t numDelObj = mDeletedDirectories.size(); archive.Write(numDelObj); +>>>>>>> 0.12 // Write the deleted object list if(mDeletedDirectories.size() > 0) { @@ -397,12 +592,15 @@ void BackupStoreInfo::Save(bool allowOverwrite) tosave -= b; } } +<<<<<<< HEAD +======= archive.Write(mAccountEnabled); mExtraData.Seek(0, IOStream::SeekType_Absolute); mExtraData.CopyStreamTo(rf); mExtraData.Seek(0, IOStream::SeekType_Absolute); +>>>>>>> 0.12 // Commit it to disc, converting it to RAID now rf.Commit(true); @@ -411,6 +609,9 @@ void BackupStoreInfo::Save(bool allowOverwrite) mIsModified = false; } +<<<<<<< HEAD + +======= int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) { int numChanges = 0; @@ -460,6 +661,7 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) \ field += delta; \ mIsModified = true; +>>>>>>> 0.12 // -------------------------------------------------------------------------- // @@ -471,6 +673,20 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) // -------------------------------------------------------------------------- void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta) { +<<<<<<< HEAD + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksUsed + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksUsed += Delta; + + mIsModified = true; +======= APPLY_DELTA(mBlocksUsed, Delta); } @@ -486,6 +702,7 @@ void BackupStoreInfo::ChangeBlocksUsed(int64_t Delta) void BackupStoreInfo::ChangeBlocksInCurrentFiles(int64_t Delta) { APPLY_DELTA(mBlocksInCurrentFiles, Delta); +>>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -498,7 +715,22 @@ void BackupStoreInfo::ChangeBlocksInCurrentFiles(int64_t Delta) // -------------------------------------------------------------------------- void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta) { +<<<<<<< HEAD + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInOldFiles + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInOldFiles += Delta; + + mIsModified = true; +======= APPLY_DELTA(mBlocksInOldFiles, Delta); +>>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -511,7 +743,22 @@ void BackupStoreInfo::ChangeBlocksInOldFiles(int64_t Delta) // -------------------------------------------------------------------------- void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta) { +<<<<<<< HEAD + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInDeletedFiles + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInDeletedFiles += Delta; + + mIsModified = true; +======= APPLY_DELTA(mBlocksInDeletedFiles, Delta); +>>>>>>> 0.12 } // -------------------------------------------------------------------------- @@ -524,6 +771,22 @@ void BackupStoreInfo::ChangeBlocksInDeletedFiles(int64_t Delta) // -------------------------------------------------------------------------- void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta) { +<<<<<<< HEAD + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + if((mBlocksInDirectories + Delta) < 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoBlockDeltaMakesValueNegative) + } + + mBlocksInDirectories += Delta; + + mIsModified = true; +} + +======= APPLY_DELTA(mBlocksInDirectories, Delta); } @@ -546,6 +809,7 @@ void BackupStoreInfo::AdjustNumDirectories(int64_t increase) { APPLY_DELTA(mNumDirectories, increase); } +>>>>>>> 0.12 // -------------------------------------------------------------------------- // @@ -689,6 +953,8 @@ void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker) } +<<<<<<< HEAD +======= // -------------------------------------------------------------------------- // // Function @@ -708,4 +974,5 @@ void BackupStoreInfo::SetAccountName(const std::string& rName) mIsModified = true; } +>>>>>>> 0.12 diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h index 752cc44a..db2a62ef 100644 --- a/lib/backupstore/BackupStoreInfo.h +++ b/lib/backupstore/BackupStoreInfo.h @@ -14,6 +14,10 @@ #include <string> #include <vector> +<<<<<<< HEAD +class BackupStoreCheck; + +======= #include "CollectInBufferStream.h" class BackupStoreCheck; @@ -59,6 +63,7 @@ END_STRUCTURE_PACKING_FOR_WIRE #define INFO_FILENAME "info" +>>>>>>> 0.12 // -------------------------------------------------------------------------- // // Class @@ -89,19 +94,33 @@ public: bool IsModified() const {return mIsModified;} // Save modified infomation back to store +<<<<<<< HEAD + void Save(); +======= void Save(bool allowOverwrite = true); +>>>>>>> 0.12 // Data access functions int32_t GetAccountID() const {return mAccountID;} int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;} int64_t GetBlocksUsed() const {return mBlocksUsed;} +<<<<<<< HEAD +======= int64_t GetBlocksInCurrentFiles() const {return mBlocksInCurrentFiles;} +>>>>>>> 0.12 int64_t GetBlocksInOldFiles() const {return mBlocksInOldFiles;} int64_t GetBlocksInDeletedFiles() const {return mBlocksInDeletedFiles;} int64_t GetBlocksInDirectories() const {return mBlocksInDirectories;} const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;} int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;} int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;} +<<<<<<< HEAD + bool IsReadOnly() const {return mReadOnly;} + int GetDiscSetNumber() const {return mDiscSet;} + + // Data modification functions + void ChangeBlocksUsed(int64_t Delta); +======= int64_t GetNumFiles() const {return mNumFiles;} int64_t GetNumOldFiles() const {return mNumOldFiles;} int64_t GetNumDeletedFiles() const {return mNumDeletedFiles;} @@ -115,6 +134,7 @@ public: // Data modification functions void ChangeBlocksUsed(int64_t Delta); void ChangeBlocksInCurrentFiles(int64_t Delta); +>>>>>>> 0.12 void ChangeBlocksInOldFiles(int64_t Delta); void ChangeBlocksInDeletedFiles(int64_t Delta); void ChangeBlocksInDirectories(int64_t Delta); @@ -122,15 +142,31 @@ public: void AddDeletedDirectory(int64_t DirID); void RemovedDeletedDirectory(int64_t DirID); void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit); +<<<<<<< HEAD +======= void AdjustNumFiles(int64_t increase); void AdjustNumOldFiles(int64_t increase); void AdjustNumDeletedFiles(int64_t increase); void AdjustNumDirectories(int64_t increase); +>>>>>>> 0.12 // Object IDs int64_t AllocateObjectID(); // Client marker set and get +<<<<<<< HEAD + int64_t GetClientStoreMarker() {return mClientStoreMarker;} + void SetClientStoreMarker(int64_t ClientStoreMarker); + +private: + static std::auto_ptr<BackupStoreInfo> CreateForRegeneration(int32_t AccountID, const std::string &rRootDir, + int DiscSet, int64_t LastObjectID, int64_t BlocksUsed, int64_t BlocksInOldFiles, + int64_t BlocksInDeletedFiles, int64_t BlocksInDirectories, int64_t BlockSoftLimit, int64_t BlockHardLimit); + +private: + // Location information + int32_t mAccountID; +======= int64_t GetClientStoreMarker() const {return mClientStoreMarker;} void SetClientStoreMarker(int64_t ClientStoreMarker); @@ -158,6 +194,7 @@ private: // they now define the sizes of fields on disk (via Archive). int32_t mAccountID; std::string mAccountName; +>>>>>>> 0.12 int mDiscSet; std::string mFilename; bool mReadOnly; @@ -169,12 +206,21 @@ private: // Account information int64_t mLastObjectIDUsed; int64_t mBlocksUsed; +<<<<<<< HEAD +======= int64_t mBlocksInCurrentFiles; +>>>>>>> 0.12 int64_t mBlocksInOldFiles; int64_t mBlocksInDeletedFiles; int64_t mBlocksInDirectories; int64_t mBlocksSoftLimit; int64_t mBlocksHardLimit; +<<<<<<< HEAD + std::vector<int64_t> mDeletedDirectories; +}; + + +======= int64_t mNumFiles; int64_t mNumOldFiles; int64_t mNumDeletedFiles; @@ -184,6 +230,7 @@ private: CollectInBufferStream mExtraData; }; +>>>>>>> 0.12 #endif // BACKUPSTOREINFO__H diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp index 26f9acca..642e260c 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.cpp +++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp @@ -61,7 +61,11 @@ std::string BackupStoreRefCountDatabase::GetFilename(const ASSERT(RootDir[RootDir.size() - 1] == '/' || RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); +<<<<<<< HEAD + std::string fn(RootDir + "refcount.db"); +======= std::string fn(RootDir + REFCOUNT_FILENAME ".db"); +>>>>>>> 0.12 RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); @@ -91,9 +95,15 @@ void BackupStoreRefCountDatabase::Create(const // Open the file for writing if (FileExists(Filename) && !AllowOverwrite) { +<<<<<<< HEAD + BOX_ERROR("Attempted to overwrite refcount database file: " << + Filename); + THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile); +======= THROW_FILE_ERROR("Failed to overwrite refcount database: " "not allowed here", Filename, RaidFileException, CannotOverwriteExistingFile); +>>>>>>> 0.12 } int flags = O_CREAT | O_BINARY | O_RDWR; @@ -134,18 +144,26 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( refcount_StreamFormat hdr; if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) { +<<<<<<< HEAD + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) +======= THROW_FILE_ERROR("Failed to read refcount database: " "short read", filename, BackupStoreException, CouldNotLoadStoreInfo); +>>>>>>> 0.12 } // Check it if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE || (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) { +<<<<<<< HEAD + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) +======= THROW_FILE_ERROR("Failed to read refcount database: " "bad magic number", filename, BackupStoreException, BadStoreInfoOnLoad); +>>>>>>> 0.12 } // Make new object @@ -162,6 +180,92 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( // -------------------------------------------------------------------------- // // Function +<<<<<<< HEAD +// Name: BackupStoreRefCountDatabase::Save() +// Purpose: Save modified info back to disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +/* +void BackupStoreRefCountDatabase::Save() +{ + // Make sure we're initialised (although should never come to this) + if(mFilename.empty() || mAccount.GetID() == 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Can we do this? + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Then... open a write file + RaidFileWrite rf(mAccount.GetDiscSet(), mFilename); + rf.Open(true); // allow overwriting + + // Make header + info_StreamFormat hdr; + hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); + hdr.mAccountID = htonl(mAccountID); + hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); + hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); + hdr.mBlocksUsed = box_hton64(mBlocksUsed); + hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); + hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); + hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); + hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); + hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); + hdr.mCurrentMarkNumber = 0; + hdr.mOptionsPresent = 0; + hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Write the deleted object list + if(mDeletedDirectories.size() > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int tosave = mDeletedDirectories.size(); + std::vector<int64_t>::iterator i(mDeletedDirectories.begin()); + while(tosave > 0) + { + // How many in this one? + int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); + + // Add them + for(int t = 0; t < b; ++t) + { + ASSERT(i != mDeletedDirectories.end()); + objs[t] = box_hton64((*i)); + i++; + } + + // Write + rf.Write(objs, b * sizeof(int64_t)); + + // Number saved + tosave -= b; + } + } + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Mark is as not modified + mIsModified = false; +} +*/ + + +// -------------------------------------------------------------------------- +// +// Function +======= +>>>>>>> 0.12 // Name: BackupStoreRefCountDatabase::GetRefCount(int64_t // ObjectID) // Purpose: Get the number of references to the specified object @@ -176,10 +280,17 @@ BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const if (GetSize() < offset + GetEntrySize()) { +<<<<<<< HEAD + BOX_ERROR("attempted read of unknown refcount for object " << + BOX_FORMAT_OBJECTID(ObjectID)); + THROW_EXCEPTION(BackupStoreException, + UnknownObjectRefCountRequested); +======= THROW_FILE_ERROR("Failed to read refcount database: " "attempted read of unknown refcount for object " << BOX_FORMAT_OBJECTID(ObjectID), mFilename, BackupStoreException, UnknownObjectRefCountRequested); +>>>>>>> 0.12 } mapDatabaseFile->Seek(offset, SEEK_SET); @@ -188,9 +299,15 @@ BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) != sizeof(refcount)) { +<<<<<<< HEAD + BOX_LOG_SYS_ERROR("short read on refcount database: " << + mFilename); + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo); +======= THROW_FILE_ERROR("Failed to read refcount database: " "short read at offset " << offset, mFilename, BackupStoreException, CouldNotLoadStoreInfo); +>>>>>>> 0.12 } return ntohl(refcount); |