summaryrefslogtreecommitdiff
path: root/lib/backupstore/BackupStoreCheck.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/backupstore/BackupStoreCheck.cpp')
-rw-r--r--lib/backupstore/BackupStoreCheck.cpp383
1 files changed, 225 insertions, 158 deletions
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
index f2302337..b53ebf6d 100644
--- a/lib/backupstore/BackupStoreCheck.cpp
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -17,11 +17,13 @@
#endif
#include "autogen_BackupStoreException.h"
+#include "BackupStoreAccountDatabase.h"
#include "BackupStoreCheck.h"
#include "BackupStoreConstants.h"
#include "BackupStoreDirectory.h"
#include "BackupStoreFile.h"
#include "BackupStoreObjectMagic.h"
+#include "BackupStoreRefCountDatabase.h"
#include "RaidFileController.h"
#include "RaidFileException.h"
#include "RaidFileRead.h"
@@ -58,7 +60,7 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum
mBlocksInOldFiles(0),
mBlocksInDeletedFiles(0),
mBlocksInDirectories(0),
- mNumFiles(0),
+ mNumCurrentFiles(0),
mNumOldFiles(0),
mNumDeletedFiles(0),
mNumDirectories(0)
@@ -78,6 +80,24 @@ BackupStoreCheck::~BackupStoreCheck()
{
// Clean up
FreeInfo();
+
+ // Avoid an exception if we forget to discard mapNewRefs
+ if (mapNewRefs.get())
+ {
+ // Discard() can throw exception, but destructors aren't supposed to do that, so
+ // just catch and log them.
+ try
+ {
+ mapNewRefs->Discard();
+ }
+ catch(BoxException &e)
+ {
+ BOX_ERROR("Error while destroying BackupStoreCheck: discarding "
+ "the refcount database threw an exception: " << e.what());
+ }
+
+ mapNewRefs.reset();
+ }
}
@@ -92,15 +112,21 @@ BackupStoreCheck::~BackupStoreCheck()
// --------------------------------------------------------------------------
void BackupStoreCheck::Check()
{
- std::string writeLockFilename;
- StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename);
- ASSERT(FileExists(writeLockFilename));
+ if(mFixErrors)
+ {
+ std::string writeLockFilename;
+ StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename);
+ ASSERT(FileExists(writeLockFilename));
+ }
if(!mQuiet && mFixErrors)
{
- BOX_NOTICE("Will fix errors encountered during checking.");
+ BOX_INFO("Will fix errors encountered during checking.");
}
+ BackupStoreAccountDatabase::Entry account(mAccountID, mDiscSetNumber);
+ mapNewRefs = BackupStoreRefCountDatabase::Create(account);
+
// Phase 1, check objects
if(!mQuiet)
{
@@ -109,14 +135,14 @@ void BackupStoreCheck::Check()
BOX_INFO("Phase 1, check objects...");
}
CheckObjects();
-
+
// Phase 2, check directories
if(!mQuiet)
{
BOX_INFO("Phase 2, check directories...");
}
CheckDirectories();
-
+
// Phase 3, check root
if(!mQuiet)
{
@@ -138,16 +164,38 @@ void BackupStoreCheck::Check()
}
FixDirsWithWrongContainerID();
FixDirsWithLostDirs();
-
+
// Phase 6, regenerate store info
if(!mQuiet)
{
BOX_INFO("Phase 6, regenerate store info...");
}
WriteNewStoreInfo();
-
-// DUMP_OBJECT_INFO
-
+
+ try
+ {
+ std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs =
+ BackupStoreRefCountDatabase::Load(account, false);
+ mNumberErrorsFound += mapNewRefs->ReportChangesTo(*apOldRefs);
+ }
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database was missing or "
+ "corrupted, cannot check it for errors.");
+ mNumberErrorsFound++;
+ }
+
+ // force file to be saved and closed before releasing the lock below
+ if(mFixErrors)
+ {
+ mapNewRefs->Commit();
+ }
+ else
+ {
+ mapNewRefs->Discard();
+ }
+ mapNewRefs.reset();
+
if(mNumberErrorsFound > 0)
{
BOX_WARNING("Finished checking store account ID " <<
@@ -250,16 +298,18 @@ void BackupStoreCheck::CheckObjects()
{
// Make sure the starting root dir doesn't end with '/'.
std::string start(mStoreRoot);
- if(start.size() > 0 && start[start.size() - 1] == '/')
+ if(start.size() > 0 && (
+ start[start.size() - 1] == '/' ||
+ start[start.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR))
{
start.resize(start.size() - 1);
}
-
- maxDir = CheckObjectsScanDir(0, 1, mStoreRoot);
+
+ maxDir = CheckObjectsScanDir(0, 1, start);
BOX_TRACE("Max dir starting ID is " <<
BOX_FORMAT_OBJECTID(maxDir));
}
-
+
// Then go through and scan all the objects within those directories
for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH))
{
@@ -287,24 +337,25 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const
// If any of the directories is missing, create it.
RaidFileController &rcontroller(RaidFileController::GetController());
RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber));
-
+
if(!rdiscSet.IsNonRaidSet())
{
unsigned int numDiscs = rdiscSet.size();
-
+
for(unsigned int l = 0; l < numDiscs; ++l)
{
// build name
std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName);
- struct stat st;
+ EMU_STRUCT_STAT st;
- if(stat(dn.c_str(), &st) != 0 && errno == ENOENT)
+ if(EMU_STAT(dn.c_str(), &st) != 0 &&
+ errno == ENOENT)
{
if(mkdir(dn.c_str(), 0755) != 0)
{
THROW_SYS_FILE_ERROR("Failed to "
"create missing RaidFile "
- "directory", dn,
+ "directory", dn,
RaidFileException, OSError);
}
}
@@ -334,7 +385,7 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const
else
{
BOX_ERROR("Spurious or invalid directory " <<
- rDirName << DIRECTORY_SEPARATOR <<
+ rDirName << DIRECTORY_SEPARATOR <<
(*i) << " found, " <<
(mFixErrors?"deleting":"delete manually"));
++mNumberErrorsFound;
@@ -361,11 +412,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
std::string dirName;
StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */);
// Check expectations
- ASSERT(dirName.size() > 4 &&
+ ASSERT(dirName.size() > 4 &&
dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR);
// Remove the filename from it
dirName.resize(dirName.size() - 4); // four chars for "/o00"
-
+
// Check directory exists
if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName))
{
@@ -377,14 +428,14 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
std::vector<std::string> files;
RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName,
RaidFileRead::DirReadType_FilesOnly, files);
-
+
// Array of things present
bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)];
for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l)
{
idsPresent[l] = false;
}
-
+
// Parse each entry, building up a list of object IDs which are present in the dir.
// This is done so that whatever order is retured from the directory, objects are scanned
// in order.
@@ -405,7 +456,8 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
fileOK = false;
}
// info and refcount databases are OK in the root directory
- else if(*i == "info" || *i == "refcount.db")
+ else if(*i == "info" || *i == "refcount.db" ||
+ *i == "refcount.rdb" || *i == "refcount.rdbX")
{
fileOK = true;
}
@@ -413,11 +465,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
{
fileOK = false;
}
-
+
if(!fileOK)
{
// Unexpected or bad file, delete it
- BOX_ERROR("Spurious file " << dirName <<
+ BOX_ERROR("Spurious file " << dirName <<
DIRECTORY_SEPARATOR << (*i) << " found" <<
(mFixErrors?", deleting":""));
++mNumberErrorsFound;
@@ -428,7 +480,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
}
}
}
-
+
// Check all the objects found in this directory
for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i)
{
@@ -436,7 +488,8 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
{
// Check the object is OK, and add entry
char leaf[8];
- ::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i);
+ ::snprintf(leaf, sizeof(leaf),
+ DIRECTORY_SEPARATOR "o%02x", i);
if(!CheckAndAddObject(StartID | i, dirName + leaf))
{
// File was bad, delete it
@@ -480,7 +533,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
std::auto_ptr<RaidFileRead> file(
RaidFileRead::Open(mDiscSetNumber, rFilename));
size = file->GetDiscUsageInBlocks();
-
+
// Read in first four bytes -- don't have to worry about
// retrying if not all bytes read as is RaidFile
uint32_t signature;
@@ -491,7 +544,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
}
// Seek back to beginning
file->Seek(0, IOStream::SeekType_Absolute);
-
+
// Then... check depending on the type
switch(ntohl(signature))
{
@@ -519,7 +572,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
// Error caught, not a good file then, let it be deleted
return false;
}
-
+
// Got a container ID? (ie check was successful)
if(containerID == -1)
{
@@ -538,13 +591,13 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
// If it looks like a good object, and it's non-RAID, and
// this is a RAID set, then convert it to RAID.
-
+
RaidFileController &rcontroller(RaidFileController::GetController());
RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber));
if(!rdiscSet.IsNonRaidSet())
{
// See if the file exists
- RaidFileUtil::ExistType existance =
+ RaidFileUtil::ExistType existance =
RaidFileUtil::RaidFileExists(rdiscSet, rFilename);
if(existance == RaidFileUtil::NonRaid)
{
@@ -565,7 +618,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
if(mFixErrors)
{
std::auto_ptr<RaidFileRead> read(
- RaidFileRead::Open(mDiscSetNumber,
+ RaidFileRead::Open(mDiscSetNumber,
rFilename));
RaidFileWrite write(mDiscSetNumber, rFilename);
write.Open(true /* overwrite */);
@@ -575,7 +628,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
}
}
}
-
+
// Report success
return true;
}
@@ -636,7 +689,7 @@ int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream)
// Wrong object ID
return -1;
}
-
+
// Return container ID
return dir.GetContainerID();
}
@@ -669,7 +722,7 @@ void BackupStoreCheck::CheckDirectories()
{
IDBlock *pblock = i->second;
int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
-
+
for(int e = 0; e < bentries; ++e)
{
uint8_t flags = GetFlags(pblock, e);
@@ -703,9 +756,9 @@ void BackupStoreCheck::CheckDirectories()
BOX_FORMAT_OBJECTID(pblock->mID[e]) <<
" was OK after fixing");
}
-
+
if(isModified && mFixErrors)
- {
+ {
BOX_WARNING("Writing modified directory to disk: " <<
BOX_FORMAT_OBJECTID(pblock->mID[e]));
RaidFileWrite fixed(mDiscSetNumber, filename);
@@ -714,64 +767,7 @@ void BackupStoreCheck::CheckDirectories()
fixed.Commit(true /* convert to raid representation now */);
}
- // Count valid entries
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *en = 0;
- while((en = i.Next()) != 0)
- {
- int32_t iIndex;
- IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
-
- ASSERT(piBlock != 0 ||
- mDirsWhichContainLostDirs.find(en->GetObjectID())
- != mDirsWhichContainLostDirs.end());
- if (piBlock)
- {
- // Normally it would exist and this
- // check would not be necessary, but
- // we might have missing directories
- // that we will recreate later.
- // cf mDirsWhichContainLostDirs.
- uint8_t iflags = GetFlags(piBlock, iIndex);
- SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
- }
-
- if(en->IsDir())
- {
- mNumDirectories++;
- }
- else if(!en->IsFile())
- {
- BOX_TRACE("Not counting object " <<
- BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
- " with flags " << en->GetFlags());
- }
- else // it's a good file, add to sizes
- if(en->IsOld() && en->IsDeleted())
- {
- BOX_WARNING("File " <<
- BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
- " is both old and deleted, "
- "this should not happen!");
- }
- else if(en->IsOld())
- {
- mNumFiles++;
- mNumOldFiles++;
- mBlocksInOldFiles += en->GetSizeInBlocks();
- }
- else if(en->IsDeleted())
- {
- mNumFiles++;
- mNumDeletedFiles++;
- mBlocksInDeletedFiles += en->GetSizeInBlocks();
- }
- else
- {
- mNumFiles++;
- mBlocksInCurrentFiles += en->GetSizeInBlocks();
- }
- }
+ CountDirectoryEntries(dir);
}
}
}
@@ -798,8 +794,8 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir)
++mNumberErrorsFound;
isModified = true;
}
-
- // Go through, and check that everything in that directory exists and is valid
+
+ // Go through, and check that every entry exists and is valid
BackupStoreDirectory::Iterator i(dir);
BackupStoreDirectory::Entry *en = 0;
while((en = i.Next()) != 0)
@@ -824,14 +820,14 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir)
{
// Just remove the entry
badEntry = true;
- BOX_ERROR("Directory ID " <<
+ BOX_ERROR("Directory ID " <<
BOX_FORMAT_OBJECTID(dir.GetObjectID()) <<
- " references object " <<
+ " references object " <<
BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
" which does not exist.");
++mNumberErrorsFound;
}
-
+
// Is this entry worth keeping?
if(badEntry)
{
@@ -846,16 +842,16 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir)
{
BOX_ERROR("Removing directory entry " <<
BOX_FORMAT_OBJECTID(*d) << " from "
- "directory " <<
+ "directory " <<
BOX_FORMAT_OBJECTID(dir.GetObjectID()));
++mNumberErrorsFound;
dir.DeleteEntry(*d);
}
-
+
// Mark as modified
restart = true;
isModified = true;
-
+
// Errors found
}
}
@@ -863,6 +859,84 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir)
return isModified;
}
+// Count valid remaining entries and the number of blocks in them.
+void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir)
+{
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next()) != 0)
+ {
+ int32_t iIndex;
+ IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
+ bool badEntry = false;
+ bool wasAlreadyContained = false;
+
+ ASSERT(piBlock != 0 ||
+ mDirsWhichContainLostDirs.find(en->GetObjectID())
+ != mDirsWhichContainLostDirs.end());
+
+ if (piBlock)
+ {
+ // Normally it would exist and this
+ // check would not be necessary, but
+ // we might have missing directories
+ // that we will recreate later.
+ // cf mDirsWhichContainLostDirs.
+ uint8_t iflags = GetFlags(piBlock, iIndex);
+ wasAlreadyContained = (iflags & Flags_IsContained);
+ SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
+ }
+
+ if(wasAlreadyContained)
+ {
+ // don't double-count objects that are
+ // contained by another directory as well.
+ }
+ else if(en->IsDir())
+ {
+ mNumDirectories++;
+ }
+ else if(!en->IsFile())
+ {
+ BOX_TRACE("Not counting object " <<
+ BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
+ " with flags " << en->GetFlags());
+ }
+ else // it's a file
+ {
+ // Add to sizes?
+ // If piBlock was zero, then wasAlreadyContained
+ // might be uninitialized; but we only process
+ // files here, and if a file's piBlock was zero
+ // then badEntry would be set above, so we
+ // wouldn't be here.
+ ASSERT(!badEntry)
+
+ // It can be both old and deleted.
+ // If neither, then it's current.
+ if(en->IsDeleted())
+ {
+ mNumDeletedFiles++;
+ mBlocksInDeletedFiles += en->GetSizeInBlocks();
+ }
+
+ if(en->IsOld())
+ {
+ mNumOldFiles++;
+ mBlocksInOldFiles += en->GetSizeInBlocks();
+ }
+
+ if(!en->IsDeleted() && !en->IsOld())
+ {
+ mNumCurrentFiles++;
+ mBlocksInCurrentFiles += en->GetSizeInBlocks();
+ }
+ }
+
+ mapNewRefs->AddReference(en->GetObjectID());
+ }
+}
+
bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry,
int64_t DirectoryID, bool& rIsModified)
{
@@ -871,8 +945,7 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry,
ASSERT(piBlock != 0);
uint8_t iflags = GetFlags(piBlock, IndexInDirBlock);
- bool badEntry = false;
-
+
// Is the type the same?
if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir())
{
@@ -882,73 +955,67 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry,
" references object " <<
BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
" which has a different type than expected.");
- badEntry = true;
++mNumberErrorsFound;
+ return false; // remove this entry
}
+
// Check that the entry is not already contained.
- else if(iflags & Flags_IsContained)
+ if(iflags & Flags_IsContained)
{
BOX_ERROR("Directory ID " <<
BOX_FORMAT_OBJECTID(DirectoryID) <<
" references object " <<
BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
" which is already contained.");
- badEntry = true;
++mNumberErrorsFound;
+ return false; // remove this entry
}
- else
+
+ // Not already contained by another directory.
+ // Don't set the flag until later, after we finish repairing
+ // the directory and removing all bad entries.
+
+ // Check that the container ID of the object is correct
+ if(piBlock->mContainer[IndexInDirBlock] != DirectoryID)
{
- // Not already contained by another directory.
- // Don't set the flag until later, after we finish repairing
- // the directory and removing all bad entries.
-
- // Check that the container ID of the object is correct
- if(piBlock->mContainer[IndexInDirBlock] != DirectoryID)
+ // Needs fixing...
+ if(iflags & Flags_IsDir)
{
- // Needs fixing...
- if(iflags & Flags_IsDir)
- {
- // Add to will fix later list
- BOX_ERROR("Directory ID " <<
- BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
- << " has wrong container ID.");
- mDirsWithWrongContainerID.push_back(rEntry.GetObjectID());
- ++mNumberErrorsFound;
- }
- else
- {
- // This is OK for files, they might move
- BOX_NOTICE("File ID " <<
- BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
- << " has different container ID, "
- "probably moved");
- }
-
- // Fix entry for now
- piBlock->mContainer[IndexInDirBlock] = DirectoryID;
+ // Add to will fix later list
+ BOX_ERROR("Directory ID " <<
+ BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
+ << " has wrong container ID.");
+ mDirsWithWrongContainerID.push_back(rEntry.GetObjectID());
+ ++mNumberErrorsFound;
+ }
+ else
+ {
+ // This is OK for files, they might move
+ BOX_INFO("File ID " <<
+ BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
+ << " has different container ID, "
+ "probably moved");
}
+
+ // Fix entry for now
+ piBlock->mContainer[IndexInDirBlock] = DirectoryID;
}
-
- // Check the object size, if it's OK and a file
- if(!badEntry && !rEntry.IsDir())
+
+ // Check the object size
+ if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock])
{
- if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock])
- {
- // Wrong size, correct it.
- rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]);
+ // Wrong size, correct it.
+ BOX_ERROR("Directory " << BOX_FORMAT_OBJECTID(DirectoryID) <<
+ " entry for " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
+ " has wrong size " << rEntry.GetSizeInBlocks() <<
+ ", should be " << piBlock->mObjectSizeInBlocks[IndexInDirBlock]);
- // Mark as changed
- rIsModified = true;
- ++mNumberErrorsFound;
+ rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]);
- // Tell user
- BOX_ERROR("Directory ID " <<
- BOX_FORMAT_OBJECTID(DirectoryID) <<
- " has wrong size for object " <<
- BOX_FORMAT_OBJECTID(rEntry.GetObjectID()));
- }
+ // Mark as changed
+ rIsModified = true;
+ ++mNumberErrorsFound;
}
- return !badEntry;
+ return true; // don't delete this entry
}
-
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-24 09:14:30 +0000