summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/backupstore/BackupStoreCheck.cpp181
-rw-r--r--lib/backupstore/BackupStoreCheck.h5
-rw-r--r--lib/backupstore/BackupStoreCheck2.cpp2
-rw-r--r--lib/backupstore/BackupStoreRefCountDatabase.cpp28
-rw-r--r--lib/backupstore/BackupStoreRefCountDatabase.h1
-rw-r--r--lib/backupstore/HousekeepStoreAccount.cpp31
-rw-r--r--test/backupstorefix/testbackupstorefix.cpp16
7 files changed, 175 insertions, 89 deletions
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
index 1b314eec..33660ffc 100644
--- a/lib/backupstore/BackupStoreCheck.cpp
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -17,11 +17,13 @@
#endif
#include "autogen_BackupStoreException.h"
+#include "BackupStoreAccountDatabase.h"
#include "BackupStoreCheck.h"
#include "BackupStoreConstants.h"
#include "BackupStoreDirectory.h"
#include "BackupStoreFile.h"
#include "BackupStoreObjectMagic.h"
+#include "BackupStoreRefCountDatabase.h"
#include "RaidFileController.h"
#include "RaidFileException.h"
#include "RaidFileRead.h"
@@ -78,6 +80,13 @@ BackupStoreCheck::~BackupStoreCheck()
{
// Clean up
FreeInfo();
+
+ // Avoid an exception if we forget to discard mapNewRefs
+ if (mapNewRefs.get())
+ {
+ mapNewRefs->Discard();
+ mapNewRefs.reset();
+ }
}
@@ -104,6 +113,9 @@ void BackupStoreCheck::Check()
BOX_INFO("Will fix errors encountered during checking.");
}
+ BackupStoreAccountDatabase::Entry account(mAccountID, mDiscSetNumber);
+ mapNewRefs = BackupStoreRefCountDatabase::Create(account);
+
// Phase 1, check objects
if(!mQuiet)
{
@@ -148,9 +160,31 @@ void BackupStoreCheck::Check()
BOX_INFO("Phase 6, regenerate store info...");
}
WriteNewStoreInfo();
-
-// DUMP_OBJECT_INFO
-
+
+ try
+ {
+ std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs =
+ BackupStoreRefCountDatabase::Load(account, false);
+ mNumberErrorsFound += mapNewRefs->ReportChangesTo(*apOldRefs);
+ }
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database was missing or "
+ "corrupted, cannot check it for errors.");
+ mNumberErrorsFound++;
+ }
+
+ // force file to be saved and closed before releasing the lock below
+ if(mFixErrors)
+ {
+ mapNewRefs->Commit();
+ }
+ else
+ {
+ mapNewRefs->Discard();
+ }
+ mapNewRefs.reset();
+
if(mNumberErrorsFound > 0)
{
BOX_WARNING("Finished checking store account ID " <<
@@ -408,7 +442,8 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
fileOK = false;
}
// info and refcount databases are OK in the root directory
- else if(*i == "info" || *i == "refcount.db" || *i == "refcount.rdb")
+ else if(*i == "info" || *i == "refcount.db" ||
+ *i == "refcount.rdb" || *i == "refcount.rdbX")
{
fileOK = true;
}
@@ -717,61 +752,7 @@ void BackupStoreCheck::CheckDirectories()
fixed.Commit(true /* convert to raid representation now */);
}
- // Count valid entries
- BackupStoreDirectory::Iterator i(dir);
- BackupStoreDirectory::Entry *en = 0;
- while((en = i.Next()) != 0)
- {
- int32_t iIndex;
- IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
-
- ASSERT(piBlock != 0 ||
- mDirsWhichContainLostDirs.find(en->GetObjectID())
- != mDirsWhichContainLostDirs.end());
- if (piBlock)
- {
- // Normally it would exist and this
- // check would not be necessary, but
- // we might have missing directories
- // that we will recreate later.
- // cf mDirsWhichContainLostDirs.
- uint8_t iflags = GetFlags(piBlock, iIndex);
- SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
- }
-
- if(en->IsDir())
- {
- mNumDirectories++;
- }
- else if(!en->IsFile())
- {
- BOX_TRACE("Not counting object " <<
- BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
- " with flags " << en->GetFlags());
- }
- else // it's a good file, add to sizes
- {
- // It can be both old and deleted.
- // If neither, then it's current.
- if(en->IsDeleted())
- {
- mNumDeletedFiles++;
- mBlocksInDeletedFiles += en->GetSizeInBlocks();
- }
-
- if(en->IsOld())
- {
- mNumOldFiles++;
- mBlocksInOldFiles += en->GetSizeInBlocks();
- }
-
- if(!en->IsDeleted() && !en->IsOld())
- {
- mNumCurrentFiles++;
- mBlocksInCurrentFiles += en->GetSizeInBlocks();
- }
- }
- }
+ CountDirectoryEntries(dir);
}
}
}
@@ -798,8 +779,8 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir)
++mNumberErrorsFound;
isModified = true;
}
-
- // Go through, and check that everything in that directory exists and is valid
+
+ // Go through, and check that every entry exists and is valid
BackupStoreDirectory::Iterator i(dir);
BackupStoreDirectory::Entry *en = 0;
while((en = i.Next()) != 0)
@@ -863,6 +844,84 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir)
return isModified;
}
+// Count valid remaining entries and the number of blocks in them.
+void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir)
+{
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next()) != 0)
+ {
+ int32_t iIndex;
+ IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
+ bool badEntry = false;
+ bool wasAlreadyContained = false;
+
+ ASSERT(piBlock != 0 ||
+ mDirsWhichContainLostDirs.find(en->GetObjectID())
+ != mDirsWhichContainLostDirs.end());
+
+ if (piBlock)
+ {
+ // Normally it would exist and this
+ // check would not be necessary, but
+ // we might have missing directories
+ // that we will recreate later.
+ // cf mDirsWhichContainLostDirs.
+ uint8_t iflags = GetFlags(piBlock, iIndex);
+ wasAlreadyContained = (iflags & Flags_IsContained);
+ SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
+ }
+
+ if(wasAlreadyContained)
+ {
+ // don't double-count objects that are
+ // contained by another directory as well.
+ }
+ else if(en->IsDir())
+ {
+ mNumDirectories++;
+ }
+ else if(!en->IsFile())
+ {
+ BOX_TRACE("Not counting object " <<
+ BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
+ " with flags " << en->GetFlags());
+ }
+ else // it's a file
+ {
+ // Add to sizes?
+ // If piBlock was zero, then wasAlreadyContained
+ // might be uninitialized; but we only process
+ // files here, and if a file's piBlock was zero
+ // then badEntry would be set above, so we
+ // wouldn't be here.
+ ASSERT(!badEntry)
+
+ // It can be both old and deleted.
+ // If neither, then it's current.
+ if(en->IsDeleted())
+ {
+ mNumDeletedFiles++;
+ mBlocksInDeletedFiles += en->GetSizeInBlocks();
+ }
+
+ if(en->IsOld())
+ {
+ mNumOldFiles++;
+ mBlocksInOldFiles += en->GetSizeInBlocks();
+ }
+
+ if(!en->IsDeleted() && !en->IsOld())
+ {
+ mNumCurrentFiles++;
+ mBlocksInCurrentFiles += en->GetSizeInBlocks();
+ }
+ }
+
+ mapNewRefs->AddReference(en->GetObjectID());
+ }
+}
+
bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry,
int64_t DirectoryID, bool& rIsModified)
{
diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h
index 22473169..5353c968 100644
--- a/lib/backupstore/BackupStoreCheck.h
+++ b/lib/backupstore/BackupStoreCheck.h
@@ -20,6 +20,7 @@
class IOStream;
class BackupStoreFilename;
+class BackupStoreRefCountDatabase;
/*
@@ -130,6 +131,7 @@ private:
bool CheckDirectory(BackupStoreDirectory& dir);
bool CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry,
int64_t DirectoryID, bool& rIsModified);
+ void CountDirectoryEntries(BackupStoreDirectory& dir);
int64_t CheckFile(int64_t ObjectID, IOStream &rStream);
int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream);
@@ -195,6 +197,9 @@ private:
// Set of extra directories added
std::set<BackupStoreCheck_ID_t> mDirsAdded;
+
+ // The refcount database, being reconstructed as the check/fix progresses
+ std::auto_ptr<BackupStoreRefCountDatabase> mapNewRefs;
// Misc stuff
int32_t mLostDirNameSerial;
diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp
index 1743d420..e5728fa9 100644
--- a/lib/backupstore/BackupStoreCheck2.cpp
+++ b/lib/backupstore/BackupStoreCheck2.cpp
@@ -20,6 +20,7 @@
#include "BackupStoreFileWire.h"
#include "BackupStoreInfo.h"
#include "BackupStoreObjectMagic.h"
+#include "BackupStoreRefCountDatabase.h"
#include "MemBlockStream.h"
#include "RaidFileRead.h"
#include "RaidFileWrite.h"
@@ -259,6 +260,7 @@ void BackupStoreCheck::CheckUnattachedObjects()
pFixer->InsertObject(ObjectID,
((flags & Flags_IsDir) == Flags_IsDir),
lostDirNameSerial);
+ mapNewRefs->AddReference(ObjectID);
}
}
}
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp
index 3f3eb94b..e561f52b 100644
--- a/lib/backupstore/BackupStoreRefCountDatabase.cpp
+++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp
@@ -336,3 +336,31 @@ bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID)
return (refcount > 0);
}
+int BackupStoreRefCountDatabase::ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs)
+{
+ int ErrorCount = 0;
+ int64_t MaxOldObjectId = rOldRefs.GetLastObjectIDUsed();
+ int64_t MaxNewObjectId = GetLastObjectIDUsed();
+
+ for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
+ ObjectID < std::max(MaxOldObjectId, MaxNewObjectId);
+ ObjectID++)
+ {
+ typedef BackupStoreRefCountDatabase::refcount_t refcount_t;
+ refcount_t OldRefs = (ObjectID <= MaxOldObjectId) ?
+ rOldRefs.GetRefCount(ObjectID) : 0;
+ refcount_t NewRefs = (ObjectID <= MaxNewObjectId) ?
+ this->GetRefCount(ObjectID) : 0;
+
+ if (OldRefs != NewRefs)
+ {
+ BOX_WARNING("Reference count of object " <<
+ BOX_FORMAT_OBJECTID(ObjectID) <<
+ " changed from " << OldRefs <<
+ " to " << NewRefs);
+ ErrorCount++;
+ }
+ }
+
+ return ErrorCount;
+}
diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h
index 797f05dc..915653a4 100644
--- a/lib/backupstore/BackupStoreRefCountDatabase.h
+++ b/lib/backupstore/BackupStoreRefCountDatabase.h
@@ -87,6 +87,7 @@ public:
void AddReference(int64_t ObjectID);
// RemoveReference returns false if refcount drops to zero
bool RemoveReference(int64_t ObjectID);
+ int ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs);
private:
static std::string GetFilename(const BackupStoreAccountDatabase::Entry&
diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp
index 8defcab1..0259461e 100644
--- a/lib/backupstore/HousekeepStoreAccount.cpp
+++ b/lib/backupstore/HousekeepStoreAccount.cpp
@@ -201,37 +201,16 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
mErrorCount += info->ReportChangesTo(*pOldInfo);
info->Save();
- // We want to compare the mapNewRefs to apOldRefs before we delete any
- // files, because that will also change the reference count in a way that's not an error.
+ // Try to load the old reference count database and check whether
+ // any counts have changed. We want to compare the mapNewRefs to
+ // apOldRefs before we delete any files, because that will also change
+ // the reference count in a way that's not an error.
- // try to load the reference count database
try
{
std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs =
BackupStoreRefCountDatabase::Load(account, false);
-
- int64_t MaxOldObjectId = apOldRefs->GetLastObjectIDUsed();
- int64_t MaxNewObjectId = mapNewRefs->GetLastObjectIDUsed();
-
- for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
- ObjectID < std::max(MaxOldObjectId, MaxNewObjectId);
- ObjectID++)
- {
- typedef BackupStoreRefCountDatabase::refcount_t refcount_t;
- refcount_t OldRefs = (ObjectID <= MaxOldObjectId) ?
- apOldRefs->GetRefCount(ObjectID) : 0;
- refcount_t NewRefs = (ObjectID <= MaxNewObjectId) ?
- mapNewRefs->GetRefCount(ObjectID) : 0;
-
- if (OldRefs != NewRefs)
- {
- BOX_WARNING("Reference count of object " <<
- BOX_FORMAT_OBJECTID(ObjectID) <<
- " changed from " << OldRefs <<
- " to " << NewRefs);
- mErrorCount++;
- }
- }
+ mErrorCount += mapNewRefs->ReportChangesTo(*apOldRefs);
}
catch(BoxException &e)
{
diff --git a/test/backupstorefix/testbackupstorefix.cpp b/test/backupstorefix/testbackupstorefix.cpp
index 31cffc78..048251c2 100644
--- a/test/backupstorefix/testbackupstorefix.cpp
+++ b/test/backupstorefix/testbackupstorefix.cpp
@@ -845,7 +845,7 @@ int test(int argc, const char *argv[])
// spotting errors! But asserting an exact number will help us catch
// changes in checker behaviour, so it's not a bad thing to test.
- // The 11 errors are:
+ // The 12 errors are:
// ERROR: Directory ID 0xb references object 0x3e which does not exist.
// ERROR: Removing directory entry 0x3e from directory 0xb
// ERROR: Directory ID 0xc had invalid entries, fixed
@@ -857,8 +857,20 @@ int test(int argc, const char *argv[])
// ERROR: BlocksInCurrentFiles changed from 226 to 220
// ERROR: BlocksInDirectories changed from 56 to 54
// ERROR: NumFiles changed from 113 to 110
+ // WARNING: Reference count of object 0x3e changed from 1 to 0
- RUN_CHECK_INTERNAL(11);
+ RUN_CHECK_INTERNAL(12);
+
+ {
+ std::auto_ptr<BackupProtocolAccountUsage2> usage =
+ BackupProtocolLocal2(0x01234567, "test",
+ "backup/01234567/", 0,
+ false).QueryGetAccountUsage2();
+ TEST_EQUAL(usage->GetBlocksUsed(), 278);
+ TEST_EQUAL(usage->GetBlocksInCurrentFiles(), 220);
+ TEST_EQUAL(usage->GetBlocksInDirectories(), 54);
+ TEST_EQUAL(usage->GetNumCurrentFiles(), 110);
+ }
// Check everything is as it should be
TEST_THAT(::system(PERL_EXECUTABLE
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-25 00:33:30 +0000