summaryrefslogtreecommitdiff
path: root/lib/backupstore/BackupStoreCheck.cpp
diff options
context:
space:
mode:
authorReinhard Tartler <siretart@tauware.de>2017-06-11 21:52:33 -0400
committerReinhard Tartler <siretart@tauware.de>2017-06-11 21:52:33 -0400
commite0eb815b67734abd09ff41e2271630d4b2a6d760 (patch)
tree0df971c34f98d2a1dfd0921524a17d561a4a6536 /lib/backupstore/BackupStoreCheck.cpp
parent676c9e1c9d4ac8eb8a440d7f11c4ac44f98f4a6a (diff)
parente19a5db232e1ef90e9a02159d2fbd9707ffe4373 (diff)
merge upstream version 0.12
Diffstat (limited to 'lib/backupstore/BackupStoreCheck.cpp')
-rw-r--r--lib/backupstore/BackupStoreCheck.cpp482
1 files changed, 482 insertions, 0 deletions
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
index 7598094e..79a61a77 100644
--- a/lib/backupstore/BackupStoreCheck.cpp
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -11,6 +11,7 @@
#include <stdio.h>
#include <string.h>
+<<<<<<< HEAD
#include <unistd.h>
#include "BackupStoreCheck.h"
@@ -22,6 +23,26 @@
#include "BackupStoreFile.h"
#include "BackupStoreDirectory.h"
#include "BackupStoreConstants.h"
+=======
+
+#ifdef HAVE_UNISTD_H
+# include <unistd.h>
+#endif
+
+#include "autogen_BackupStoreException.h"
+#include "BackupStoreCheck.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreObjectMagic.h"
+#include "RaidFileController.h"
+#include "RaidFileException.h"
+#include "RaidFileRead.h"
+#include "RaidFileUtil.h"
+#include "RaidFileWrite.h"
+#include "StoreStructure.h"
+#include "Utils.h"
+>>>>>>> 0.12
#include "MemLeakFindOn.h"
@@ -47,9 +68,20 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum
mLostDirNameSerial(0),
mLostAndFoundDirectoryID(0),
mBlocksUsed(0),
+<<<<<<< HEAD
mBlocksInOldFiles(0),
mBlocksInDeletedFiles(0),
mBlocksInDirectories(0)
+=======
+ mBlocksInCurrentFiles(0),
+ mBlocksInOldFiles(0),
+ mBlocksInDeletedFiles(0),
+ mBlocksInDirectories(0),
+ mNumFiles(0),
+ mNumOldFiles(0),
+ mNumDeletedFiles(0),
+ mNumDirectories(0)
+>>>>>>> 0.12
{
}
@@ -73,12 +105,18 @@ BackupStoreCheck::~BackupStoreCheck()
//
// Function
// Name: BackupStoreCheck::Check()
+<<<<<<< HEAD
// Purpose: Perform the check on the given account
+=======
+// Purpose: Perform the check on the given account. You need to
+// hold a lock on the account before calling this!
+>>>>>>> 0.12
// Created: 21/4/04
//
// --------------------------------------------------------------------------
void BackupStoreCheck::Check()
{
+<<<<<<< HEAD
// Lock the account
{
std::string writeLockFilename;
@@ -107,6 +145,11 @@ void BackupStoreCheck::Check()
THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount)
}
}
+=======
+ std::string writeLockFilename;
+ StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename);
+ ASSERT(FileExists(writeLockFilename));
+>>>>>>> 0.12
if(!mQuiet && mFixErrors)
{
@@ -296,6 +339,36 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const
// Read in all the directories, and recurse downwards
{
+<<<<<<< HEAD
+=======
+ // If any of the directories is missing, create it.
+ RaidFileController &rcontroller(RaidFileController::GetController());
+ RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber));
+
+ if(!rdiscSet.IsNonRaidSet())
+ {
+ unsigned int numDiscs = rdiscSet.size();
+
+ for(unsigned int l = 0; l < numDiscs; ++l)
+ {
+ // build name
+ std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName);
+ struct stat st;
+
+ if(stat(dn.c_str(), &st) != 0 && errno == ENOENT)
+ {
+ if(mkdir(dn.c_str(), 0755) != 0)
+ {
+ THROW_SYS_FILE_ERROR("Failed to "
+ "create missing RaidFile "
+ "directory", dn,
+ RaidFileException, OSError);
+ }
+ }
+ }
+ }
+
+>>>>>>> 0.12
std::vector<std::string> dirs;
RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName,
RaidFileRead::DirReadType_DirsOnly, dirs);
@@ -318,7 +391,11 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const
}
else
{
+<<<<<<< HEAD
BOX_WARNING("Spurious or invalid directory " <<
+=======
+ BOX_ERROR("Spurious or invalid directory " <<
+>>>>>>> 0.12
rDirName << DIRECTORY_SEPARATOR <<
(*i) << " found, " <<
(mFixErrors?"deleting":"delete manually"));
@@ -335,7 +412,12 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const
//
// Function
// Name: BackupStoreCheck::CheckObjectsDir(int64_t)
+<<<<<<< HEAD
// Purpose: Check all the files within this directory which has the given starting ID.
+=======
+// Purpose: Check all the files within this directory which has
+// the given starting ID.
+>>>>>>> 0.12
// Created: 22/4/04
//
// --------------------------------------------------------------------------
@@ -383,6 +465,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
// Filename is valid, mark as existing
idsPresent[n] = true;
}
+<<<<<<< HEAD
else
{
// info file in root dir is OK!
@@ -390,12 +473,31 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
{
fileOK = false;
}
+=======
+ // No other files should be present in subdirectories
+ else if(StartID != 0)
+ {
+ fileOK = false;
+ }
+ // info and refcount databases are OK in the root directory
+ else if(*i == "info" || *i == "refcount.db")
+ {
+ fileOK = true;
+ }
+ else
+ {
+ fileOK = false;
+>>>>>>> 0.12
}
if(!fileOK)
{
// Unexpected or bad file, delete it
+<<<<<<< HEAD
BOX_WARNING("Spurious file " << dirName <<
+=======
+ BOX_ERROR("Spurious file " << dirName <<
+>>>>>>> 0.12
DIRECTORY_SEPARATOR << (*i) << " found" <<
(mFixErrors?", deleting":""));
++mNumberErrorsFound;
@@ -418,7 +520,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
if(!CheckAndAddObject(StartID | i, dirName + leaf))
{
// File was bad, delete it
+<<<<<<< HEAD
BOX_WARNING("Corrupted file " << dirName <<
+=======
+ BOX_ERROR("Corrupted file " << dirName <<
+>>>>>>> 0.12
leaf << " found" <<
(mFixErrors?", deleting":""));
++mNumberErrorsFound;
@@ -436,6 +542,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
// --------------------------------------------------------------------------
//
// Function
+<<<<<<< HEAD
// Name: BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &)
// Purpose: Check a specific object and add it to the list if it's OK -- if
// there are any errors with the reading, return false and it'll be deleted.
@@ -443,6 +550,18 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
//
// --------------------------------------------------------------------------
bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename)
+=======
+// Name: BackupStoreCheck::CheckAndAddObject(int64_t,
+// const std::string &)
+// Purpose: Check a specific object and add it to the list
+// if it's OK. If there are any errors with the
+// reading, return false and it'll be deleted.
+// Created: 21/4/04
+//
+// --------------------------------------------------------------------------
+bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
+ const std::string &rFilename)
+>>>>>>> 0.12
{
// Info on object...
bool isFile = true;
@@ -452,10 +571,19 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF
try
{
// Open file
+<<<<<<< HEAD
std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename));
size = file->GetDiscUsageInBlocks();
// Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile
+=======
+ std::auto_ptr<RaidFileRead> file(
+ RaidFileRead::Open(mDiscSetNumber, rFilename));
+ size = file->GetDiscUsageInBlocks();
+
+ // Read in first four bytes -- don't have to worry about
+ // retrying if not all bytes read as is RaidFile
+>>>>>>> 0.12
uint32_t signature;
if(file->Read(&signature, sizeof(signature)) != sizeof(signature))
{
@@ -486,6 +614,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF
return false;
break;
}
+<<<<<<< HEAD
// Add to usage counts
mBlocksUsed += size;
@@ -493,6 +622,8 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF
{
mBlocksInDirectories += size;
}
+=======
+>>>>>>> 0.12
}
catch(...)
{
@@ -505,10 +636,64 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF
{
return false;
}
+<<<<<<< HEAD
// Add to list of IDs known about
AddID(ObjectID, containerID, size, isFile);
+=======
+
+ // Add to list of IDs known about
+ AddID(ObjectID, containerID, size, isFile);
+
+ // Add to usage counts
+ mBlocksUsed += size;
+ if(!isFile)
+ {
+ mBlocksInDirectories += size;
+ }
+
+ // If it looks like a good object, and it's non-RAID, and
+ // this is a RAID set, then convert it to RAID.
+
+ RaidFileController &rcontroller(RaidFileController::GetController());
+ RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber));
+ if(!rdiscSet.IsNonRaidSet())
+ {
+ // See if the file exists
+ RaidFileUtil::ExistType existance =
+ RaidFileUtil::RaidFileExists(rdiscSet, rFilename);
+ if(existance == RaidFileUtil::NonRaid)
+ {
+ BOX_WARNING("Found non-RAID write file in RAID set" <<
+ (mFixErrors?", transforming to RAID: ":"") <<
+ (mFixErrors?rFilename:""));
+ if(mFixErrors)
+ {
+ RaidFileWrite write(mDiscSetNumber, rFilename);
+ write.TransformToRaidStorage();
+ }
+ }
+ else if(existance == RaidFileUtil::AsRaidWithMissingReadable)
+ {
+ BOX_WARNING("Found damaged but repairable RAID file" <<
+ (mFixErrors?", repairing: ":"") <<
+ (mFixErrors?rFilename:""));
+ if(mFixErrors)
+ {
+ std::auto_ptr<RaidFileRead> read(
+ RaidFileRead::Open(mDiscSetNumber,
+ rFilename));
+ RaidFileWrite write(mDiscSetNumber, rFilename);
+ write.Open(true /* overwrite */);
+ read->CopyStreamTo(write);
+ read.reset();
+ write.Commit(true /* transform to RAID */);
+ }
+ }
+ }
+
+>>>>>>> 0.12
// Report success
return true;
}
@@ -518,13 +703,23 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF
//
// Function
// Name: BackupStoreCheck::CheckFile(int64_t, IOStream &)
+<<<<<<< HEAD
// Purpose: Do check on file, return original container ID if OK, or -1 on error
+=======
+// Purpose: Do check on file, return original container ID
+// if OK, or -1 on error
+>>>>>>> 0.12
// Created: 22/4/04
//
// --------------------------------------------------------------------------
int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream)
{
+<<<<<<< HEAD
// Check that it's not the root directory ID. Having a file as the root directory would be bad.
+=======
+ // Check that it's not the root directory ID. Having a file as
+ // the root directory would be bad.
+>>>>>>> 0.12
if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
{
// Get that dodgy thing deleted!
@@ -534,7 +729,12 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream)
// Check the format of the file, and obtain the container ID
int64_t originalContainerID = -1;
+<<<<<<< HEAD
if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */,
+=======
+ if(!BackupStoreFile::VerifyEncodedFileFormat(rStream,
+ 0 /* don't want diffing from ID */,
+>>>>>>> 0.12
&originalContainerID))
{
// Didn't verify
@@ -549,7 +749,12 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream)
//
// Function
// Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &)
+<<<<<<< HEAD
// Purpose: Do initial check on directory, return container ID if OK, or -1 on error
+=======
+// Purpose: Do initial check on directory, return container ID
+// if OK, or -1 on error
+>>>>>>> 0.12
// Created: 22/4/04
//
// --------------------------------------------------------------------------
@@ -588,7 +793,16 @@ void BackupStoreCheck::CheckDirectories()
// This phase will check all the files in the directories, make
// a note of all directories which are missing, and do initial fixing.
+<<<<<<< HEAD
// Scan all objects
+=======
+ // The root directory is not contained inside another directory, so
+ // it has no directory entry to scan, but we have to count it
+ // somewhere, so we'll count it here.
+ mNumDirectories++;
+
+ // Scan all objects.
+>>>>>>> 0.12
for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
{
IDBlock *pblock = i->second;
@@ -609,6 +823,7 @@ void BackupStoreCheck::CheckDirectories()
}
// Flag for modifications
+<<<<<<< HEAD
bool isModified = false;
// Check for validity
@@ -625,10 +840,43 @@ void BackupStoreCheck::CheckDirectories()
// Go through, and check that everything in that directory exists and is valid
std::vector<int64_t> toDelete;
+=======
+ bool isModified = CheckDirectory(dir);
+
+ // Check the directory again, now that entries have been removed
+ if(dir.CheckAndFix())
+ {
+ // Wasn't quite right, and has been modified
+ BOX_ERROR("Directory ID " <<
+ BOX_FORMAT_OBJECTID(pblock->mID[e]) <<
+ " was still bad after all checks");
+ ++mNumberErrorsFound;
+ isModified = true;
+ }
+ else if(isModified)
+ {
+ BOX_INFO("Directory ID " <<
+ BOX_FORMAT_OBJECTID(pblock->mID[e]) <<
+ " was OK after fixing");
+ }
+
+ if(isModified && mFixErrors)
+ {
+ BOX_WARNING("Writing modified directory to disk: " <<
+ BOX_FORMAT_OBJECTID(pblock->mID[e]));
+ RaidFileWrite fixed(mDiscSetNumber, filename);
+ fixed.Open(true /* allow overwriting */);
+ dir.WriteToStream(fixed);
+ fixed.Commit(true /* convert to raid representation now */);
+ }
+
+ // Count valid entries
+>>>>>>> 0.12
BackupStoreDirectory::Iterator i(dir);
BackupStoreDirectory::Entry *en = 0;
while((en = i.Next()) != 0)
{
+<<<<<<< HEAD
// Lookup the item
int32_t iIndex;
IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
@@ -773,4 +1021,238 @@ void BackupStoreCheck::CheckDirectories()
}
+=======
+ int32_t iIndex;
+ IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
+
+ ASSERT(piBlock != 0 ||
+ mDirsWhichContainLostDirs.find(en->GetObjectID())
+ != mDirsWhichContainLostDirs.end());
+ if (piBlock)
+ {
+ // Normally it would exist and this
+ // check would not be necessary, but
+ // we might have missing directories
+ // that we will recreate later.
+ // cf mDirsWhichContainLostDirs.
+ uint8_t iflags = GetFlags(piBlock, iIndex);
+ SetFlags(piBlock, iIndex, iflags | Flags_IsContained);
+ }
+
+ if(en->IsDir())
+ {
+ mNumDirectories++;
+ }
+ else if(!en->IsFile())
+ {
+ BOX_TRACE("Not counting object " <<
+ BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
+ " with flags " << en->GetFlags());
+ }
+ else // it's a good file, add to sizes
+ if(en->IsOld() && en->IsDeleted())
+ {
+ BOX_WARNING("File " <<
+ BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
+ " is both old and deleted, "
+ "this should not happen!");
+ }
+ else if(en->IsOld())
+ {
+ mNumFiles++;
+ mNumOldFiles++;
+ mBlocksInOldFiles += en->GetSizeInBlocks();
+ }
+ else if(en->IsDeleted())
+ {
+ mNumFiles++;
+ mNumDeletedFiles++;
+ mBlocksInDeletedFiles += en->GetSizeInBlocks();
+ }
+ else
+ {
+ mNumFiles++;
+ mBlocksInCurrentFiles += en->GetSizeInBlocks();
+ }
+ }
+ }
+ }
+ }
+}
+
+bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir)
+{
+ bool restart = true;
+ bool isModified = false;
+
+ while(restart)
+ {
+ std::vector<int64_t> toDelete;
+ restart = false;
+
+ // Check for validity
+ if(dir.CheckAndFix())
+ {
+ // Wasn't quite right, and has been modified
+ BOX_ERROR("Directory ID " <<
+ BOX_FORMAT_OBJECTID(dir.GetObjectID()) <<
+ " had invalid entries" <<
+ (mFixErrors ? ", fixed" : ""));
+ ++mNumberErrorsFound;
+ isModified = true;
+ }
+
+ // Go through, and check that everything in that directory exists and is valid
+ BackupStoreDirectory::Iterator i(dir);
+ BackupStoreDirectory::Entry *en = 0;
+ while((en = i.Next()) != 0)
+ {
+ // Lookup the item
+ int32_t iIndex;
+ IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
+ bool badEntry = false;
+ if(piBlock != 0)
+ {
+ badEntry = !CheckDirectoryEntry(*en,
+ dir.GetObjectID(), isModified);
+ }
+ // Item can't be found. Is it a directory?
+ else if(en->IsDir())
+ {
+ // Store the directory for later attention
+ mDirsWhichContainLostDirs[en->GetObjectID()] =
+ dir.GetObjectID();
+ }
+ else
+ {
+ // Just remove the entry
+ badEntry = true;
+ BOX_ERROR("Directory ID " <<
+ BOX_FORMAT_OBJECTID(dir.GetObjectID()) <<
+ " references object " <<
+ BOX_FORMAT_OBJECTID(en->GetObjectID()) <<
+ " which does not exist.");
+ ++mNumberErrorsFound;
+ }
+
+ // Is this entry worth keeping?
+ if(badEntry)
+ {
+ toDelete.push_back(en->GetObjectID());
+ }
+ }
+
+ if(toDelete.size() > 0)
+ {
+ // Delete entries from directory
+ for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
+ {
+ BOX_ERROR("Removing directory entry " <<
+ BOX_FORMAT_OBJECTID(*d) << " from "
+ "directory " <<
+ BOX_FORMAT_OBJECTID(dir.GetObjectID()));
+ ++mNumberErrorsFound;
+ dir.DeleteEntry(*d);
+ }
+
+ // Mark as modified
+ restart = true;
+ isModified = true;
+
+ // Errors found
+ }
+ }
+
+ return isModified;
+}
+
+bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry,
+ int64_t DirectoryID, bool& rIsModified)
+{
+ int32_t IndexInDirBlock;
+ IDBlock *piBlock = LookupID(rEntry.GetObjectID(), IndexInDirBlock);
+ ASSERT(piBlock != 0);
+
+ uint8_t iflags = GetFlags(piBlock, IndexInDirBlock);
+ bool badEntry = false;
+
+ // Is the type the same?
+ if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir())
+ {
+ // Entry is of wrong type
+ BOX_ERROR("Directory ID " <<
+ BOX_FORMAT_OBJECTID(DirectoryID) <<
+ " references object " <<
+ BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
+ " which has a different type than expected.");
+ badEntry = true;
+ ++mNumberErrorsFound;
+ }
+ // Check that the entry is not already contained.
+ else if(iflags & Flags_IsContained)
+ {
+ BOX_ERROR("Directory ID " <<
+ BOX_FORMAT_OBJECTID(DirectoryID) <<
+ " references object " <<
+ BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
+ " which is already contained.");
+ badEntry = true;
+ ++mNumberErrorsFound;
+ }
+ else
+ {
+ // Not already contained by another directory.
+ // Don't set the flag until later, after we finish repairing
+ // the directory and removing all bad entries.
+
+ // Check that the container ID of the object is correct
+ if(piBlock->mContainer[IndexInDirBlock] != DirectoryID)
+ {
+ // Needs fixing...
+ if(iflags & Flags_IsDir)
+ {
+ // Add to will fix later list
+ BOX_ERROR("Directory ID " <<
+ BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
+ << " has wrong container ID.");
+ mDirsWithWrongContainerID.push_back(rEntry.GetObjectID());
+ ++mNumberErrorsFound;
+ }
+ else
+ {
+ // This is OK for files, they might move
+ BOX_NOTICE("File ID " <<
+ BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
+ << " has different container ID, "
+ "probably moved");
+ }
+
+ // Fix entry for now
+ piBlock->mContainer[IndexInDirBlock] = DirectoryID;
+ }
+ }
+
+ // Check the object size, if it's OK and a file
+ if(!badEntry && !rEntry.IsDir())
+ {
+ if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock])
+ {
+ // Wrong size, correct it.
+ rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]);
+
+ // Mark as changed
+ rIsModified = true;
+ ++mNumberErrorsFound;
+
+ // Tell user
+ BOX_ERROR("Directory ID " <<
+ BOX_FORMAT_OBJECTID(DirectoryID) <<
+ " has wrong size for object " <<
+ BOX_FORMAT_OBJECTID(rEntry.GetObjectID()));
+ }
+ }
+
+ return !badEntry;
+}
+>>>>>>> 0.12