diff options
author | Reinhard Tartler <siretart@tauware.de> | 2017-06-11 21:52:33 -0400 |
---|---|---|
committer | Reinhard Tartler <siretart@tauware.de> | 2017-06-11 21:52:33 -0400 |
commit | e0eb815b67734abd09ff41e2271630d4b2a6d760 (patch) | |
tree | 0df971c34f98d2a1dfd0921524a17d561a4a6536 /lib/backupstore/BackupStoreCheck.cpp | |
parent | 676c9e1c9d4ac8eb8a440d7f11c4ac44f98f4a6a (diff) | |
parent | e19a5db232e1ef90e9a02159d2fbd9707ffe4373 (diff) |
merge upstream version 0.12
Diffstat (limited to 'lib/backupstore/BackupStoreCheck.cpp')
-rw-r--r-- | lib/backupstore/BackupStoreCheck.cpp | 482 |
1 files changed, 482 insertions, 0 deletions
diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index 7598094e..79a61a77 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -11,6 +11,7 @@ #include <stdio.h> #include <string.h> +<<<<<<< HEAD #include <unistd.h> #include "BackupStoreCheck.h" @@ -22,6 +23,26 @@ #include "BackupStoreFile.h" #include "BackupStoreDirectory.h" #include "BackupStoreConstants.h" +======= + +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + +#include "autogen_BackupStoreException.h" +#include "BackupStoreCheck.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreFile.h" +#include "BackupStoreObjectMagic.h" +#include "RaidFileController.h" +#include "RaidFileException.h" +#include "RaidFileRead.h" +#include "RaidFileUtil.h" +#include "RaidFileWrite.h" +#include "StoreStructure.h" +#include "Utils.h" +>>>>>>> 0.12 #include "MemLeakFindOn.h" @@ -47,9 +68,20 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum mLostDirNameSerial(0), mLostAndFoundDirectoryID(0), mBlocksUsed(0), +<<<<<<< HEAD mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), mBlocksInDirectories(0) +======= + mBlocksInCurrentFiles(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0), + mNumFiles(0), + mNumOldFiles(0), + mNumDeletedFiles(0), + mNumDirectories(0) +>>>>>>> 0.12 { } @@ -73,12 +105,18 @@ BackupStoreCheck::~BackupStoreCheck() // // Function // Name: BackupStoreCheck::Check() +<<<<<<< HEAD // Purpose: Perform the check on the given account +======= +// Purpose: Perform the check on the given account. You need to +// hold a lock on the account before calling this! +>>>>>>> 0.12 // Created: 21/4/04 // // -------------------------------------------------------------------------- void BackupStoreCheck::Check() { +<<<<<<< HEAD // Lock the account { std::string writeLockFilename; @@ -107,6 +145,11 @@ void BackupStoreCheck::Check() THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount) } } +======= + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + ASSERT(FileExists(writeLockFilename)); +>>>>>>> 0.12 if(!mQuiet && mFixErrors) { @@ -296,6 +339,36 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const // Read in all the directories, and recurse downwards { +<<<<<<< HEAD +======= + // If any of the directories is missing, create it. + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); + + if(!rdiscSet.IsNonRaidSet()) + { + unsigned int numDiscs = rdiscSet.size(); + + for(unsigned int l = 0; l < numDiscs; ++l) + { + // build name + std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName); + struct stat st; + + if(stat(dn.c_str(), &st) != 0 && errno == ENOENT) + { + if(mkdir(dn.c_str(), 0755) != 0) + { + THROW_SYS_FILE_ERROR("Failed to " + "create missing RaidFile " + "directory", dn, + RaidFileException, OSError); + } + } + } + } + +>>>>>>> 0.12 std::vector<std::string> dirs; RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName, RaidFileRead::DirReadType_DirsOnly, dirs); @@ -318,7 +391,11 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const } else { +<<<<<<< HEAD BOX_WARNING("Spurious or invalid directory " << +======= + BOX_ERROR("Spurious or invalid directory " << +>>>>>>> 0.12 rDirName << DIRECTORY_SEPARATOR << (*i) << " found, " << (mFixErrors?"deleting":"delete manually")); @@ -335,7 +412,12 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const // // Function // Name: BackupStoreCheck::CheckObjectsDir(int64_t) +<<<<<<< HEAD // Purpose: Check all the files within this directory which has the given starting ID. +======= +// Purpose: Check all the files within this directory which has +// the given starting ID. +>>>>>>> 0.12 // Created: 22/4/04 // // -------------------------------------------------------------------------- @@ -383,6 +465,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) // Filename is valid, mark as existing idsPresent[n] = true; } +<<<<<<< HEAD else { // info file in root dir is OK! @@ -390,12 +473,31 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) { fileOK = false; } +======= + // No other files should be present in subdirectories + else if(StartID != 0) + { + fileOK = false; + } + // info and refcount databases are OK in the root directory + else if(*i == "info" || *i == "refcount.db") + { + fileOK = true; + } + else + { + fileOK = false; +>>>>>>> 0.12 } if(!fileOK) { // Unexpected or bad file, delete it +<<<<<<< HEAD BOX_WARNING("Spurious file " << dirName << +======= + BOX_ERROR("Spurious file " << dirName << +>>>>>>> 0.12 DIRECTORY_SEPARATOR << (*i) << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; @@ -418,7 +520,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) if(!CheckAndAddObject(StartID | i, dirName + leaf)) { // File was bad, delete it +<<<<<<< HEAD BOX_WARNING("Corrupted file " << dirName << +======= + BOX_ERROR("Corrupted file " << dirName << +>>>>>>> 0.12 leaf << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; @@ -436,6 +542,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) // -------------------------------------------------------------------------- // // Function +<<<<<<< HEAD // Name: BackupStoreCheck::CheckAndAddObject(int64_t, const std::string &) // Purpose: Check a specific object and add it to the list if it's OK -- if // there are any errors with the reading, return false and it'll be deleted. @@ -443,6 +550,18 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) // // -------------------------------------------------------------------------- bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rFilename) +======= +// Name: BackupStoreCheck::CheckAndAddObject(int64_t, +// const std::string &) +// Purpose: Check a specific object and add it to the list +// if it's OK. If there are any errors with the +// reading, return false and it'll be deleted. +// Created: 21/4/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, + const std::string &rFilename) +>>>>>>> 0.12 { // Info on object... bool isFile = true; @@ -452,10 +571,19 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF try { // Open file +<<<<<<< HEAD std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, rFilename)); size = file->GetDiscUsageInBlocks(); // Read in first four bytes -- don't have to worry about retrying if not all bytes read as is RaidFile +======= + std::auto_ptr<RaidFileRead> file( + RaidFileRead::Open(mDiscSetNumber, rFilename)); + size = file->GetDiscUsageInBlocks(); + + // Read in first four bytes -- don't have to worry about + // retrying if not all bytes read as is RaidFile +>>>>>>> 0.12 uint32_t signature; if(file->Read(&signature, sizeof(signature)) != sizeof(signature)) { @@ -486,6 +614,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF return false; break; } +<<<<<<< HEAD // Add to usage counts mBlocksUsed += size; @@ -493,6 +622,8 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF { mBlocksInDirectories += size; } +======= +>>>>>>> 0.12 } catch(...) { @@ -505,10 +636,64 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF { return false; } +<<<<<<< HEAD // Add to list of IDs known about AddID(ObjectID, containerID, size, isFile); +======= + + // Add to list of IDs known about + AddID(ObjectID, containerID, size, isFile); + + // Add to usage counts + mBlocksUsed += size; + if(!isFile) + { + mBlocksInDirectories += size; + } + + // If it looks like a good object, and it's non-RAID, and + // this is a RAID set, then convert it to RAID. + + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); + if(!rdiscSet.IsNonRaidSet()) + { + // See if the file exists + RaidFileUtil::ExistType existance = + RaidFileUtil::RaidFileExists(rdiscSet, rFilename); + if(existance == RaidFileUtil::NonRaid) + { + BOX_WARNING("Found non-RAID write file in RAID set" << + (mFixErrors?", transforming to RAID: ":"") << + (mFixErrors?rFilename:"")); + if(mFixErrors) + { + RaidFileWrite write(mDiscSetNumber, rFilename); + write.TransformToRaidStorage(); + } + } + else if(existance == RaidFileUtil::AsRaidWithMissingReadable) + { + BOX_WARNING("Found damaged but repairable RAID file" << + (mFixErrors?", repairing: ":"") << + (mFixErrors?rFilename:"")); + if(mFixErrors) + { + std::auto_ptr<RaidFileRead> read( + RaidFileRead::Open(mDiscSetNumber, + rFilename)); + RaidFileWrite write(mDiscSetNumber, rFilename); + write.Open(true /* overwrite */); + read->CopyStreamTo(write); + read.reset(); + write.Commit(true /* transform to RAID */); + } + } + } + +>>>>>>> 0.12 // Report success return true; } @@ -518,13 +703,23 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, const std::string &rF // // Function // Name: BackupStoreCheck::CheckFile(int64_t, IOStream &) +<<<<<<< HEAD // Purpose: Do check on file, return original container ID if OK, or -1 on error +======= +// Purpose: Do check on file, return original container ID +// if OK, or -1 on error +>>>>>>> 0.12 // Created: 22/4/04 // // -------------------------------------------------------------------------- int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) { +<<<<<<< HEAD // Check that it's not the root directory ID. Having a file as the root directory would be bad. +======= + // Check that it's not the root directory ID. Having a file as + // the root directory would be bad. +>>>>>>> 0.12 if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID) { // Get that dodgy thing deleted! @@ -534,7 +729,12 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) // Check the format of the file, and obtain the container ID int64_t originalContainerID = -1; +<<<<<<< HEAD if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, 0 /* don't want diffing from ID */, +======= + if(!BackupStoreFile::VerifyEncodedFileFormat(rStream, + 0 /* don't want diffing from ID */, +>>>>>>> 0.12 &originalContainerID)) { // Didn't verify @@ -549,7 +749,12 @@ int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream) // // Function // Name: BackupStoreCheck::CheckDirInitial(int64_t, IOStream &) +<<<<<<< HEAD // Purpose: Do initial check on directory, return container ID if OK, or -1 on error +======= +// Purpose: Do initial check on directory, return container ID +// if OK, or -1 on error +>>>>>>> 0.12 // Created: 22/4/04 // // -------------------------------------------------------------------------- @@ -588,7 +793,16 @@ void BackupStoreCheck::CheckDirectories() // This phase will check all the files in the directories, make // a note of all directories which are missing, and do initial fixing. +<<<<<<< HEAD // Scan all objects +======= + // The root directory is not contained inside another directory, so + // it has no directory entry to scan, but we have to count it + // somewhere, so we'll count it here. + mNumDirectories++; + + // Scan all objects. +>>>>>>> 0.12 for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) { IDBlock *pblock = i->second; @@ -609,6 +823,7 @@ void BackupStoreCheck::CheckDirectories() } // Flag for modifications +<<<<<<< HEAD bool isModified = false; // Check for validity @@ -625,10 +840,43 @@ void BackupStoreCheck::CheckDirectories() // Go through, and check that everything in that directory exists and is valid std::vector<int64_t> toDelete; +======= + bool isModified = CheckDirectory(dir); + + // Check the directory again, now that entries have been removed + if(dir.CheckAndFix()) + { + // Wasn't quite right, and has been modified + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " was still bad after all checks"); + ++mNumberErrorsFound; + isModified = true; + } + else if(isModified) + { + BOX_INFO("Directory ID " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " was OK after fixing"); + } + + if(isModified && mFixErrors) + { + BOX_WARNING("Writing modified directory to disk: " << + BOX_FORMAT_OBJECTID(pblock->mID[e])); + RaidFileWrite fixed(mDiscSetNumber, filename); + fixed.Open(true /* allow overwriting */); + dir.WriteToStream(fixed); + fixed.Commit(true /* convert to raid representation now */); + } + + // Count valid entries +>>>>>>> 0.12 BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { +<<<<<<< HEAD // Lookup the item int32_t iIndex; IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); @@ -773,4 +1021,238 @@ void BackupStoreCheck::CheckDirectories() } +======= + int32_t iIndex; + IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); + + ASSERT(piBlock != 0 || + mDirsWhichContainLostDirs.find(en->GetObjectID()) + != mDirsWhichContainLostDirs.end()); + if (piBlock) + { + // Normally it would exist and this + // check would not be necessary, but + // we might have missing directories + // that we will recreate later. + // cf mDirsWhichContainLostDirs. + uint8_t iflags = GetFlags(piBlock, iIndex); + SetFlags(piBlock, iIndex, iflags | Flags_IsContained); + } + + if(en->IsDir()) + { + mNumDirectories++; + } + else if(!en->IsFile()) + { + BOX_TRACE("Not counting object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " with flags " << en->GetFlags()); + } + else // it's a good file, add to sizes + if(en->IsOld() && en->IsDeleted()) + { + BOX_WARNING("File " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " is both old and deleted, " + "this should not happen!"); + } + else if(en->IsOld()) + { + mNumFiles++; + mNumOldFiles++; + mBlocksInOldFiles += en->GetSizeInBlocks(); + } + else if(en->IsDeleted()) + { + mNumFiles++; + mNumDeletedFiles++; + mBlocksInDeletedFiles += en->GetSizeInBlocks(); + } + else + { + mNumFiles++; + mBlocksInCurrentFiles += en->GetSizeInBlocks(); + } + } + } + } + } +} + +bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) +{ + bool restart = true; + bool isModified = false; + + while(restart) + { + std::vector<int64_t> toDelete; + restart = false; + + // Check for validity + if(dir.CheckAndFix()) + { + // Wasn't quite right, and has been modified + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(dir.GetObjectID()) << + " had invalid entries" << + (mFixErrors ? ", fixed" : "")); + ++mNumberErrorsFound; + isModified = true; + } + + // Go through, and check that everything in that directory exists and is valid + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + // Lookup the item + int32_t iIndex; + IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); + bool badEntry = false; + if(piBlock != 0) + { + badEntry = !CheckDirectoryEntry(*en, + dir.GetObjectID(), isModified); + } + // Item can't be found. Is it a directory? + else if(en->IsDir()) + { + // Store the directory for later attention + mDirsWhichContainLostDirs[en->GetObjectID()] = + dir.GetObjectID(); + } + else + { + // Just remove the entry + badEntry = true; + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(dir.GetObjectID()) << + " references object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " which does not exist."); + ++mNumberErrorsFound; + } + + // Is this entry worth keeping? + if(badEntry) + { + toDelete.push_back(en->GetObjectID()); + } + } + + if(toDelete.size() > 0) + { + // Delete entries from directory + for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) + { + BOX_ERROR("Removing directory entry " << + BOX_FORMAT_OBJECTID(*d) << " from " + "directory " << + BOX_FORMAT_OBJECTID(dir.GetObjectID())); + ++mNumberErrorsFound; + dir.DeleteEntry(*d); + } + + // Mark as modified + restart = true; + isModified = true; + + // Errors found + } + } + + return isModified; +} + +bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, + int64_t DirectoryID, bool& rIsModified) +{ + int32_t IndexInDirBlock; + IDBlock *piBlock = LookupID(rEntry.GetObjectID(), IndexInDirBlock); + ASSERT(piBlock != 0); + + uint8_t iflags = GetFlags(piBlock, IndexInDirBlock); + bool badEntry = false; + + // Is the type the same? + if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir()) + { + // Entry is of wrong type + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(DirectoryID) << + " references object " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << + " which has a different type than expected."); + badEntry = true; + ++mNumberErrorsFound; + } + // Check that the entry is not already contained. + else if(iflags & Flags_IsContained) + { + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(DirectoryID) << + " references object " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << + " which is already contained."); + badEntry = true; + ++mNumberErrorsFound; + } + else + { + // Not already contained by another directory. + // Don't set the flag until later, after we finish repairing + // the directory and removing all bad entries. + + // Check that the container ID of the object is correct + if(piBlock->mContainer[IndexInDirBlock] != DirectoryID) + { + // Needs fixing... + if(iflags & Flags_IsDir) + { + // Add to will fix later list + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) + << " has wrong container ID."); + mDirsWithWrongContainerID.push_back(rEntry.GetObjectID()); + ++mNumberErrorsFound; + } + else + { + // This is OK for files, they might move + BOX_NOTICE("File ID " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) + << " has different container ID, " + "probably moved"); + } + + // Fix entry for now + piBlock->mContainer[IndexInDirBlock] = DirectoryID; + } + } + + // Check the object size, if it's OK and a file + if(!badEntry && !rEntry.IsDir()) + { + if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock]) + { + // Wrong size, correct it. + rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]); + + // Mark as changed + rIsModified = true; + ++mNumberErrorsFound; + + // Tell user + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(DirectoryID) << + " has wrong size for object " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID())); + } + } + + return !badEntry; +} +>>>>>>> 0.12 |