diff options
Diffstat (limited to 'lib/backupstore')
-rw-r--r-- | lib/backupstore/BackupStoreAccountDatabase.cpp | 4 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreCheck.cpp | 3 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreCheck.h | 4 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreCheck2.cpp | 271 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreCheckData.cpp | 15 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreConfigVerify.cpp | 17 | ||||
-rw-r--r-- | lib/backupstore/BackupStoreInfo.cpp | 4 | ||||
-rw-r--r-- | lib/backupstore/StoreStructure.h | 2 |
8 files changed, 199 insertions, 121 deletions
diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp index 91a6b758..46cab68f 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.cpp +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -208,8 +208,8 @@ void BackupStoreAccountDatabase::CheckUpToDate() const // -------------------------------------------------------------------------- box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const { - struct stat st; - if(::stat(pImpl->mFilename.c_str(), &st) == -1) + EMU_STRUCT_STAT st; + if(EMU_STAT(pImpl->mFilename.c_str(), &st) == -1) { THROW_EXCEPTION(CommonException, OSFileError) } diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index 176ece8f..7598094e 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -268,7 +268,8 @@ void BackupStoreCheck::CheckObjects() } maxDir = CheckObjectsScanDir(0, 1, mStoreRoot); - TRACE1("Max dir starting ID is %llx\n", maxDir); + BOX_TRACE("Max dir starting ID is " << + BOX_FORMAT_OBJECTID(maxDir)); } // Then go through and scan all the objects within those directories diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h index 3f48312a..1d5c1b1e 100644 --- a/lib/backupstore/BackupStoreCheck.h +++ b/lib/backupstore/BackupStoreCheck.h @@ -48,7 +48,7 @@ The following problems can be fixed: // Size of blocks in the list of IDs -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define BACKUPSTORECHECK_BLOCK_SIZE (64*1024) #else #define BACKUPSTORECHECK_BLOCK_SIZE 8 @@ -150,7 +150,7 @@ private: return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK; } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD void DumpObjectInfo(); #define DUMP_OBJECT_INFO DumpObjectInfo(); #else diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 9c6f2452..bcb5c5e9 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -95,6 +95,21 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain mBlocksInDirectories += size; } +class BackupStoreDirectoryFixer +{ + private: + BackupStoreDirectory mDirectory; + std::string mFilename; + std::string mStoreRoot; + int mDiscSetNumber; + + public: + BackupStoreDirectoryFixer(std::string storeRoot, int discSetNumber, + int64_t ID); + void InsertObject(int64_t ObjectID, bool IsDirectory, + int32_t lostDirNameSerial); + ~BackupStoreDirectoryFixer(); +}; // -------------------------------------------------------------------------- // @@ -106,6 +121,10 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain // -------------------------------------------------------------------------- void BackupStoreCheck::CheckUnattachedObjects() { + typedef std::map<int64_t, BackupStoreDirectoryFixer*> fixers_t; + typedef std::pair<int64_t, BackupStoreDirectoryFixer*> fixer_pair_t; + fixers_t fixers; + // Scan all objects, finding ones which have no container for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) { @@ -118,7 +137,9 @@ void BackupStoreCheck::CheckUnattachedObjects() if((flags & Flags_IsContained) == 0) { // Unattached object... - BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached."); + BOX_WARNING("Object " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " is unattached."); ++mNumberErrorsFound; // What's to be done? @@ -196,14 +217,50 @@ void BackupStoreCheck::CheckUnattachedObjects() } ASSERT(putIntoDirectoryID != 0); + if (!mFixErrors) + { + continue; + } + + BackupStoreDirectoryFixer* pFixer; + fixers_t::iterator fi = + fixers.find(putIntoDirectoryID); + if (fi == fixers.end()) + { + // no match, create a new one + pFixer = new BackupStoreDirectoryFixer( + mStoreRoot, mDiscSetNumber, + putIntoDirectoryID); + fixers.insert(fixer_pair_t( + putIntoDirectoryID, pFixer)); + } + else + { + pFixer = fi->second; + } + + int32_t lostDirNameSerial = 0; + + if(flags & Flags_IsDir) + { + lostDirNameSerial = mLostDirNameSerial++; + } + // Add it to the directory - InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID, - ((flags & Flags_IsDir) == Flags_IsDir)); + pFixer->InsertObject(pblock->mID[e], + ((flags & Flags_IsDir) == Flags_IsDir), + lostDirNameSerial); } } } -} + // clean up all the fixers. Deleting them commits them automatically. + for (fixers_t::iterator i = fixers.begin(); i != fixers.end(); i++) + { + BackupStoreDirectoryFixer* pFixer = i->second; + delete pFixer; + } +} // -------------------------------------------------------------------------- // @@ -261,6 +318,86 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) return true; } +BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(std::string storeRoot, + int discSetNumber, int64_t ID) +: mStoreRoot(storeRoot), + mDiscSetNumber(discSetNumber) +{ + // Generate filename + StoreStructure::MakeObjectFilename(ID, mStoreRoot, mDiscSetNumber, + mFilename, false /* don't make sure the dir exists */); + + // Read it in + std::auto_ptr<RaidFileRead> file( + RaidFileRead::Open(mDiscSetNumber, mFilename)); + mDirectory.ReadFromStream(*file, IOStream::TimeOutInfinite); +} + +void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, + int32_t lostDirNameSerial) +{ + // Data for the object + BackupStoreFilename objectStoreFilename; + int64_t modTime = 100; // something which isn't zero or a special time + int32_t sizeInBlocks = 0; // suitable for directories + + if(IsDirectory) + { + // Directory -- simply generate a name for it. + char name[32]; + ::sprintf(name, "dir%08x", lostDirNameSerial); + objectStoreFilename.SetAsClearFilename(name); + } + else + { + // Files require a little more work... + // Open file + std::string fileFilename; + StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, + mDiscSetNumber, fileFilename, + false /* don't make sure the dir exists */); + std::auto_ptr<RaidFileRead> file( + RaidFileRead::Open(mDiscSetNumber, fileFilename)); + + // Fill in size information + sizeInBlocks = file->GetDiscUsageInBlocks(); + + // Read in header + file_StreamFormat hdr; + if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || + (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + )) + { + // This should never happen, everything has been + // checked before. + THROW_EXCEPTION(BackupStoreException, Internal) + } + // This tells us nice things + modTime = box_ntoh64(hdr.mModificationTime); + // And the filename comes next + objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Add a new entry in an appropriate place + mDirectory.AddUnattactedObject(objectStoreFilename, modTime, + ObjectID, sizeInBlocks, + IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); +} + +BackupStoreDirectoryFixer::~BackupStoreDirectoryFixer() +{ + // Fix any flags which have been broken, which there's a good chance of doing + mDirectory.CheckAndFix(); + + // Write it out + RaidFileWrite root(mDiscSetNumber, mFilename); + root.Open(true /* allow overwriting */); + mDirectory.WriteToStream(root); + root.Commit(true /* convert to raid now */); +} // -------------------------------------------------------------------------- // @@ -335,91 +472,6 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool) -// Purpose: -// Created: 22/4/04 -// -// -------------------------------------------------------------------------- -void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory) -{ - if(!mFixErrors) - { - // Don't do anything if we're not supposed to fix errors - return; - } - - // Data for the object - BackupStoreFilename objectStoreFilename; - int64_t modTime = 100; // something which isn't zero or a special time - int32_t sizeInBlocks = 0; // suitable for directories - - if(IsDirectory) - { - // Directory -- simply generate a name for it. - char name[32]; - ::sprintf(name, "dir%08x", mLostDirNameSerial++); - objectStoreFilename.SetAsClearFilename(name); - } - else - { - // Files require a little more work... - // Open file - std::string fileFilename; - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */); - std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, fileFilename)); - // Fill in size information - sizeInBlocks = file->GetDiscUsageInBlocks(); - // Read in header - file_StreamFormat hdr; - if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - )) - { - // This should never happen, everything has been checked before. - THROW_EXCEPTION(BackupStoreException, Internal) - } - // This tells us nice things - modTime = box_ntoh64(hdr.mModificationTime); - // And the filename comes next - objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite); - } - - // Directory object - BackupStoreDirectory dir; - - // Generate filename - std::string filename; - StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); - - // Read it in - { - std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); - dir.ReadFromStream(*file, IOStream::TimeOutInfinite); - } - - // Add a new entry in an appropraite place - dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks, - IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); - - // Fix any flags which have been broken, which there's a good change of going - dir.CheckAndFix(); - - // Write it out - if(mFixErrors) - { - RaidFileWrite root(mDiscSetNumber, filename); - root.Open(true /* allow overwriting */); - dir.WriteToStream(root); - root.Commit(true /* convert to raid now */); - } -} - - -// -------------------------------------------------------------------------- -// -// Function // Name: BackupStoreCheck::FixDirsWithWrongContainerID() // Purpose: Rewrites container IDs where required // Created: 22/4/04 @@ -594,6 +646,8 @@ void BackupStoreCheck::WriteNewStoreInfo() } } +#define FMT_OID(x) BOX_FORMAT_OBJECTID(x) +#define FMT_i BOX_FORMAT_OBJECTID((*i)->GetObjectID()) // -------------------------------------------------------------------------- // @@ -620,7 +674,11 @@ bool BackupStoreDirectory::CheckAndFix() if(newerEn == 0) { // Depends on something, but it isn't there. - TRACE2("Entry id %llx removed because depends on newer version %llx which doesn't exist\n", (*i)->GetObjectID(), dependsNewer); + BOX_TRACE("Entry id " << FMT_i << + " removed because depends " + "on newer version " << + FMT_OID(dependsNewer) << + " which doesn't exist"); // Remove delete *i; @@ -638,7 +696,12 @@ bool BackupStoreDirectory::CheckAndFix() if(newerEn->GetDependsOlder() != (*i)->GetObjectID()) { // Wrong entry - TRACE3("Entry id %llx, correcting DependsOlder to %llx, was %llx\n", dependsNewer, (*i)->GetObjectID(), newerEn->GetDependsOlder()); + BOX_TRACE("Entry id " << + FMT_OID(dependsNewer) << + ", correcting DependsOlder to " << + FMT_i << + ", was " << + FMT_OID(newerEn->GetDependsOlder())); newerEn->SetDependsOlder((*i)->GetObjectID()); // Mark as changed changed = true; @@ -657,7 +720,11 @@ bool BackupStoreDirectory::CheckAndFix() if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0) { // Has an older version marked, but this doesn't exist. Remove this mark - TRACE2("Entry id %llx was marked that %llx depended on it, which doesn't exist, dependency info cleared\n", (*i)->GetObjectID(), dependsOlder); + BOX_TRACE("Entry id " << FMT_i << + " was marked as depended on by " << + FMT_OID(dependsOlder) << ", " + "which doesn't exist, dependency " + "info cleared"); (*i)->SetDependsOlder(0); @@ -683,7 +750,7 @@ bool BackupStoreDirectory::CheckAndFix() // Records of things seen std::set<int64_t> idsEncountered; - std::set<BackupStoreFilename> filenamesEncountered; + std::set<std::string> filenamesEncountered; do { @@ -693,7 +760,7 @@ bool BackupStoreDirectory::CheckAndFix() bool removeEntry = false; if((*i) == 0) { - TRACE0("Remove because null pointer found\n"); + BOX_TRACE("Remove because null pointer found"); removeEntry = true; } else @@ -704,7 +771,8 @@ bool BackupStoreDirectory::CheckAndFix() if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File)) { // Bad! Unset the file flag - TRACE1("Entry %llx: File flag set when dir flag set\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": File flag and dir flag both set"); (*i)->RemoveFlags(Entry::Flags_File); changed = true; } @@ -713,7 +781,8 @@ bool BackupStoreDirectory::CheckAndFix() if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end()) { // ID already seen, or type doesn't match - TRACE1("Entry %llx: Remove because ID already seen\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": Remove because ID already seen"); removeEntry = true; } else @@ -723,14 +792,15 @@ bool BackupStoreDirectory::CheckAndFix() // Check to see if the name has already been encountered -- if not, then it // needs to have the old version flag set - if(filenamesEncountered.find((*i)->GetName()) != filenamesEncountered.end()) + if(filenamesEncountered.find((*i)->GetName().GetEncodedFilename()) != filenamesEncountered.end()) { // Seen before -- check old version flag set if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion && ((*i)->GetFlags() & Entry::Flags_Deleted) == 0) { // Not set, set it - TRACE1("Entry %llx: Set old flag\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": Set old flag"); (*i)->AddFlags(Entry::Flags_OldVersion); changed = true; } @@ -741,13 +811,14 @@ bool BackupStoreDirectory::CheckAndFix() if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion) { // Set, unset it - TRACE1("Entry %llx: Old flag unset\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": Old flag unset"); (*i)->RemoveFlags(Entry::Flags_OldVersion); changed = true; } // Remember filename - filenamesEncountered.insert((*i)->GetName()); + filenamesEncountered.insert((*i)->GetName().GetEncodedFilename()); } } } diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp index f22c8339..fed0c3f1 100644 --- a/lib/backupstore/BackupStoreCheckData.cpp +++ b/lib/backupstore/BackupStoreCheckData.cpp @@ -174,7 +174,7 @@ BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD // -------------------------------------------------------------------------- // // Function @@ -189,15 +189,18 @@ void BackupStoreCheck::DumpObjectInfo() { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; - TRACE2("BLOCK @ 0x%08x, %d entries\n", pblock, bentries); + BOX_TRACE("BLOCK @ " << BOX_FORMAT_HEX32(pblock) << + ", " << bentries << " entries"); for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); - TRACE4("id %llx, c %llx, %s, %s\n", - pblock->mID[e], pblock->mContainer[e], - (flags & Flags_IsDir)?"dir":"file", - (flags & Flags_IsContained)?"contained":"unattached"); + BOX_TRACE(std::hex << + "id " << pblock->mID[e] << + ", c " << pblock->mContainer[e] << + ", " << ((flags & Flags_IsDir)?"dir":"file") << + ", " << ((flags & Flags_IsContained) ? + "contained":"unattached")); } } } diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp index 784adfb8..cc6efcf5 100644 --- a/lib/backupstore/BackupStoreConfigVerify.cpp +++ b/lib/backupstore/BackupStoreConfigVerify.cpp @@ -16,7 +16,8 @@ static const ConfigurationVerifyKey verifyserverkeys[] = { - SERVERTLS_VERIFY_SERVER_KEYS(0) // no default listen addresses + SERVERTLS_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) + // no default listen addresses }; static const ConfigurationVerify verifyserver[] = @@ -32,16 +33,18 @@ static const ConfigurationVerify verifyserver[] = static const ConfigurationVerifyKey verifyrootkeys[] = { - {"AccountDatabase", 0, ConfigTest_Exists, 0}, - {"TimeBetweenHousekeeping", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file + ConfigurationVerifyKey("AccountDatabase", ConfigTest_Exists), + ConfigurationVerifyKey("TimeBetweenHousekeeping", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), + // make value "yes" to enable in config file #ifdef WIN32 - {"RaidFileConf", "", ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry) #else - {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry, + BOX_FILE_RAIDFILE_DEFAULT_CONFIG) #endif - }; const ConfigurationVerify BackupConfigFileVerify = diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index 3588cc00..1d55fdf0 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -55,7 +55,7 @@ typedef struct END_STRUCTURE_PACKING_FOR_WIRE #endif -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define NUM_DELETED_DIRS_BLOCK 256 #else #define NUM_DELETED_DIRS_BLOCK 2 @@ -182,7 +182,7 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const st // Insert info from file info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker); info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed); - info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); + info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); info->mBlocksInOldFiles = box_ntoh64(hdr.mBlocksInOldFiles); info->mBlocksInDeletedFiles = box_ntoh64(hdr.mBlocksInDeletedFiles); info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories); diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h index 094c0deb..ffbe83dd 100644 --- a/lib/backupstore/StoreStructure.h +++ b/lib/backupstore/StoreStructure.h @@ -12,7 +12,7 @@ #include <string> -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define STORE_ID_SEGMENT_LENGTH 8 #define STORE_ID_SEGMENT_MASK 0xff #else |