Repair damaged Box RAID objects and non-transformed .rfw files.

5 files changed, 99 insertions, 35 deletions

diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp
index b8b8cc40..f2302337 100644
--- a/lib/backupstore/BackupStoreCheck.cpp
+++ b/lib/backupstore/BackupStoreCheck.cpp
@@ -16,15 +16,18 @@
 #	include <unistd.h>
 #endif
 
+#include "autogen_BackupStoreException.h"
 #include "BackupStoreCheck.h"
-#include "StoreStructure.h"
+#include "BackupStoreConstants.h"
+#include "BackupStoreDirectory.h"
+#include "BackupStoreFile.h"
+#include "BackupStoreObjectMagic.h"
+#include "RaidFileController.h"
+#include "RaidFileException.h"
 #include "RaidFileRead.h"
+#include "RaidFileUtil.h"
 #include "RaidFileWrite.h"
-#include "autogen_BackupStoreException.h"
-#include "BackupStoreObjectMagic.h"
-#include "BackupStoreFile.h"
-#include "BackupStoreDirectory.h"
-#include "BackupStoreConstants.h"
+#include "StoreStructure.h"
 #include "Utils.h"
 
 #include "MemLeakFindOn.h"
@@ -281,6 +284,33 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const
 
 	// Read in all the directories, and recurse downwards
 	{
+		// If any of the directories is missing, create it.
+		RaidFileController &rcontroller(RaidFileController::GetController());
+		RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber));
+		
+		if(!rdiscSet.IsNonRaidSet())
+		{
+			unsigned int numDiscs = rdiscSet.size();
+			
+			for(unsigned int l = 0; l < numDiscs; ++l)
+			{
+				// build name
+				std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName);
+				struct stat st;
+
+				if(stat(dn.c_str(), &st) != 0 && errno == ENOENT)
+				{
+					if(mkdir(dn.c_str(), 0755) != 0)
+					{
+						THROW_SYS_FILE_ERROR("Failed to "
+							"create missing RaidFile "
+							"directory", dn, 
+							RaidFileException, OSError);
+					}
+				}
+			}
+		}
+
 		std::vector<std::string> dirs;
 		RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName,
 			RaidFileRead::DirReadType_DirsOnly, dirs);
@@ -483,13 +513,6 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
 			return false;
 			break;
 		}
-		
-		// Add to usage counts
-		mBlocksUsed += size;
-		if(!isFile)
-		{
-			mBlocksInDirectories += size;
-		}
 	}
 	catch(...)
 	{
@@ -503,18 +526,56 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
 		return false;
 	}
 
-	// Debugging for Sune Molgaard's issue with non-existent files being
-	// detected as unattached and crashing later in CheckUnattachedObjects()
-	if (ObjectID == 0x90c1a)
-	{
-		BOX_INFO("Adding ID " << BOX_FORMAT_OBJECTID(ObjectID) <<
-			" contained by " << BOX_FORMAT_OBJECTID(containerID) <<
-			" with size " << size << " and isFile " << isFile);
-	}
-
 	// Add to list of IDs known about
 	AddID(ObjectID, containerID, size, isFile);
 
+	// Add to usage counts
+	mBlocksUsed += size;
+	if(!isFile)
+	{
+		mBlocksInDirectories += size;
+	}
+
+	// If it looks like a good object, and it's non-RAID, and
+	// this is a RAID set, then convert it to RAID.
+		
+	RaidFileController &rcontroller(RaidFileController::GetController());
+	RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber));
+	if(!rdiscSet.IsNonRaidSet())
+	{
+		// See if the file exists
+		RaidFileUtil::ExistType existance = 
+			RaidFileUtil::RaidFileExists(rdiscSet, rFilename);
+		if(existance == RaidFileUtil::NonRaid)
+		{
+			BOX_WARNING("Found non-RAID write file in RAID set" <<
+				(mFixErrors?", transforming to RAID: ":"") <<
+				(mFixErrors?rFilename:""));
+			if(mFixErrors)
+			{
+				RaidFileWrite write(mDiscSetNumber, rFilename);
+				write.TransformToRaidStorage();
+			}
+		}
+		else if(existance == RaidFileUtil::AsRaidWithMissingReadable)
+		{
+			BOX_WARNING("Found damaged but repairable RAID file" <<
+				(mFixErrors?", repairing: ":"") << 
+				(mFixErrors?rFilename:""));
+			if(mFixErrors)
+			{
+				std::auto_ptr<RaidFileRead> read(
+					RaidFileRead::Open(mDiscSetNumber, 
+						rFilename));
+				RaidFileWrite write(mDiscSetNumber, rFilename);
+				write.Open(true /* overwrite */);
+				read->CopyStreamTo(write);
+				read.reset();
+				write.Commit(true /* transform to RAID */);
+			}
+		}
+	}
+			
 	// Report success
 	return true;
 }
@@ -645,13 +706,11 @@ void BackupStoreCheck::CheckDirectories()
 				
 				if(isModified && mFixErrors)
 				{	
-					BOX_WARNING("Fixing directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]));
-
-					// Save back to disc
+					BOX_WARNING("Writing modified directory to disk: " <<
+						BOX_FORMAT_OBJECTID(pblock->mID[e]));
 					RaidFileWrite fixed(mDiscSetNumber, filename);
 					fixed.Open(true /* allow overwriting */);
 					dir.WriteToStream(fixed);
-					// Commit it
 					fixed.Commit(true /* convert to raid representation now */);
 				}
 
diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h
index e091432c..178a873a 100644
--- a/lib/backupstore/BackupStoreCheck.h
+++ b/lib/backupstore/BackupStoreCheck.h
@@ -28,9 +28,8 @@ The following problems can be fixed:
 	* Spurious files deleted
 	* Corrupted files deleted
 	* Root ID as file, deleted
-	* Dirs with wrong object id inside, deleted
-	* Direcetory entries pointing to non-existant files, deleted
-	* Doubly references files have second reference deleted
+	* Dirs with wrong object id in header, deleted
+	* Doubly referenced files have second reference deleted
 	* Wrong directory container IDs fixed
 	* Missing root recreated
 	* Reattach files which exist, but aren't referenced
@@ -43,7 +42,9 @@ The following problems can be fixed:
 	* Inside directories,
 		- only one object per name has old version clear
 		- IDs aren't duplicated
-	* Bad store info files regenerated
+		- entries pointing to non-existant files are deleted
+		- patches depending on non-existent objects are deleted
+	* Bad store info and refcount files regenerated
 	* Bad sizes of files in directories fixed
 
 */
diff --git a/lib/raidfile/RaidFileController.cpp b/lib/raidfile/RaidFileController.cpp
index e1305d9a..cf93947f 100644
--- a/lib/raidfile/RaidFileController.cpp
+++ b/lib/raidfile/RaidFileController.cpp
@@ -171,7 +171,11 @@ RaidFileDiscSet &RaidFileController::GetDiscSet(unsigned int DiscSetNum)
 	return mSetList[DiscSetNum];
 }
 
-
+// Overload to make usable in gdb debugger.
+int RaidFileDiscSet::GetSetNumForWriteFiles(const char* filename) const
+{
+	return GetSetNumForWriteFiles(std::string(filename));
+}
 
 // --------------------------------------------------------------------------
 //
diff --git a/lib/raidfile/RaidFileController.h b/lib/raidfile/RaidFileController.h
index 216bdf3a..601cca22 100644
--- a/lib/raidfile/RaidFileController.h
+++ b/lib/raidfile/RaidFileController.h
@@ -49,7 +49,7 @@ public:
 	int GetSetID() const {return mSetID;}
 	
 	int GetSetNumForWriteFiles(const std::string &rFilename) const;
-	
+	int GetSetNumForWriteFiles(const char* filename) const;
 	unsigned int GetBlockSize() const {return mBlockSize;}
 
 	// Is this disc set a non-RAID disc set? (ie files never get transformed to raid storage)
diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp
index 3dab69e5..bcff54c6 100644
--- a/lib/raidfile/RaidFileRead.cpp
+++ b/lib/raidfile/RaidFileRead.cpp
@@ -1025,8 +1025,8 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string
 	RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, Filename, &startDisc, &existingFiles, pRevisionID);
 	if(existance == RaidFileUtil::NoFile)
 	{
-		BOX_ERROR("Expected raidfile " << Filename << " does not exist");
-		THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist)
+		THROW_FILE_ERROR("Expected raidfile does not exist",
+			Filename, RaidFileException, RaidFileDoesntExist);
 	}
 	else if(existance == RaidFileUtil::NonRaid)
 	{
@@ -1588,7 +1588,7 @@ bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirN
 	{
 		// build name
 		std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName);
-		
+
 		// read the contents...
 		DIR *dirHandle = 0;
 		try