18 files changed, 392 insertions, 86 deletions
diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp
index b8d42d47..e03d53c2 100644
--- a/bin/bbackupd/BackupClientDirectoryRecord.cpp
+++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp
@@ -294,14 +294,21 @@ void BackupClientDirectoryRecord::SyncDirectory(
 				#else
 				if(EMU_LSTAT(filename.c_str(), &file_st) != 0)
 				{
-					// Report the error (logs and 
-					// eventual email to administrator)
- 					rNotifier.NotifyFileStatFailed(this, 
- 						filename, strerror(errno));
+					if(!(rParams.mrContext.ExcludeDir(
+						filename)))
+					{
+						// Report the error (logs and 
+						// eventual email to
+						// administrator)
+ 						rNotifier.NotifyFileStatFailed(
+							this, filename,
+							strerror(errno));
 					
-					// FIXME move to NotifyFileStatFailed()
-					SetErrorWhenReadingFilesystemObject(
-						rParams, filename.c_str());
+						// FIXME move to
+						// NotifyFileStatFailed()
+						SetErrorWhenReadingFilesystemObject(
+							rParams, filename.c_str());
+					}
 
 					// Ignore this entry for now.
 					continue;
@@ -606,8 +613,10 @@ void BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord::
 		const StreamableMemBlock &storeAttrEnc(pDirOnStore->GetAttributes());
 		// Explict decryption
 		BackupClientFileAttributes storeAttr(storeAttrEnc);
+		
 		// Compare the attributes
-		if(attr.Compare(storeAttr, true, true /* ignore both modification times */))
+		if(attr.Compare(storeAttr, true,
+			true /* ignore both modification times */))
 		{
 			// No update necessary
 			updateAttr = false;
@@ -918,7 +927,7 @@ bool BackupClientDirectoryRecord::UpdateItems(
 				BOX_TRACE("Upload decision: " <<
 					filename << ": will not upload "
 					"(modified too recently: "
-					"only " << age << "seconds ago)");
+					"only " << age << " seconds ago)");
 			}
 			else
 			{
@@ -1041,6 +1050,9 @@ bool BackupClientDirectoryRecord::UpdateItems(
 			{
 				try
 				{
+					rNotifier.NotifyFileUploadingAttributes(
+						this, filename);
+					
 					// Update store
 					BackupClientFileAttributes attr;
 					attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */);
diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp
index 3615b848..5134d234 100644
--- a/bin/bbackupd/BackupDaemon.cpp
+++ b/bin/bbackupd/BackupDaemon.cpp
@@ -1017,12 +1017,15 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed()
 	// If there's no result, try again in five minutes
 	int waitInSeconds = (60*5);
 
+	std::string script(conf.GetKeyValue("SyncAllowScript") + 
+		" \"" + GetConfigFileName() + "\"");
+
 	// Run it?
 	pid_t pid = 0;
 	try
 	{
-		std::auto_ptr<IOStream> pscript(LocalProcessStream(
-			conf.GetKeyValue("SyncAllowScript").c_str(), pid));
+		std::auto_ptr<IOStream> pscript(LocalProcessStream(script,
+			pid));
 
 		// Read in the result
 		IOStreamGetLine getLine(*pscript);
@@ -1044,17 +1047,15 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed()
 				}
 				catch(ConversionException &e)
 				{
-					BOX_ERROR("Invalid output "
-						"from SyncAllowScript '"
-						<< conf.GetKeyValue("SyncAllowScript")
-						<< "': '" << line << "'");
+					BOX_ERROR("Invalid output from "
+						"SyncAllowScript: '" <<
+						line << "' (" << script << ")");
 					throw;
 				}
 
 				BOX_NOTICE("Delaying sync by " << waitInSeconds
-					<< " seconds (SyncAllowScript '"
-					<< conf.GetKeyValue("SyncAllowScript")
-					<< "')");
+					<< " seconds due to SyncAllowScript "
+					<< "(" << script << ")");
 			}
 		}
 		
@@ -1062,14 +1063,14 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed()
 	catch(std::exception &e)
 	{
 		BOX_ERROR("Internal error running SyncAllowScript: "
-			<< e.what());
+			<< e.what() << " (" << script << ")");
 	}
 	catch(...)
 	{
 		// Ignore any exceptions
 		// Log that something bad happened
-		BOX_ERROR("Error running SyncAllowScript '"
-			<< conf.GetKeyValue("SyncAllowScript") << "'");
+		BOX_ERROR("Unknown error running SyncAllowScript (" <<
+			script << ")");
 	}
 
 	// Wait and then cleanup child process, if any
@@ -2186,27 +2187,26 @@ void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event)
 			Event != SysadminNotifier::BackupFinish)
 		{
 			BOX_INFO("Not notifying administrator about event "
-				<< sEventNames[Event] << " -- set NotifyScript "
+				<< sEventNames[Event] << ", set NotifyScript "
 				"to do this in future");
 		}
 		return;
 	}
 
 	// Script to run
-	std::string script(conf.GetKeyValue("NotifyScript") + ' ' +
-		sEventNames[Event]);
+	std::string script(conf.GetKeyValue("NotifyScript") + " " +
+		sEventNames[Event] + " \"" + GetConfigFileName() + "\"");
 	
 	// Log what we're about to do
 	BOX_INFO("About to notify administrator about event "
-		<< sEventNames[Event] << ", running script '"
-		<< script << "'");
+		<< sEventNames[Event] << ", running script '" << script << "'");
 	
 	// Then do it
 	int returnCode = ::system(script.c_str());
 	if(returnCode != 0)
 	{
 		BOX_WARNING("Notify script returned error code: " <<
-			returnCode << " ('" << script << "')");
+			returnCode << " (" << script << ")");
 	}
 	else if(Event != SysadminNotifier::BackupStart &&
 		Event != SysadminNotifier::BackupFinish)
diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h
index 0c864abd..b41c6508 100644
--- a/bin/bbackupd/BackupDaemon.h
+++ b/bin/bbackupd/BackupDaemon.h
@@ -444,6 +444,16 @@ public:
 			BOX_NOTICE("Uploading patch to file: " << rLocalPath);
 		} 
 	}
+ 	virtual void NotifyFileUploadingAttributes(
+ 		const BackupClientDirectoryRecord* pDirRecord,
+ 		const std::string& rLocalPath) 
+	{
+		if (mLogAllFileAccess)
+		{
+			BOX_NOTICE("Uploading new file attributes: " << 
+				rLocalPath);
+		} 
+	}
  	virtual void NotifyFileUploaded(
  		const BackupClientDirectoryRecord* pDirRecord,
  		const std::string& rLocalPath,
diff --git a/bin/bbackupd/BackupDaemonInterface.h b/bin/bbackupd/BackupDaemonInterface.h
index 5bbdd427..2a2d8d4b 100644
--- a/bin/bbackupd/BackupDaemonInterface.h
+++ b/bin/bbackupd/BackupDaemonInterface.h
@@ -123,6 +123,9 @@ class ProgressNotifier
 	virtual void NotifyFileUploadingPatch(
 		const BackupClientDirectoryRecord* pDirRecord,
 		const std::string& rLocalPath) = 0;
+	virtual void NotifyFileUploadingAttributes(
+ 		const BackupClientDirectoryRecord* pDirRecord,
+ 		const std::string& rLocalPath) = 0;
 	virtual void NotifyFileUploaded(
 		const BackupClientDirectoryRecord* pDirRecord,
 		const std::string& rLocalPath,
diff --git a/bin/bbackupd/Makefile.extra b/bin/bbackupd/Makefile.extra
index 52e21366..25ceb1e7 100644
--- a/bin/bbackupd/Makefile.extra
+++ b/bin/bbackupd/Makefile.extra
@@ -3,5 +3,5 @@ MAKEEXCEPTION = ../../lib/common/makeexception.pl
 
 # AUTOGEN SEEDING
 autogen_ClientException.h autogen_ClientException.cpp:	$(MAKEEXCEPTION) ClientException.txt
-	$(PERL) $(MAKEEXCEPTION) ClientException.txt
+	$(_PERL) $(MAKEEXCEPTION) ClientException.txt
 
diff --git a/bin/bbackupquery/BackupQueries.cpp b/bin/bbackupquery/BackupQueries.cpp
index 687dcb05..f799ab43 100644
--- a/bin/bbackupquery/BackupQueries.cpp
+++ b/bin/bbackupquery/BackupQueries.cpp
@@ -212,7 +212,7 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine)
 	{
 		{ "quit", "" },
 		{ "exit", "" },
-		{ "list", "rodIFtTsh", },
+		{ "list", "rodIFtTash", },
 		{ "pwd",  "" },
 		{ "cd",   "od" },
 		{ "lcd",  "" },
@@ -397,6 +397,7 @@ void BackupQueries::CommandList(const std::vector<std::string> &args, const bool
 	#define LIST_OPTION_NOFLAGS		'F'
 	#define LIST_OPTION_TIMES_LOCAL		't'
 	#define LIST_OPTION_TIMES_UTC		'T'
+	#define LIST_OPTION_TIMES_ATTRIBS	'a'
 	#define LIST_OPTION_SIZEINBLOCKS	's'
 	#define LIST_OPTION_DISPLAY_HASH	'h'
 
@@ -435,6 +436,52 @@ void BackupQueries::CommandList(const std::vector<std::string> &args, const bool
 	List(rootDir, listRoot, opts, true /* first level to list */);
 }
 
+static std::string GetTimeString(BackupStoreDirectory::Entry& en,
+	bool useLocalTime, bool showAttrModificationTimes)
+{
+	std::ostringstream out;
+	box_time_t originalTime, newAttributesTime;
+
+	// there is no attribute modification time in the directory
+	// entry, unfortunately, so we can't display it.
+	originalTime = en.GetModificationTime();
+	out << BoxTimeToISO8601String(originalTime, useLocalTime);
+
+	if(en.HasAttributes())
+	{
+		const StreamableMemBlock &storeAttr(en.GetAttributes());
+		BackupClientFileAttributes attr(storeAttr);
+		
+		box_time_t NewModificationTime, NewAttrModificationTime;
+		attr.GetModificationTimes(&NewModificationTime,
+			&NewAttrModificationTime);
+		
+		if (showAttrModificationTimes)
+		{
+			newAttributesTime = NewAttrModificationTime;
+		}
+		else
+		{
+			newAttributesTime = NewModificationTime;
+		}
+		
+		if (newAttributesTime == originalTime)
+		{
+			out << "*";
+		}
+		else
+		{
+			out << "~" << BoxTimeToISO8601String(newAttributesTime,
+				useLocalTime);
+		}
+	}
+	else
+	{
+		out << " ";
+	}
+	
+	return out.str();
+}
 
 // --------------------------------------------------------------------------
 //
@@ -534,17 +581,15 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool
 		if(opts[LIST_OPTION_TIMES_UTC])
 		{
 			// Show UTC times...
-			std::string time = BoxTimeToISO8601String(
-				en->GetModificationTime(), false);
-			printf("%s ", time.c_str());
+			printf("%s ", GetTimeString(*en, false,
+				opts[LIST_OPTION_TIMES_ATTRIBS]).c_str());
 		}
 
 		if(opts[LIST_OPTION_TIMES_LOCAL])
 		{
 			// Show local times...
-			std::string time = BoxTimeToISO8601String(
-				en->GetModificationTime(), true);
-			printf("%s ", time.c_str());
+			printf("%s ", GetTimeString(*en, true,
+				opts[LIST_OPTION_TIMES_ATTRIBS]).c_str());
 		}
 		
 		if(opts[LIST_OPTION_DISPLAY_HASH])
@@ -1544,13 +1589,31 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir,
 				continue;
 			}
 
+			std::string localDirPath(MakeFullPath(rLocalDir,
+				localDirEn->d_name));
+			std::string storeDirPath(rStoreDir + "/" +
+				localDirEn->d_name);
+
 #ifndef HAVE_VALID_DIRENT_D_TYPE
-			std::string fn(MakeFullPath
-				(rLocalDir, localDirEn->d_name));
 			EMU_STRUCT_STAT st;
-			if(EMU_LSTAT(fn.c_str(), &st) != 0)
+			if(EMU_LSTAT(localDirPath.c_str(), &st) != 0)
 			{
-			    THROW_EXCEPTION(CommonException, OSFileError)
+				// Check whether dir is excluded before trying
+				// to stat it, to fix problems with .gvfs
+				// directories that are not readable by root
+				// causing compare to crash:
+				// http://lists.boxbackup.org/pipermail/boxbackup/2010-January/000013.html
+				if(rParams.IsExcludedDir(localDirPath))
+				{
+					rParams.NotifyExcludedDir(localDirPath,
+						storeDirPath);
+					continue;
+				}
+				else
+				{
+					THROW_EXCEPTION_MESSAGE(CommonException,
+						OSFileError, localDirPath);
+				}
 			}
 			
 			// Entry -- file or dir?
diff --git a/bin/bbackupquery/Makefile.extra b/bin/bbackupquery/Makefile.extra
index f347c451..e1049b6d 100644
--- a/bin/bbackupquery/Makefile.extra
+++ b/bin/bbackupquery/Makefile.extra
@@ -1,6 +1,6 @@
 
 # AUTOGEN SEEDING
 autogen_Documentation.cpp:	makedocumentation.pl documentation.txt
-	$(PERL) makedocumentation.pl
+	$(_PERL) makedocumentation.pl
 
 
diff --git a/bin/bbackupquery/bbackupquery.cpp b/bin/bbackupquery/bbackupquery.cpp
index bab697b8..5aa7e97e 100644
--- a/bin/bbackupquery/bbackupquery.cpp
+++ b/bin/bbackupquery/bbackupquery.cpp
@@ -60,7 +60,7 @@
 
 void PrintUsageAndExit()
 {
-	printf("Usage: bbackupquery [-q] [-w] "
+	printf("Usage: bbackupquery [-q*|v*|V|W<level>] [-w] "
 #ifdef WIN32
 	"[-u] "
 #endif
@@ -123,10 +123,10 @@ int main(int argc, const char *argv[])
 	#endif
 
 #ifdef WIN32
-	const char* validOpts = "qvwuc:l:o:O:W:";
+	const char* validOpts = "qvVwuc:l:o:O:W:";
 	bool unicodeConsole = false;
 #else
-	const char* validOpts = "qvwc:l:o:O:W:";
+	const char* validOpts = "qvVwc:l:o:O:W:";
 #endif
 
 	std::string fileLogFile;
@@ -138,7 +138,7 @@ int main(int argc, const char *argv[])
 	{
 		switch(c)
 		{
-			case 'q':
+		case 'q':
 			{
 				if(masterLevel == Log::NOTHING)
 				{
@@ -151,7 +151,7 @@ int main(int argc, const char *argv[])
 			}
 			break;
 
-			case 'v':
+		case 'v':
 			{
 				if(masterLevel == Log::EVERYTHING)
 				{
@@ -164,6 +164,12 @@ int main(int argc, const char *argv[])
 			}
 			break;
 
+		case 'V':
+			{
+				masterLevel = Log::EVERYTHING;
+			}
+			break;
+
 		case 'W':
 			{
 				masterLevel = Logging::GetNamedLevel(optarg);
diff --git a/bin/bbackupquery/documentation.txt b/bin/bbackupquery/documentation.txt
index d32bf200..96eda158 100644
--- a/bin/bbackupquery/documentation.txt
+++ b/bin/bbackupquery/documentation.txt
@@ -38,10 +38,13 @@ start with '/'. The initial directory is always the root directory.
 	-o -- list old versions of files/directories
 	-I -- don't display object ID
 	-F -- don't display flags
-	-t -- show file modification time
-			(and attr mod time if has the object has attributes, ~ separated)
+	-t -- show file modification time in local time
+	      (and attr mod time if has the object has attributes, ~ separated)
+	-T -- show file modification time in GMT
+	-a -- show updated attribute instead of file modification time
 	-s -- show file size in blocks used on server
-			(only very approximate indication of size locally)
+	      (only very approximate indication of size locally)
+	-h -- show file attributes hash
 
 ls can be used as an alias.
 <
diff --git a/bin/bbstoreaccounts/bbstoreaccounts.cpp b/bin/bbstoreaccounts/bbstoreaccounts.cpp
index e377b661..a9d2b0af 100644
--- a/bin/bbstoreaccounts/bbstoreaccounts.cpp
+++ b/bin/bbstoreaccounts/bbstoreaccounts.cpp
@@ -45,16 +45,17 @@ bool sMachineReadableOutput = false;
 
 void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit)
 {
-	if(SoftLimit >= HardLimit)
+	if(SoftLimit > HardLimit)
 	{
 		BOX_FATAL("Soft limit must be less than the hard limit.");
 		exit(1);
 	}
 	if(SoftLimit > ((HardLimit * MAX_SOFT_LIMIT_SIZE) / 100))
 	{
-		BOX_FATAL("Soft limit must be no more than " << 
-			MAX_SOFT_LIMIT_SIZE << "% of the hard limit.");
-		exit(1);
+		BOX_WARNING("We recommend setting the soft limit below " <<
+			MAX_SOFT_LIMIT_SIZE << "% of the hard limit, or " <<
+			HumanReadableSize((HardLimit * MAX_SOFT_LIMIT_SIZE)
+				/ 100) << " in this case.");
 	}
 }
 
diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp
index 1c1767ca..7f799008 100644
--- a/bin/bbstored/BBStoreDHousekeeping.cpp
+++ b/bin/bbstored/BBStoreDHousekeeping.cpp
@@ -108,7 +108,8 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded()
 				mpAccounts->GetAccountRoot(*i, rootDir, discSet);
 				
 				// Do housekeeping on this account
-				HousekeepStoreAccount housekeeping(*i, rootDir, discSet, *this);
+				HousekeepStoreAccount housekeeping(*i, rootDir,
+					discSet, this);
 				housekeeping.DoHousekeeping();
 			}
 		}
diff --git a/bin/bbstored/BackupStoreContext.cpp b/bin/bbstored/BackupStoreContext.cpp
index 990be05d..2e915f57 100644
--- a/bin/bbstored/BackupStoreContext.cpp
+++ b/bin/bbstored/BackupStoreContext.cpp
@@ -191,6 +191,23 @@ void BackupStoreContext::LoadStoreInfo()
 	
 	// Keep the pointer to it
 	mpStoreInfo = i;
+
+	BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet);
+
+	// try to load the reference count database
+	try
+	{
+		mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+	}
+	catch(BoxException &e)
+	{
+		BOX_WARNING("Reference count database is missing or corrupted, "
+			"creating a new one, expect housekeeping to find and "
+			"fix problems with reference counts later.");
+		
+		BackupStoreRefCountDatabase::CreateForRegeneration(account);
+		mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+	}
 }
 
 
@@ -395,15 +412,18 @@ int64_t BackupStoreContext::AllocateObjectID()
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
-//		Purpose: Add a file to the store, from a given stream, into a specified directory.
-//				 Returns object ID of the new file.
+//		Name:    BackupStoreContext::AddFile(IOStream &, int64_t,
+//			 int64_t, int64_t, const BackupStoreFilename &, bool)
+//		Purpose: Add a file to the store, from a given stream, into
+//			 a specified directory. Returns object ID of the new
+//			 file.
 //		Created: 2003/09/03
 //
 // --------------------------------------------------------------------------
-int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime,
-		int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
-		bool MarkFileWithSameNameAsOldVersions)
+int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory,
+	int64_t ModificationTime, int64_t AttributesHash,
+	int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
+	bool MarkFileWithSameNameAsOldVersions)
 {
 	if(mpStoreInfo.get() == 0)
 	{
@@ -670,6 +690,9 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_
 	mpStoreInfo->ChangeBlocksUsed(blocksUsed);
 	mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
 	
+	// Increment reference count on the new directory to one
+	mapRefCount->AddReference(id);
+	
 	// Save the store info -- can cope if this exceptions because infomation
 	// will be rebuilt by housekeeping, and ID allocation can recover.
 	SaveStoreInfo();
@@ -768,8 +791,9 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
-//		Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
+//		Name:    BackupStoreContext::UndeleteFile(int64_t, int64_t)
+//		Purpose: Undeletes a file, if it exists, returning true if
+//			 the file existed.
 //		Created: 2003/10/21
 //
 // --------------------------------------------------------------------------
@@ -933,8 +957,10 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec
 // --------------------------------------------------------------------------
 //
 // Function
-//		Name:    BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &)
-//		Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely.
+//		Name:    BackupStoreContext::AddDirectory(int64_t,
+//			 const BackupStoreFilename &, bool &)
+//		Purpose: Creates a directory (or just returns the ID of an
+//			 existing one). rAlreadyExists set appropraitely.
 //		Created: 2003/09/04
 //
 // --------------------------------------------------------------------------
@@ -974,7 +1000,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF
 	// Allocate the next ID
 	int64_t id = AllocateObjectID();
 
-	// Create a blank directory with the given attributes on disc
+	// Create an empty directory with the given attributes on disc
 	std::string fn;
 	MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
 	{
@@ -998,11 +1024,14 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF
 		// Not added to cache, so don't set the size in the directory
 	}
 	
-	// Then add it into the directory
+	// Then add it into the parent directory
 	try
 	{
 		dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
 		SaveDirectory(dir, InDirectory);
+
+		// Increment reference count on the new directory to one
+		mapRefCount->AddReference(id);
 	}
 	catch(...)
 	{
@@ -1016,7 +1045,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF
 		// Don't worry about the incremented number in the store info
 		throw;	
 	}
-	
+
 	// Save the store info (may be postponed)
 	SaveStoreInfo();
 
diff --git a/bin/bbstored/BackupStoreContext.h b/bin/bbstored/BackupStoreContext.h
index 4cfdb601..6053e4b8 100644
--- a/bin/bbstored/BackupStoreContext.h
+++ b/bin/bbstored/BackupStoreContext.h
@@ -14,6 +14,7 @@
 #include <map>
 #include <memory>
 
+#include "BackupStoreRefCountDatabase.h"
 #include "NamedLock.h"
 #include "ProtocolObject.h"
 #include "Utils.h"
@@ -137,7 +138,6 @@ private:
 	void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
 	int64_t AllocateObjectID();
 
-private:
 	int32_t mClientID;
 	HousekeepingInterface &mrDaemon;
 	int mProtocolPhase;
@@ -150,6 +150,9 @@ private:
 	
 	// Store info
 	std::auto_ptr<BackupStoreInfo> mpStoreInfo;
+
+	// Refcount database
+	std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount;
 	
 	// Directory cache
 	std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;
diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h
index a5d216f4..49af5b81 100644
--- a/bin/bbstored/BackupStoreDaemon.h
+++ b/bin/bbstored/BackupStoreDaemon.h
@@ -14,11 +14,11 @@
 #include "BoxPortsAndFiles.h"
 #include "BackupConstants.h"
 #include "BackupStoreContext.h"
+#include "HousekeepStoreAccount.h"
 #include "IOStreamGetLine.h"
 
 class BackupStoreAccounts;
 class BackupStoreAccountDatabase;
-class HousekeepStoreAccount;
 
 // --------------------------------------------------------------------------
 //
@@ -29,10 +29,8 @@ class HousekeepStoreAccount;
 //
 // --------------------------------------------------------------------------
 class BackupStoreDaemon : public ServerTLS<BOX_PORT_BBSTORED>,
-	HousekeepingInterface
+	HousekeepingInterface, HousekeepingCallback
 {
-	friend class HousekeepStoreAccount;
-
 public:
 	BackupStoreDaemon();
 	~BackupStoreDaemon();
@@ -64,10 +62,13 @@ protected:
 	
 	// Housekeeping functions
 	void HousekeepingProcess();
-	bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0);
 
 	void LogConnectionStats(const char *commonName, const SocketStreamTLS &s);
 
+public:
+	// HousekeepingInterface implementation
+	virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0);
+
 private:
 	BackupStoreAccountDatabase *mpAccountDatabase;
 	BackupStoreAccounts *mpAccounts;
diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp
index dbb9b544..0ccbcf23 100644
--- a/bin/bbstored/HousekeepStoreAccount.cpp
+++ b/bin/bbstored/HousekeepStoreAccount.cpp
@@ -39,11 +39,13 @@
 //		Created: 11/12/03
 //
 // --------------------------------------------------------------------------
-HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon)
+HousekeepStoreAccount::HousekeepStoreAccount(int AccountID,
+	const std::string &rStoreRoot, int StoreDiscSet,
+	HousekeepingCallback* pHousekeepingCallback)
 	: mAccountID(AccountID),
 	  mStoreRoot(rStoreRoot),
 	  mStoreDiscSet(StoreDiscSet),
-	  mrDaemon(rDaemon),
+	  mpHousekeepingCallback(pHousekeepingCallback),
 	  mDeletionSizeTarget(0),
   	  mPotentialDeletionsTotalSize(0),
 	  mMaxSizeInPotentialDeletions(0),
@@ -57,6 +59,7 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, const std::string &r
 	  mBlocksInDirectoriesDelta(0),
 	  mFilesDeleted(0),
 	  mEmptyDirectoriesDeleted(0),
+	  mSuppressRefCountChangeWarnings(false),
 	  mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY)
 {
 }
@@ -81,7 +84,7 @@ HousekeepStoreAccount::~HousekeepStoreAccount()
 //		Created: 11/12/03
 //
 // --------------------------------------------------------------------------
-void HousekeepStoreAccount::DoHousekeeping()
+void HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever)
 {
 	// Attempt to lock the account
 	std::string writeLockFilename;
@@ -91,8 +94,21 @@ void HousekeepStoreAccount::DoHousekeeping()
 	if(!writeLock.TryAndGetLock(writeLockFilename.c_str(),
 		0600 /* restrictive file permissions */))
 	{
-		// Couldn't lock the account -- just stop now
-		return;
+		if(KeepTryingForever)
+		{
+			BOX_WARNING("Failed to lock account for housekeeping, "
+				"still trying...");
+			while(!writeLock.TryAndGetLock(writeLockFilename,
+				0600 /* restrictive file permissions */))
+			{
+				sleep(1);
+			}
+		}
+		else
+		{
+			// Couldn't lock the account -- just stop now
+			return;
+		}
 	}
 
 	// Load the store info to find necessary info for the housekeeping
@@ -106,6 +122,14 @@ void HousekeepStoreAccount::DoHousekeeping()
 		mDeletionSizeTarget = 0;
 	}
 
+	// initialise the refcount database
+	mNewRefCounts.clear();
+	// try to pre-allocate as much memory as we need
+	mNewRefCounts.reserve(info->GetLastObjectIDUsed());
+	// initialise the refcount of the root entry
+	mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0);
+	mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1;
+
 	// Scan the directory for potential things to delete
 	// This will also remove eligible items marked with RemoveASAP
 	bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID);
@@ -207,6 +231,86 @@ void HousekeepStoreAccount::DoHousekeeping()
 			mEmptyDirectoriesDeleted << " dirs)" <<
 			(deleteInterrupted?" and was interrupted":""));
 	}
+
+	// We can only update the refcount database if we successfully
+	// finished our scan of all directories, otherwise we don't actually
+	// know which of the new counts are valid and which aren't
+	// (we might not have seen second references to some objects, etc.)
+
+	BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet);
+	std::auto_ptr<BackupStoreRefCountDatabase> apReferences;
+
+	// try to load the reference count database
+	try
+	{
+		apReferences = BackupStoreRefCountDatabase::Load(account,
+			false);
+	}
+	catch(BoxException &e)
+	{
+		BOX_WARNING("Reference count database is missing or corrupted "
+			"during housekeeping, creating a new one.");
+		mSuppressRefCountChangeWarnings = true;
+		BackupStoreRefCountDatabase::CreateForRegeneration(account);
+		apReferences = BackupStoreRefCountDatabase::Load(account,
+			false);
+	}
+
+	int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed();
+
+	for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID;
+		ObjectID < mNewRefCounts.size(); ObjectID++)
+	{
+		if (ObjectID > LastUsedObjectIdOnDisk)
+		{
+			if (!mSuppressRefCountChangeWarnings)
+			{
+				BOX_WARNING("Reference count of object " <<
+					BOX_FORMAT_OBJECTID(ObjectID) <<
+					" not found in database, added"
+					" with " << mNewRefCounts[ObjectID] <<
+					" references");
+			}
+			apReferences->SetRefCount(ObjectID,
+				mNewRefCounts[ObjectID]);
+			LastUsedObjectIdOnDisk = ObjectID;
+			continue;
+		}
+
+		BackupStoreRefCountDatabase::refcount_t OldRefCount =
+			apReferences->GetRefCount(ObjectID);
+
+		if (OldRefCount != mNewRefCounts[ObjectID])
+		{
+			BOX_WARNING("Reference count of object " <<
+				BOX_FORMAT_OBJECTID(ObjectID) <<
+				" changed from " << OldRefCount <<
+				" to " << mNewRefCounts[ObjectID]);
+			apReferences->SetRefCount(ObjectID,
+				mNewRefCounts[ObjectID]);
+		}
+	}
+
+	// zero excess references in the database
+	for (int64_t ObjectID = mNewRefCounts.size(); 
+		ObjectID <= LastUsedObjectIdOnDisk; ObjectID++)
+	{
+		BackupStoreRefCountDatabase::refcount_t OldRefCount =
+			apReferences->GetRefCount(ObjectID);
+		BackupStoreRefCountDatabase::refcount_t NewRefCount = 0;
+
+		if (OldRefCount != NewRefCount)
+		{
+			BOX_WARNING("Reference count of object " <<
+				BOX_FORMAT_OBJECTID(ObjectID) <<
+				" changed from " << OldRefCount <<
+				" to " << NewRefCount << " (not found)");
+			apReferences->SetRefCount(ObjectID, NewRefCount);
+		}
+	}
+
+	// force file to be saved and closed before releasing the lock below
+	apReferences.reset();
 	
 	// Make sure the delta's won't cause problems if the counts are 
 	// really wrong, and it wasn't fixed because the store was 
@@ -279,7 +383,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
 
 		// Check for having to stop
 		// Include account ID here as the specified account is locked
-		if(mrDaemon.CheckForInterProcessMsg(mAccountID))
+		if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))
 		{
 			// Need to abort now
 			return false;
@@ -312,7 +416,25 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID)
 		// Add it to the list of directories to potentially delete
 		mEmptyDirectories.push_back(dir.GetObjectID());
 	}
-	
+
+	// Calculate reference counts first, before we start requesting
+	// files to be deleted.
+	// BLOCK
+	{
+		BackupStoreDirectory::Iterator i(dir);
+		BackupStoreDirectory::Entry *en = 0;
+
+		while((en = i.Next()) != 0)
+		{
+			// This directory references this object
+			if (mNewRefCounts.size() <= en->GetObjectID())
+			{
+				mNewRefCounts.resize(en->GetObjectID() + 1, 0);
+			}
+			mNewRefCounts[en->GetObjectID()]++;
+		}
+	}
+
 	// BLOCK
 	{
 		// Remove any files which are marked for removal as soon
@@ -551,7 +673,7 @@ bool HousekeepStoreAccount::DeleteFiles()
 		{
 			mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
 			// Check for having to stop
-			if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+			if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
 			{
 				// Need to abort now
 				return true;
@@ -761,13 +883,22 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba
 		padjustedEntry.reset();	// delete it now
 	}
 
-	// Delete from disc
+	// Drop reference count by one. If it reaches zero, delete the file.
+	if(--mNewRefCounts[ObjectID] == 0)
 	{
+		BOX_TRACE("Removing unreferenced object " <<
+			BOX_FORMAT_OBJECTID(ObjectID));
 		std::string objFilename;
 		MakeObjectFilename(ObjectID, objFilename);
 		RaidFileWrite del(mStoreDiscSet, objFilename);
 		del.Delete();
 	}
+	else
+	{
+		BOX_TRACE("Preserving object " <<
+			BOX_FORMAT_OBJECTID(ObjectID) << " with " <<
+			mNewRefCounts[ObjectID] << " references");
+	}
 
 	// Adjust counts for the file
 	++mFilesDeleted;
@@ -808,7 +939,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories()
 			{
 				mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY;
 				// Check for having to stop
-				if(mrDaemon.CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
+				if(mpHousekeepingCallback && mpHousekeepingCallback->CheckForInterProcessMsg(mAccountID))	// include account ID here as the specified account is now locked
 				{
 					// Need to abort now
 					return true;
diff --git a/bin/bbstored/HousekeepStoreAccount.h b/bin/bbstored/HousekeepStoreAccount.h
index 5c2a9885..1dd6d79c 100644
--- a/bin/bbstored/HousekeepStoreAccount.h
+++ b/bin/bbstored/HousekeepStoreAccount.h
@@ -17,6 +17,12 @@
 class BackupStoreDaemon;
 class BackupStoreDirectory;
 
+class HousekeepingCallback
+{
+	public:
+	virtual ~HousekeepingCallback() {}
+	virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0) = 0;
+};
 
 // --------------------------------------------------------------------------
 //
@@ -29,10 +35,11 @@ class BackupStoreDirectory;
 class HousekeepStoreAccount
 {
 public:
-	HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot, int StoreDiscSet, BackupStoreDaemon &rDaemon);
+	HousekeepStoreAccount(int AccountID, const std::string &rStoreRoot,
+		int StoreDiscSet, HousekeepingCallback* pHousekeepingCallback);
 	~HousekeepStoreAccount();
 	
-	void DoHousekeeping();
+	void DoHousekeeping(bool KeepTryingForever = false);
 	
 	
 private:
@@ -65,7 +72,7 @@ private:
 	int mAccountID;
 	std::string mStoreRoot;
 	int mStoreDiscSet;
-	BackupStoreDaemon &mrDaemon;
+	HousekeepingCallback* mpHousekeepingCallback;
 	
 	int64_t mDeletionSizeTarget;
 	
@@ -91,6 +98,10 @@ private:
 	// Deletion count
 	int64_t mFilesDeleted;
 	int64_t mEmptyDirectoriesDeleted;
+
+	// New reference count list
+	std::vector<uint32_t> mNewRefCounts;
+	bool mSuppressRefCountChangeWarnings;
 	
 	// Poll frequency
 	int mCountUntilNextInterprocessMsgCheck;
diff --git a/bin/bbstored/Makefile.extra b/bin/bbstored/Makefile.extra
index 94bc3fb9..6562647d 100644
--- a/bin/bbstored/Makefile.extra
+++ b/bin/bbstored/Makefile.extra
@@ -5,5 +5,5 @@ GEN_CMD_SRV = $(MAKEPROTOCOL) Server backupprotocol.txt
 
 # AUTOGEN SEEDING
 autogen_BackupProtocolServer.cpp autogen_BackupProtocolServer.h:	$(MAKEPROTOCOL) backupprotocol.txt
-	$(PERL) $(GEN_CMD_SRV)
+	$(_PERL) $(GEN_CMD_SRV)
 
diff --git a/bin/s3simulator/s3simulator.cpp b/bin/s3simulator/s3simulator.cpp
new file mode 100644
index 00000000..9a10635c
--- /dev/null
+++ b/bin/s3simulator/s3simulator.cpp
@@ -0,0 +1,32 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    s3simulator.cpp
+//		Purpose: main file for S3 simulator daemon
+//		Created: 2003/10/11
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+#include "S3Simulator.h"
+#include "MainHelper.h"
+
+#include "MemLeakFindOn.h"
+
+int main(int argc, const char *argv[])
+{
+	int ExitCode = 0;
+
+	MAINHELPER_START
+
+	Logging::SetProgramName("s3simulator");
+	Logging::ToConsole(true);
+	Logging::ToSyslog (true);
+	
+	S3Simulator daemon;
+	ExitCode = daemon.Main("s3simulator.conf", argc, argv);
+
+	MAINHELPER_END
+
+	return ExitCode;
+}