Move reusable files from bin/bbstored to lib/bbstored.

Break dependendency of test/bbackupd on individual files from other modules.

3 files changed, 739 insertions, 0 deletions

diff --git a/lib/bbstored/BBStoreDHousekeeping.cpp b/lib/bbstored/BBStoreDHousekeeping.cpp
new file mode 100644
index 00000000..86d6409c
--- /dev/null
+++ b/lib/bbstored/BBStoreDHousekeeping.cpp
@@ -0,0 +1,261 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BBStoreDHousekeeping.cpp
+//		Purpose: Implementation of housekeeping functions for bbstored
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdio.h>
+
+#include "BackupStoreDaemon.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "HousekeepStoreAccount.h"
+#include "BoxTime.h"
+#include "Configuration.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::HousekeepingProcess()
+//		Purpose: Do housekeeping
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::HousekeepingInit()
+{
+
+	mLastHousekeepingRun = 0;
+}
+
+void BackupStoreDaemon::HousekeepingProcess()
+{
+	HousekeepingInit();
+
+	// Get the time between housekeeping runs
+	const Configuration &rconfig(GetConfiguration());
+	int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping"));
+
+	while(!StopRun())
+	{
+		RunHousekeepingIfNeeded();
+
+		// Stop early?
+		if(StopRun())
+		{
+			break;
+		}
+
+		// Calculate how long should wait before doing the next 
+		// housekeeping run
+		int64_t timeNow = GetCurrentBoxTime();
+		time_t secondsToGo = BoxTimeToSeconds(
+			(mLastHousekeepingRun + housekeepingInterval) - 
+			timeNow);
+		if(secondsToGo < 1) secondsToGo = 1;
+		if(secondsToGo > 60) secondsToGo = 60;
+		int32_t millisecondsToGo = ((int)secondsToGo) * 1000;
+	
+		// Check to see if there's any message pending
+		CheckForInterProcessMsg(0 /* no account */, millisecondsToGo);
+	}
+}
+
+void BackupStoreDaemon::RunHousekeepingIfNeeded()
+{
+	// Get the time between housekeeping runs
+	const Configuration &rconfig(GetConfiguration());
+	int64_t housekeepingInterval = SecondsToBoxTime(rconfig.GetKeyValueInt("TimeBetweenHousekeeping"));
+
+	// Time now
+	int64_t timeNow = GetCurrentBoxTime();
+
+	// Do housekeeping if the time interval has elapsed since the last check
+	if((timeNow - mLastHousekeepingRun) < housekeepingInterval)
+	{
+		BOX_TRACE("No need for housekeeping, " <<
+			BoxTimeToSeconds(timeNow - mLastHousekeepingRun) <<
+			" seconds since last run is less than " <<
+			BoxTimeToSeconds(housekeepingInterval));
+		return;
+	}
+	else
+	{
+		BOX_TRACE("Running housekeeping now, because " <<
+			BoxTimeToSeconds(timeNow - mLastHousekeepingRun) <<
+			" seconds since last run is more than " <<
+			BoxTimeToSeconds(housekeepingInterval));
+	}
+
+	// Store the time
+	mLastHousekeepingRun = timeNow;
+	BOX_INFO("Starting housekeeping");
+
+	// Get the list of accounts
+	std::vector<int32_t> accounts;
+	if(mpAccountDatabase)
+	{
+		mpAccountDatabase->GetAllAccountIDs(accounts);
+	}
+			
+	SetProcessTitle("housekeeping, active");
+			
+	// Check them all
+	for(std::vector<int32_t>::const_iterator i = accounts.begin(); i != accounts.end(); ++i)
+	{
+		try
+		{
+			std::string rootDir;
+			int discSet = 0;
+
+			{
+				// Tag log output to identify account
+				std::ostringstream tag;
+				tag << "hk/" << BOX_FORMAT_ACCOUNT(*i);
+				Logging::Tagger tagWithClientID(tag.str());
+
+				// Get the account root
+				mpAccounts->GetAccountRoot(*i, rootDir, discSet);
+
+				// Reset tagging as HousekeepStoreAccount will
+				// do that itself, to avoid duplicate tagging.
+				// Happens automatically when tagWithClientID
+				// goes out of scope.
+			}
+			
+			// Do housekeeping on this account
+			HousekeepStoreAccount housekeeping(*i, rootDir,
+				discSet, this);
+			housekeeping.DoHousekeeping();
+		}
+		catch(BoxException &e)
+		{
+			BOX_ERROR("Housekeeping on account " <<
+				BOX_FORMAT_ACCOUNT(*i) << " threw exception, "
+				"aborting run for this account: " <<
+				e.what() << " (" <<
+				e.GetType() << "/" << e.GetSubType() << ")");
+		}
+		catch(std::exception &e)
+		{
+			BOX_ERROR("Housekeeping on account " <<
+				BOX_FORMAT_ACCOUNT(*i) << " threw exception, "
+				"aborting run for this account: " <<
+				e.what());
+		}
+		catch(...)
+		{
+			BOX_ERROR("Housekeeping on account " <<
+				BOX_FORMAT_ACCOUNT(*i) << " threw exception, "
+				"aborting run for this account: "
+				"unknown exception");
+		}
+	
+		int64_t timeNow = GetCurrentBoxTime();
+		time_t secondsToGo = BoxTimeToSeconds(
+			(mLastHousekeepingRun + housekeepingInterval) - 
+			timeNow);
+		if(secondsToGo < 1) secondsToGo = 1;
+		if(secondsToGo > 60) secondsToGo = 60;
+		int32_t millisecondsToGo = ((int)secondsToGo) * 1000;
+
+		// Check to see if there's any message pending
+		CheckForInterProcessMsg(0 /* no account */, millisecondsToGo);
+
+		// Stop early?
+		if(StopRun())
+		{
+			break;
+		}
+	}
+		
+	BOX_INFO("Finished housekeeping");
+
+	// Placed here for accuracy, if StopRun() is true, for example.
+	SetProcessTitle("housekeeping, idle");
+}
+
+void BackupStoreDaemon::OnIdle()
+{
+	if (!IsSingleProcess())
+	{
+		return;
+	}
+
+	if (!mHousekeepingInited)
+	{
+		HousekeepingInit();
+		mHousekeepingInited = true;
+	}
+
+	RunHousekeepingIfNeeded();
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::CheckForInterProcessMsg(int, int)
+//		Purpose: Process a message, returning true if the housekeeping process
+//				 should abort for the specified account.
+//		Created: 11/12/03
+//
+// --------------------------------------------------------------------------
+bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitTime)
+{
+	if(!mInterProcessCommsSocket.IsOpened())
+	{
+		return false;
+	}
+
+	// First, check to see if it's EOF -- this means something has gone wrong, and the housekeeping should terminate.
+	if(mInterProcessComms.IsEOF())
+	{
+		SetTerminateWanted();
+		return true;
+	}
+
+	// Get a line, and process the message
+	std::string line;
+	if(mInterProcessComms.GetLine(line, false /* no pre-processing */, MaximumWaitTime))
+	{
+		BOX_TRACE("Housekeeping received command '" << line <<
+			"' over interprocess comms");
+	
+		int account = 0;
+	
+		if(line == "h")
+		{
+			// HUP signal received by main process
+			SetReloadConfigWanted();
+			return true;
+		}
+		else if(line == "t")
+		{
+			// Terminate signal received by main process
+			SetTerminateWanted();
+			return true;
+		}
+		else if(sscanf(line.c_str(), "r%x", &account) == 1)
+		{
+			// Main process is trying to lock an account -- are we processing it?
+			if(account == AccountNum)
+			{
+				// Yes! -- need to stop now so when it retries to get the lock, it will succeed
+				BOX_INFO("Housekeeping on account " <<
+					BOX_FORMAT_ACCOUNT(AccountNum) <<
+					"giving way to client connection");
+				return true;
+			}
+		}
+	}
+	
+	return false;
+}
+
+
diff --git a/lib/bbstored/BackupStoreDaemon.cpp b/lib/bbstored/BackupStoreDaemon.cpp
new file mode 100644
index 00000000..8fddf125
--- /dev/null
+++ b/lib/bbstored/BackupStoreDaemon.cpp
@@ -0,0 +1,377 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreDaemon.cpp
+//		Purpose: Backup store daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#include "Box.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <signal.h>
+
+#ifdef HAVE_SYSLOG_H
+	#include <syslog.h>
+#endif
+
+#include "BackupStoreContext.h"
+#include "BackupStoreDaemon.h"
+#include "BackupStoreConfigVerify.h"
+#include "autogen_BackupProtocol.h"
+#include "RaidFileController.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreAccounts.h"
+#include "BannerText.h"
+
+#include "MemLeakFindOn.h"
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::BackupStoreDaemon()
+//		Purpose: Constructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreDaemon::BackupStoreDaemon()
+	: mpAccountDatabase(0),
+	  mpAccounts(0),
+	  mExtendedLogging(false),
+	  mHaveForkedHousekeeping(false),
+	  mIsHousekeepingProcess(false),
+	  mHousekeepingInited(false),
+	  mInterProcessComms(mInterProcessCommsSocket),
+	  mpTestHook(NULL)
+{
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::~BackupStoreDaemon()
+//		Purpose: Destructor
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+BackupStoreDaemon::~BackupStoreDaemon()
+{
+	// Must delete this one before the database ...
+	if(mpAccounts != 0)
+	{
+		delete mpAccounts;
+		mpAccounts = 0;
+	}
+	// ... as the mpAccounts object has a reference to it
+	if(mpAccountDatabase != 0)
+	{
+		delete mpAccountDatabase;
+		mpAccountDatabase = 0;
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::DaemonName()
+//		Purpose: Name of daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+const char *BackupStoreDaemon::DaemonName() const
+{
+	return "bbstored";
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::DaemonBanner()
+//		Purpose: Daemon banner
+//		Created: 1/1/04
+//
+// --------------------------------------------------------------------------
+std::string BackupStoreDaemon::DaemonBanner() const
+{
+	return BANNER_TEXT("Backup Store Server");
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::GetConfigVerify()
+//		Purpose: Configuration definition
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+const ConfigurationVerify *BackupStoreDaemon::GetConfigVerify() const
+{
+	return &BackupConfigFileVerify;
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::SetupInInitialProcess()
+//		Purpose: Setup before we fork -- get raid file controller going
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::SetupInInitialProcess()
+{
+	const Configuration &config(GetConfiguration());
+	
+	// Initialise the raid files controller
+	RaidFileController &rcontroller = RaidFileController::GetController();
+
+	std::string raidFileConfig;
+
+	#ifdef WIN32
+		if (!config.KeyExists("RaidFileConf"))
+		{
+			raidFileConfig = BOX_GET_DEFAULT_RAIDFILE_CONFIG_FILE;
+		}
+		else
+		{
+			raidFileConfig = config.GetKeyValue("RaidFileConf");
+		}
+	#else
+		raidFileConfig = config.GetKeyValue("RaidFileConf");
+	#endif
+
+	rcontroller.Initialise(raidFileConfig);
+	
+	// Load the account database
+	std::auto_ptr<BackupStoreAccountDatabase> pdb(BackupStoreAccountDatabase::Read(config.GetKeyValue("AccountDatabase").c_str()));
+	mpAccountDatabase = pdb.release();
+	
+	// Create a accounts object
+	mpAccounts = new BackupStoreAccounts(*mpAccountDatabase);
+	
+	// Ready to go!
+}
+
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::Run()
+//		Purpose: Run shim for the store daemon -- read some config details
+//		Created: 2003/10/24
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::Run()
+{
+	// Get extended logging flag
+	mExtendedLogging = false;
+	const Configuration &config(GetConfiguration());
+	mExtendedLogging = config.GetKeyValueBool("ExtendedLogging");
+	
+	// Fork off housekeeping daemon -- must only do this the first
+	// time Run() is called.  Housekeeping runs synchronously on Win32
+	// because IsSingleProcess() is always true
+	
+#ifndef WIN32
+	if(!IsSingleProcess() && !mHaveForkedHousekeeping)
+	{
+		// Open a socket pair for communication
+		int sv[2] = {-1,-1};
+		if(::socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, sv) != 0)
+		{
+			THROW_EXCEPTION(ServerException, SocketPairFailed)
+		}
+		int whichSocket = 0;
+		
+		// Fork
+		switch(::fork())
+		{
+		case -1:
+			{
+				// Error
+				THROW_EXCEPTION(ServerException, ServerForkError)
+			}
+			break;
+		case 0:
+			{
+				// In child process
+				mIsHousekeepingProcess = true;
+				SetProcessTitle("housekeeping, idle");
+				whichSocket = 1;
+				// Change the log name
+				::openlog("bbstored/hk", LOG_PID, LOG_LOCAL6);
+				// Log that housekeeping started
+				BOX_INFO("Housekeeping process started");
+				// Ignore term and hup
+				// Parent will handle these and alert the
+				// child via the socket, don't want to
+				// randomly die!
+				::signal(SIGHUP, SIG_IGN);
+				::signal(SIGTERM, SIG_IGN);
+			}
+			break;
+		default:
+			{
+				// Parent process
+				whichSocket = 0;
+			}
+			break;
+		}
+		
+		// Mark that this has been, so -HUP doesn't try and do this again
+		mHaveForkedHousekeeping = true;
+		
+		// Attach the comms thing to the right socket, and close the other one
+		mInterProcessCommsSocket.Attach(sv[whichSocket]);
+		
+		if(::close(sv[(whichSocket == 0)?1:0]) != 0)
+		{
+			THROW_EXCEPTION(ServerException, SocketCloseError)
+		}
+	}
+#endif // WIN32
+
+	if(mIsHousekeepingProcess)
+	{
+		// Housekeeping process -- do other stuff
+		HousekeepingProcess();
+	}
+	else
+	{
+		// In server process -- use the base class to do the magic
+		ServerTLS<BOX_PORT_BBSTORED>::Run();
+
+		if (!mInterProcessCommsSocket.IsOpened())
+		{
+			return;
+		}
+
+		// Why did it stop? Tell the housekeeping process to do the same
+		if(IsReloadConfigWanted())
+		{
+			mInterProcessCommsSocket.Write("h\n", 2);
+		}
+
+		if(IsTerminateWanted())
+		{
+			mInterProcessCommsSocket.Write("t\n", 2);
+		}
+	}
+}
+
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::Connection(SocketStreamTLS &)
+//		Purpose: Handles a connection, by catching exceptions and
+//			 delegating to Connection2
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::Connection(std::auto_ptr<SocketStreamTLS> apStream)
+{
+	try
+	{
+		Connection2(apStream);
+	}
+	catch(BoxException &e)
+	{
+		BOX_ERROR("Error in child process, terminating connection: " <<
+			e.what() << " (" << e.GetType() << "/" << 
+			e.GetSubType() << ")");
+	}
+	catch(std::exception &e)
+	{
+		BOX_ERROR("Error in child process, terminating connection: " <<
+			e.what());
+	}
+	catch(...)
+	{
+		BOX_ERROR("Error in child process, terminating connection: " <<
+			"unknown exception");
+	}
+}
+	
+// --------------------------------------------------------------------------
+//
+// Function
+//		Name:    BackupStoreDaemon::Connection2(SocketStreamTLS &)
+//		Purpose: Handles a connection from bbackupd
+//		Created: 2006/11/12
+//
+// --------------------------------------------------------------------------
+void BackupStoreDaemon::Connection2(std::auto_ptr<SocketStreamTLS> apStream)
+{
+	// Get the common name from the certificate
+	std::string clientCommonName(apStream->GetPeerCommonName());
+	
+	// Log the name
+	BOX_INFO("Client certificate CN: " << clientCommonName);
+	
+	// Check it
+	int32_t id;
+	if(::sscanf(clientCommonName.c_str(), "BACKUP-%x", &id) != 1)
+	{
+		// Bad! Disconnect immediately
+		BOX_WARNING("Failed login: invalid client common name: " <<
+			clientCommonName);
+		return;
+	}
+
+	// Make ps listings clearer
+	std::ostringstream tag;
+	tag << "client=" << BOX_FORMAT_ACCOUNT(id);
+	SetProcessTitle(tag.str().c_str());
+	Logging::Tagger tagWithClientID(tag.str());
+
+	// Create a context, using this ID
+	BackupStoreContext context(id, this, GetConnectionDetails());
+
+	if (mpTestHook)
+	{
+		context.SetTestHook(*mpTestHook);
+	}
+	
+	// See if the client has an account?
+	if(mpAccounts && mpAccounts->AccountExists(id))
+	{
+		std::string root;
+		int discSet;
+		mpAccounts->GetAccountRoot(id, root, discSet);
+		context.SetClientHasAccount(root, discSet);
+	}
+
+	// Handle a connection with the backup protocol
+	std::auto_ptr<SocketStream> apPlainStream(apStream);
+	BackupProtocolServer server(apPlainStream);
+	server.SetLogToSysLog(mExtendedLogging);
+	server.SetTimeout(BACKUP_STORE_TIMEOUT);
+	try
+	{
+		server.DoServer(context);
+	}
+	catch(...)
+	{
+		LogConnectionStats(id, context.GetAccountName(), server);
+		throw;
+	}
+	LogConnectionStats(id, context.GetAccountName(), server);
+	context.CleanUp();
+}
+
+void BackupStoreDaemon::LogConnectionStats(uint32_t accountId,
+	const std::string& accountName, const BackupProtocolServer &server)
+{
+	// Log the amount of data transferred
+	BOX_NOTICE("Connection statistics for " << 
+		BOX_FORMAT_ACCOUNT(accountId) << " "
+		"(name=" << accountName << "):"
+		" IN="  << server.GetBytesRead() <<
+		" OUT=" << server.GetBytesWritten() <<
+		" NET_IN=" << (server.GetBytesRead() - server.GetBytesWritten()) <<
+		" TOTAL=" << (server.GetBytesRead() + server.GetBytesWritten()));
+}
diff --git a/lib/bbstored/BackupStoreDaemon.h b/lib/bbstored/BackupStoreDaemon.h
new file mode 100644
index 00000000..a2dab5e5
--- /dev/null
+++ b/lib/bbstored/BackupStoreDaemon.h
@@ -0,0 +1,101 @@
+// --------------------------------------------------------------------------
+//
+// File
+//		Name:    BackupStoreDaemon.h
+//		Purpose: Backup store daemon
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+
+#ifndef BACKUPSTOREDAEMON__H
+#define BACKUPSTOREDAEMON__H
+
+#include "ServerTLS.h"
+#include "BoxPortsAndFiles.h"
+#include "BackupConstants.h"
+#include "BackupStoreContext.h"
+#include "HousekeepStoreAccount.h"
+#include "IOStreamGetLine.h"
+
+class BackupStoreAccounts;
+class BackupStoreAccountDatabase;
+
+// --------------------------------------------------------------------------
+//
+// Class
+//		Name:    BackupStoreDaemon
+//		Purpose: Backup store daemon implementation
+//		Created: 2003/08/20
+//
+// --------------------------------------------------------------------------
+class BackupStoreDaemon : public ServerTLS<BOX_PORT_BBSTORED>,
+	HousekeepingInterface, HousekeepingCallback
+{
+public:
+	BackupStoreDaemon();
+	~BackupStoreDaemon();
+private:
+	BackupStoreDaemon(const BackupStoreDaemon &rToCopy);
+public:
+
+	// For BackupStoreContext to communicate with housekeeping process
+	void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen)
+	{
+#ifndef WIN32
+		mInterProcessCommsSocket.Write(Msg, MsgLen);
+#endif
+	}
+
+protected:
+	
+	virtual void SetupInInitialProcess();
+
+	virtual void Run();
+
+	virtual void Connection(std::auto_ptr<SocketStreamTLS> apStream);
+	void Connection2(std::auto_ptr<SocketStreamTLS> apStream);
+	
+	virtual const char *DaemonName() const;
+	virtual std::string DaemonBanner() const;
+
+	const ConfigurationVerify *GetConfigVerify() const;
+	
+	// Housekeeping functions
+	void HousekeepingProcess();
+
+	void LogConnectionStats(uint32_t accountId,
+		const std::string& accountName, const BackupProtocolServer &server);
+
+public:
+	// HousekeepingInterface implementation
+	virtual bool CheckForInterProcessMsg(int AccountNum = 0, int MaximumWaitTime = 0);
+	void RunHousekeepingIfNeeded();
+
+private:
+	BackupStoreAccountDatabase *mpAccountDatabase;
+	BackupStoreAccounts *mpAccounts;
+	bool mExtendedLogging;
+	bool mHaveForkedHousekeeping;
+	bool mIsHousekeepingProcess;
+	bool mHousekeepingInited;
+	
+	SocketStream mInterProcessCommsSocket;
+	IOStreamGetLine mInterProcessComms;
+
+	virtual void OnIdle();
+	void HousekeepingInit();
+	int64_t mLastHousekeepingRun;
+
+public:
+	void SetTestHook(BackupStoreContext::TestHook& rTestHook)
+	{
+		mpTestHook = &rTestHook;
+	}
+
+private:
+	BackupStoreContext::TestHook* mpTestHook;
+};
+
+
+#endif // BACKUPSTOREDAEMON__H
+