summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Wilson <chris+github@qwirx.com>2009-06-28 12:04:25 +0000
committerChris Wilson <chris+github@qwirx.com>2009-06-28 12:04:25 +0000
commit95433c3e00a790d6c16c8da5171b354e9497b5ef (patch)
tree4ca1477baf74f47a89064aec61d1c88cbc105025
parentcac51d0017b98ea98a16cdd2100e4fe20d3a74ac (diff)
Update reference counts when files and directories are uploaded.
-rw-r--r--bin/bbstored/BackupStoreContext.cpp55
-rw-r--r--bin/bbstored/BackupStoreContext.h5
-rw-r--r--test/backupstore/testbackupstore.cpp174
3 files changed, 160 insertions, 74 deletions
diff --git a/bin/bbstored/BackupStoreContext.cpp b/bin/bbstored/BackupStoreContext.cpp
index 990be05d..2e915f57 100644
--- a/bin/bbstored/BackupStoreContext.cpp
+++ b/bin/bbstored/BackupStoreContext.cpp
@@ -191,6 +191,23 @@ void BackupStoreContext::LoadStoreInfo()
// Keep the pointer to it
mpStoreInfo = i;
+
+ BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet);
+
+ // try to load the reference count database
+ try
+ {
+ mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+ }
+ catch(BoxException &e)
+ {
+ BOX_WARNING("Reference count database is missing or corrupted, "
+ "creating a new one, expect housekeeping to find and "
+ "fix problems with reference counts later.");
+
+ BackupStoreRefCountDatabase::CreateForRegeneration(account);
+ mapRefCount = BackupStoreRefCountDatabase::Load(account, false);
+ }
}
@@ -395,15 +412,18 @@ int64_t BackupStoreContext::AllocateObjectID()
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool)
-// Purpose: Add a file to the store, from a given stream, into a specified directory.
-// Returns object ID of the new file.
+// Name: BackupStoreContext::AddFile(IOStream &, int64_t,
+// int64_t, int64_t, const BackupStoreFilename &, bool)
+// Purpose: Add a file to the store, from a given stream, into
+// a specified directory. Returns object ID of the new
+// file.
// Created: 2003/09/03
//
// --------------------------------------------------------------------------
-int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime,
- int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
- bool MarkFileWithSameNameAsOldVersions)
+int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory,
+ int64_t ModificationTime, int64_t AttributesHash,
+ int64_t DiffFromFileID, const BackupStoreFilename &rFilename,
+ bool MarkFileWithSameNameAsOldVersions)
{
if(mpStoreInfo.get() == 0)
{
@@ -670,6 +690,9 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_
mpStoreInfo->ChangeBlocksUsed(blocksUsed);
mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles);
+ // Increment reference count on the new directory to one
+ mapRefCount->AddReference(id);
+
// Save the store info -- can cope if this exceptions because infomation
// will be rebuilt by housekeeping, and ID allocation can recover.
SaveStoreInfo();
@@ -768,8 +791,9 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &)
-// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found.
+// Name: BackupStoreContext::UndeleteFile(int64_t, int64_t)
+// Purpose: Undeletes a file, if it exists, returning true if
+// the file existed.
// Created: 2003/10/21
//
// --------------------------------------------------------------------------
@@ -933,8 +957,10 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec
// --------------------------------------------------------------------------
//
// Function
-// Name: BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &)
-// Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely.
+// Name: BackupStoreContext::AddDirectory(int64_t,
+// const BackupStoreFilename &, bool &)
+// Purpose: Creates a directory (or just returns the ID of an
+// existing one). rAlreadyExists set appropraitely.
// Created: 2003/09/04
//
// --------------------------------------------------------------------------
@@ -974,7 +1000,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF
// Allocate the next ID
int64_t id = AllocateObjectID();
- // Create a blank directory with the given attributes on disc
+ // Create an empty directory with the given attributes on disc
std::string fn;
MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */);
{
@@ -998,11 +1024,14 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF
// Not added to cache, so don't set the size in the directory
}
- // Then add it into the directory
+ // Then add it into the parent directory
try
{
dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */);
SaveDirectory(dir, InDirectory);
+
+ // Increment reference count on the new directory to one
+ mapRefCount->AddReference(id);
}
catch(...)
{
@@ -1016,7 +1045,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF
// Don't worry about the incremented number in the store info
throw;
}
-
+
// Save the store info (may be postponed)
SaveStoreInfo();
diff --git a/bin/bbstored/BackupStoreContext.h b/bin/bbstored/BackupStoreContext.h
index 4cfdb601..6053e4b8 100644
--- a/bin/bbstored/BackupStoreContext.h
+++ b/bin/bbstored/BackupStoreContext.h
@@ -14,6 +14,7 @@
#include <map>
#include <memory>
+#include "BackupStoreRefCountDatabase.h"
#include "NamedLock.h"
#include "ProtocolObject.h"
#include "Utils.h"
@@ -137,7 +138,6 @@ private:
void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete);
int64_t AllocateObjectID();
-private:
int32_t mClientID;
HousekeepingInterface &mrDaemon;
int mProtocolPhase;
@@ -150,6 +150,9 @@ private:
// Store info
std::auto_ptr<BackupStoreInfo> mpStoreInfo;
+
+ // Refcount database
+ std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount;
// Directory cache
std::map<int64_t, BackupStoreDirectory*> mDirectoryCache;
diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp
index 0266d097..58de7ca9 100644
--- a/test/backupstore/testbackupstore.cpp
+++ b/test/backupstore/testbackupstore.cpp
@@ -34,6 +34,8 @@
#include "BackupClientFileAttributes.h"
#include "BackupClientCryptoKeys.h"
#include "ServerControl.h"
+#include "BackupStoreAccountDatabase.h"
+#include "BackupStoreRefCountDatabase.h"
#include "MemLeakFindOn.h"
@@ -470,7 +472,27 @@ void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID)
TEST_THAT(dirs == 0 || dirs == 2);
}
-int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, const char *name, int depth)
+void create_file_in_dir(std::string name, std::string source, int64_t parentId,
+ BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount)
+{
+ BackupStoreFilenameClear name_encoded("file_One");
+ std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(
+ source.c_str(), parentId, name_encoded));
+ std::auto_ptr<BackupProtocolClientSuccess> stored(
+ protocol.QueryStoreFile(
+ parentId,
+ 0x123456789abcdefLL, /* modification time */
+ 0x7362383249872dfLL, /* attr hash */
+ 0, /* diff from ID */
+ name_encoded,
+ *upload));
+ int64_t objectId = stored->GetObjectID();
+ TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed());
+ TEST_EQUAL(1, rRefCount.GetRefCount(objectId))
+}
+
+int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir,
+ const char *name, int depth, BackupStoreRefCountDatabase& rRefCount)
{
// Create a directory
int64_t subdirid = 0;
@@ -486,49 +508,31 @@ int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir,
}
printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid);
+
+ std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+ BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
+ std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
+ BackupStoreRefCountDatabase::Load(
+ apAccounts->GetEntry(0x1234567), true));
+ TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed());
+ TEST_EQUAL(1, rRefCount.GetRefCount(subdirid))
// Put more directories in it, if we haven't gone down too far
if(depth > 0)
{
- create_test_data_subdirs(protocol, subdirid, "dir_One", depth - 1);
- create_test_data_subdirs(protocol, subdirid, "dir_Two", depth - 1);
+ create_test_data_subdirs(protocol, subdirid, "dir_One",
+ depth - 1, rRefCount);
+ create_test_data_subdirs(protocol, subdirid, "dir_Two",
+ depth - 1, rRefCount);
}
// Stick some files in it
- {
- BackupStoreFilenameClear name("file_One");
- std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
- std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
- subdirid,
- 0x123456789abcdefLL, /* modification time */
- 0x7362383249872dfLL, /* attr hash */
- 0, /* diff from ID */
- name,
- *upload));
- }
- {
- BackupStoreFilenameClear name("file_Two");
- std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
- std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
- subdirid,
- 0x123456789abcdefLL, /* modification time */
- 0x7362383249872dfLL, /* attr hash */
- 0, /* diff from ID */
- name,
- *upload));
- }
- {
- BackupStoreFilenameClear name("file_Three");
- std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name));
- std::auto_ptr<BackupProtocolClientSuccess> stored(protocol.QueryStoreFile(
- subdirid,
- 0x123456789abcdefLL, /* modification time */
- 0x7362383249872dfLL, /* attr hash */
- 0, /* diff from ID */
- name,
- *upload));
- }
-
+ create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol,
+ rRefCount);
+ create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol,
+ rRefCount);
+ create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol,
+ rRefCount);
return subdirid;
}
@@ -885,15 +889,48 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco
}
}
-
-int test_server(const char *hostname)
+void init_context(TLSContext& rContext)
{
- // Context
- TLSContext context;
- context.Initialise(false /* client */,
+ rContext.Initialise(false /* client */,
"testfiles/clientCerts.pem",
"testfiles/clientPrivKey.pem",
"testfiles/clientTrustedCAs.pem");
+}
+
+std::auto_ptr<SocketStreamTLS> open_conn(const char *hostname,
+ TLSContext& rContext)
+{
+ init_context(rContext);
+ std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS);
+ conn->Open(rContext, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
+ return conn;
+}
+
+std::auto_ptr<BackupProtocolClient> test_server_login(SocketStreamTLS& rConn)
+{
+ // Make a protocol
+ std::auto_ptr<BackupProtocolClient> protocol(new
+ BackupProtocolClient(rConn));
+
+ // Check the version
+ std::auto_ptr<BackupProtocolClientVersion> serverVersion(
+ protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION));
+ TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
+
+ // Login
+ std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(
+ protocol->QueryLogin(0x01234567, 0));
+
+ return protocol;
+}
+
+int test_server(const char *hostname)
+{
+ TLSContext context;
+ std::auto_ptr<SocketStreamTLS> conn = open_conn(hostname, context);
+ std::auto_ptr<BackupProtocolClient> apProtocol(
+ test_server_login(*conn));
+ BackupProtocolClient& protocol(*apProtocol);
// Make some test attributes
#define ATTR1_SIZE 245
@@ -911,28 +948,11 @@ int test_server(const char *hostname)
// BLOCK
{
- // Open a connection to the server
- SocketStreamTLS conn;
- conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED);
-
- // Make a protocol
- BackupProtocolClient protocol(conn);
-
// Get it logging
FILE *protocolLog = ::fopen("testfiles/protocol.log", "w");
TEST_THAT(protocolLog != 0);
protocol.SetLogToFile(protocolLog);
- // Check the version
- std::auto_ptr<BackupProtocolClientVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION));
- TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION);
-
- // Login
- std::auto_ptr<BackupProtocolClientLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0));
-
- // Check marker is 0
- TEST_THAT(loginConf->GetClientStoreMarker() == 0);
-
#ifndef WIN32
// Check that we can't open a new connection which requests write permissions
{
@@ -1435,10 +1455,18 @@ int test_server(const char *hostname)
}
//} skip:
+
+ std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+ BackupStoreAccountDatabase::Read(
+ "testfiles/accounts.txt"));
+ std::auto_ptr<BackupStoreRefCountDatabase> apRefCount(
+ BackupStoreRefCountDatabase::Load(
+ apAccounts->GetEntry(0x1234567), true));
// Create some nice recursive directories
int64_t dirtodelete = create_test_data_subdirs(protocol,
- BackupProtocolClientListDirectory::RootDirectory, "test_delete", 6 /* depth */);
+ BackupProtocolClientListDirectory::RootDirectory,
+ "test_delete", 6 /* depth */, *apRefCount);
// And delete them
{
@@ -1760,6 +1788,32 @@ int test3(int argc, const char *argv[])
TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567"));
TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8);
// make sure something is written to it
+
+ std::auto_ptr<BackupStoreAccountDatabase> apAccounts(
+ BackupStoreAccountDatabase::Read("testfiles/accounts.txt"));
+
+ std::auto_ptr<BackupStoreRefCountDatabase> apReferences(
+ BackupStoreRefCountDatabase::Load(
+ apAccounts->GetEntry(0x1234567), true));
+ TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID,
+ apReferences->GetLastObjectIDUsed());
+ TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID))
+ apReferences.reset();
+
+ // Delete the refcount database and log in again,
+ // check that it is recreated automatically but with
+ // no objects in it, to ensure seamless upgrade.
+ TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw"));
+
+ // Context
+ TLSContext context;
+ std::auto_ptr<SocketStreamTLS> conn = open_conn("localhost",
+ context);
+ test_server_login(*conn)->QueryFinished();
+
+ apReferences = BackupStoreRefCountDatabase::Load(
+ apAccounts->GetEntry(0x1234567), true);
+ TEST_EQUAL(0, apReferences->GetLastObjectIDUsed());
TEST_THAT(ServerIsAlive(pid));