From 95433c3e00a790d6c16c8da5171b354e9497b5ef Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sun, 28 Jun 2009 12:04:25 +0000 Subject: Update reference counts when files and directories are uploaded. --- bin/bbstored/BackupStoreContext.cpp | 55 ++++++++--- bin/bbstored/BackupStoreContext.h | 5 +- test/backupstore/testbackupstore.cpp | 174 +++++++++++++++++++++++------------ 3 files changed, 160 insertions(+), 74 deletions(-) diff --git a/bin/bbstored/BackupStoreContext.cpp b/bin/bbstored/BackupStoreContext.cpp index 990be05d..2e915f57 100644 --- a/bin/bbstored/BackupStoreContext.cpp +++ b/bin/bbstored/BackupStoreContext.cpp @@ -191,6 +191,23 @@ void BackupStoreContext::LoadStoreInfo() // Keep the pointer to it mpStoreInfo = i; + + BackupStoreAccountDatabase::Entry account(mClientID, mStoreDiscSet); + + // try to load the reference count database + try + { + mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + } + catch(BoxException &e) + { + BOX_WARNING("Reference count database is missing or corrupted, " + "creating a new one, expect housekeeping to find and " + "fix problems with reference counts later."); + + BackupStoreRefCountDatabase::CreateForRegeneration(account); + mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + } } @@ -395,15 +412,18 @@ int64_t BackupStoreContext::AllocateObjectID() // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) -// Purpose: Add a file to the store, from a given stream, into a specified directory. -// Returns object ID of the new file. +// Name: BackupStoreContext::AddFile(IOStream &, int64_t, +// int64_t, int64_t, const BackupStoreFilename &, bool) +// Purpose: Add a file to the store, from a given stream, into +// a specified directory. Returns object ID of the new +// file. // Created: 2003/09/03 // // -------------------------------------------------------------------------- -int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, - int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, - bool MarkFileWithSameNameAsOldVersions) +int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, + int64_t ModificationTime, int64_t AttributesHash, + int64_t DiffFromFileID, const BackupStoreFilename &rFilename, + bool MarkFileWithSameNameAsOldVersions) { if(mpStoreInfo.get() == 0) { @@ -670,6 +690,9 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_ mpStoreInfo->ChangeBlocksUsed(blocksUsed); mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); + // Increment reference count on the new directory to one + mapRefCount->AddReference(id); + // Save the store info -- can cope if this exceptions because infomation // will be rebuilt by housekeeping, and ID allocation can recover. SaveStoreInfo(); @@ -768,8 +791,9 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) -// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Name: BackupStoreContext::UndeleteFile(int64_t, int64_t) +// Purpose: Undeletes a file, if it exists, returning true if +// the file existed. // Created: 2003/10/21 // // -------------------------------------------------------------------------- @@ -933,8 +957,10 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &) -// Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely. +// Name: BackupStoreContext::AddDirectory(int64_t, +// const BackupStoreFilename &, bool &) +// Purpose: Creates a directory (or just returns the ID of an +// existing one). rAlreadyExists set appropraitely. // Created: 2003/09/04 // // -------------------------------------------------------------------------- @@ -974,7 +1000,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF // Allocate the next ID int64_t id = AllocateObjectID(); - // Create a blank directory with the given attributes on disc + // Create an empty directory with the given attributes on disc std::string fn; MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); { @@ -998,11 +1024,14 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF // Not added to cache, so don't set the size in the directory } - // Then add it into the directory + // Then add it into the parent directory try { dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); SaveDirectory(dir, InDirectory); + + // Increment reference count on the new directory to one + mapRefCount->AddReference(id); } catch(...) { @@ -1016,7 +1045,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF // Don't worry about the incremented number in the store info throw; } - + // Save the store info (may be postponed) SaveStoreInfo(); diff --git a/bin/bbstored/BackupStoreContext.h b/bin/bbstored/BackupStoreContext.h index 4cfdb601..6053e4b8 100644 --- a/bin/bbstored/BackupStoreContext.h +++ b/bin/bbstored/BackupStoreContext.h @@ -14,6 +14,7 @@ #include #include +#include "BackupStoreRefCountDatabase.h" #include "NamedLock.h" #include "ProtocolObject.h" #include "Utils.h" @@ -137,7 +138,6 @@ private: void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); int64_t AllocateObjectID(); -private: int32_t mClientID; HousekeepingInterface &mrDaemon; int mProtocolPhase; @@ -150,6 +150,9 @@ private: // Store info std::auto_ptr mpStoreInfo; + + // Refcount database + std::auto_ptr mapRefCount; // Directory cache std::map mDirectoryCache; diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp index 0266d097..58de7ca9 100644 --- a/test/backupstore/testbackupstore.cpp +++ b/test/backupstore/testbackupstore.cpp @@ -34,6 +34,8 @@ #include "BackupClientFileAttributes.h" #include "BackupClientCryptoKeys.h" #include "ServerControl.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreRefCountDatabase.h" #include "MemLeakFindOn.h" @@ -470,7 +472,27 @@ void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID) TEST_THAT(dirs == 0 || dirs == 2); } -int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, const char *name, int depth) +void create_file_in_dir(std::string name, std::string source, int64_t parentId, + BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount) +{ + BackupStoreFilenameClear name_encoded("file_One"); + std::auto_ptr upload(BackupStoreFile::EncodeFile( + source.c_str(), parentId, name_encoded)); + std::auto_ptr stored( + protocol.QueryStoreFile( + parentId, + 0x123456789abcdefLL, /* modification time */ + 0x7362383249872dfLL, /* attr hash */ + 0, /* diff from ID */ + name_encoded, + *upload)); + int64_t objectId = stored->GetObjectID(); + TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed()); + TEST_EQUAL(1, rRefCount.GetRefCount(objectId)) +} + +int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, + const char *name, int depth, BackupStoreRefCountDatabase& rRefCount) { // Create a directory int64_t subdirid = 0; @@ -486,49 +508,31 @@ int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, } printf("Create subdirs, depth = %d, dirid = %llx\n", depth, subdirid); + + std::auto_ptr apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + std::auto_ptr apReferences( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true)); + TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed()); + TEST_EQUAL(1, rRefCount.GetRefCount(subdirid)) // Put more directories in it, if we haven't gone down too far if(depth > 0) { - create_test_data_subdirs(protocol, subdirid, "dir_One", depth - 1); - create_test_data_subdirs(protocol, subdirid, "dir_Two", depth - 1); + create_test_data_subdirs(protocol, subdirid, "dir_One", + depth - 1, rRefCount); + create_test_data_subdirs(protocol, subdirid, "dir_Two", + depth - 1, rRefCount); } // Stick some files in it - { - BackupStoreFilenameClear name("file_One"); - std::auto_ptr upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name)); - std::auto_ptr stored(protocol.QueryStoreFile( - subdirid, - 0x123456789abcdefLL, /* modification time */ - 0x7362383249872dfLL, /* attr hash */ - 0, /* diff from ID */ - name, - *upload)); - } - { - BackupStoreFilenameClear name("file_Two"); - std::auto_ptr upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name)); - std::auto_ptr stored(protocol.QueryStoreFile( - subdirid, - 0x123456789abcdefLL, /* modification time */ - 0x7362383249872dfLL, /* attr hash */ - 0, /* diff from ID */ - name, - *upload)); - } - { - BackupStoreFilenameClear name("file_Three"); - std::auto_ptr upload(BackupStoreFile::EncodeFile("testfiles/file1", subdirid, name)); - std::auto_ptr stored(protocol.QueryStoreFile( - subdirid, - 0x123456789abcdefLL, /* modification time */ - 0x7362383249872dfLL, /* attr hash */ - 0, /* diff from ID */ - name, - *upload)); - } - + create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol, + rRefCount); + create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol, + rRefCount); + create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol, + rRefCount); return subdirid; } @@ -885,15 +889,48 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco } } - -int test_server(const char *hostname) +void init_context(TLSContext& rContext) { - // Context - TLSContext context; - context.Initialise(false /* client */, + rContext.Initialise(false /* client */, "testfiles/clientCerts.pem", "testfiles/clientPrivKey.pem", "testfiles/clientTrustedCAs.pem"); +} + +std::auto_ptr open_conn(const char *hostname, + TLSContext& rContext) +{ + init_context(rContext); + std::auto_ptr conn(new SocketStreamTLS); + conn->Open(rContext, Socket::TypeINET, hostname, BOX_PORT_BBSTORED); + return conn; +} + +std::auto_ptr test_server_login(SocketStreamTLS& rConn) +{ + // Make a protocol + std::auto_ptr protocol(new + BackupProtocolClient(rConn)); + + // Check the version + std::auto_ptr serverVersion( + protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION)); + TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + + // Login + std::auto_ptr loginConf( + protocol->QueryLogin(0x01234567, 0)); + + return protocol; +} + +int test_server(const char *hostname) +{ + TLSContext context; + std::auto_ptr conn = open_conn(hostname, context); + std::auto_ptr apProtocol( + test_server_login(*conn)); + BackupProtocolClient& protocol(*apProtocol); // Make some test attributes #define ATTR1_SIZE 245 @@ -911,28 +948,11 @@ int test_server(const char *hostname) // BLOCK { - // Open a connection to the server - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED); - - // Make a protocol - BackupProtocolClient protocol(conn); - // Get it logging FILE *protocolLog = ::fopen("testfiles/protocol.log", "w"); TEST_THAT(protocolLog != 0); protocol.SetLogToFile(protocolLog); - // Check the version - std::auto_ptr serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); - - // Login - std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, 0)); - - // Check marker is 0 - TEST_THAT(loginConf->GetClientStoreMarker() == 0); - #ifndef WIN32 // Check that we can't open a new connection which requests write permissions { @@ -1435,10 +1455,18 @@ int test_server(const char *hostname) } //} skip: + + std::auto_ptr apAccounts( + BackupStoreAccountDatabase::Read( + "testfiles/accounts.txt")); + std::auto_ptr apRefCount( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true)); // Create some nice recursive directories int64_t dirtodelete = create_test_data_subdirs(protocol, - BackupProtocolClientListDirectory::RootDirectory, "test_delete", 6 /* depth */); + BackupProtocolClientListDirectory::RootDirectory, + "test_delete", 6 /* depth */, *apRefCount); // And delete them { @@ -1760,6 +1788,32 @@ int test3(int argc, const char *argv[]) TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567")); TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8); // make sure something is written to it + + std::auto_ptr apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + + std::auto_ptr apReferences( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true)); + TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, + apReferences->GetLastObjectIDUsed()); + TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) + apReferences.reset(); + + // Delete the refcount database and log in again, + // check that it is recreated automatically but with + // no objects in it, to ensure seamless upgrade. + TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw")); + + // Context + TLSContext context; + std::auto_ptr conn = open_conn("localhost", + context); + test_server_login(*conn)->QueryFinished(); + + apReferences = BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true); + TEST_EQUAL(0, apReferences->GetLastObjectIDUsed()); TEST_THAT(ServerIsAlive(pid)); -- cgit v1.2.3