diff options
Diffstat (limited to 'test/backupstore/testbackupstore.cpp')
| -rw-r--r-- | test/backupstore/testbackupstore.cpp | 2470 |
1 files changed, 1598 insertions, 872 deletions
diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp index 661973e0..6441d66c 100644 --- a/test/backupstore/testbackupstore.cpp +++ b/test/backupstore/testbackupstore.cpp @@ -15,17 +15,22 @@ #include "Archive.h" #include "BackupClientCryptoKeys.h" #include "BackupClientFileAttributes.h" +#include "BackupProtocol.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreAccounts.h" +#include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" -#include "BackupStoreInfo.h" +#include "BackupStoreFile.h" #include "BackupStoreFilenameClear.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreInfo.h" +#include "BackupStoreObjectMagic.h" #include "BackupStoreRefCountDatabase.h" -#include "BackupStoreFile.h" #include "BoxPortsAndFiles.h" #include "CollectInBufferStream.h" +#include "Configuration.h" #include "FileStream.h" #include "HousekeepStoreAccount.h" #include "MemBlockStream.h" @@ -37,15 +42,27 @@ #include "ServerControl.h" #include "Socket.h" #include "SocketStreamTLS.h" +#include "StoreStructure.h" +#include "StoreTestUtils.h" #include "TLSContext.h" #include "Test.h" -#include "autogen_BackupProtocol.h" +#include "ZeroStream.h" #include "MemLeakFindOn.h" - #define ENCFILE_SIZE 2765 +// Make some test attributes +#define ATTR1_SIZE 245 +#define ATTR2_SIZE 23 +#define ATTR3_SIZE 122 + +#define SHORT_TIMEOUT 5000 + +int attr1[ATTR1_SIZE]; +int attr2[ATTR2_SIZE]; +int attr3[ATTR3_SIZE]; + typedef struct { BackupStoreFilenameClear fn; @@ -94,7 +111,7 @@ typedef struct #define TEST_FILE_FOR_PATCHING_SIZE ((128*1024)+2564) #define UPLOAD_PATCH_EN 2 -uploadtest uploads[] = +uploadtest uploads[] = { {"0", BackupStoreFilenameClear(), 324, 455, 0, 0, false, false}, {"1", BackupStoreFilenameClear(), 3232432, 2674, 0, 0, true, false}, // old ver @@ -121,11 +138,42 @@ static const char *uploads_filenames[] = {"49587fds", "cvhjhj324", "sdfcscs324", // file which will be moved (as well as it's old version) #define UPLOAD_FILE_TO_MOVE 8 +#define UNLINK_IF_EXISTS(filename) \ + if (FileExists(filename)) { TEST_THAT(unlink(filename) == 0); } + +//! Simplifies calling setUp() with the current function name in each test. +#define SETUP_TEST_BACKUPSTORE() \ + SETUP(); \ + if (ServerIsAlive(bbstored_pid)) \ + TEST_THAT_OR(StopServer(), FAIL); \ + ExpectedRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1); \ + set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); \ + TEST_THAT_OR(create_account(10000, 20000), FAIL); + +//! Checks account for errors and shuts down daemons at end of every test. +bool teardown_test_backupstore() +{ + bool status = true; + + if (FileExists("testfiles/0_0/backup/01234567/info.rf")) + { + TEST_THAT_OR(check_reference_counts(), status = false); + TEST_THAT_OR(check_account(), status = false); + } + + return status; +} + +#define TEARDOWN_TEST_BACKUPSTORE() \ + if (ServerIsAlive(bbstored_pid)) \ + StopServer(); \ + TEST_THAT(teardown_test_backupstore()); \ + TEARDOWN(); // Nice random data for testing written files class R250 { public: - // Set up internal state table with 32-bit random numbers. + // Set up internal state table with 32-bit random numbers. // The bizarre bit-twiddling is because rand() returns 16 bits of which // the bottom bit is always zero! Hence, I use only some of the bits. // You might want to do something better than this.... @@ -142,7 +190,7 @@ public: // stir up the numbers to ensure they're random - for (int j = 0; j != stateLen * 4; ++j) + for (int j = 0; j != stateLen * 4; ++j) (void) next(); } @@ -174,7 +222,7 @@ int SkipEntries(int e, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) do { skip = false; - + if(FlagsMustBeSet != BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING) { if((ens[e].flags & FlagsMustBeSet) != FlagsMustBeSet) @@ -186,20 +234,20 @@ int SkipEntries(int e, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { skip = true; } - + if(skip) { ++e; } } while(skip && e < DIR_NUM); - + return e; } void CheckEntries(BackupStoreDirectory &rDir, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { int e = 0; - + BackupStoreDirectory::Iterator i(rDir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) @@ -213,16 +261,18 @@ void CheckEntries(BackupStoreDirectory &rDir, int16_t FlagsMustBeSet, int16_t Fl TEST_THAT(en->GetName() == ens[e].fn && en->GetModificationTime() == ens[e].mod && en->GetObjectID() == ens[e].id && en->GetFlags() == ens[e].flags && en->GetSizeInBlocks() == ens[e].size); // next - ++e; + ++e; } - + // Got them all? TEST_THAT(en == 0); TEST_THAT(DIR_NUM == SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet)); } -int test1(int argc, const char *argv[]) +bool test_filename_encoding() { + SETUP_TEST_BACKUPSTORE(); + // test some basics -- encoding and decoding filenames { // Make some filenames in various ways @@ -232,11 +282,11 @@ int test1(int argc, const char *argv[]) BackupStoreFilenameClear fn3(fn1); TEST_THAT(fn1 == fn2); TEST_THAT(fn1 == fn3); - + // Check that it's been encrypted std::string name(fn2.GetEncodedFilename()); TEST_THAT(name.find("name") == name.npos); - + // Bung it in a stream, get it out in a Clear filename { CollectInBufferStream stream; @@ -247,6 +297,7 @@ int test1(int argc, const char *argv[]) TEST_THAT(fn4.GetClearFilename() == "filenameXYZ"); TEST_THAT(fn4 == fn1); } + // Bung it in a stream, get it out in a server non-Clear filename (two of them into the same var) { BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf "); @@ -260,6 +311,7 @@ int test1(int argc, const char *argv[]) fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5 == fno); } + // Same again with clear strings { BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf "); @@ -273,6 +325,7 @@ int test1(int argc, const char *argv[]) fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5.GetClearFilename() == "pinglet dksfnsf jksjdf "); } + // Test a very big filename { const char *fnr = "01234567890123456789012345678901234567890123456789" @@ -292,19 +345,23 @@ int test1(int argc, const char *argv[]) TEST_THAT(fn9.GetClearFilename() == fnr); TEST_THAT(fn9 == fnLong); } + // Test a filename which went wrong once { BackupStoreFilenameClear dodgy("content-negotiation.html"); } } - return 0; + + TEARDOWN_TEST_BACKUPSTORE(); } -int test2(int argc, const char *argv[]) +bool test_backupstore_directory() { + SETUP_TEST_BACKUPSTORE(); + { // Now play with directories - + // Fill in... BackupStoreDirectory dir1(12, 98); for(int e = 0; e < DIR_NUM; ++e) @@ -313,27 +370,25 @@ int test2(int argc, const char *argv[]) } // Got the right number TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM); - + // Stick it into a stream and get it out again { CollectInBufferStream stream; dir1.WriteToStream(stream); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_NUM); TEST_THAT(dir2.GetObjectID() == 12); TEST_THAT(dir2.GetContainerID() == 98); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } - + // Then do selective writes and reads { CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } @@ -341,8 +396,7 @@ int test2(int argc, const char *argv[]) CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_DIRS); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_Dir, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } @@ -350,12 +404,11 @@ int test2(int argc, const char *argv[]) CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - DIR_OLD); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion); } - + // Finally test deleting items { dir1.DeleteEntry(12312312321LL); @@ -364,11 +417,10 @@ int test2(int argc, const char *argv[]) CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - 1); } - + // Check attributes { int attrI[4] = {1, 2, 3, 4}; @@ -380,73 +432,70 @@ int test2(int argc, const char *argv[]) CollectInBufferStream stream; d1.WriteToStream(stream); stream.SetForReading(); - BackupStoreDirectory d2; - d2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory d2(stream); TEST_THAT(d2.GetAttributes() == attr); TEST_THAT(d2.GetAttributesModTime() == 56234987324232LL); } } - return 0; + + TEARDOWN_TEST_BACKUPSTORE(); } void write_test_file(int t) { std::string filename("testfiles/test"); filename += uploads[t].fnextra; - printf("%s\n", filename.c_str()); - + BOX_TRACE("Writing " << filename); + FileStream write(filename.c_str(), O_WRONLY | O_CREAT); - + R250 r(uploads[t].seed); - + unsigned char *data = (unsigned char*)malloc(uploads[t].size); for(int l = 0; l < uploads[t].size; ++l) { data[l] = r.next() & 0xff; } write.Write(data, uploads[t].size); - + free(data); } void test_test_file(int t, IOStream &rStream) { // Decode to a file - BackupStoreFile::DecodeFile(rStream, "testfiles/test_download", IOStream::TimeOutInfinite); - + BackupStoreFile::DecodeFile(rStream, "testfiles/test_download", SHORT_TIMEOUT); + // Compare... FileStream in("testfiles/test_download"); TEST_THAT(in.BytesLeftToRead() == uploads[t].size); - + R250 r(uploads[t].seed); - + unsigned char *data = (unsigned char*)malloc(uploads[t].size); TEST_THAT(in.ReadFullBuffer(data, uploads[t].size, 0 /* not interested in bytes read if this fails */)); - + for(int l = 0; l < uploads[t].size; ++l) { TEST_THAT(data[l] == (r.next() & 0xff)); } - + free(data); in.Close(); TEST_THAT(unlink("testfiles/test_download") == 0); } -void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID) +void assert_everything_deleted(BackupProtocolCallable &protocol, int64_t DirID) { - printf("Test for del: %llx\n", (unsigned long long)DirID); - + BOX_TRACE("Test for del: " << BOX_FORMAT_OBJECTID(DirID)); + // Command std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( DirID, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - + BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; int files = 0; @@ -457,34 +506,24 @@ void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID) { dirs++; // Recurse - test_everything_deleted(protocol, en->GetObjectID()); + assert_everything_deleted(protocol, en->GetObjectID()); } else { files++; } + // Check it's deleted - TEST_THAT(en->GetFlags() & BackupProtocolListDirectory::Flags_Deleted); + TEST_THAT(en->IsDeleted()); } - + // Check there were the right number of files and directories TEST_THAT(files == 3); TEST_THAT(dirs == 0 || dirs == 2); } -std::vector<uint32_t> ExpectedRefCounts; - -void set_refcount(int64_t ObjectID, uint32_t RefCount = 1) -{ - if ((int64_t)ExpectedRefCounts.size() <= ObjectID); - { - ExpectedRefCounts.resize(ObjectID + 1, 0); - } - ExpectedRefCounts[ObjectID] = RefCount; -} - void create_file_in_dir(std::string name, std::string source, int64_t parentId, - BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount) + BackupProtocolCallable &protocol, BackupStoreRefCountDatabase* pRefCount) { BackupStoreFilenameClear name_encoded("file_One"); std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile( @@ -496,15 +535,22 @@ void create_file_in_dir(std::string name, std::string source, int64_t parentId, 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ name_encoded, - *upload)); + upload)); int64_t objectId = stored->GetObjectID(); - TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed()); - TEST_EQUAL(1, rRefCount.GetRefCount(objectId)) + if (pRefCount) + { + TEST_EQUAL(objectId, pRefCount->GetLastObjectIDUsed()); + TEST_EQUAL(1, pRefCount->GetRefCount(objectId)) + } set_refcount(objectId, 1); } -int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, - const char *name, int depth, BackupStoreRefCountDatabase& rRefCount) +const box_time_t FAKE_MODIFICATION_TIME = 0xfeedfacedeadbeefLL; +const box_time_t FAKE_ATTR_MODIFICATION_TIME = 0xdeadbeefcafebabeLL; + +int64_t create_test_data_subdirs(BackupProtocolCallable &protocol, + int64_t indir, const char *name, int depth, + BackupStoreRefCountDatabase* pRefCount) { // Create a directory int64_t subdirid = 0; @@ -512,72 +558,80 @@ int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, { // Create with dummy attributes int attrS = 0; - MemBlockStream attr(&attrS, sizeof(attrS)); + std::auto_ptr<IOStream> attr(new MemBlockStream(&attrS, sizeof(attrS))); std::auto_ptr<BackupProtocolSuccess> dirCreate(protocol.QueryCreateDirectory( - indir, - 9837429842987984LL, dirname, attr)); - subdirid = dirCreate->GetObjectID(); + indir, FAKE_ATTR_MODIFICATION_TIME, dirname, attr)); + subdirid = dirCreate->GetObjectID(); + } + + BOX_TRACE("Creating subdirs, depth = " << depth << ", dirid = " << + BOX_FORMAT_OBJECTID(subdirid)); + + if (pRefCount) + { + TEST_EQUAL(subdirid, pRefCount->GetLastObjectIDUsed()); + TEST_EQUAL(1, pRefCount->GetRefCount(subdirid)) } - - printf("Create subdirs, depth = %d, dirid = %llx\n", depth, - (unsigned long long)subdirid); - TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed()); - TEST_EQUAL(1, rRefCount.GetRefCount(subdirid)) set_refcount(subdirid, 1); - + // Put more directories in it, if we haven't gone down too far if(depth > 0) { create_test_data_subdirs(protocol, subdirid, "dir_One", - depth - 1, rRefCount); + depth - 1, pRefCount); create_test_data_subdirs(protocol, subdirid, "dir_Two", - depth - 1, rRefCount); + depth - 1, pRefCount); } - + // Stick some files in it - create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol, - rRefCount); - create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol, - rRefCount); - create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol, - rRefCount); + create_file_in_dir("file_One", "testfiles/test1", subdirid, protocol, + pRefCount); + create_file_in_dir("file_Two", "testfiles/test1", subdirid, protocol, + pRefCount); + create_file_in_dir("file_Three", "testfiles/test1", subdirid, protocol, + pRefCount); return subdirid; } - -void check_dir_after_uploads(BackupProtocolClient &protocol, const StreamableMemBlock &Attributes) +void check_dir_after_uploads(BackupProtocolCallable &protocol, + const StreamableMemBlock &Attributes) { // Command std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); - TEST_THAT(dirreply->GetObjectID() == BackupProtocolListDirectory::RootDirectory); + TEST_THAT(dirreply->GetObjectID() == BACKUPSTORE_ROOT_DIRECTORY_ID); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - TEST_THAT(dir.GetNumberOfEntries() == UPLOAD_NUM + 1 /* for the first test file */); + BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); + TEST_EQUAL(UPLOAD_NUM, dir.GetNumberOfEntries()); TEST_THAT(!dir.HasAttributes()); // Check them! BackupStoreDirectory::Iterator i(dir); - // Discard first - BackupStoreDirectory::Entry *en = i.Next(); - TEST_THAT(en != 0); + BackupStoreDirectory::Entry *en; for(int t = 0; t < UPLOAD_NUM; ++t) { en = i.Next(); TEST_THAT(en != 0); - TEST_THAT(en->GetName() == uploads[t].name); - TEST_THAT(en->GetObjectID() == uploads[t].allocated_objid); - TEST_THAT(en->GetModificationTime() == uploads[t].mod_time); + if (en == 0) continue; + TEST_LINE(uploads[t].name == en->GetName(), + "uploaded file " << t << " name"); + BackupStoreFilenameClear clear(en->GetName()); + TEST_EQUAL_LINE(uploads[t].name.GetClearFilename(), + clear.GetClearFilename(), + "uploaded file " << t << " name"); + TEST_EQUAL_LINE(uploads[t].allocated_objid, en->GetObjectID(), + "uploaded file " << t << " ID"); + TEST_EQUAL_LINE(uploads[t].mod_time, en->GetModificationTime(), + "uploaded file " << t << " modtime"); int correct_flags = BackupProtocolListDirectory::Flags_File; if(uploads[t].should_be_old_version) correct_flags |= BackupProtocolListDirectory::Flags_OldVersion; if(uploads[t].delete_file) correct_flags |= BackupProtocolListDirectory::Flags_Deleted; - TEST_THAT(en->GetFlags() == correct_flags); + TEST_EQUAL_LINE(correct_flags, en->GetFlags(), + "uploaded file " << t << " flags"); if(t == UPLOAD_ATTRS_EN) { TEST_THAT(en->HasAttributes()); @@ -589,12 +643,11 @@ void check_dir_after_uploads(BackupProtocolClient &protocol, const StreamableMem // No attributes on this one TEST_THAT(!en->HasAttributes()); } - } + } en = i.Next(); TEST_THAT(en == 0); } - typedef struct { int objectsNotDel; @@ -602,7 +655,8 @@ typedef struct int old; } recursive_count_objects_results; -void recursive_count_objects_r(BackupProtocolClient &protocol, int64_t id, recursive_count_objects_results &results) +void recursive_count_objects_r(BackupProtocolCallable &protocol, int64_t id, + recursive_count_objects_results &results) { // Command std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( @@ -610,21 +664,19 @@ void recursive_count_objects_r(BackupProtocolClient &protocol, int64_t id, recur BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); // Check them! BackupStoreDirectory::Iterator i(dir); // Discard first BackupStoreDirectory::Entry *en = 0; - + while((en = i.Next()) != 0) { if((en->GetFlags() & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) == 0) results.objectsNotDel++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) results.deleted++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) results.old++; - + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) { recursive_count_objects_r(protocol, en->GetObjectID(), results); @@ -632,27 +684,14 @@ void recursive_count_objects_r(BackupProtocolClient &protocol, int64_t id, recur } } -void recursive_count_objects(const char *hostname, int64_t id, recursive_count_objects_results &results) -{ - // Context - TLSContext context; - context.Initialise(false /* client */, - "testfiles/clientCerts.pem", - "testfiles/clientPrivKey.pem", - "testfiles/clientTrustedCAs.pem"); +TLSContext context; +void recursive_count_objects(int64_t id, recursive_count_objects_results &results) +{ // Get a connection - SocketStreamTLS connReadOnly; - connReadOnly.Open(context, Socket::TypeINET, hostname, - BOX_PORT_BBSTORED_TEST); - BackupProtocolClient protocolReadOnly(connReadOnly); + BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", + "backup/01234567/", 0, false); - { - std::auto_ptr<BackupProtocolVersion> serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); - } - // Count objects recursive_count_objects_r(protocolReadOnly, id, results); @@ -673,8 +712,8 @@ bool check_block_index(const char *encoded_file, IOStream &rBlockIndex) { char buffer1[2048]; char buffer2[2048]; - int s = enc.Read(buffer1, sizeof(buffer1)); - if(rBlockIndex.Read(buffer2, s) != s) + int s = enc.Read(buffer1, sizeof(buffer1), SHORT_TIMEOUT); + if(rBlockIndex.Read(buffer2, s, SHORT_TIMEOUT) != s) { same = false; break; @@ -685,19 +724,19 @@ bool check_block_index(const char *encoded_file, IOStream &rBlockIndex) break; } } - + if(rBlockIndex.StreamDataLeft()) { same = false; - + // Absorb all this excess data so procotol is in the first state char buffer[2048]; while(rBlockIndex.StreamDataLeft()) { - rBlockIndex.Read(buffer, sizeof(buffer)); + rBlockIndex.Read(buffer, sizeof(buffer), SHORT_TIMEOUT); } } - + return same; } @@ -726,18 +765,77 @@ bool check_files_same(const char *f1, const char *f2) break; } } - + if(f2s.StreamDataLeft()) { same = false; } - + return same; } +std::auto_ptr<RaidFileRead> get_raid_file(int64_t ObjectID) +{ + std::string filename; + StoreStructure::MakeObjectFilename(ObjectID, + "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, + filename, false /* EnsureDirectoryExists */); + return RaidFileRead::Open(0, filename); +} + +int64_t create_directory(BackupProtocolCallable& protocol, + int64_t parent_dir_id = BACKUPSTORE_ROOT_DIRECTORY_ID); +int64_t create_file(BackupProtocolCallable& protocol, int64_t subdirid, + const std::string& remote_filename = ""); + +bool run_housekeeping_and_check_account(BackupProtocolLocal2& protocol) +{ + protocol.QueryFinished(); + bool check_account_status; + TEST_THAT(check_account_status = run_housekeeping_and_check_account()); + bool check_refcount_status; + TEST_THAT(check_refcount_status = check_reference_counts()); + protocol.Reopen(); + return check_account_status & check_refcount_status; +} -void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protocolReadOnly) +bool test_temporary_refcount_db_is_independent() { + SETUP_TEST_BACKUPSTORE(); + + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + std::auto_ptr<BackupStoreRefCountDatabase> temp( + BackupStoreRefCountDatabase::Create( + apAccounts->GetEntry(0x1234567))); + std::auto_ptr<BackupStoreRefCountDatabase> perm( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), + true // ReadOnly + )); + + TEST_CHECK_THROWS(temp->GetRefCount(2), + BackupStoreException, UnknownObjectRefCountRequested); + TEST_CHECK_THROWS(perm->GetRefCount(2), + BackupStoreException, UnknownObjectRefCountRequested); + temp->AddReference(2); + TEST_EQUAL(1, temp->GetRefCount(2)); + TEST_CHECK_THROWS(perm->GetRefCount(2), + BackupStoreException, UnknownObjectRefCountRequested); + temp->Discard(); + + // Need to delete perm object so it doesn't keep a filehandle open, + // preventing tearDown from rewriting the refcount DB and thus causing + // test failure. + perm.reset(); + + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_server_housekeeping() +{ + SETUP_TEST_BACKUPSTORE(); + int encfile[ENCFILE_SIZE]; { for(int l = 0; l < ENCFILE_SIZE; ++l) @@ -747,71 +845,64 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco // Write this to a file { - FileStream f("testfiles/file1", O_WRONLY | O_CREAT | O_EXCL); + FileStream f("testfiles/file1", O_WRONLY | O_CREAT); f.Write(encfile, sizeof(encfile)); } - } - // Read the root directory a few times (as it's cached, so make sure it doesn't hurt anything) - for(int l = 0; l < 3; ++l) - { - // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); - // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - TEST_THAT(dir.GetNumberOfEntries() == 0); - } + // We need complete control over housekeeping, so use a local client + // instead of a network client + daemon. - // Read the dir from the readonly connection (make sure it gets in the cache) - { - // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); - // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - TEST_THAT(dir.GetNumberOfEntries() == 0); - } + BackupProtocolLocal2 protocol(0x01234567, "test", "backup/01234567/", + 0, false); + + int root_dir_blocks = get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID)->GetDiscUsageInBlocks(); + TEST_THAT(check_num_files(0, 0, 0, 1)); + TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, + root_dir_blocks)); // Store a file -- first make the encoded file - BackupStoreFilenameClear store1name("testfiles/file1"); + BackupStoreFilenameClear store1name("file1"); { - FileStream out("testfiles/file1_upload1", O_WRONLY | O_CREAT | O_EXCL); - std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/file1", BackupProtocolListDirectory::RootDirectory, store1name)); + FileStream out("testfiles/file1_upload1", O_WRONLY | O_CREAT); + std::auto_ptr<IOStream> encoded( + BackupStoreFile::EncodeFile("testfiles/file1", + BACKUPSTORE_ROOT_DIRECTORY_ID, store1name)); encoded->CopyStreamTo(out); } -// printf("SKIPPING\n"); -// goto skip; { + // TODO FIXME move most of the code immediately below into + // test_server_commands. + + // TODO FIXME use COMMAND macro for all commands to check the returned + // object ID. + #define COMMAND(command, objectid) \ + TEST_EQUAL(objectid, protocol.command->GetObjectID()); + // Then send it - int64_t store1objid = 0; + int64_t store1objid; { - FileStream upload("testfiles/file1_upload1"); + std::auto_ptr<IOStream> upload(new FileStream("testfiles/file1_upload1")); std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ store1name, upload)); store1objid = stored->GetObjectID(); - TEST_THAT(store1objid == 2); + TEST_EQUAL_LINE(2, store1objid, "wrong ObjectID for newly " + "uploaded file"); } + + // Update expected reference count of this new object set_refcount(store1objid, 1); + // And retrieve it { // Retrieve as object - std::auto_ptr<BackupProtocolSuccess> getfile(protocol.QueryGetObject(store1objid)); - TEST_THAT(getfile->GetObjectID() == store1objid); + COMMAND(QueryGetObject(store1objid), store1objid); + // BLOCK { // Get stream @@ -821,14 +912,17 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco filestream->CopyStreamTo(f); f.SetForReading(); // Get and decode + UNLINK_IF_EXISTS("testfiles/file1_upload_retrieved"); BackupStoreFile::DecodeFile(f, "testfiles/file1_upload_retrieved", IOStream::TimeOutInfinite); } // Retrieve as file - std::auto_ptr<BackupProtocolSuccess> getobj(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, store1objid)); - TEST_THAT(getobj->GetObjectID() == store1objid); + COMMAND(QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, store1objid), store1objid); + // BLOCK { + UNLINK_IF_EXISTS("testfiles/file1_upload_retrieved_str"); + // Get stream std::auto_ptr<IOStream> filestream(protocol.ReceiveStream()); // Get and decode @@ -842,13 +936,14 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } + { FileStream in("testfiles/file1_upload_retrieved_str"); int encfile_i[ENCFILE_SIZE]; in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } - + // Retrieve the block index, by ID { std::auto_ptr<BackupProtocolSuccess> getblockindex(protocol.QueryGetBlockIndexByID(store1objid)); @@ -857,26 +952,26 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco // Check against uploaded file TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream)); } + // and again, by name { - std::auto_ptr<BackupProtocolSuccess> getblockindex(protocol.QueryGetBlockIndexByName(BackupProtocolListDirectory::RootDirectory, store1name)); + std::auto_ptr<BackupProtocolSuccess> getblockindex(protocol.QueryGetBlockIndexByName(BACKUPSTORE_ROOT_DIRECTORY_ID, store1name)); TEST_THAT(getblockindex->GetObjectID() == store1objid); std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream()); // Check against uploaded file TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream)); } } + // Get the directory again, and see if the entry is in it { // Command std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); TEST_THAT(dir.GetNumberOfEntries() == 1); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(); @@ -888,182 +983,303 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco TEST_THAT(en->GetModificationTime() == 0x123456789abcdefLL); TEST_THAT(en->GetAttributesHash() == 0x7362383249872dfLL); TEST_THAT(en->GetObjectID() == store1objid); - TEST_THAT(en->GetSizeInBlocks() < ((ENCFILE_SIZE * 4 * 3) / 2 / 2048)+2); + TEST_EQUAL(6, en->GetSizeInBlocks()); TEST_THAT(en->GetFlags() == BackupStoreDirectory::Entry::Flags_File); } } + int file1_blocks = get_raid_file(store1objid)->GetDiscUsageInBlocks(); + TEST_THAT(check_num_files(1, 0, 0, 1)); + TEST_THAT(check_num_blocks(protocol, file1_blocks, 0, 0, root_dir_blocks, + file1_blocks + root_dir_blocks)); + + // Upload again, as a patch to the original file. + int64_t patch1_id = BackupStoreFile::QueryStoreFileDiff(protocol, + "testfiles/file1", // LocalFilename + BACKUPSTORE_ROOT_DIRECTORY_ID, // DirectoryObjectID + store1objid, // DiffFromFileID + 0x7362383249872dfLL, // AttributesHash + store1name // StoreFilename + ); + TEST_EQUAL_LINE(3, patch1_id, "wrong ObjectID for newly uploaded " + "patch file"); + + // We need to check the old file's size, because it's been replaced + // by a reverse diff, and patch1_id is a complete file, not a diff. + int patch1_blocks = get_raid_file(store1objid)->GetDiscUsageInBlocks(); + + // It will take extra blocks, even though there are no changes, because + // the server code is not smart enough to realise that the file + // contents are identical, so it will create an empty patch. + + TEST_THAT(check_num_files(1, 1, 0, 1)); + TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks, 0, + root_dir_blocks, file1_blocks + patch1_blocks + root_dir_blocks)); + + // Change the file and upload again, as a patch to the original file. + { + FileStream out("testfiles/file1", O_WRONLY | O_APPEND); + std::string appendix = "appendix!"; + out.Write(appendix.c_str(), appendix.size()); + out.Close(); + } + + int64_t patch2_id = BackupStoreFile::QueryStoreFileDiff(protocol, + "testfiles/file1", // LocalFilename + BACKUPSTORE_ROOT_DIRECTORY_ID, // DirectoryObjectID + patch1_id, // DiffFromFileID + 0x7362383249872dfLL, // AttributesHash + store1name // StoreFilename + ); + TEST_EQUAL_LINE(4, patch2_id, "wrong ObjectID for newly uploaded " + "patch file"); + + // How many blocks used by the new file? + // We need to check the old file's size, because it's been replaced + // by a reverse diff, and patch1_id is a complete file, not a diff. + int patch2_blocks = get_raid_file(patch1_id)->GetDiscUsageInBlocks(); + + TEST_THAT(check_num_files(1, 2, 0, 1)); + TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks + patch2_blocks, 0, + root_dir_blocks, file1_blocks + patch1_blocks + patch2_blocks + + root_dir_blocks)); + + // Housekeeping should not change anything just yet + protocol.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + + TEST_THAT(check_num_files(1, 2, 0, 1)); + TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks + patch2_blocks, 0, + root_dir_blocks, file1_blocks + patch1_blocks + patch2_blocks + + root_dir_blocks)); + + // Upload not as a patch, but as a completely different file. This + // marks the previous file as old (because the filename is the same) + // but used to not adjust the number of old/deleted files properly. + int64_t replaced_id = BackupStoreFile::QueryStoreFileDiff(protocol, + "testfiles/file1", // LocalFilename + BACKUPSTORE_ROOT_DIRECTORY_ID, // DirectoryObjectID + 0, // DiffFromFileID + 0x7362383249872dfLL, // AttributesHash + store1name // StoreFilename + ); + TEST_EQUAL_LINE(5, replaced_id, "wrong ObjectID for newly uploaded " + "full file"); + + // How many blocks used by the new file? This time we need to check + // the new file, because it's not a patch. + int replaced_blocks = get_raid_file(replaced_id)->GetDiscUsageInBlocks(); + + TEST_THAT(check_num_files(1, 3, 0, 1)); + TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current + file1_blocks + patch1_blocks + patch2_blocks, // old + 0, // deleted + root_dir_blocks, // directories + file1_blocks + patch1_blocks + patch2_blocks + replaced_blocks + + root_dir_blocks)); // total + + // Housekeeping should not change anything just yet + protocol.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + + TEST_THAT(check_num_files(1, 3, 0, 1)); + TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current + file1_blocks + patch1_blocks + patch2_blocks, // old + 0, // deleted + root_dir_blocks, // directories + file1_blocks + patch1_blocks + patch2_blocks + replaced_blocks + + root_dir_blocks)); // total + + // But if we reduce the limits, then it will + protocol.QueryFinished(); + TEST_THAT(change_account_limits( + "14B", // replaced_blocks + file1_blocks + root_dir_blocks + "2000B")); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + + TEST_THAT(check_num_files(1, 1, 0, 1)); + TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current + file1_blocks, // old + 0, // deleted + root_dir_blocks, // directories + file1_blocks + replaced_blocks + root_dir_blocks)); // total + + // Check that deleting files is accounted for as well + protocol.QueryDeleteFile( + BACKUPSTORE_ROOT_DIRECTORY_ID, // InDirectory + store1name); // Filename + + // The old version file is deleted as well! + TEST_THAT(check_num_files(0, 1, 2, 1)); + TEST_THAT(check_num_blocks(protocol, 0, // current + file1_blocks, // old + replaced_blocks + file1_blocks, // deleted + root_dir_blocks, // directories + file1_blocks + replaced_blocks + root_dir_blocks)); + + // Reduce limits again, check that removed files are subtracted from + // block counts. + protocol.QueryFinished(); + TEST_THAT(change_account_limits("0B", "2000B")); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + + TEST_THAT(check_num_files(0, 0, 0, 1)); + TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, root_dir_blocks)); + + // Used to not consume the stream + std::auto_ptr<IOStream> upload(new ZeroStream(1000)); + TEST_COMMAND_RETURNS_ERROR(protocol, QueryStoreFile( + BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, + 0, /* use for attr hash too */ + 99999, /* diff from ID */ + uploads[0].name, + upload), + Err_DiffFromFileDoesNotExist); + + // TODO FIXME These tests should not be here, but in + // test_server_commands. But make sure you use a network protocol, + // not a local one, when you move them. + // Try using GetFile on a directory { - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolSuccess> getFile(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::RootDirectory)), - ConnectionException, Conn_Protocol_UnexpectedReply); + int64_t subdirid = create_directory(protocol); + TEST_COMMAND_RETURNS_ERROR(protocol, + QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, subdirid), + Err_FileDoesNotVerify); } -} -void init_context(TLSContext& rContext) -{ - rContext.Initialise(false /* client */, - "testfiles/clientCerts.pem", - "testfiles/clientPrivKey.pem", - "testfiles/clientTrustedCAs.pem"); + // Try retrieving an object that doesn't exist. That used to return + // BackupProtocolSuccess(NoObject) for no apparent reason. + TEST_COMMAND_RETURNS_ERROR(protocol, QueryGetObject(store1objid + 1), + Err_DoesNotExist); + + // Close the protocol, so we can housekeep the account + protocol.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + + ExpectedRefCounts.resize(3); // stop test failure in teardown_test_backupstore() + TEARDOWN_TEST_BACKUPSTORE(); } -std::auto_ptr<SocketStreamTLS> open_conn(const char *hostname, - TLSContext& rContext) +int64_t create_directory(BackupProtocolCallable& protocol, int64_t parent_dir_id) { - init_context(rContext); - std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS); - conn->Open(rContext, Socket::TypeINET, hostname, - BOX_PORT_BBSTORED_TEST); - return conn; + // Create a directory + BackupStoreFilenameClear dirname("lovely_directory"); + // Attributes + std::auto_ptr<IOStream> attr(new MemBlockStream(attr1, sizeof(attr1))); + + std::auto_ptr<BackupProtocolSuccess> dirCreate( + protocol.QueryCreateDirectory2( + parent_dir_id, FAKE_ATTR_MODIFICATION_TIME, + FAKE_MODIFICATION_TIME, dirname, attr)); + + int64_t subdirid = dirCreate->GetObjectID(); + set_refcount(subdirid, 1); + return subdirid; } -std::auto_ptr<BackupProtocolClient> test_server_login(const char *hostname, - TLSContext& rContext, std::auto_ptr<SocketStreamTLS>& rapConn) +int64_t create_file(BackupProtocolCallable& protocol, int64_t subdirid, + const std::string& remote_filename) { - rapConn = open_conn(hostname, rContext); - - // Make a protocol - std::auto_ptr<BackupProtocolClient> protocol(new - BackupProtocolClient(*rapConn)); - - // Check the version - std::auto_ptr<BackupProtocolVersion> serverVersion( - protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + // Stick a file in it + write_test_file(0); - // Login - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf( - protocol->QueryLogin(0x01234567, 0)); + BackupStoreFilenameClear remote_filename_encoded; + if (remote_filename.empty()) + { + remote_filename_encoded = uploads[0].name; + } + else + { + remote_filename_encoded = remote_filename; + } - return protocol; + std::string filename("testfiles/test0"); + int64_t modtime; + std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename, + subdirid, remote_filename_encoded, &modtime)); + + std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile( + subdirid, + modtime, + modtime, /* use for attr hash too */ + 0, /* diff from ID */ + remote_filename_encoded, + upload)); + + int64_t subdirfileid = stored->GetObjectID(); + set_refcount(subdirfileid, 1); + return subdirfileid; } -void run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount) +bool assert_writable_connection_fails(BackupProtocolCallable& protocol) { - std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount); - int discSet = rAccount.GetDiscSet(); - - // Do housekeeping on this account - HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir, - discSet, NULL); - housekeeping.DoHousekeeping(true /* keep trying forever */); + std::auto_ptr<BackupProtocolVersion> serverVersion + (protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); + TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + TEST_COMMAND_RETURNS_ERROR_OR(protocol, QueryLogin(0x01234567, 0), + Err_CannotLockStoreForWriting, return false); + protocol.QueryFinished(); + return true; } -// Run housekeeping (for which we need to disconnect ourselves) and check -// that it doesn't change the numbers of files. -// -// Also check that bbstoreaccounts doesn't change anything +int64_t assert_readonly_connection_succeeds(BackupProtocolCallable& protocol) +{ + // TODO FIXME share code with test_server_login + std::auto_ptr<BackupProtocolVersion> serverVersion + (protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); + TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + std::auto_ptr<BackupProtocolLoginConfirmed> loginConf + (protocol.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); + return loginConf->GetClientStoreMarker(); +} -void run_housekeeping_and_check_account(const char *hostname, - TLSContext& rContext, std::auto_ptr<SocketStreamTLS>& rapConn, - std::auto_ptr<BackupProtocolClient>& rapProtocol) +bool test_multiple_uploads() { - rapProtocol->QueryFinished(); - std::auto_ptr<BackupStoreAccountDatabase> apAccounts( - BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - BackupStoreAccountDatabase::Entry account = - apAccounts->GetEntry(0x1234567); - run_housekeeping(account); + SETUP_TEST_BACKUPSTORE(); + TEST_THAT_OR(StartServer(), FAIL); - TEST_THAT(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf check 01234567 fix") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + std::auto_ptr<BackupProtocolCallable> apProtocol = + connect_and_login(context); - rapProtocol = test_server_login(hostname, rContext, rapConn); -} + // TODO FIXME replace protocolReadOnly with apProtocolReadOnly. + BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", + "backup/01234567/", 0, true); // ReadOnly -int test_server(const char *hostname) -{ - TLSContext context; - std::auto_ptr<SocketStreamTLS> conn; - std::auto_ptr<BackupProtocolClient> apProtocol = - test_server_login(hostname, context, conn); - - // Make some test attributes - #define ATTR1_SIZE 245 - #define ATTR2_SIZE 23 - #define ATTR3_SIZE 122 - int attr1[ATTR1_SIZE]; - int attr2[ATTR2_SIZE]; - int attr3[ATTR3_SIZE]; + // Read the root directory a few times (as it's cached, so make sure it doesn't hurt anything) + for(int l = 0; l < 3; ++l) { - R250 r(3465657); - for(int l = 0; l < ATTR1_SIZE; ++l) {attr1[l] = r.next();} - for(int l = 0; l < ATTR2_SIZE; ++l) {attr2[l] = r.next();} - for(int l = 0; l < ATTR3_SIZE; ++l) {attr3[l] = r.next();} + // Command + apProtocol->QueryListDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */); + // Stream + BackupStoreDirectory dir(apProtocol->ReceiveStream(), + apProtocol->GetTimeout()); + TEST_THAT(dir.GetNumberOfEntries() == 0); } - // BLOCK - { - // Get it logging - FILE *protocolLog = ::fopen("testfiles/protocol.log", "w"); - TEST_THAT(protocolLog != 0); - apProtocol->SetLogToFile(protocolLog); - -#ifndef WIN32 - // Check that we can't open a new connection which requests write permissions - { - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, hostname, - BOX_PORT_BBSTORED_TEST); - BackupProtocolClient protocol(conn); - std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)), - ConnectionException, Conn_Protocol_UnexpectedReply); - protocol.QueryFinished(); - } -#endif - - // Set the client store marker - apProtocol->QuerySetClientStoreMarker(0x8732523ab23aLL); - -#ifndef WIN32 - // Open a new connection which is read only - SocketStreamTLS connReadOnly; - connReadOnly.Open(context, Socket::TypeINET, hostname, - BOX_PORT_BBSTORED_TEST); - BackupProtocolClient protocolReadOnly(connReadOnly); - - // Get it logging - FILE *protocolReadOnlyLog = ::fopen("testfiles/protocolReadOnly.log", "w"); - TEST_THAT(protocolReadOnlyLog != 0); - protocolReadOnly.SetLogToFile(protocolReadOnlyLog); - - { - std::auto_ptr<BackupProtocolVersion> serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); - - // Check client store marker - TEST_THAT(loginConf->GetClientStoreMarker() == 0x8732523ab23aLL); - } -#else // WIN32 - #define protocolReadOnly (*apProtocol) -#endif - - test_server_1(*apProtocol, protocolReadOnly); - - #define TEST_NUM_FILES(files, old, deleted, dirs) \ - { \ - std::auto_ptr<BackupStoreInfo> apInfo = \ - BackupStoreInfo::Load(0x1234567, \ - "backup/01234567/", 0, true); \ - TEST_EQUAL_LINE(files, apInfo->GetNumFiles(), \ - "num files"); \ - TEST_EQUAL_LINE(old, apInfo->GetNumOldFiles(), \ - "old files"); \ - TEST_EQUAL_LINE(deleted, apInfo->GetNumDeletedFiles(), \ - "deleted files"); \ - TEST_EQUAL_LINE(dirs, apInfo->GetNumDirectories(), \ - "directories"); \ - } + // Read the dir from the readonly connection (make sure it gets in the cache) + // Command + protocolReadOnly.QueryListDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes */); + // Stream + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + protocolReadOnly.GetTimeout()); + TEST_THAT(dir.GetNumberOfEntries() == 0); - TEST_NUM_FILES(1, 0, 0, 1); - run_housekeeping_and_check_account(hostname, context, - conn, apProtocol); - TEST_NUM_FILES(1, 0, 0, 1); + // TODO FIXME dedent + { + TEST_THAT(check_num_files(0, 0, 0, 1)); // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); @@ -1078,16 +1294,16 @@ int test_server(const char *hostname) filename += uploads[t].fnextra; int64_t modtime = 0; - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), BackupProtocolListDirectory::RootDirectory, uploads[t].name, &modtime)); + std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[t].name, &modtime)); TEST_THAT(modtime != 0); - + std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, modtime, modtime, /* use it for attr hash too */ 0, /* diff from ID */ uploads[t].name, - *upload)); + upload)); uploads[t].allocated_objid = stored->GetObjectID(); uploads[t].mod_time = modtime; if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID(); @@ -1095,35 +1311,73 @@ int test_server(const char *hostname) BOX_TRACE("wrote file " << filename << " to server " "as object " << BOX_FORMAT_OBJECTID(stored->GetObjectID())); - TEST_NUM_FILES(t + 2, 0, 0, 1); - run_housekeeping_and_check_account(hostname, context, - conn, apProtocol); - TEST_NUM_FILES(t + 2, 0, 0, 1); + // Some of the uploaded files replace old ones, increasing + // the old file count instead of the current file count. + int expected_num_old_files = 0; + if (t >= 8) expected_num_old_files++; + if (t >= 12) expected_num_old_files++; + if (t >= 13) expected_num_old_files++; + int expected_num_current_files = t + 1 - expected_num_old_files; + + TEST_THAT(check_num_files(expected_num_current_files, + expected_num_old_files, 0, 1)); + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + apProtocol = connect_and_login(context); + protocolReadOnly.Reopen(); + + TEST_THAT(check_num_files(expected_num_current_files, + expected_num_old_files, 0, 1)); } // Add some attributes onto one of them { - TEST_NUM_FILES(UPLOAD_NUM + 1, 0, 0, 1); - MemBlockStream attrnew(attr3, sizeof(attr3)); + TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 0, 1)); + std::auto_ptr<IOStream> attrnew( + new MemBlockStream(attr3, sizeof(attr3))); std::auto_ptr<BackupProtocolSuccess> set(apProtocol->QuerySetReplacementFileAttributes( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, 32498749832475LL, uploads[UPLOAD_ATTRS_EN].name, attrnew)); TEST_THAT(set->GetObjectID() == uploads[UPLOAD_ATTRS_EN].allocated_objid); - TEST_NUM_FILES(UPLOAD_NUM + 1, 0, 0, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 0, 1)); } - + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + apProtocol = connect_and_login(context); + protocolReadOnly.Reopen(); + // Delete one of them (will implicitly delete an old version) { std::auto_ptr<BackupProtocolSuccess> del(apProtocol->QueryDeleteFile( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[UPLOAD_DELETE_EN].name)); TEST_THAT(del->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid); - TEST_NUM_FILES(UPLOAD_NUM, 0, 1, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); } +#ifdef _MSC_VER + BOX_TRACE("1"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + apProtocol = connect_and_login(context); + protocolReadOnly.Reopen(); + +#ifdef _MSC_VER + BOX_TRACE("2"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + // Check that the block index can be obtained by name even though it's been deleted { // Fetch the raw object @@ -1131,11 +1385,11 @@ int test_server(const char *hostname) FileStream out("testfiles/downloaddelobj", O_WRONLY | O_CREAT); std::auto_ptr<BackupProtocolSuccess> getobj(apProtocol->QueryGetObject(uploads[UPLOAD_DELETE_EN].allocated_objid)); std::auto_ptr<IOStream> objstream(apProtocol->ReceiveStream()); - objstream->CopyStreamTo(out); + objstream->CopyStreamTo(out, apProtocol->GetTimeout()); } // query index and test std::auto_ptr<BackupProtocolSuccess> getblockindex(apProtocol->QueryGetBlockIndexByName( - BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_DELETE_EN].name)); + BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[UPLOAD_DELETE_EN].name)); TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid); std::auto_ptr<IOStream> blockIndexStream(apProtocol->ReceiveStream()); TEST_THAT(check_block_index("testfiles/downloaddelobj", *blockIndexStream)); @@ -1145,12 +1399,17 @@ int test_server(const char *hostname) for(int t = 0; t < UPLOAD_NUM; ++t) { printf("%d\n", t); - std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, uploads[t].allocated_objid)); + std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[t].allocated_objid)); TEST_THAT(getFile->GetObjectID() == uploads[t].allocated_objid); std::auto_ptr<IOStream> filestream(apProtocol->ReceiveStream()); test_test_file(t, *filestream); } +#ifdef _MSC_VER + BOX_TRACE("3"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + { StreamableMemBlock attrtest(attr3, sizeof(attr3)); @@ -1161,10 +1420,15 @@ int test_server(const char *hostname) // And on the read/write one check_dir_after_uploads(*apProtocol, attrtest); } - + // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); +#ifdef _MSC_VER + BOX_TRACE("4"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + // Check diffing and rsync like stuff... // Build a modified file { @@ -1172,7 +1436,7 @@ int test_server(const char *hostname) TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING) == TEST_FILE_FOR_PATCHING_SIZE); FileStream in(TEST_FILE_FOR_PATCHING); void *buf = ::malloc(TEST_FILE_FOR_PATCHING_SIZE); - FileStream out(TEST_FILE_FOR_PATCHING ".mod", O_WRONLY | O_CREAT | O_EXCL); + FileStream out(TEST_FILE_FOR_PATCHING ".mod", O_WRONLY | O_CREAT); TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_PATCH_AT); out.Write(buf, TEST_FILE_FOR_PATCHING_PATCH_AT); char insert[13] = "INSERTINSERT"; @@ -1182,51 +1446,66 @@ int test_server(const char *hostname) ::free(buf); } - TEST_NUM_FILES(UPLOAD_NUM, 0, 1, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); // Run housekeeping (for which we need to disconnect // ourselves) and check that it doesn't change the numbers // of files + +#ifdef _MSC_VER + BOX_TRACE("5"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); - run_housekeeping(account); +#ifdef _MSC_VER + BOX_TRACE("6"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + TEST_EQUAL(0, run_housekeeping(account)); - // Also check that bbstoreaccounts doesn't change anything - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf check 01234567 fix") == 0); + // Also check that bbstoreaccounts doesn't change anything, + // using an external process instead of the internal one. + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf check 01234567 fix") == 0, + FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - apProtocol = test_server_login(hostname, context, conn); + apProtocol = connect_and_login(context); + protocolReadOnly.Reopen(); - TEST_NUM_FILES(UPLOAD_NUM, 0, 1, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); { // Fetch the block index for this one std::auto_ptr<BackupProtocolSuccess> getblockindex(apProtocol->QueryGetBlockIndexByName( - BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_PATCH_EN].name)); + BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[UPLOAD_PATCH_EN].name)); TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_PATCH_EN].allocated_objid); std::auto_ptr<IOStream> blockIndexStream(apProtocol->ReceiveStream()); - + // Do the patching bool isCompletelyDifferent = false; int64_t modtime; std::auto_ptr<IOStream> patchstream( BackupStoreFile::EncodeFileDiff( - TEST_FILE_FOR_PATCHING ".mod", - BackupProtocolListDirectory::RootDirectory, - uploads[UPLOAD_PATCH_EN].name, - uploads[UPLOAD_PATCH_EN].allocated_objid, + TEST_FILE_FOR_PATCHING ".mod", + BACKUPSTORE_ROOT_DIRECTORY_ID, + uploads[UPLOAD_PATCH_EN].name, + uploads[UPLOAD_PATCH_EN].allocated_objid, *blockIndexStream, - IOStream::TimeOutInfinite, + SHORT_TIMEOUT, NULL, // pointer to DiffTimer impl &modtime, &isCompletelyDifferent)); TEST_THAT(isCompletelyDifferent == false); // Sent this to a file, so we can check the size, rather than uploading it directly { - FileStream patch(TEST_FILE_FOR_PATCHING ".patch", O_WRONLY | O_CREAT | O_EXCL); + FileStream patch(TEST_FILE_FOR_PATCHING ".patch", O_WRONLY | O_CREAT); patchstream->CopyStreamTo(patch); } // Make sure the stream is a plausible size for a patch containing only one new block @@ -1234,9 +1513,9 @@ int test_server(const char *hostname) // Upload it int64_t patchedID = 0; { - FileStream uploadpatch(TEST_FILE_FOR_PATCHING ".patch"); + std::auto_ptr<IOStream> uploadpatch(new FileStream(TEST_FILE_FOR_PATCHING ".patch")); std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, modtime, modtime, /* use it for attr hash too */ uploads[UPLOAD_PATCH_EN].allocated_objid, /* diff from ID */ @@ -1250,71 +1529,81 @@ int test_server(const char *hostname) set_refcount(patchedID, 1); // Then download it to check it's OK - std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, patchedID)); + std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, patchedID)); TEST_THAT(getFile->GetObjectID() == patchedID); std::auto_ptr<IOStream> filestream(apProtocol->ReceiveStream()); - BackupStoreFile::DecodeFile(*filestream, TEST_FILE_FOR_PATCHING ".downloaded", IOStream::TimeOutInfinite); + BackupStoreFile::DecodeFile(*filestream, + TEST_FILE_FOR_PATCHING ".downloaded", SHORT_TIMEOUT); // Check it's the same TEST_THAT(check_files_same(TEST_FILE_FOR_PATCHING ".downloaded", TEST_FILE_FOR_PATCHING ".mod")); - - TEST_NUM_FILES(UPLOAD_NUM, 1, 1, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 4, 4, 2, 1)); } + } - // Create a directory - int64_t subdirid = 0; - BackupStoreFilenameClear dirname("lovely_directory"); - { - // Attributes - MemBlockStream attr(attr1, sizeof(attr1)); - std::auto_ptr<BackupProtocolSuccess> dirCreate(apProtocol->QueryCreateDirectory( - BackupProtocolListDirectory::RootDirectory, - 9837429842987984LL, dirname, attr)); - subdirid = dirCreate->GetObjectID(); - TEST_THAT(subdirid == maxID + 1); + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); - TEST_NUM_FILES(UPLOAD_NUM, 1, 1, 2); - } + TEARDOWN_TEST_BACKUPSTORE(); +} - set_refcount(subdirid, 1); +bool test_server_commands() +{ + SETUP_TEST_BACKUPSTORE(); - // Stick a file in it - int64_t subdirfileid = 0; - { - std::string filename("testfiles/test0"); - int64_t modtime; - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), subdirid, uploads[0].name, &modtime)); + std::auto_ptr<BackupProtocolLocal2> apProtocol( + new BackupProtocolLocal2(0x01234567, "test", + "backup/01234567/", 0, false)); - std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile( - subdirid, - modtime, - modtime, /* use for attr hash too */ - 0, /* diff from ID */ - uploads[0].name, - *upload)); - subdirfileid = stored->GetObjectID(); + // Try using GetFile on an object ID that doesn't exist in the directory + { + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, + BACKUPSTORE_ROOT_DIRECTORY_ID), + Err_DoesNotExistInDirectory); + } - TEST_NUM_FILES(UPLOAD_NUM + 1, 1, 1, 2); + // BLOCK + // TODO FIXME dedent this block. + { + // Create a directory + int64_t subdirid = create_directory(*apProtocol); + TEST_THAT(check_num_files(0, 0, 0, 2)); + + // Try using GetFile on the directory + { + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, + subdirid), + Err_FileDoesNotVerify); } - set_refcount(subdirfileid, 1); + // Stick a file in it + int64_t subdirfileid = create_file(*apProtocol, subdirid); + TEST_THAT(check_num_files(1, 0, 0, 2)); + + BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", + "backup/01234567/", 0, true); // read-only - printf("\n==== Checking upload using read-only connection\n"); - // Check the directories on the read only connection + BOX_TRACE("Checking root directory using read-only connection"); { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - TEST_THAT(dir.GetNumberOfEntries() == UPLOAD_NUM + 3 /* for the first test file, the patched upload, and this new dir */); + protocolReadOnly.QueryListDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes! */); // Stream + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); + + // UPLOAD_NUM test files, patch uploaded and new dir + TEST_EQUAL(1, dir.GetNumberOfEntries()); // Check the last one... BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; BackupStoreDirectory::Entry *t = 0; + BackupStoreFilenameClear dirname("lovely_directory"); + while((t = i.Next()) != 0) { if(en != 0) @@ -1325,158 +1614,214 @@ int test_server(const char *hostname) } en = t; } - // Does it look right? + + // Check that the last entry looks right + TEST_EQUAL(subdirid, en->GetObjectID()); TEST_THAT(en->GetName() == dirname); - TEST_THAT(en->GetFlags() == BackupProtocolListDirectory::Flags_Dir); - TEST_THAT(en->GetObjectID() == subdirid); - TEST_THAT(en->GetModificationTime() == 0); // dirs don't have modification times. + TEST_EQUAL(BackupProtocolListDirectory::Flags_Dir, en->GetFlags()); + int64_t actual_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); + TEST_EQUAL(actual_size, en->GetSizeInBlocks()); + TEST_EQUAL(FAKE_MODIFICATION_TIME, en->GetModificationTime()); } + BOX_TRACE("Checking subdirectory using read-only connection"); { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( + TEST_EQUAL(subdirid, + protocolReadOnly.QueryListDirectory( subdirid, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, true /* get attributes */)); - TEST_THAT(dirreply->GetObjectID() == subdirid); - // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + true /* get attributes */)->GetObjectID()); + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); TEST_THAT(dir.GetNumberOfEntries() == 1); - // Check the last one... + // Check the (only) one... BackupStoreDirectory::Iterator i(dir); - // Discard first BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); - // Does it look right? + + // Check that it looks right + TEST_EQUAL(subdirfileid, en->GetObjectID()); TEST_THAT(en->GetName() == uploads[0].name); - TEST_THAT(en->GetFlags() == BackupProtocolListDirectory::Flags_File); - TEST_THAT(en->GetObjectID() == subdirfileid); + TEST_EQUAL(BackupProtocolListDirectory::Flags_File, en->GetFlags()); + int64_t actual_size = get_raid_file(subdirfileid)->GetDiscUsageInBlocks(); + TEST_EQUAL(actual_size, en->GetSizeInBlocks()); TEST_THAT(en->GetModificationTime() != 0); // Attributes TEST_THAT(dir.HasAttributes()); - TEST_THAT(dir.GetAttributesModTime() == 9837429842987984LL); + TEST_EQUAL(FAKE_ATTR_MODIFICATION_TIME, + dir.GetAttributesModTime()); StreamableMemBlock attr(attr1, sizeof(attr1)); TEST_THAT(dir.GetAttributes() == attr); } - printf("done.\n\n"); - // Check that we don't get attributes if we don't ask for them + BOX_TRACE("Checking that we don't get attributes if we don't ask for them"); { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - subdirid, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); + protocolReadOnly.QueryListDirectory( + subdirid, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes! */); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); TEST_THAT(!dir.HasAttributes()); } - // sleep to ensure that the timestamp on the file will change + // Sleep to ensure that the timestamp on the file will change, + // invalidating the read-only connection's cache of the + // directory, and forcing it to be reloaded. ::safe_sleep(1); // Change attributes on the directory { - MemBlockStream attrnew(attr2, sizeof(attr2)); + std::auto_ptr<IOStream> attrnew( + new MemBlockStream(attr2, sizeof(attr2))); std::auto_ptr<BackupProtocolSuccess> changereply(apProtocol->QueryChangeDirAttributes( subdirid, 329483209443598LL, attrnew)); TEST_THAT(changereply->GetObjectID() == subdirid); } + // Check the new attributes { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - subdirid, - 0, // no flags - BackupProtocolListDirectory::Flags_EXCLUDE_EVERYTHING, true /* get attributes */)); + protocolReadOnly.QueryListDirectory( + subdirid, + 0, // no flags + BackupProtocolListDirectory::Flags_EXCLUDE_EVERYTHING, + true /* get attributes */); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); TEST_THAT(dir.GetNumberOfEntries() == 0); // Attributes TEST_THAT(dir.HasAttributes()); - TEST_THAT(dir.GetAttributesModTime() == 329483209443598LL); + TEST_EQUAL(329483209443598LL, dir.GetAttributesModTime()); StreamableMemBlock attrtest(attr2, sizeof(attr2)); TEST_THAT(dir.GetAttributes() == attrtest); } - - // sleep to ensure that the timestamp on the file will change + + BackupStoreFilenameClear& oldName(uploads[0].name); + int64_t root_file_id = create_file(*apProtocol, BACKUPSTORE_ROOT_DIRECTORY_ID); + TEST_THAT(check_num_files(2, 0, 0, 2)); + + // Upload a new version of the file as well, to ensure that the + // old version is moved along with the current version. + root_file_id = BackupStoreFile::QueryStoreFileDiff(*apProtocol, + "testfiles/test0", BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, // DiffFromFileID + 0, // AttributesHash + oldName); + set_refcount(root_file_id, 1); + TEST_THAT(check_num_files(2, 1, 0, 2)); + + // Check that it's in the root directory (it won't be for long) + protocolReadOnly.QueryListDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, 0, false); + TEST_THAT(BackupStoreDirectory(protocolReadOnly.ReceiveStream()) + .FindEntryByID(root_file_id) != NULL); + + BackupStoreFilenameClear newName("moved-files"); + + // Sleep before modifying the root directory, to ensure that + // the timestamp on the file it's stored in will change when + // we modify it, invalidating the read-only connection's cache + // and forcing it to reload the root directory, next time we + // ask for its contents. ::safe_sleep(1); // Test moving a file { - BackupStoreFilenameClear newName("moved-files"); - - std::auto_ptr<BackupProtocolSuccess> rep(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, - BackupProtocolListDirectory::RootDirectory, + std::auto_ptr<BackupProtocolSuccess> rep(apProtocol->QueryMoveObject(root_file_id, + BACKUPSTORE_ROOT_DIRECTORY_ID, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName)); - TEST_THAT(rep->GetObjectID() == uploads[UPLOAD_FILE_TO_MOVE].allocated_objid); + TEST_EQUAL(root_file_id, rep->GetObjectID()); } // Try some dodgy renames { + // File doesn't exist at all + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject(-1, + BACKUPSTORE_ROOT_DIRECTORY_ID, subdirid, + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_DoesNotExistInDirectory); BackupStoreFilenameClear newName("moved-files"); - TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject( + uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, BackupProtocolListDirectory::RootDirectory, - subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName), - ConnectionException, Conn_Protocol_UnexpectedReply); - TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, subdirid, - subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName), - ConnectionException, Conn_Protocol_UnexpectedReply); + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_DoesNotExistInDirectory); + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject( + uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, + subdirid, + subdirid, + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_DoesNotExistInDirectory); } - // Rename within a directory - { - BackupStoreFilenameClear newName("moved-files-x"); - apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, + // File exists, but not in this directory (we just moved it) + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject(root_file_id, + BACKUPSTORE_ROOT_DIRECTORY_ID, subdirid, - subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName); - } + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_DoesNotExistInDirectory); + + // Moving file to same directory that it's already in, + // with the same name + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject(root_file_id, + subdirid, + subdirid, + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_TargetNameExists); - // Check it's all gone from the root directory... + // Rename within a directory (successfully) { - // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); - // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - // Read all entries - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next()) != 0) - { - TEST_THAT(en->GetName() != uploads[UPLOAD_FILE_TO_MOVE].name); - } + BackupStoreFilenameClear newName2("moved-files-x"); + apProtocol->QueryMoveObject(root_file_id, subdirid, + subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName2); } + // Check it's all gone from the root directory... + protocolReadOnly.QueryListDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, 0, false); + TEST_THAT(BackupStoreDirectory(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT).FindEntryByID(root_file_id) == NULL); + // Check the old and new versions are in the other directory { + BackupStoreFilenameClear notThere("moved-files"); BackupStoreFilenameClear lookFor("moved-files-x"); // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - subdirid, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); + protocolReadOnly.QueryListDirectory( + subdirid, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes */); + // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); + // Check entries BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; @@ -1484,6 +1829,10 @@ int test_server(const char *hostname) bool foundOld = false; while((en = i.Next()) != 0) { + // If we find the old name, then the rename + // operation didn't work. + TEST_THAT(en->GetName() != notThere); + if(en->GetName() == lookFor) { if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File)) foundCurrent = true; @@ -1501,52 +1850,63 @@ int test_server(const char *hostname) int64_t subsubdirid = 0; int64_t subsubfileid = 0; { + // TODO FIXME use create_dir() and create_file() instead. BackupStoreFilenameClear nd("sub2"); // Attributes - MemBlockStream attr(attr1, sizeof(attr1)); - std::auto_ptr<BackupProtocolSuccess> dirCreate(apProtocol->QueryCreateDirectory( - subdirid, - 9837429842987984LL, nd, attr)); - subsubdirid = dirCreate->GetObjectID(); + std::auto_ptr<IOStream> attr(new MemBlockStream(attr1, + sizeof(attr1))); + subsubdirid = apProtocol->QueryCreateDirectory(subdirid, + FAKE_ATTR_MODIFICATION_TIME, nd, attr)->GetObjectID(); + + write_test_file(2); - FileStream upload("testfiles/file1_upload1"); - BackupStoreFilenameClear nf("file2"); + BackupStoreFilenameClear file2("file2"); + std::auto_ptr<IOStream> upload( + BackupStoreFile::EncodeFile("testfiles/test2", + BACKUPSTORE_ROOT_DIRECTORY_ID, file2)); std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile( subsubdirid, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ - nf, + file2, upload)); subsubfileid = stored->GetObjectID(); } set_refcount(subsubdirid, 1); set_refcount(subsubfileid, 1); + TEST_THAT(check_num_files(3, 1, 0, 3)); + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + apProtocol->Reopen(); + protocolReadOnly.Reopen(); // Query names -- test that invalid stuff returns not found OK { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(3248972347823478927LL, subsubdirid)); - TEST_THAT(nameRep->GetNumNameElements() == 0); + TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(subsubfileid, 2342378424LL)); - TEST_THAT(nameRep->GetNumNameElements() == 0); + TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(38947234789LL, 2342378424LL)); - TEST_THAT(nameRep->GetNumNameElements() == 0); + TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, 2234342378424LL)); - TEST_THAT(nameRep->GetNumNameElements() == 0); + TEST_THAT(nameRep->GetNumNameElements() == 0); } // Query names... first, get info for the file { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(subsubfileid, subsubdirid)); std::auto_ptr<IOStream> namestream(apProtocol->ReceiveStream()); - + TEST_THAT(nameRep->GetNumNameElements() == 3); TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_File); TEST_THAT(nameRep->GetModificationTime() == 0x123456789abcdefLL); @@ -1564,7 +1924,7 @@ int test_server(const char *hostname) { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, subsubdirid)); std::auto_ptr<IOStream> namestream(apProtocol->ReceiveStream()); - + TEST_THAT(nameRep->GetNumNameElements() == 2); TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_Dir); static const char *testnames[] = {"sub2","lovely_directory"}; @@ -1575,82 +1935,348 @@ int test_server(const char *hostname) TEST_THAT(fn.GetClearFilename() == testnames[l]); } } - + //} skip: - std::auto_ptr<BackupStoreRefCountDatabase> apRefCount( - BackupStoreRefCountDatabase::Load( - apAccounts->GetEntry(0x1234567), true)); - // Create some nice recursive directories - int64_t dirtodelete = create_test_data_subdirs(*apProtocol, - BackupProtocolListDirectory::RootDirectory, - "test_delete", 6 /* depth */, *apRefCount); - + TEST_THAT(check_reference_counts()); + + write_test_file(1); + int64_t dirtodelete; + + { + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + std::auto_ptr<BackupStoreRefCountDatabase> apRefCount( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true)); + + + dirtodelete = create_test_data_subdirs(*apProtocol, + BACKUPSTORE_ROOT_DIRECTORY_ID, + "test_delete", 6 /* depth */, apRefCount.get()); + } + + TEST_THAT(check_reference_counts()); + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(check_reference_counts()); + // And delete them + apProtocol->Reopen(); + protocolReadOnly.Reopen(); + { std::auto_ptr<BackupProtocolSuccess> dirdel(apProtocol->QueryDeleteDirectory( dirtodelete)); TEST_THAT(dirdel->GetObjectID() == dirtodelete); } + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(check_reference_counts()); + protocolReadOnly.Reopen(); + // Get the root dir, checking for deleted items { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_Dir | BackupProtocolListDirectory::Flags_Deleted, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); + protocolReadOnly.QueryListDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + BackupProtocolListDirectory::Flags_Dir | + BackupProtocolListDirectory::Flags_Deleted, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes */); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); + // Check there's only that one entry TEST_THAT(dir.GetNumberOfEntries() == 1); - + BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); if(en) { - TEST_THAT(en->GetObjectID() == dirtodelete); + TEST_EQUAL(dirtodelete, en->GetObjectID()); BackupStoreFilenameClear n("test_delete"); TEST_THAT(en->GetName() == n); } - + // Then... check everything's deleted - test_everything_deleted(protocolReadOnly, dirtodelete); + assert_everything_deleted(protocolReadOnly, dirtodelete); } - + + // Undelete and check that block counts are restored properly + apProtocol->Reopen(); + TEST_EQUAL(dirtodelete, + apProtocol->QueryUndeleteDirectory(dirtodelete)->GetObjectID()); + // Finish the connections -#ifndef WIN32 - protocolReadOnly.QueryFinished(); -#endif apProtocol->QueryFinished(); - - // Close logs -#ifndef WIN32 - ::fclose(protocolReadOnlyLog); -#endif - ::fclose(protocolLog); + protocolReadOnly.QueryFinished(); + + TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(check_reference_counts()); } - - return 0; + + TEARDOWN_TEST_BACKUPSTORE(); +} + +int get_object_size(BackupProtocolCallable& protocol, int64_t ObjectID, + int64_t ContainerID) +{ + // Get the root directory cached in the read-only connection + protocol.QueryListDirectory(ContainerID, 0, // FlagsMustBeSet + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes */); + + BackupStoreDirectory dir(protocol.ReceiveStream()); + BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); + TEST_THAT_OR(en != 0, return -1); + TEST_EQUAL_OR(ObjectID, en->GetObjectID(), return -1); + return en->GetSizeInBlocks(); +} + +bool write_dir(BackupStoreDirectory& dir) +{ + std::string rfn; + StoreStructure::MakeObjectFilename(dir.GetObjectID(), + "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, + rfn, false); // EnsureDirectoryExists + RaidFileWrite rfw(0, rfn); + rfw.Open(true); // AllowOverwrite + dir.WriteToStream(rfw); + rfw.Commit(/* ConvertToRaidNow */ true); + return true; +} + +bool test_directory_parent_entry_tracks_directory_size() +{ + SETUP_TEST_BACKUPSTORE(); + + BackupProtocolLocal2 protocol(0x01234567, "test", "backup/01234567/", + 0, false); + BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", + "backup/01234567/", 0, true); // read only + + int64_t subdirid = create_directory(protocol); + + // Get the root directory cached in the read-only connection, and + // test that the initial size is correct. + int old_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); + TEST_THAT(old_size > 0); + TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Sleep to ensure that the directory file timestamp changes, so that + // the read-only connection will discard its cached copy. + safe_sleep(1); + + // Start adding files until the size on disk increases. This is + // guaranteed to happen eventually :) + int new_size = old_size; + int64_t last_added_file_id = 0; + std::string last_added_filename; + + for (int i = 0; new_size == old_size; i++) + { + std::ostringstream name; + name << "testfile_" << i; + last_added_filename = name.str(); + last_added_file_id = create_file(protocol, subdirid, name.str()); + new_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); + } + + // Check that the root directory entry has been updated + TEST_EQUAL(new_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Now delete an entry, and check that the size is reduced + protocol.QueryDeleteFile(subdirid, + BackupStoreFilenameClear(last_added_filename)); + ExpectedRefCounts[last_added_file_id] = 0; + + // Reduce the limits, to remove it permanently from the store + protocol.QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(change_account_limits("0B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account()); + set_refcount(last_added_file_id, 0); + protocol.Reopen(); + protocolReadOnly.Reopen(); + + TEST_EQUAL(old_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); + + // Check that the entry in the root directory was updated too + TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Push the limits back up + protocol.QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(change_account_limits("1000B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + protocolReadOnly.Reopen(); + + // Now modify the root directory to remove its entry for this one + BackupStoreDirectory root(*get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID), + IOStream::TimeOutInfinite); + BackupStoreDirectory::Entry *en = root.FindEntryByID(subdirid); + TEST_THAT_OR(en, return false); + BackupStoreDirectory::Entry enCopy(*en); + root.DeleteEntry(subdirid); + TEST_THAT(write_dir(root)); + + // Add a directory, this should try to push the object size back up, + // which will try to modify the subdir's entry in its parent, which + // no longer exists, which should just log an error instead of + // aborting/segfaulting. + create_directory(protocol, subdirid); + + // Repair the error ourselves, as bbstoreaccounts can't. + protocol.QueryFinished(); + enCopy.SetSizeInBlocks(get_raid_file(subdirid)->GetDiscUsageInBlocks()); + root.AddEntry(enCopy); + TEST_THAT(write_dir(root)); + + // We also have to remove the entry for lovely_directory created by + // create_directory(), because otherwise we can't create it again. + // (Perhaps it should not have been committed because we failed to + // update the parent, but currently it is.) + BackupStoreDirectory subdir(*get_raid_file(subdirid), + IOStream::TimeOutInfinite); + { + BackupStoreDirectory::Iterator i(subdir); + en = i.FindMatchingClearName( + BackupStoreFilenameClear("lovely_directory")); + } + TEST_THAT_OR(en, return false); + protocol.Reopen(); + protocol.QueryDeleteDirectory(en->GetObjectID()); + set_refcount(en->GetObjectID(), 0); + + // This should have fixed the error, so we should be able to add the + // entry now. This should push the object size back up. + int64_t dir2id = create_directory(protocol, subdirid); + TEST_EQUAL(new_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); + TEST_EQUAL(new_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Delete it again, which should reduce the object size again + protocol.QueryDeleteDirectory(dir2id); + set_refcount(dir2id, 0); + + // Reduce the limits, to remove it permanently from the store + protocol.QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(change_account_limits("0B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + protocolReadOnly.Reopen(); + + // Check that the entry in the root directory was updated + TEST_EQUAL(old_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); + TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Check that bbstoreaccounts check fix will detect and repair when + // a directory's parent entry has the wrong size for the directory. + + protocol.QueryFinished(); + + root.ReadFromStream(*get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID), + IOStream::TimeOutInfinite); + en = root.FindEntryByID(subdirid); + TEST_THAT_OR(en != 0, return false); + en->SetSizeInBlocks(1234); + + // Sleep to ensure that the directory file timestamp changes, so that + // the read-only connection will discard its cached copy. + safe_sleep(1); + TEST_THAT(write_dir(root)); + + TEST_EQUAL(1234, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Sleep to ensure that the directory file timestamp changes, so that + // the read-only connection will discard its cached copy. + safe_sleep(1); + + protocolReadOnly.QueryFinished(); + TEST_EQUAL(1, check_account_for_errors()); + + protocolReadOnly.Reopen(); + TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + protocolReadOnly.QueryFinished(); + + TEARDOWN_TEST_BACKUPSTORE(); } -int test3(int argc, const char *argv[]) +bool test_cannot_open_multiple_writable_connections() +{ + SETUP_TEST_BACKUPSTORE(); + + // First try a local protocol. This works even on Windows. + BackupProtocolLocal2 protocolWritable(0x01234567, "test", + "backup/01234567/", 0, false); // Not read-only + + // Set the client store marker + protocolWritable.QuerySetClientStoreMarker(0x8732523ab23aLL); + + // First try a local protocol. This works even on Windows. + { + BackupStoreContext bsContext(0x01234567, (HousekeepingInterface *)NULL, "test"); + bsContext.SetClientHasAccount("backup/01234567/", 0); + BackupProtocolLocal protocolWritable2(bsContext); + TEST_THAT(assert_writable_connection_fails(protocolWritable2)); + } + + { + BackupStoreContext bsContext(0x01234567, (HousekeepingInterface *)NULL, "test"); + bsContext.SetClientHasAccount("backup/01234567/", 0); + BackupProtocolLocal protocolReadOnly(bsContext); + TEST_EQUAL(0x8732523ab23aLL, + assert_readonly_connection_succeeds(protocolReadOnly)); + } + + // Try network connections too. + TEST_THAT_OR(StartServer(), return false); + + BackupProtocolClient protocolWritable3(open_conn("localhost", context)); + TEST_THAT(assert_writable_connection_fails(protocolWritable3)); + + // Do not dedent. Object needs to go out of scope to release lock + { + BackupProtocolClient protocolReadOnly2(open_conn("localhost", context)); + TEST_EQUAL(0x8732523ab23aLL, + assert_readonly_connection_succeeds(protocolReadOnly2)); + } + + protocolWritable.QueryFinished(); + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_encoding() { // Now test encoded files // TODO: This test needs to check failure situations as well as everything working, // but this will be saved for the full implementation. + + SETUP_TEST_BACKUPSTORE(); + int encfile[ENCFILE_SIZE]; { for(int l = 0; l < ENCFILE_SIZE; ++l) { encfile[l] = l * 173; } - + // Encode and decode a small block (shouldn't be compressed) { #define SMALL_BLOCK_SIZE 251 @@ -1658,14 +2284,14 @@ int test3(int argc, const char *argv[]) TEST_THAT(encBlockSize > SMALL_BLOCK_SIZE); BackupStoreFile::EncodingBuffer encoded; encoded.Allocate(encBlockSize / 8); // make sure reallocation happens - + // Encode! int encSize = BackupStoreFile::EncodeChunk(encfile, SMALL_BLOCK_SIZE, encoded); // Check the header says it's not been compressed TEST_THAT((encoded.mpBuffer[0] & 1) == 0); // Check the output size has been inflated (no compression) TEST_THAT(encSize > SMALL_BLOCK_SIZE); - + // Decode it int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(SMALL_BLOCK_SIZE); TEST_THAT(decBlockSize > SMALL_BLOCK_SIZE); @@ -1673,10 +2299,10 @@ int test3(int argc, const char *argv[]) int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize); TEST_THAT(decSize < decBlockSize); TEST_THAT(decSize == SMALL_BLOCK_SIZE); - + // Check it came out of the wash the same TEST_THAT(::memcmp(encfile, decoded, SMALL_BLOCK_SIZE) == 0); - + free(decoded); } @@ -1686,14 +2312,14 @@ int test3(int argc, const char *argv[]) TEST_THAT(encBlockSize > ENCFILE_SIZE); BackupStoreFile::EncodingBuffer encoded; encoded.Allocate(encBlockSize / 8); // make sure reallocation happens - + // Encode! int encSize = BackupStoreFile::EncodeChunk(encfile, ENCFILE_SIZE, encoded); // Check the header says it's compressed TEST_THAT((encoded.mpBuffer[0] & 1) == 1); // Check the output size make it likely that it's compressed (is very compressible data) TEST_THAT(encSize < ENCFILE_SIZE); - + // Decode it int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(ENCFILE_SIZE); TEST_THAT(decBlockSize > ENCFILE_SIZE); @@ -1704,37 +2330,74 @@ int test3(int argc, const char *argv[]) // Check it came out of the wash the same TEST_THAT(::memcmp(encfile, decoded, ENCFILE_SIZE) == 0); - + free(decoded); } - + // The test block to a file { - FileStream f("testfiles/testenc1", O_WRONLY | O_CREAT | O_EXCL); + FileStream f("testfiles/testenc1", O_WRONLY | O_CREAT); f.Write(encfile, sizeof(encfile)); } - + // Encode it { - FileStream out("testfiles/testenc1_enc", O_WRONLY | O_CREAT | O_EXCL); + FileStream out("testfiles/testenc1_enc", O_WRONLY | O_CREAT); BackupStoreFilenameClear name("testfiles/testenc1"); std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testenc1", 32, name)); encoded->CopyStreamTo(out); } - + // Verify it { FileStream enc("testfiles/testenc1_enc"); TEST_THAT(BackupStoreFile::VerifyEncodedFileFormat(enc) == true); + + // And using the stream-based interface, writing different + // block sizes at a time. + CollectInBufferStream contents; + enc.Seek(0, IOStream::SeekType_Absolute); + enc.CopyStreamTo(contents); + contents.SetForReading(); + + enc.Seek(0, IOStream::SeekType_End); + size_t file_size = enc.GetPosition(); + TEST_EQUAL(file_size, contents.GetSize()); + + for(int buffer_size = 1; ; buffer_size <<= 1) + { + enc.Seek(0, IOStream::SeekType_Absolute); + CollectInBufferStream temp_copy; + BackupStoreFile::VerifyStream verifier(&temp_copy); + enc.CopyStreamTo(verifier, IOStream::TimeOutInfinite, + buffer_size); + + // The block index is only validated on Close(), which + // CopyStreamTo() doesn't do. + verifier.Close(); + + temp_copy.SetForReading(); + TEST_EQUAL(file_size, temp_copy.GetSize()); + TEST_THAT(memcmp(contents.GetBuffer(), + temp_copy.GetBuffer(), file_size) == 0); + + // Keep doubling buffer size until we've copied the + // entire encoded file in a single pass, then stop. + if(buffer_size > file_size) + { + break; + } + } } - + // Decode it { + UNLINK_IF_EXISTS("testfiles/testenc1_orig"); FileStream enc("testfiles/testenc1_enc"); BackupStoreFile::DecodeFile(enc, "testfiles/testenc1_orig", IOStream::TimeOutInfinite); } - + // Read in rebuilt original, and compare contents { TEST_THAT(TestGetFileSize("testfiles/testenc1_orig") == sizeof(encfile)); @@ -1743,7 +2406,7 @@ int test3(int argc, const char *argv[]) in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } - + // Check how many blocks it had, and test the stream based interface { FileStream enc("testfiles/testenc1_enc"); @@ -1756,11 +2419,11 @@ int test3(int argc, const char *argv[]) TEST_THAT(decoded->GetNumBlocks() == 3); } - + // Test that the last block in a file, if less than 256 bytes, gets put into the last block { #define FILE_SIZE_JUST_OVER ((4096*2)+58) - FileStream f("testfiles/testenc2", O_WRONLY | O_CREAT | O_EXCL); + FileStream f("testfiles/testenc2", O_WRONLY | O_CREAT); f.Write(encfile + 2, FILE_SIZE_JUST_OVER); BackupStoreFilenameClear name("testenc2"); std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testenc2", 32, name)); @@ -1772,11 +2435,11 @@ int test3(int argc, const char *argv[]) decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 879 /* buffer block size */); d.SetForReading(); TEST_THAT(d.GetSize() == FILE_SIZE_JUST_OVER); - TEST_THAT(memcmp(encfile + 2, d.GetBuffer(), FILE_SIZE_JUST_OVER) == 0); + TEST_THAT(memcmp(encfile + 2, d.GetBuffer(), FILE_SIZE_JUST_OVER) == 0); TEST_THAT(decoded->GetNumBlocks() == 2); } - + // Test that reordered streams work too { FileStream enc("testfiles/testenc1_enc"); @@ -1790,25 +2453,39 @@ int test3(int argc, const char *argv[]) TEST_THAT(decoded->GetNumBlocks() == 3); } - + } + + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_symlinks() +{ + SETUP_TEST_BACKUPSTORE(); + #ifndef WIN32 // no symlinks on Win32 - // Try out doing this on a symlink - { - TEST_THAT(::symlink("does/not/exist", "testfiles/testsymlink") == 0); - BackupStoreFilenameClear name("testsymlink"); - std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testsymlink", 32, name)); - // Can't decode it from the stream, because it's in file order, and doesn't have the - // required properties to be able to reorder it. So buffer it... - CollectInBufferStream b; - encoded->CopyStreamTo(b); - b.SetForReading(); - // Decode it - BackupStoreFile::DecodeFile(b, "testfiles/testsymlink_2", IOStream::TimeOutInfinite); - } + UNLINK_IF_EXISTS("testfiles/testsymlink"); + TEST_THAT(::symlink("does/not/exist", "testfiles/testsymlink") == 0); + BackupStoreFilenameClear name("testsymlink"); + std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testsymlink", 32, name)); + + // Can't decode it from the stream, because it's in file order, and doesn't have the + // required properties to be able to reorder it. So buffer it... + CollectInBufferStream b; + encoded->CopyStreamTo(b); + b.SetForReading(); + + // Decode it + UNLINK_IF_EXISTS("testfiles/testsymlink_2"); + BackupStoreFile::DecodeFile(b, "testfiles/testsymlink_2", IOStream::TimeOutInfinite); #endif - } - // Store info + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_store_info() +{ + SETUP_TEST_BACKUPSTORE(); + { RaidFileWrite::CreateDirectory(0, "test-info"); BackupStoreInfo::CreateNew(76, "test-info/", 0, 3461231233455433LL, 2934852487LL); @@ -1841,6 +2518,7 @@ int test3(int argc, const char *argv[]) TEST_CHECK_THROWS(info->RemovedDeletedDirectory(9), BackupStoreException, StoreInfoDirNotInList); info->Save(); } + { std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(76, "test-info/", 0, true)); TEST_THAT(info->GetBlocksUsed() == 7); @@ -1855,68 +2533,79 @@ int test3(int argc, const char *argv[]) TEST_THAT(delfiles[1] == 4); } -//printf("SKIPPINGTESTS---------\n"); -//return 0; - - // Context - TLSContext context; - context.Initialise(false /* client */, - "testfiles/clientCerts.pem", - "testfiles/clientPrivKey.pem", - "testfiles/clientTrustedCAs.pem"); + TEARDOWN_TEST_BACKUPSTORE(); +} +bool test_login_without_account() +{ // First, try logging in without an account having been created... just make sure login fails. - std::string cmd = BBSTORED " " + bbstored_args + - " testfiles/bbstored.conf"; - int pid = LaunchServer(cmd.c_str(), "testfiles/bbstored.pid"); - - TEST_THAT(pid != -1 && pid != 0); - if(pid <= 0) - { - return 1; - } - - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); + SETUP_TEST_BACKUPSTORE(); + delete_account(); + TEST_THAT_OR(StartServer(), FAIL); // BLOCK { // Open a connection to the server - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", - BOX_PORT_BBSTORED_TEST); - - // Make a protocol - BackupProtocolClient protocol(conn); + BackupProtocolClient protocol(open_conn("localhost", context)); // Check the version std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)), - ConnectionException, Conn_Protocol_UnexpectedReply); - + TEST_COMMAND_RETURNS_ERROR(protocol, QueryLogin(0x01234567, 0), + Err_BadLogin); + // Finish the connection protocol.QueryFinished(); } - // Create an account for the test client - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf create 01234567 0 " - "10000B 20000B") == 0); + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_bbstoreaccounts_create() +{ + SETUP_TEST_BACKUPSTORE(); + + // Delete the account, and create it again using bbstoreaccounts + delete_account(); + + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf -Wwarning create 01234567 0 " + "10000B 20000B") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_bbstoreaccounts_delete() +{ + SETUP_TEST_BACKUPSTORE(); + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf -Wwarning delete 01234567 yes") == 0, FAIL); + TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + + // Recreate the account so that teardown_test_backupstore() doesn't freak out + TEST_THAT(create_account(10000, 20000)); + + TEARDOWN_TEST_BACKUPSTORE(); +} + +// Test that login fails on a disabled account +bool test_login_with_disabled_account() +{ + SETUP_TEST_BACKUPSTORE(); + TEST_THAT_OR(StartServer(), FAIL); + TEST_THAT(TestDirExists("testfiles/0_0/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_1/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567")); TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8); + // make sure something is written to it - std::auto_ptr<BackupStoreAccountDatabase> apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - std::auto_ptr<BackupStoreRefCountDatabase> apReferences( BackupStoreRefCountDatabase::Load( apAccounts->GetEntry(0x1234567), true)); @@ -1924,224 +2613,193 @@ int test3(int argc, const char *argv[]) apReferences->GetLastObjectIDUsed()); TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) apReferences.reset(); - - // Test that login fails on a disabled account - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf enabled 01234567 no") == 0); + + // Test that login fails on a disabled account + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf enabled 01234567 no") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + // BLOCK { // Open a connection to the server - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", - BOX_PORT_BBSTORED_TEST); - - // Make a protocol - BackupProtocolClient protocol(conn); + BackupProtocolClient protocol(open_conn("localhost", context)); // Check the version std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolLoginConfirmed> - loginConf(protocol.QueryLogin(0x01234567, 0)), - ConnectionException, Conn_Protocol_UnexpectedReply); - int type, subType; - TEST_EQUAL_LINE(true, protocol.GetLastError(type, subType), - "expected a protocol error"); - TEST_EQUAL_LINE(BackupProtocolError::ErrorType, type, - "expected a BackupProtocolError"); - TEST_EQUAL_LINE(BackupProtocolError::Err_DisabledAccount, subType, - "expected an Err_DisabledAccount"); - + TEST_COMMAND_RETURNS_ERROR(protocol, QueryLogin(0x01234567, 0), + Err_DisabledAccount); + // Finish the connection protocol.QueryFinished(); } - // Re-enable the account so that subsequent logins should succeed - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf enabled 01234567 yes") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + TEARDOWN_TEST_BACKUPSTORE(); +} - // Delete the refcount database and log in again, - // check that it is recreated automatically but with - // no objects in it, to ensure seamless upgrade. - TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw")); +bool test_login_with_no_refcount_db() +{ + SETUP_TEST_BACKUPSTORE(); - std::auto_ptr<SocketStreamTLS> conn; - test_server_login("localhost", context, conn)->QueryFinished(); + // The account is already enabled, but doing it again shouldn't hurt + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf enabled 01234567 yes") == 0, FAIL); + TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + // Delete the refcount database and try to log in again. Check that + // we're locked out of the account until housekeeping has recreated + // the refcount db. + TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); + TEST_CHECK_THROWS(BackupProtocolLocal2 protocolLocal(0x01234567, + "test", "backup/01234567/", 0, false), // Not read-only + BackupStoreException, CorruptReferenceCountDatabase); + + // Run housekeeping, check that it fixes the refcount db + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); - apReferences = BackupStoreRefCountDatabase::Load(account, true); - TEST_EQUAL(0, apReferences->GetLastObjectIDUsed()); - - TEST_THAT(ServerIsAlive(pid)); + TEST_EQUAL_LINE(1, run_housekeeping(account), + "Housekeeping should report 1 error if the refcount db is missing"); + TEST_THAT(FileExists("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); - run_housekeeping(account); + // And that we can log in afterwards + BackupProtocolLocal2(0x01234567, "test", "backup/01234567/", 0, + false).QueryFinished(); // Not read-only // Check that housekeeping fixed the ref counts - TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, - apReferences->GetLastObjectIDUsed()); - TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) + TEST_THAT(check_reference_counts()); - TEST_THAT(ServerIsAlive(pid)); + // Start a server and try again, remotely. This is difficult to debug + // because housekeeping may fix the refcount database while we're + // stepping through. + TEST_THAT_THROWONFAIL(StartServer()); + TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); + TEST_CHECK_THROWS(connect_and_login(context), + ConnectionException, Protocol_UnexpectedReply); - set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); + TEST_THAT(ServerIsAlive(bbstored_pid)); - TEST_THAT(test_server("localhost") == 0); + // Run housekeeping, check that it fixes the refcount db + TEST_EQUAL_LINE(1, run_housekeeping(account), + "Housekeeping should report 1 error if the refcount db is missing"); + TEST_THAT(FileExists("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); + TEST_THAT(check_reference_counts()); - // test that all object reference counts have the - // expected values - TEST_EQUAL(ExpectedRefCounts.size() - 1, - apReferences->GetLastObjectIDUsed()); - for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID; - i < ExpectedRefCounts.size(); i++) - { - TEST_EQUAL_LINE(ExpectedRefCounts[i], - apReferences->GetRefCount(i), - "object " << BOX_FORMAT_OBJECTID(i)); - } + // And that we can log in afterwards + connect_and_login(context)->QueryFinished(); - // Delete the refcount database again, and let - // housekeeping recreate it and fix the ref counts. - // This could also happen after upgrade, if a housekeeping - // runs before the user logs in. - apReferences.reset(); - TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw")); - run_housekeeping(account); - apReferences = BackupStoreRefCountDatabase::Load(account, true); + TEARDOWN_TEST_BACKUPSTORE(); +} - TEST_EQUAL(ExpectedRefCounts.size() - 1, - apReferences->GetLastObjectIDUsed()); - for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID; - i < ExpectedRefCounts.size(); i++) - { - TEST_EQUAL_LINE(ExpectedRefCounts[i], - apReferences->GetRefCount(i), - "object " << BOX_FORMAT_OBJECTID(i)); - } - +bool test_housekeeping_deletes_files() +{ // Test the deletion of objects by the housekeeping system + + SETUP_TEST_BACKUPSTORE(); + + BackupProtocolLocal2 protocolLocal(0x01234567, "test", + "backup/01234567/", 0, false); // Not read-only + + // Create some nice recursive directories + write_test_file(1); + int64_t dirtodelete = create_test_data_subdirs(protocolLocal, + BACKUPSTORE_ROOT_DIRECTORY_ID, "test_delete", 6 /* depth */, + NULL /* pRefCount */); + + TEST_EQUAL(dirtodelete, + protocolLocal.QueryDeleteDirectory(dirtodelete)->GetObjectID()); + assert_everything_deleted(protocolLocal, dirtodelete); + protocolLocal.QueryFinished(); + // First, things as they are now. + TEST_THAT_OR(StartServer(), FAIL); recursive_count_objects_results before = {0,0,0}; + recursive_count_objects(BACKUPSTORE_ROOT_DIRECTORY_ID, before); - recursive_count_objects("localhost", BackupProtocolListDirectory::RootDirectory, before); - - TEST_THAT(before.objectsNotDel != 0); + TEST_EQUAL(0, before.objectsNotDel); TEST_THAT(before.deleted != 0); TEST_THAT(before.old != 0); // Kill it - TEST_THAT(KillServer(pid)); - ::sleep(1); - TEST_THAT(!ServerIsAlive(pid)); - - #ifdef WIN32 - TEST_THAT(unlink("testfiles/bbstored.pid") == 0); - #else - TestRemoteProcessMemLeaks("bbstored.memleaks"); - #endif - - // Set a new limit on the account -- leave the hard limit - // high to make sure the target for freeing space is the - // soft limit. - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf setlimit 01234567 " - "10B 20000B") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - - // Start things up - pid = LaunchServer(BBSTORED " testfiles/bbstored.conf", - "testfiles/bbstored.pid"); + TEST_THAT(StopServer()); - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); - - // wait for housekeeping to happen - printf("waiting for housekeeping:\n"); - for(int l = 0; l < 30; ++l) - { - ::sleep(1); - printf("."); - fflush(stdout); - } - printf("\n"); + // Reduce the store limits, so housekeeping will remove all old files. + // Leave the hard limit high, so we know that housekeeping's target + // for freeing space is the soft limit. + TEST_THAT(change_account_limits("0B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account()); // Count the objects again recursive_count_objects_results after = {0,0,0}; - recursive_count_objects("localhost", - BackupProtocolListDirectory::RootDirectory, - after); - - // If these tests fail then try increasing the timeout above - TEST_THAT(after.objectsNotDel == before.objectsNotDel); - TEST_THAT(after.deleted == 0); - TEST_THAT(after.old == 0); - + recursive_count_objects(BACKUPSTORE_ROOT_DIRECTORY_ID, after); + TEST_EQUAL(before.objectsNotDel, after.objectsNotDel); + TEST_EQUAL(0, after.deleted); + TEST_EQUAL(0, after.old); + + // Adjust reference counts on deleted files, so that the final checks in + // teardown_test_backupstore() don't fail. + ExpectedRefCounts.resize(2); + + // Delete the account to stop teardown_test_backupstore from checking it. + // TODO FIXME investigate the block count mismatch that teardown_test_backupstore + // catches if we don't delete the account. + delete_account(); + + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_account_limits_respected() +{ + SETUP_TEST_BACKUPSTORE(); + TEST_THAT_OR(StartServer(), FAIL); + // Set a really small hard limit - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + TEST_THAT_OR(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf setlimit 01234567 " - "10B 20B") == 0); + "2B 2B") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - // Try to upload a file and create a directory, and check an error is generated + // Try to upload a file and create a directory, both of which would exceed the + // current account limits, and check that each command returns an error. { - // Open a connection to the server - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", - BOX_PORT_BBSTORED_TEST); - - // Make a protocol - BackupProtocolClient protocol(conn); - - // Check the version - std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + write_test_file(3); - // Login - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)); - - int64_t modtime = 0; - + // Open a connection to the server + std::auto_ptr<BackupProtocolCallable> apProtocol( + connect_and_login(context)); BackupStoreFilenameClear fnx("exceed-limit"); - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/test3", BackupProtocolListDirectory::RootDirectory, fnx, &modtime)); + int64_t modtime = 0; + std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/test3", BACKUPSTORE_ROOT_DIRECTORY_ID, fnx, &modtime)); TEST_THAT(modtime != 0); - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile( - BackupProtocolListDirectory::RootDirectory, + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryStoreFile( + BACKUPSTORE_ROOT_DIRECTORY_ID, modtime, modtime, /* use it for attr hash too */ - 0, /* diff from ID */ + 0, /* diff from ID */ fnx, - *upload)), - ConnectionException, Conn_Protocol_UnexpectedReply); + upload), + Err_StorageLimitExceeded); - MemBlockStream attr(&modtime, sizeof(modtime)); + // This currently causes a fatal error on the server, which + // kills the connection. TODO FIXME return an error instead. + std::auto_ptr<IOStream> attr(new MemBlockStream(&modtime, sizeof(modtime))); BackupStoreFilenameClear fnxd("exceed-limit-dir"); - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolSuccess> dirCreate(protocol.QueryCreateDirectory( - BackupProtocolListDirectory::RootDirectory, - 9837429842987984LL, fnxd, attr)), - ConnectionException, Conn_Protocol_UnexpectedReply); - + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryCreateDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + FAKE_ATTR_MODIFICATION_TIME, fnxd, attr), + Err_StorageLimitExceeded); - // Finish the connection - protocol.QueryFinished(); + // Finish the connection. + apProtocol->QueryFinished(); } - // Kill it again - TEST_THAT(KillServer(pid)); - ::sleep(1); - TEST_THAT(!ServerIsAlive(pid)); - - #ifdef WIN32 - TEST_THAT(unlink("testfiles/bbstored.pid") == 0); - #else - TestRemoteProcessMemLeaks("bbstored.memleaks"); - #endif - - return 0; + TEARDOWN_TEST_BACKUPSTORE(); } int multi_server() @@ -2149,14 +2807,14 @@ int multi_server() printf("Starting server for connection from remote machines...\n"); // Create an account for the test client - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + TEST_THAT_OR(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf create 01234567 0 " - "30000B 40000B") == 0); + "30000B 40000B") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // First, try logging in without an account having been created... just make sure login fails. - int pid = LaunchServer(BBSTORED " testfiles/bbstored_multi.conf", + int pid = LaunchServer(BBSTORED " testfiles/bbstored_multi.conf", "testfiles/bbstored.pid"); TEST_THAT(pid != -1 && pid != 0); @@ -2172,43 +2830,33 @@ int multi_server() printf("Terminating server...\n"); // Kill it - TEST_THAT(KillServer(pid)); - ::sleep(1); - TEST_THAT(!ServerIsAlive(pid)); - - #ifdef WIN32 - TEST_THAT(unlink("testfiles/bbstored.pid") == 0); - #else - TestRemoteProcessMemLeaks("bbstored.memleaks"); - #endif + TEST_THAT(StopServer()); } - return 0; } -void test_open_files_with_limited_win32_permissions() +bool test_open_files_with_limited_win32_permissions() { #ifdef WIN32 // this had better work, or bbstored will die when combining diffs - char* file = "foo"; + const char* file = "foo"; - DWORD accessRights = FILE_READ_ATTRIBUTES | + DWORD accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY | FILE_READ_EA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_DATA | FILE_WRITE_EA /*| FILE_ALL_ACCESS*/; DWORD shareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; HANDLE h1 = CreateFileA(file, accessRights, shareMode, - NULL, OPEN_ALWAYS, FILE_FLAG_BACKUP_SEMANTICS, NULL); - assert(h1 != INVALID_HANDLE_VALUE); + NULL, OPEN_ALWAYS, // create file if it doesn't exist + FILE_FLAG_BACKUP_SEMANTICS, NULL); TEST_THAT(h1 != INVALID_HANDLE_VALUE); - accessRights = FILE_READ_ATTRIBUTES | + accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY | FILE_READ_EA; HANDLE h2 = CreateFileA(file, accessRights, shareMode, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - assert(h2 != INVALID_HANDLE_VALUE); TEST_THAT(h2 != INVALID_HANDLE_VALUE); CloseHandle(h2); @@ -2221,6 +2869,8 @@ void test_open_files_with_limited_win32_permissions() CloseHandle(h2); CloseHandle(h1); #endif + + return true; } void compare_backupstoreinfo_values_to_expected @@ -2280,20 +2930,18 @@ void compare_backupstoreinfo_values_to_expected TEST_EQUAL_LINE(expected_account_enabled, actual.IsAccountEnabled(), test_phase << " AccountEnabled"); - TEST_EQUAL_LINE(extra_data.GetSize(), actual.GetExtraData().GetSize(), + TEST_EQUAL_LINE(extra_data.GetSize(), actual.GetExtraData().GetSize(), test_phase << " extra data has wrong size"); TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), actual.GetExtraData().GetBuffer(), extra_data.GetSize()), test_phase << " extra data has wrong contents"); } -int test_read_old_backupstoreinfo_files() +bool test_read_old_backupstoreinfo_files() { + SETUP_TEST_BACKUPSTORE(); + // Create an account for the test client - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf create 01234567 0 " - "10000B 20000B") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); std::auto_ptr<BackupStoreInfo> apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), @@ -2326,7 +2974,7 @@ int test_read_old_backupstoreinfo_files() apArchive->Write((int64_t) 14); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); - + apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded from v1", info_v1, @@ -2334,10 +2982,10 @@ int test_read_old_backupstoreinfo_files() true /* enabled by default */); apInfo->SetAccountName("bonk"); - + // Save the info again apInfo->Save(/* allowOverwrite */ true); - + // Check that it was saved in the new Archive format std::auto_ptr<RaidFileRead> rfr(RaidFileRead::Open(0, info_filename, 0)); int32_t magic; @@ -2356,7 +3004,7 @@ int test_read_old_backupstoreinfo_files() /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded in v1, resaved in v2", info_v1, *apInfo, "bonk", true); - + // Check that the new AccountEnabled flag is saved properly apInfo->SetAccountEnabled(false); apInfo->Save(/* allowOverwrite */ true); @@ -2375,7 +3023,7 @@ int test_read_old_backupstoreinfo_files() // Now save the info in v2 format without the AccountEnabled flag // (boxbackup 0.11 format) and check that the flag is set to true // for backwards compatibility - + rfw.reset(new RaidFileWrite(0, info_filename)); rfw->Open(/* allowOverwrite */ true); magic = htonl(INFO_MAGIC_VALUE_2); @@ -2392,7 +3040,7 @@ int test_read_old_backupstoreinfo_files() apArchive->Write(apInfo->GetBlocksInDirectories()); apArchive->Write(apInfo->GetBlocksSoftLimit()); apArchive->Write(apInfo->GetBlocksHardLimit()); - apArchive->Write(apInfo->GetNumFiles()); + apArchive->Write(apInfo->GetNumCurrentFiles()); apArchive->Write(apInfo->GetNumOldFiles()); apArchive->Write(apInfo->GetNumDeletedFiles()); apArchive->Write(apInfo->GetNumDirectories()); @@ -2401,7 +3049,7 @@ int test_read_old_backupstoreinfo_files() apArchive->Write(apInfo->GetDeletedDirectories()[1]); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); - + apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("saved in v2 without " @@ -2411,7 +3059,7 @@ int test_read_old_backupstoreinfo_files() // Rewrite using full length, so that the first 4 bytes of extra data // doesn't get swallowed by "extra data". apInfo->Save(/* allowOverwrite */ true); - + // Append some extra data after the known account values, to simulate a // new addition to the store format. Check that this extra data is loaded // and resaved with the info file. We made the mistake of deleting it in @@ -2431,7 +3079,7 @@ int test_read_old_backupstoreinfo_files() rfw.reset(); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); - TEST_EQUAL_LINE(extra_data.GetSize(), apInfo->GetExtraData().GetSize(), + TEST_EQUAL_LINE(extra_data.GetSize(), apInfo->GetExtraData().GetSize(), "wrong amount of extra data loaded from info file"); TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), apInfo->GetExtraData().GetBuffer(), extra_data.GetSize()), @@ -2441,16 +3089,16 @@ int test_read_old_backupstoreinfo_files() apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); compare_backupstoreinfo_values_to_expected("saved in future format " "with extra_data", info_v1, *apInfo, "test", true, extra_data); - + // Check that the new bbstoreaccounts command sets the flag properly - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf enabled 01234567 no") == 0); + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf enabled 01234567 no") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); TEST_EQUAL_LINE(false, apInfo->IsAccountEnabled(), "'bbstoreaccounts disabled no' should have reset AccountEnabled flag"); - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf enabled 01234567 yes") == 0); + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf enabled 01234567 yes") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), @@ -2489,48 +3137,119 @@ int test_read_old_backupstoreinfo_files() "BackupStoreInfo::CreateForRegeneration and reloaded", info_v1, *apInfo, "spurtle", false /* AccountEnabled */, extra_data); - // Delete the account to leave the store in the same state as before + // Delete the account to stop teardown_test_backupstore checking it for errors. apInfo.reset(); - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf delete 01234567 yes") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - - return 0; + TEST_THAT(delete_account()); + + TEARDOWN_TEST_BACKUPSTORE(); +} + +// Test that attributes can be correctly read from and written to the standard +// format, for compatibility with other servers and clients. See +// http://mailman.uk.freebsd.org/pipermail../public/boxbackup/2010-November/005818.html and +// http://lists.boxbackup.org/pipermail/boxbackup/2011-February/005978.html for +// details of the problems with packed structs. +bool test_read_write_attr_streamformat() +{ + SETUP_TEST_BACKUPSTORE(); + + // Construct a minimal valid directory with 1 entry in memory using Archive, and + // try to read it back. + CollectInBufferStream buf; + Archive darc(buf, IOStream::TimeOutInfinite); + + // Write a dir_StreamFormat structure + darc.Write((int32_t)OBJECTMAGIC_DIR_MAGIC_VALUE); // mMagicValue + darc.Write((int32_t)1); // mNumEntries + darc.Write((int64_t)0x0123456789abcdef); // mObjectID + darc.Write((int64_t)0x0000000000000001); // mContainerID + darc.Write((uint64_t)0x23456789abcdef01); // mAttributesModTime + darc.Write((int32_t)BackupStoreDirectory::Option_DependencyInfoPresent); + // mOptionsPresent + // 36 bytes to here + + // Write fake attributes to make it valid. + StreamableMemBlock attr; + attr.WriteToStream(buf); + // 40 bytes to here + + // Write a single entry in an en_StreamFormat structure + darc.Write((uint64_t)0x3456789012345678); // mModificationTime + darc.Write((int64_t)0x0000000000000002); // mObjectID + darc.Write((int64_t)0x0000000000000003); // mSizeInBlocks + darc.Write((uint64_t)0x0000000000000004); // mAttributesHash + darc.WriteInt16((int16_t)0x3141); // mFlags + // 74 bytes to here + + // Then a BackupStoreFilename + BackupStoreFilename fn; + fn.SetAsClearFilename("hello"); + fn.WriteToStream(buf); + // 81 bytes to here + + // Then a StreamableMemBlock for attributes + attr.WriteToStream(buf); + // 85 bytes to here + + // Then an en_StreamFormatDepends for dependency info. + darc.Write((uint64_t)0x0000000000000005); // mDependsNewer + darc.Write((uint64_t)0x0000000000000006); // mDependsOlder + // 101 bytes to here + + // Make sure none of the fields was expanded in transit by Archive. + TEST_EQUAL(101, buf.GetSize()); + + buf.SetForReading(); + BackupStoreDirectory dir(buf); + + TEST_EQUAL(1, dir.GetNumberOfEntries()); + TEST_EQUAL(0x0123456789abcdef, dir.GetObjectID()); + TEST_EQUAL(0x0000000000000001, dir.GetContainerID()); + TEST_EQUAL(0x23456789abcdef01, dir.GetAttributesModTime()); + + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry* pen = i.Next(); + TEST_THAT_OR(pen != NULL, FAIL); + TEST_EQUAL(0x3456789012345678, pen->GetModificationTime()); + TEST_EQUAL(0x0000000000000002, pen->GetObjectID()); + TEST_EQUAL(0x0000000000000003, pen->GetSizeInBlocks()); + TEST_EQUAL(0x0000000000000004, pen->GetAttributesHash()); + TEST_EQUAL(0x0000000000000005, pen->GetDependsNewer()); + TEST_EQUAL(0x0000000000000006, pen->GetDependsOlder()); + TEST_EQUAL(0x3141, pen->GetFlags()); + + CollectInBufferStream buf2; + dir.WriteToStream(buf2); + buf2.SetForReading(); + buf.Seek(0, IOStream::SeekType_Absolute); + TEST_EQUAL(101, buf2.GetSize()); + TEST_EQUAL(buf.GetSize(), buf2.GetSize()); + + // Test that the stream written out for the Directory is exactly the same as the + // one we hand-crafted earlier. + TEST_EQUAL(0, memcmp(buf.GetBuffer(), buf2.GetBuffer(), buf.GetSize())); + + TEARDOWN_TEST_BACKUPSTORE(); } int test(int argc, const char *argv[]) { - test_open_files_with_limited_win32_permissions(); + TEST_THAT(test_open_files_with_limited_win32_permissions()); // Initialise the raid file controller RaidFileController &rcontroller = RaidFileController::GetController(); rcontroller.Initialise("testfiles/raidfile.conf"); - - int ret = test_read_old_backupstoreinfo_files(); - if (ret != 0) - { - return ret; - } - + + TEST_THAT(test_read_old_backupstoreinfo_files()); + // SSL library SSLLib::Initialise(); - - // Give a test key for the filenames -// BackupStoreFilenameClear::SetBlowfishKey(FilenameEncodingKey, sizeof(FilenameEncodingKey)); - // And set the encoding to blowfish -// BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish); - - // And for directory attributes -- need to set it, as used in file encoding -// BackupClientFileAttributes::SetBlowfishKey(AttributesEncodingKey, sizeof(AttributesEncodingKey)); - - // And finally for file encoding -// BackupStoreFile::SetBlowfishKeys(FileEncodingKey, sizeof(FileEncodingKey), FileBlockEntryEncodingKey, sizeof(FileBlockEntryEncodingKey)); // Use the setup crypto command to set up all these keys, so that the bbackupquery command can be used // for seeing what's going on. - BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); - - // encode in some filenames -- can't do static initialisation + BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); + + // encode in some filenames -- can't do static initialisation // because the key won't be set up when these are initialised { MEMLEAKFINDER_NO_LEAKS @@ -2544,36 +3263,43 @@ int test(int argc, const char *argv[]) uploads[l].name = BackupStoreFilenameClear(uploads_filenames[l]); } } - + // Trace errors out SET_DEBUG_SSLLIB_TRACE_ERRORS - if(argc == 2 && strcmp(argv[1], "server") == 0) - { - return multi_server(); - } - if(argc == 3 && strcmp(argv[1], "client") == 0) { - return test_server(argv[2]); + R250 r(3465657); + for(int l = 0; l < ATTR1_SIZE; ++l) {attr1[l] = r.next();} + for(int l = 0; l < ATTR2_SIZE; ++l) {attr2[l] = r.next();} + for(int l = 0; l < ATTR3_SIZE; ++l) {attr3[l] = r.next();} } -// large file test -/* { - int64_t modtime = 0; - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("/Users/ben/temp/large.tar", - BackupProtocolListDirectory::RootDirectory, uploads[0].name, &modtime)); - TEST_THAT(modtime != 0); - FileStream write("testfiles/large.enc", O_WRONLY | O_CREAT); - upload->CopyStreamTo(write); - } -printf("SKIPPING TESTS ------------------------------------------------------\n"); -return 0;*/ - int r = 0; - r = test1(argc, argv); - if(r != 0) return r; - r = test2(argc, argv); - if(r != 0) return r; - r = test3(argc, argv); - if(r != 0) return r; - return 0; + + TEST_THAT(test_filename_encoding()); + TEST_THAT(test_temporary_refcount_db_is_independent()); + TEST_THAT(test_bbstoreaccounts_create()); + TEST_THAT(test_bbstoreaccounts_delete()); + TEST_THAT(test_backupstore_directory()); + TEST_THAT(test_directory_parent_entry_tracks_directory_size()); + TEST_THAT(test_cannot_open_multiple_writable_connections()); + TEST_THAT(test_encoding()); + TEST_THAT(test_symlinks()); + TEST_THAT(test_store_info()); + + context.Initialise(false /* client */, + "testfiles/clientCerts.pem", + "testfiles/clientPrivKey.pem", + "testfiles/clientTrustedCAs.pem"); + + TEST_THAT(test_login_without_account()); + TEST_THAT(test_login_with_disabled_account()); + TEST_THAT(test_login_with_no_refcount_db()); + TEST_THAT(test_server_housekeeping()); + TEST_THAT(test_server_commands()); + TEST_THAT(test_account_limits_respected()); + TEST_THAT(test_multiple_uploads()); + TEST_THAT(test_housekeeping_deletes_files()); + TEST_THAT(test_read_write_attr_streamformat()); + + return finish_test_suite(); } |