// -------------------------------------------------------------------------- // // File // Name: testbackupstore.cpp // Purpose: Test backup store server // Created: 2003/08/20 // // -------------------------------------------------------------------------- #include "Box.h" #include #include #include "Archive.h" #include "BackupClientCryptoKeys.h" #include "BackupClientFileAttributes.h" #include "BackupProtocol.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreAccounts.h" #include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreInfo.h" #include "BackupStoreFilenameClear.h" #include "BackupStoreFileEncodeStream.h" #include "BackupStoreRefCountDatabase.h" #include "BackupStoreFile.h" #include "BoxPortsAndFiles.h" #include "CollectInBufferStream.h" #include "Configuration.h" #include "FileStream.h" #include "HousekeepStoreAccount.h" #include "MemBlockStream.h" #include "RaidFileController.h" #include "RaidFileException.h" #include "RaidFileRead.h" #include "RaidFileWrite.h" #include "SSLLib.h" #include "ServerControl.h" #include "Socket.h" #include "SocketStreamTLS.h" #include "StoreStructure.h" #include "StoreTestUtils.h" #include "TLSContext.h" #include "Test.h" #include "ZeroStream.h" #include "MemLeakFindOn.h" #define ENCFILE_SIZE 2765 // Make some test attributes #define ATTR1_SIZE 245 #define ATTR2_SIZE 23 #define ATTR3_SIZE 122 #define SHORT_TIMEOUT 5000 int attr1[ATTR1_SIZE]; int attr2[ATTR2_SIZE]; int attr3[ATTR3_SIZE]; typedef struct { BackupStoreFilenameClear fn; box_time_t mod; int64_t id; int64_t size; int16_t flags; box_time_t attrmod; } dirtest; static dirtest ens[] = { {BackupStoreFilenameClear(), 324324, 3432, 324, BackupStoreDirectory::Entry::Flags_File, 458763243422LL}, {BackupStoreFilenameClear(), 3432, 32443245645LL, 78, BackupStoreDirectory::Entry::Flags_Dir, 3248972347LL}, {BackupStoreFilenameClear(), 544435, 234234, 23324, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_Deleted, 2348974782LL}, {BackupStoreFilenameClear(), 234, 235436, 6523, BackupStoreDirectory::Entry::Flags_File, 32458923175634LL}, {BackupStoreFilenameClear(), 0x3242343532144LL, 8978979789LL, 21345, BackupStoreDirectory::Entry::Flags_File, 329483243432LL}, {BackupStoreFilenameClear(), 324265765734LL, 12312312321LL, 324987324329874LL, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_Deleted, 32489747234LL}, {BackupStoreFilenameClear(), 3452134, 7868578768LL, 324243, BackupStoreDirectory::Entry::Flags_Dir, 34786457432LL}, {BackupStoreFilenameClear(), 43543543, 324234, 21432, BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, 3489723478327LL}, {BackupStoreFilenameClear(), 325654765874324LL, 4353543, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 32489734789237LL}, {BackupStoreFilenameClear(), 32144325, 436547657, 9, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 234897347234LL} }; static const char *ens_filenames[] = {"obj1ertewt", "obj2", "obj3", "obj4dfedfg43", "obj5", "obj6dfgs", "obj7", "obj8xcvbcx", "obj9", "obj10fgjhfg"}; #define DIR_NUM 10 #define DIR_DIRS 3 #define DIR_FILES 7 #define DIR_OLD 2 #define DIR_DELETED 3 typedef struct { const char *fnextra; BackupStoreFilenameClear name; int seed; int size; box_time_t mod_time; int64_t allocated_objid; bool should_be_old_version; bool delete_file; } uploadtest; #define TEST_FILE_FOR_PATCHING "testfiles/test2" // a few bytes will be inserted at this point: #define TEST_FILE_FOR_PATCHING_PATCH_AT ((64*1024)-128) #define TEST_FILE_FOR_PATCHING_SIZE ((128*1024)+2564) #define UPLOAD_PATCH_EN 2 uploadtest uploads[] = { {"0", BackupStoreFilenameClear(), 324, 455, 0, 0, false, false}, {"1", BackupStoreFilenameClear(), 3232432, 2674, 0, 0, true, false}, // old ver {"2", BackupStoreFilenameClear(), 234, TEST_FILE_FOR_PATCHING_SIZE, 0, 0, false, false}, {"3", BackupStoreFilenameClear(), 324324, 6763, 0, 0, false, false}, {"4", BackupStoreFilenameClear(), 23456, 124, 0, 0, true, false}, // old ver {"5", BackupStoreFilenameClear(), 675745, 1, 0, 0, false, false}, // will upload new attrs for this one! {"6", BackupStoreFilenameClear(), 345213, 0, 0, 0, false, false}, {"7", BackupStoreFilenameClear(), 12313, 3246, 0, 0, true, true}, // old ver, will get deleted {"8", BackupStoreFilenameClear(), 457, 3434, 0, 0, false, false}, // overwrites {"9", BackupStoreFilenameClear(), 12315, 446, 0, 0, false, false}, {"a", BackupStoreFilenameClear(), 3476, 2466, 0, 0, false, false}, {"b", BackupStoreFilenameClear(), 124334, 4562, 0, 0, false, false}, {"c", BackupStoreFilenameClear(), 45778, 234, 0, 0, false, false}, // overwrites {"d", BackupStoreFilenameClear(), 2423425, 435, 0, 0, false, true} // overwrites, will be deleted }; static const char *uploads_filenames[] = {"49587fds", "cvhjhj324", "sdfcscs324", "dsfdsvsdc3214", "XXsfdsdf2342", "dsfdsc232", "sfdsdce2345", "YYstfbdtrdf76", "cvhjhj324", "fbfd098.ycy", "dfs98732hj", "svd987kjsad", "XXsfdsdf2342", "YYstfbdtrdf76"}; #define UPLOAD_NUM 14 #define UPLOAD_LATEST_FILES 12 // file we'll upload some new attributes for #define UPLOAD_ATTRS_EN 5 #define UPLOAD_DELETE_EN 13 // file which will be moved (as well as it's old version) #define UPLOAD_FILE_TO_MOVE 8 #define UNLINK_IF_EXISTS(filename) \ if (FileExists(filename)) { TEST_THAT(unlink(filename) == 0); } // Nice random data for testing written files class R250 { public: // Set up internal state table with 32-bit random numbers. // The bizarre bit-twiddling is because rand() returns 16 bits of which // the bottom bit is always zero! Hence, I use only some of the bits. // You might want to do something better than this.... R250(int seed) : posn1(0), posn2(103) { // populate the state and incr tables srand(seed); for (int i = 0; i != stateLen; ++i) { state[i] = ((rand() >> 2) << 19) ^ ((rand() >> 2) << 11) ^ (rand() >> 2); incrTable[i] = i == stateLen - 1 ? 0 : i + 1; } // stir up the numbers to ensure they're random for (int j = 0; j != stateLen * 4; ++j) (void) next(); } // Returns the next random number. Xor together two elements separated // by 103 mod 250, replacing the first element with the result. Then // increment the two indices mod 250. inline int next() { int ret = (state[posn1] ^= state[posn2]); // xor and replace element posn1 = incrTable[posn1]; // increment indices using lookup table posn2 = incrTable[posn2]; return ret; } private: enum { stateLen = 250 }; // length of the state table int state[stateLen]; // holds the random number state int incrTable[stateLen]; // lookup table: maps i to (i+1) % stateLen int posn1, posn2; // indices into the state table }; int SkipEntries(int e, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { if(e >= DIR_NUM) return e; bool skip = false; do { skip = false; if(FlagsMustBeSet != BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING) { if((ens[e].flags & FlagsMustBeSet) != FlagsMustBeSet) { skip = true; } } if((ens[e].flags & FlagsNotToBeSet) != 0) { skip = true; } if(skip) { ++e; } } while(skip && e < DIR_NUM); return e; } void CheckEntries(BackupStoreDirectory &rDir, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { int e = 0; BackupStoreDirectory::Iterator i(rDir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { TEST_THAT(e < DIR_NUM); // Skip to entry in the ens array which matches e = SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet); // Does it match? TEST_THAT(en->GetName() == ens[e].fn && en->GetModificationTime() == ens[e].mod && en->GetObjectID() == ens[e].id && en->GetFlags() == ens[e].flags && en->GetSizeInBlocks() == ens[e].size); // next ++e; } // Got them all? TEST_THAT(en == 0); TEST_THAT(DIR_NUM == SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet)); } bool test_filename_encoding() { SETUP(); // test some basics -- encoding and decoding filenames { // Make some filenames in various ways BackupStoreFilenameClear fn1; fn1.SetClearFilename(std::string("filenameXYZ")); BackupStoreFilenameClear fn2(std::string("filenameXYZ")); BackupStoreFilenameClear fn3(fn1); TEST_THAT(fn1 == fn2); TEST_THAT(fn1 == fn3); // Check that it's been encrypted std::string name(fn2.GetEncodedFilename()); TEST_THAT(name.find("name") == name.npos); // Bung it in a stream, get it out in a Clear filename { CollectInBufferStream stream; fn1.WriteToStream(stream); stream.SetForReading(); BackupStoreFilenameClear fn4; fn4.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn4.GetClearFilename() == "filenameXYZ"); TEST_THAT(fn4 == fn1); } // Bung it in a stream, get it out in a server non-Clear filename (two of them into the same var) { BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf "); CollectInBufferStream stream; fn1.WriteToStream(stream); fno.WriteToStream(stream); stream.SetForReading(); BackupStoreFilename fn5; fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5 == fn1); fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5 == fno); } // Same again with clear strings { BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf "); CollectInBufferStream stream; fn1.WriteToStream(stream); fno.WriteToStream(stream); stream.SetForReading(); BackupStoreFilenameClear fn5; fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5.GetClearFilename() == "filenameXYZ"); fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5.GetClearFilename() == "pinglet dksfnsf jksjdf "); } // Test a very big filename { const char *fnr = "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789"; BackupStoreFilenameClear fnLong(fnr); CollectInBufferStream stream; fnLong.WriteToStream(stream); stream.SetForReading(); BackupStoreFilenameClear fn9; fn9.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn9.GetClearFilename() == fnr); TEST_THAT(fn9 == fnLong); } // Test a filename which went wrong once { BackupStoreFilenameClear dodgy("content-negotiation.html"); } } tearDown(); return true; } bool test_backupstore_directory() { SETUP(); { // Now play with directories // Fill in... BackupStoreDirectory dir1(12, 98); for(int e = 0; e < DIR_NUM; ++e) { dir1.AddEntry(ens[e].fn, ens[e].mod, ens[e].id, ens[e].size, ens[e].flags, ens[e].attrmod); } // Got the right number TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM); // Stick it into a stream and get it out again { CollectInBufferStream stream; dir1.WriteToStream(stream); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_NUM); TEST_THAT(dir2.GetObjectID() == 12); TEST_THAT(dir2.GetContainerID() == 98); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } // Then do selective writes and reads { CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } { CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_DIRS); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_Dir, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } { CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - DIR_OLD); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion); } // Finally test deleting items { dir1.DeleteEntry(12312312321LL); // Verify TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM - 1); CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - 1); } // Check attributes { int attrI[4] = {1, 2, 3, 4}; StreamableMemBlock attr(attrI, sizeof(attrI)); BackupStoreDirectory d1(16, 546); d1.SetAttributes(attr, 56234987324232LL); TEST_THAT(d1.GetAttributes() == attr); TEST_THAT(d1.GetAttributesModTime() == 56234987324232LL); CollectInBufferStream stream; d1.WriteToStream(stream); stream.SetForReading(); BackupStoreDirectory d2(stream); TEST_THAT(d2.GetAttributes() == attr); TEST_THAT(d2.GetAttributesModTime() == 56234987324232LL); } } tearDown(); return true; } void write_test_file(int t) { std::string filename("testfiles/test"); filename += uploads[t].fnextra; printf("%s\n", filename.c_str()); FileStream write(filename.c_str(), O_WRONLY | O_CREAT); R250 r(uploads[t].seed); unsigned char *data = (unsigned char*)malloc(uploads[t].size); for(int l = 0; l < uploads[t].size; ++l) { data[l] = r.next() & 0xff; } write.Write(data, uploads[t].size); free(data); } void test_test_file(int t, IOStream &rStream) { // Decode to a file BackupStoreFile::DecodeFile(rStream, "testfiles/test_download", SHORT_TIMEOUT); // Compare... FileStream in("testfiles/test_download"); TEST_THAT(in.BytesLeftToRead() == uploads[t].size); R250 r(uploads[t].seed); unsigned char *data = (unsigned char*)malloc(uploads[t].size); TEST_THAT(in.ReadFullBuffer(data, uploads[t].size, 0 /* not interested in bytes read if this fails */)); for(int l = 0; l < uploads[t].size; ++l) { TEST_THAT(data[l] == (r.next() & 0xff)); } free(data); in.Close(); TEST_THAT(unlink("testfiles/test_download") == 0); } void test_everything_deleted(BackupProtocolCallable &protocol, int64_t DirID) { printf("Test for del: %llx\n", (unsigned long long)DirID); // Command std::auto_ptr dirreply(protocol.QueryListDirectory( DirID, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; int files = 0; int dirs = 0; while((en = i.Next()) != 0) { if(en->GetFlags() & BackupProtocolListDirectory::Flags_Dir) { dirs++; // Recurse test_everything_deleted(protocol, en->GetObjectID()); } else { files++; } // Check it's deleted TEST_THAT(en->GetFlags() & BackupProtocolListDirectory::Flags_Deleted); } // Check there were the right number of files and directories TEST_THAT(files == 3); TEST_THAT(dirs == 0 || dirs == 2); } void create_file_in_dir(std::string name, std::string source, int64_t parentId, BackupProtocolCallable &protocol, BackupStoreRefCountDatabase* pRefCount) { BackupStoreFilenameClear name_encoded("file_One"); std::auto_ptr upload(BackupStoreFile::EncodeFile( source.c_str(), parentId, name_encoded)); std::auto_ptr stored( protocol.QueryStoreFile( parentId, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ name_encoded, upload)); int64_t objectId = stored->GetObjectID(); if (pRefCount) { TEST_EQUAL(objectId, pRefCount->GetLastObjectIDUsed()); TEST_EQUAL(1, pRefCount->GetRefCount(objectId)) } set_refcount(objectId, 1); } const box_time_t FAKE_MODIFICATION_TIME = 0xfeedfacedeadbeefLL; const box_time_t FAKE_ATTR_MODIFICATION_TIME = 0xdeadbeefcafebabeLL; int64_t create_test_data_subdirs(BackupProtocolCallable &protocol, int64_t indir, const char *name, int depth, BackupStoreRefCountDatabase* pRefCount) { // Create a directory int64_t subdirid = 0; BackupStoreFilenameClear dirname(name); { // Create with dummy attributes int attrS = 0; std::auto_ptr attr(new MemBlockStream(&attrS, sizeof(attrS))); std::auto_ptr dirCreate(protocol.QueryCreateDirectory( indir, FAKE_ATTR_MODIFICATION_TIME, dirname, attr)); subdirid = dirCreate->GetObjectID(); } BOX_TRACE("Creating subdirs, depth = " << depth << ", dirid = " << BOX_FORMAT_OBJECTID(subdirid)); if (pRefCount) { TEST_EQUAL(subdirid, pRefCount->GetLastObjectIDUsed()); TEST_EQUAL(1, pRefCount->GetRefCount(subdirid)) } set_refcount(subdirid, 1); // Put more directories in it, if we haven't gone down too far if(depth > 0) { create_test_data_subdirs(protocol, subdirid, "dir_One", depth - 1, pRefCount); create_test_data_subdirs(protocol, subdirid, "dir_Two", depth - 1, pRefCount); } // Stick some files in it create_file_in_dir("file_One", "testfiles/test1", subdirid, protocol, pRefCount); create_file_in_dir("file_Two", "testfiles/test1", subdirid, protocol, pRefCount); create_file_in_dir("file_Three", "testfiles/test1", subdirid, protocol, pRefCount); return subdirid; } void check_dir_after_uploads(BackupProtocolCallable &protocol, const StreamableMemBlock &Attributes) { // Command std::auto_ptr dirreply(protocol.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); TEST_THAT(dirreply->GetObjectID() == BackupProtocolListDirectory::RootDirectory); // Stream BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); TEST_EQUAL(UPLOAD_NUM, dir.GetNumberOfEntries()); TEST_THAT(!dir.HasAttributes()); // Check them! BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en; for(int t = 0; t < UPLOAD_NUM; ++t) { en = i.Next(); TEST_THAT(en != 0); if (en == 0) continue; TEST_LINE(uploads[t].name == en->GetName(), "uploaded file " << t << " name"); BackupStoreFilenameClear clear(en->GetName()); TEST_EQUAL_LINE(uploads[t].name.GetClearFilename(), clear.GetClearFilename(), "uploaded file " << t << " name"); TEST_EQUAL_LINE(uploads[t].allocated_objid, en->GetObjectID(), "uploaded file " << t << " ID"); TEST_EQUAL_LINE(uploads[t].mod_time, en->GetModificationTime(), "uploaded file " << t << " modtime"); int correct_flags = BackupProtocolListDirectory::Flags_File; if(uploads[t].should_be_old_version) correct_flags |= BackupProtocolListDirectory::Flags_OldVersion; if(uploads[t].delete_file) correct_flags |= BackupProtocolListDirectory::Flags_Deleted; TEST_EQUAL_LINE(correct_flags, en->GetFlags(), "uploaded file " << t << " flags"); if(t == UPLOAD_ATTRS_EN) { TEST_THAT(en->HasAttributes()); TEST_THAT(en->GetAttributesHash() == 32498749832475LL); TEST_THAT(en->GetAttributes() == Attributes); } else { // No attributes on this one TEST_THAT(!en->HasAttributes()); } } en = i.Next(); TEST_THAT(en == 0); } typedef struct { int objectsNotDel; int deleted; int old; } recursive_count_objects_results; void recursive_count_objects_r(BackupProtocolCallable &protocol, int64_t id, recursive_count_objects_results &results) { // Command std::auto_ptr dirreply(protocol.QueryListDirectory( id, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); // Check them! BackupStoreDirectory::Iterator i(dir); // Discard first BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { if((en->GetFlags() & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) == 0) results.objectsNotDel++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) results.deleted++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) results.old++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) { recursive_count_objects_r(protocol, en->GetObjectID(), results); } } } void recursive_count_objects(const char *hostname, int64_t id, recursive_count_objects_results &results) { // Context TLSContext context; context.Initialise(false /* client */, "testfiles/clientCerts.pem", "testfiles/clientPrivKey.pem", "testfiles/clientTrustedCAs.pem"); // Get a connection BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", "backup/01234567/", 0, false); // Count objects recursive_count_objects_r(protocolReadOnly, id, results); // Close it protocolReadOnly.QueryFinished(); } bool check_block_index(const char *encoded_file, IOStream &rBlockIndex) { // Open file, and move to the right position FileStream enc(encoded_file); BackupStoreFile::MoveStreamPositionToBlockIndex(enc); bool same = true; // Now compare the two... while(enc.StreamDataLeft()) { char buffer1[2048]; char buffer2[2048]; int s = enc.Read(buffer1, sizeof(buffer1), SHORT_TIMEOUT); if(rBlockIndex.Read(buffer2, s, SHORT_TIMEOUT) != s) { same = false; break; } if(::memcmp(buffer1, buffer2, s) != 0) { same = false; break; } } if(rBlockIndex.StreamDataLeft()) { same = false; // Absorb all this excess data so procotol is in the first state char buffer[2048]; while(rBlockIndex.StreamDataLeft()) { rBlockIndex.Read(buffer, sizeof(buffer), SHORT_TIMEOUT); } } return same; } bool check_files_same(const char *f1, const char *f2) { // Open file, and move to the right position FileStream f1s(f1); FileStream f2s(f2); bool same = true; // Now compare the two... while(f1s.StreamDataLeft()) { char buffer1[2048]; char buffer2[2048]; int s = f1s.Read(buffer1, sizeof(buffer1)); if(f2s.Read(buffer2, s) != s) { same = false; break; } if(::memcmp(buffer1, buffer2, s) != 0) { same = false; break; } } if(f2s.StreamDataLeft()) { same = false; } return same; } std::auto_ptr get_raid_file(int64_t ObjectID) { std::string filename; StoreStructure::MakeObjectFilename(ObjectID, "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, filename, false /* EnsureDirectoryExists */); return RaidFileRead::Open(0, filename); } bool check_num_files(int files, int old, int deleted, int dirs); bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, int Deleted, int Dirs, int Total); bool change_account_limits(const char* soft, const char* hard) { std::string errs; std::auto_ptr config( Configuration::LoadAndVerify ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); BackupStoreAccountsControl control(*config); int result = control.SetLimit(0x01234567, soft, hard); TEST_EQUAL(0, result); return (result == 0); } int64_t create_directory(BackupProtocolCallable& protocol, int64_t parent_dir_id = BACKUPSTORE_ROOT_DIRECTORY_ID); int64_t create_file(BackupProtocolCallable& protocol, int64_t subdirid, const std::string& remote_filename = ""); bool test_server_housekeeping() { SETUP(); int encfile[ENCFILE_SIZE]; { for(int l = 0; l < ENCFILE_SIZE; ++l) { encfile[l] = l * 173; } // Write this to a file { FileStream f("testfiles/file1", O_WRONLY | O_CREAT); f.Write(encfile, sizeof(encfile)); } } // We need complete control over housekeeping, so use a local client // instead of a network client + daemon. BackupProtocolLocal2 protocol(0x01234567, "test", "backup/01234567/", 0, false); std::string root_dir_fn; StoreStructure::MakeObjectFilename( BackupProtocolListDirectory::RootDirectory, "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, root_dir_fn, false /* EnsureDirectoryExists */); std::auto_ptr storedFile(RaidFileRead::Open(0, root_dir_fn)); int root_dir_blocks = storedFile->GetDiscUsageInBlocks(); TEST_THAT(check_num_files(0, 0, 0, 1)); TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, root_dir_blocks)); // Store a file -- first make the encoded file BackupStoreFilenameClear store1name("file1"); { FileStream out("testfiles/file1_upload1", O_WRONLY | O_CREAT); std::auto_ptr encoded(BackupStoreFile::EncodeFile("testfiles/file1", BackupProtocolListDirectory::RootDirectory, store1name)); encoded->CopyStreamTo(out); } // TODO FIXME move most of the code immediately below into // test_server_commands. // TODO FIXME use COMMAND macro for all commands to check the returned // object ID. #define COMMAND(command, objectid) \ TEST_EQUAL(objectid, protocol.command->GetObjectID()); // Then send it int64_t store1objid; { std::auto_ptr upload(new FileStream("testfiles/file1_upload1")); std::auto_ptr stored(protocol.QueryStoreFile( BackupProtocolListDirectory::RootDirectory, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ store1name, upload)); store1objid = stored->GetObjectID(); TEST_EQUAL_LINE(2, store1objid, "wrong ObjectID for newly " "uploaded file"); } // Update expected reference count of this new object set_refcount(store1objid, 1); // And retrieve it { // Retrieve as object std::auto_ptr getfile(protocol.QueryGetObject(store1objid)); TEST_THAT(getfile->GetObjectID() == store1objid); // BLOCK { // Get stream std::auto_ptr filestream(protocol.ReceiveStream()); // Need to put it in another stream, because it's not in stream order CollectInBufferStream f; filestream->CopyStreamTo(f); f.SetForReading(); // Get and decode UNLINK_IF_EXISTS("testfiles/file1_upload_retrieved"); BackupStoreFile::DecodeFile(f, "testfiles/file1_upload_retrieved", IOStream::TimeOutInfinite); } // Retrieve as file std::auto_ptr getobj(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, store1objid)); TEST_THAT(getobj->GetObjectID() == store1objid); // BLOCK { UNLINK_IF_EXISTS("testfiles/file1_upload_retrieved_str"); // Get stream std::auto_ptr filestream(protocol.ReceiveStream()); // Get and decode BackupStoreFile::DecodeFile(*filestream, "testfiles/file1_upload_retrieved_str", IOStream::TimeOutInfinite); } // Read in rebuilt original, and compare contents { FileStream in("testfiles/file1_upload_retrieved"); int encfile_i[ENCFILE_SIZE]; in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } { FileStream in("testfiles/file1_upload_retrieved_str"); int encfile_i[ENCFILE_SIZE]; in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } // Retrieve the block index, by ID { std::auto_ptr getblockindex(protocol.QueryGetBlockIndexByID(store1objid)); TEST_THAT(getblockindex->GetObjectID() == store1objid); std::auto_ptr blockIndexStream(protocol.ReceiveStream()); // Check against uploaded file TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream)); } // and again, by name { std::auto_ptr getblockindex(protocol.QueryGetBlockIndexByName(BackupProtocolListDirectory::RootDirectory, store1name)); TEST_THAT(getblockindex->GetObjectID() == store1objid); std::auto_ptr blockIndexStream(protocol.ReceiveStream()); // Check against uploaded file TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream)); } } // Get the directory again, and see if the entry is in it { // Command std::auto_ptr dirreply(protocol.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); TEST_THAT(dir.GetNumberOfEntries() == 1); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); TEST_THAT(i.Next() == 0); if(en != 0) { TEST_THAT(en->GetName() == store1name); TEST_THAT(en->GetModificationTime() == 0x123456789abcdefLL); TEST_THAT(en->GetAttributesHash() == 0x7362383249872dfLL); TEST_THAT(en->GetObjectID() == store1objid); TEST_EQUAL(6, en->GetSizeInBlocks()); TEST_THAT(en->GetFlags() == BackupStoreDirectory::Entry::Flags_File); } } int file1_blocks = get_raid_file(store1objid)->GetDiscUsageInBlocks(); TEST_THAT(check_num_files(1, 0, 0, 1)); TEST_THAT(check_num_blocks(protocol, file1_blocks, 0, 0, root_dir_blocks, file1_blocks + root_dir_blocks)); // Upload again, as a patch to the original file. int64_t patch1_id = BackupStoreFile::QueryStoreFileDiff(protocol, "testfiles/file1", // LocalFilename BackupProtocolListDirectory::RootDirectory, // DirectoryObjectID store1objid, // DiffFromFileID 0x7362383249872dfLL, // AttributesHash store1name // StoreFilename ); TEST_EQUAL_LINE(3, patch1_id, "wrong ObjectID for newly uploaded " "patch file"); // We need to check the old file's size, because it's been replaced // by a reverse diff, and patch1_id is a complete file, not a diff. int patch1_blocks = get_raid_file(store1objid)->GetDiscUsageInBlocks(); // It will take extra blocks, even though there are no changes, because // the server code is not smart enough to realise that the file // contents are identical, so it will create an empty patch. TEST_THAT(check_num_files(1, 1, 0, 1)); TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks, 0, root_dir_blocks, file1_blocks + patch1_blocks + root_dir_blocks)); // Change the file and upload again, as a patch to the original file. { FileStream out("testfiles/file1", O_WRONLY | O_APPEND); std::string appendix = "appendix!"; out.Write(appendix.c_str(), appendix.size()); out.Close(); } int64_t patch2_id = BackupStoreFile::QueryStoreFileDiff(protocol, "testfiles/file1", // LocalFilename BackupProtocolListDirectory::RootDirectory, // DirectoryObjectID patch1_id, // DiffFromFileID 0x7362383249872dfLL, // AttributesHash store1name // StoreFilename ); TEST_EQUAL_LINE(4, patch2_id, "wrong ObjectID for newly uploaded " "patch file"); // How many blocks used by the new file? // We need to check the old file's size, because it's been replaced // by a reverse diff, and patch1_id is a complete file, not a diff. int patch2_blocks = get_raid_file(patch1_id)->GetDiscUsageInBlocks(); TEST_THAT(check_num_files(1, 2, 0, 1)); TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks + patch2_blocks, 0, root_dir_blocks, file1_blocks + patch1_blocks + patch2_blocks + root_dir_blocks)); // Housekeeping should not change anything just yet protocol.QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); protocol.Reopen(); TEST_THAT(check_num_files(1, 2, 0, 1)); TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks + patch2_blocks, 0, root_dir_blocks, file1_blocks + patch1_blocks + patch2_blocks + root_dir_blocks)); // Upload not as a patch, but as a completely different file. This // marks the previous file as old (because the filename is the same) // but used to not adjust the number of old/deleted files properly. int64_t replaced_id = BackupStoreFile::QueryStoreFileDiff(protocol, "testfiles/file1", // LocalFilename BackupProtocolListDirectory::RootDirectory, // DirectoryObjectID 0, // DiffFromFileID 0x7362383249872dfLL, // AttributesHash store1name // StoreFilename ); TEST_EQUAL_LINE(5, replaced_id, "wrong ObjectID for newly uploaded " "full file"); // How many blocks used by the new file? This time we need to check // the new file, because it's not a patch. int replaced_blocks = get_raid_file(replaced_id)->GetDiscUsageInBlocks(); TEST_THAT(check_num_files(1, 3, 0, 1)); TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current file1_blocks + patch1_blocks + patch2_blocks, // old 0, // deleted root_dir_blocks, // directories file1_blocks + patch1_blocks + patch2_blocks + replaced_blocks + root_dir_blocks)); // total // Housekeeping should not change anything just yet protocol.QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); protocol.Reopen(); TEST_THAT(check_num_files(1, 3, 0, 1)); TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current file1_blocks + patch1_blocks + patch2_blocks, // old 0, // deleted root_dir_blocks, // directories file1_blocks + patch1_blocks + patch2_blocks + replaced_blocks + root_dir_blocks)); // total // But if we reduce the limits, then it will protocol.QueryFinished(); TEST_THAT(change_account_limits( "14B", // replaced_blocks + file1_blocks + root_dir_blocks "2000B")); TEST_THAT(run_housekeeping_and_check_account()); protocol.Reopen(); TEST_THAT(check_num_files(1, 1, 0, 1)); TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current file1_blocks, // old 0, // deleted root_dir_blocks, // directories file1_blocks + replaced_blocks + root_dir_blocks)); // total // Check that deleting files is accounted for as well protocol.QueryDeleteFile( BackupProtocolListDirectory::RootDirectory, // InDirectory store1name); // Filename // The old version file is deleted as well! TEST_THAT(check_num_files(0, 1, 2, 1)); TEST_THAT(check_num_blocks(protocol, 0, // current file1_blocks, // old replaced_blocks + file1_blocks, // deleted root_dir_blocks, // directories file1_blocks + replaced_blocks + root_dir_blocks)); // Reduce limits again, check that removed files are subtracted from // block counts. protocol.QueryFinished(); TEST_THAT(change_account_limits("0B", "2000B")); TEST_THAT(run_housekeeping_and_check_account()); protocol.Reopen(); TEST_THAT(check_num_files(0, 0, 0, 1)); TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, root_dir_blocks)); // Used to not consume the stream std::auto_ptr upload(new ZeroStream(1000)); TEST_COMMAND_RETURNS_ERROR(protocol, QueryStoreFile( BACKUPSTORE_ROOT_DIRECTORY_ID, 0, 0, /* use for attr hash too */ 99999, /* diff from ID */ uploads[0].name, upload), Err_DiffFromFileDoesNotExist); // TODO FIXME replace all other TEST_CHECK_THROWS with TEST_COMMAND_RETURNS_ERROR // TODO FIXME These tests should not be here, but in // test_server_commands. But make sure you use a network protocol, // not a local one, when you move them. // Try using GetFile on a directory { TEST_CHECK_THROWS(std::auto_ptr getFile(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::RootDirectory)), ConnectionException, Protocol_UnexpectedReply); } // Try retrieving an object that doesn't exist. That used to return // BackupProtocolSuccess(NoObject) for no apparent reason. TEST_COMMAND_RETURNS_ERROR(protocol, QueryGetObject(store1objid + 1), Err_DoesNotExist); // Close the protocol, so we can housekeep the account protocol.QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); ExpectedRefCounts.resize(2); // stop test failure in tearDown() tearDown(); return true; } int64_t create_directory(BackupProtocolCallable& protocol, int64_t parent_dir_id) { // Create a directory BackupStoreFilenameClear dirname("lovely_directory"); // Attributes std::auto_ptr attr(new MemBlockStream(attr1, sizeof(attr1))); std::auto_ptr dirCreate(protocol.QueryCreateDirectory( parent_dir_id, FAKE_ATTR_MODIFICATION_TIME, dirname, attr)); int64_t subdirid = dirCreate->GetObjectID(); set_refcount(subdirid, 1); return subdirid; } int64_t create_file(BackupProtocolCallable& protocol, int64_t subdirid, const std::string& remote_filename) { // Stick a file in it write_test_file(0); BackupStoreFilenameClear remote_filename_encoded; if (remote_filename.empty()) { remote_filename_encoded = uploads[0].name; } else { remote_filename_encoded = remote_filename; } std::string filename("testfiles/test0"); int64_t modtime; std::auto_ptr upload(BackupStoreFile::EncodeFile(filename, subdirid, remote_filename_encoded, &modtime)); std::auto_ptr stored(protocol.QueryStoreFile( subdirid, modtime, modtime, /* use for attr hash too */ 0, /* diff from ID */ remote_filename_encoded, upload)); int64_t subdirfileid = stored->GetObjectID(); set_refcount(subdirfileid, 1); return subdirfileid; } bool assert_writable_connection_fails(BackupProtocolCallable& protocol) { std::auto_ptr serverVersion (protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); TEST_CHECK_THROWS(protocol.QueryLogin(0x01234567, 0), ConnectionException, Protocol_UnexpectedReply); protocol.QueryFinished(); return true; } int64_t assert_readonly_connection_succeeds(BackupProtocolCallable& protocol) { // TODO FIXME share code with test_server_login std::auto_ptr serverVersion (protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); std::auto_ptr loginConf (protocol.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); return loginConf->GetClientStoreMarker(); } bool test_multiple_uploads() { TLSContext context; SETUP(); TEST_THAT_THROWONFAIL(StartServer()); std::auto_ptr apProtocol( test_server_login("localhost", context).release()); #ifndef WIN32 // Open a new connection which is read only // TODO FIXME replace protocolReadOnly with apProtocolReadOnly. std::auto_ptr apProtocolReadOnly = test_server_login("localhost", context, BackupProtocolLogin::Flags_ReadOnly); BackupProtocolCallable& protocolReadOnly(*apProtocolReadOnly); #else // WIN32 BackupProtocolCallable& protocolReadOnly(*apProtocol); #endif // Read the root directory a few times (as it's cached, so make sure it doesn't hurt anything) for(int l = 0; l < 3; ++l) { // Command std::auto_ptr dirreply( apProtocol->QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(apProtocol->ReceiveStream(), apProtocol->GetTimeout()); TEST_THAT(dir.GetNumberOfEntries() == 0); } // Read the dir from the readonly connection (make sure it gets in the cache) // Command std::auto_ptr dirreply( protocolReadOnly.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), protocolReadOnly.GetTimeout()); TEST_THAT(dir.GetNumberOfEntries() == 0); // BLOCK { TEST_THAT(check_num_files(0, 0, 0, 1)); // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); // Create and upload some test files int64_t maxID = 0; for(int t = 0; t < UPLOAD_NUM; ++t) { write_test_file(t); std::string filename("testfiles/test"); filename += uploads[t].fnextra; int64_t modtime = 0; std::auto_ptr upload(BackupStoreFile::EncodeFile(filename.c_str(), BackupProtocolListDirectory::RootDirectory, uploads[t].name, &modtime)); TEST_THAT(modtime != 0); std::auto_ptr stored(apProtocol->QueryStoreFile( BackupProtocolListDirectory::RootDirectory, modtime, modtime, /* use it for attr hash too */ 0, /* diff from ID */ uploads[t].name, upload)); uploads[t].allocated_objid = stored->GetObjectID(); uploads[t].mod_time = modtime; if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID(); set_refcount(stored->GetObjectID(), 1); BOX_TRACE("wrote file " << filename << " to server " "as object " << BOX_FORMAT_OBJECTID(stored->GetObjectID())); // Some of the uploaded files replace old ones, increasing // the old file count instead of the current file count. int expected_num_old_files = 0; if (t >= 8) expected_num_old_files++; if (t >= 12) expected_num_old_files++; if (t >= 13) expected_num_old_files++; int expected_num_current_files = t + 1 - expected_num_old_files; TEST_THAT(check_num_files(expected_num_current_files, expected_num_old_files, 0, 1)); apProtocol->QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); apProtocol = test_server_login("localhost", context); TEST_THAT(check_num_files(expected_num_current_files, expected_num_old_files, 0, 1)); } // Add some attributes onto one of them { TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 0, 1)); std::auto_ptr attrnew( new MemBlockStream(attr3, sizeof(attr3))); std::auto_ptr set(apProtocol->QuerySetReplacementFileAttributes( BackupProtocolListDirectory::RootDirectory, 32498749832475LL, uploads[UPLOAD_ATTRS_EN].name, attrnew)); TEST_THAT(set->GetObjectID() == uploads[UPLOAD_ATTRS_EN].allocated_objid); TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 0, 1)); } // Delete one of them (will implicitly delete an old version) { std::auto_ptr del(apProtocol->QueryDeleteFile( BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_DELETE_EN].name)); TEST_THAT(del->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid); TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); } // Check that the block index can be obtained by name even though it's been deleted { // Fetch the raw object { FileStream out("testfiles/downloaddelobj", O_WRONLY | O_CREAT); std::auto_ptr getobj(apProtocol->QueryGetObject(uploads[UPLOAD_DELETE_EN].allocated_objid)); std::auto_ptr objstream(apProtocol->ReceiveStream()); objstream->CopyStreamTo(out, apProtocol->GetTimeout()); } // query index and test std::auto_ptr getblockindex(apProtocol->QueryGetBlockIndexByName( BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_DELETE_EN].name)); TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid); std::auto_ptr blockIndexStream(apProtocol->ReceiveStream()); TEST_THAT(check_block_index("testfiles/downloaddelobj", *blockIndexStream)); } // Download them all... (even deleted files) for(int t = 0; t < UPLOAD_NUM; ++t) { printf("%d\n", t); std::auto_ptr getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, uploads[t].allocated_objid)); TEST_THAT(getFile->GetObjectID() == uploads[t].allocated_objid); std::auto_ptr filestream(apProtocol->ReceiveStream()); test_test_file(t, *filestream); } { StreamableMemBlock attrtest(attr3, sizeof(attr3)); // Use the read only connection to verify that the directory is as we expect printf("\n\n==== Reading directory using read-only connection\n"); check_dir_after_uploads(protocolReadOnly, attrtest); printf("done.\n\n"); // And on the read/write one check_dir_after_uploads(*apProtocol, attrtest); } // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); // Check diffing and rsync like stuff... // Build a modified file { // Basically just insert a bit in the middle TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING) == TEST_FILE_FOR_PATCHING_SIZE); FileStream in(TEST_FILE_FOR_PATCHING); void *buf = ::malloc(TEST_FILE_FOR_PATCHING_SIZE); FileStream out(TEST_FILE_FOR_PATCHING ".mod", O_WRONLY | O_CREAT); TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_PATCH_AT); out.Write(buf, TEST_FILE_FOR_PATCHING_PATCH_AT); char insert[13] = "INSERTINSERT"; out.Write(insert, sizeof(insert)); TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT); out.Write(buf, TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT); ::free(buf); } TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); // Run housekeeping (for which we need to disconnect // ourselves) and check that it doesn't change the numbers // of files apProtocol->QueryFinished(); std::auto_ptr apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); TEST_EQUAL(0, run_housekeeping(account)); // Also check that bbstoreaccounts doesn't change anything, // using an external process instead of the internal one. TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf check 01234567 fix") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apProtocol = test_server_login("localhost", context); TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); { // Fetch the block index for this one std::auto_ptr getblockindex(apProtocol->QueryGetBlockIndexByName( BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_PATCH_EN].name)); TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_PATCH_EN].allocated_objid); std::auto_ptr blockIndexStream(apProtocol->ReceiveStream()); // Do the patching bool isCompletelyDifferent = false; int64_t modtime; std::auto_ptr patchstream( BackupStoreFile::EncodeFileDiff( TEST_FILE_FOR_PATCHING ".mod", BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_PATCH_EN].name, uploads[UPLOAD_PATCH_EN].allocated_objid, *blockIndexStream, SHORT_TIMEOUT, NULL, // pointer to DiffTimer impl &modtime, &isCompletelyDifferent)); TEST_THAT(isCompletelyDifferent == false); // Sent this to a file, so we can check the size, rather than uploading it directly { FileStream patch(TEST_FILE_FOR_PATCHING ".patch", O_WRONLY | O_CREAT); patchstream->CopyStreamTo(patch); } // Make sure the stream is a plausible size for a patch containing only one new block TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING ".patch") < (8*1024)); // Upload it int64_t patchedID = 0; { std::auto_ptr uploadpatch(new FileStream(TEST_FILE_FOR_PATCHING ".patch")); std::auto_ptr stored(apProtocol->QueryStoreFile( BackupProtocolListDirectory::RootDirectory, modtime, modtime, /* use it for attr hash too */ uploads[UPLOAD_PATCH_EN].allocated_objid, /* diff from ID */ uploads[UPLOAD_PATCH_EN].name, uploadpatch)); TEST_THAT(stored->GetObjectID() > 0); if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID(); patchedID = stored->GetObjectID(); } set_refcount(patchedID, 1); // Then download it to check it's OK std::auto_ptr getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, patchedID)); TEST_THAT(getFile->GetObjectID() == patchedID); std::auto_ptr filestream(apProtocol->ReceiveStream()); BackupStoreFile::DecodeFile(*filestream, TEST_FILE_FOR_PATCHING ".downloaded", SHORT_TIMEOUT); // Check it's the same TEST_THAT(check_files_same(TEST_FILE_FOR_PATCHING ".downloaded", TEST_FILE_FOR_PATCHING ".mod")); TEST_THAT(check_num_files(UPLOAD_NUM - 4, 4, 2, 1)); } // Create a directory int64_t subdirid = create_directory(*apProtocol); TEST_THAT(check_num_files(UPLOAD_NUM - 4, 4, 2, 2)); // Stick a file in it int64_t subdirfileid = create_file(*apProtocol, subdirid); TEST_THAT(check_num_files(UPLOAD_NUM - 3, 4, 2, 2)); printf("\n==== Checking upload using read-only connection\n"); // Check the directories on the read only connection { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); // Stream BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), SHORT_TIMEOUT); TEST_EQUAL(UPLOAD_NUM + 2, dir.GetNumberOfEntries()); // for the patched upload, and this new dir. // Check the last one... BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; BackupStoreDirectory::Entry *t = 0; BackupStoreFilenameClear dirname("lovely_directory"); while((t = i.Next()) != 0) { if(en != 0) { // here for all but last object TEST_THAT(en->GetObjectID() != subdirid); TEST_THAT(en->GetName() != dirname); } en = t; } // Does it look right? TEST_THAT(en->GetName() == dirname); TEST_THAT(en->GetFlags() == BackupProtocolListDirectory::Flags_Dir); TEST_THAT(en->GetObjectID() == subdirid); TEST_THAT(en->GetModificationTime() == 0); // dirs don't have modification times. } { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( subdirid, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, true /* get attributes */)); TEST_THAT(dirreply->GetObjectID() == subdirid); BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), SHORT_TIMEOUT); TEST_THAT(dir.GetNumberOfEntries() == 1); // Check the (only) one... BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); // Check that it looks right TEST_EQUAL(subdirfileid, en->GetObjectID()); TEST_THAT(en->GetName() == uploads[0].name); TEST_EQUAL(BackupProtocolListDirectory::Flags_File, en->GetFlags()); int64_t actual_size = get_raid_file(subdirfileid)->GetDiscUsageInBlocks(); TEST_EQUAL(actual_size, en->GetSizeInBlocks()); TEST_THAT(en->GetModificationTime() != 0); // Attributes TEST_THAT(dir.HasAttributes()); TEST_EQUAL(FAKE_ATTR_MODIFICATION_TIME, dir.GetAttributesModTime()); StreamableMemBlock attr(attr1, sizeof(attr1)); TEST_THAT(dir.GetAttributes() == attr); } printf("done.\n\n"); // Check that we don't get attributes if we don't ask for them { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( subdirid, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); // Stream BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), SHORT_TIMEOUT); TEST_THAT(!dir.HasAttributes()); } // Sleep to ensure that the timestamp on the file will change, // invalidating the read-only connection's cache of the // directory, and forcing it to be reloaded. ::safe_sleep(1); // Change attributes on the directory { std::auto_ptr attrnew( new MemBlockStream(attr2, sizeof(attr2))); std::auto_ptr changereply(apProtocol->QueryChangeDirAttributes( subdirid, 329483209443598LL, attrnew)); TEST_THAT(changereply->GetObjectID() == subdirid); } // Check the new attributes { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( subdirid, 0, // no flags BackupProtocolListDirectory::Flags_EXCLUDE_EVERYTHING, true /* get attributes */)); // Stream BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), SHORT_TIMEOUT); TEST_THAT(dir.GetNumberOfEntries() == 0); // Attributes TEST_THAT(dir.HasAttributes()); TEST_EQUAL(329483209443598LL, dir.GetAttributesModTime()); StreamableMemBlock attrtest(attr2, sizeof(attr2)); TEST_THAT(dir.GetAttributes() == attrtest); } // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); // Test moving a file { BackupStoreFilenameClear newName("moved-files"); std::auto_ptr rep(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, BackupProtocolListDirectory::RootDirectory, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName)); TEST_THAT(rep->GetObjectID() == uploads[UPLOAD_FILE_TO_MOVE].allocated_objid); } // Try some dodgy renames { BackupStoreFilenameClear newName("moved-files"); TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, BackupProtocolListDirectory::RootDirectory, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName), ConnectionException, Protocol_UnexpectedReply); TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, subdirid, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName), ConnectionException, Protocol_UnexpectedReply); } // Rename within a directory (successfully) { BackupStoreFilenameClear newName("moved-files-x"); apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, subdirid, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName); } // Check it's all gone from the root directory... { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir; std::auto_ptr dirstream(protocolReadOnly.ReceiveStream()); dir.ReadFromStream(*dirstream, SHORT_TIMEOUT); // Read all entries BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { TEST_THAT(en->GetName() != uploads[UPLOAD_FILE_TO_MOVE].name); } } // Check the old and new versions are in the other directory { BackupStoreFilenameClear lookFor("moved-files-x"); // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( subdirid, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir; std::auto_ptr dirstream(protocolReadOnly.ReceiveStream()); dir.ReadFromStream(*dirstream, SHORT_TIMEOUT); // Check entries BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; bool foundCurrent = false; bool foundOld = false; while((en = i.Next()) != 0) { if(en->GetName() == lookFor) { if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File)) foundCurrent = true; if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion)) foundOld = true; } } TEST_THAT(foundCurrent); TEST_THAT(foundOld); } // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); // make a little bit more of a thing to look at int64_t subsubdirid = 0; int64_t subsubfileid = 0; { // TODO FIXME use create_dir() and create_file() instead. BackupStoreFilenameClear nd("sub2"); // Attributes std::auto_ptr attr(new MemBlockStream(attr1, sizeof(attr1))); subsubdirid = apProtocol->QueryCreateDirectory(subdirid, FAKE_ATTR_MODIFICATION_TIME, nd, attr)->GetObjectID(); BackupStoreFilenameClear file2("file2"); std::auto_ptr upload( BackupStoreFile::EncodeFile("testfiles/test2", BackupProtocolListDirectory::RootDirectory, file2)); std::auto_ptr stored(apProtocol->QueryStoreFile( subsubdirid, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ file2, upload)); subsubfileid = stored->GetObjectID(); } set_refcount(subsubdirid, 1); set_refcount(subsubfileid, 1); TEST_THAT(check_num_files(UPLOAD_NUM - 2, 4, 2, 3)); apProtocol->QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); apProtocol = test_server_login("localhost", context); // Query names -- test that invalid stuff returns not found OK { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(3248972347823478927LL, subsubdirid)); TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(subsubfileid, 2342378424LL)); TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(38947234789LL, 2342378424LL)); TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, 2234342378424LL)); TEST_THAT(nameRep->GetNumNameElements() == 0); } // Query names... first, get info for the file { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(subsubfileid, subsubdirid)); std::auto_ptr namestream(apProtocol->ReceiveStream()); TEST_THAT(nameRep->GetNumNameElements() == 3); TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_File); TEST_THAT(nameRep->GetModificationTime() == 0x123456789abcdefLL); TEST_THAT(nameRep->GetAttributesHash() == 0x7362383249872dfLL); static const char *testnames[] = {"file2","sub2","lovely_directory"}; for(int l = 0; l < nameRep->GetNumNameElements(); ++l) { BackupStoreFilenameClear fn; fn.ReadFromStream(*namestream, 10000); TEST_THAT(fn.GetClearFilename() == testnames[l]); } } // Query names... secondly, for the directory { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, subsubdirid)); std::auto_ptr namestream(apProtocol->ReceiveStream()); TEST_THAT(nameRep->GetNumNameElements() == 2); TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_Dir); static const char *testnames[] = {"sub2","lovely_directory"}; for(int l = 0; l < nameRep->GetNumNameElements(); ++l) { BackupStoreFilenameClear fn; fn.ReadFromStream(*namestream, 10000); TEST_THAT(fn.GetClearFilename() == testnames[l]); } } //} skip: std::auto_ptr apRefCount( BackupStoreRefCountDatabase::Load( apAccounts->GetEntry(0x1234567), true)); // Create some nice recursive directories TEST_THAT(check_reference_counts()); int64_t dirtodelete = create_test_data_subdirs(*apProtocol, BackupProtocolListDirectory::RootDirectory, "test_delete", 6 /* depth */, apRefCount.get()); TEST_THAT(check_reference_counts()); apProtocol->QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); TEST_THAT(check_reference_counts()); // And delete them apProtocol.reset(new BackupProtocolLocal2(0x01234567, "test", "backup/01234567/", 0, false)); // Not read-only { std::auto_ptr dirdel(apProtocol->QueryDeleteDirectory( dirtodelete)); TEST_THAT(dirdel->GetObjectID() == dirtodelete); } apProtocol->QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); TEST_THAT(check_reference_counts()); // Get the root dir, checking for deleted items { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_Dir | BackupProtocolListDirectory::Flags_Deleted, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), SHORT_TIMEOUT); // Check there's only that one entry TEST_THAT(dir.GetNumberOfEntries() == 1); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); if(en) { TEST_EQUAL(dirtodelete, en->GetObjectID()); BackupStoreFilenameClear n("test_delete"); TEST_THAT(en->GetName() == n); } // Then... check everything's deleted test_everything_deleted(protocolReadOnly, dirtodelete); } // Finish the connections #ifndef WIN32 protocolReadOnly.QueryFinished(); #endif apProtocol->QueryFinished(); } tearDown(); return true; } int get_object_size(BackupProtocolCallable& protocol, int64_t ObjectID, int64_t ContainerID) { // Get the root directory cached in the read-only connection protocol.QueryListDirectory(ContainerID, 0, // FlagsMustBeSet BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */); BackupStoreDirectory dir(protocol.ReceiveStream()); BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); TEST_THAT(en != 0); if (!en) return -1; TEST_EQUAL(ObjectID, en->GetObjectID()); return en->GetSizeInBlocks(); } bool write_dir(BackupStoreDirectory& dir) { std::string rfn; StoreStructure::MakeObjectFilename(dir.GetObjectID(), "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, rfn, false); // EnsureDirectoryExists RaidFileWrite rfw(0, rfn); rfw.Open(true); // AllowOverwrite dir.WriteToStream(rfw); rfw.Commit(/* ConvertToRaidNow */ true); return true; } bool test_directory_parent_entry_tracks_directory_size() { SETUP(); BackupProtocolLocal2 protocol(0x01234567, "test", "backup/01234567/", 0, false); BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", "backup/01234567/", 0, true); // read only int64_t subdirid = create_directory(protocol); // Get the root directory cached in the read-only connection, and // test that the initial size is correct. int old_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); TEST_THAT(old_size > 0); TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); // TODO FIXME Sleep to ensure that file timestamp changes (win32?) // Start adding files until the size on disk increases. This is // guaranteed to happen eventually :) int new_size = old_size; int64_t last_added_file_id = 0; std::string last_added_filename; for (int i = 0; new_size == old_size; i++) { std::ostringstream name; name << "testfile_" << i; last_added_filename = name.str(); last_added_file_id = create_file(protocol, subdirid, name.str()); new_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); } // Check that the root directory entry has been updated TEST_EQUAL(new_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); // Now delete an entry, and check that the size is reduced protocol.QueryDeleteFile(subdirid, BackupStoreFilenameClear(last_added_filename)); // Reduce the limits, to remove it permanently from the store protocol.QueryFinished(); protocolReadOnly.QueryFinished(); TEST_THAT(change_account_limits("0B", "20000B")); TEST_THAT(run_housekeeping_and_check_account()); set_refcount(last_added_file_id, 0); protocol.Reopen(); protocolReadOnly.Reopen(); TEST_EQUAL(old_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); // Check that the entry in the root directory was updated too TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); // Push the limits back up protocol.QueryFinished(); protocolReadOnly.QueryFinished(); TEST_THAT(change_account_limits("1000B", "20000B")); TEST_THAT(run_housekeeping_and_check_account()); protocol.Reopen(); protocolReadOnly.Reopen(); // Now modify the root directory to remove its entry for this one std::auto_ptr root_read(get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID)); BackupStoreDirectory root(static_cast >(root_read)); BackupStoreDirectory::Entry *en = root.FindEntryByID(subdirid); TEST_THAT_OR(en, return false); BackupStoreDirectory::Entry enCopy(*en); root.DeleteEntry(subdirid); TEST_THAT(write_dir(root)); // Add a directory, this should try to push the object size back up, // which will try to modify the subdir's entry in its parent, which // no longer exists, which should just log an error instead of // aborting/segfaulting. create_directory(protocol, subdirid); // Repair the error ourselves, as bbstoreaccounts can't. protocol.QueryFinished(); enCopy.SetSizeInBlocks(get_raid_file(subdirid)->GetDiscUsageInBlocks()); root.AddEntry(enCopy); TEST_THAT(write_dir(root)); // We also have to remove the entry for lovely_directory created by // create_directory(), because otherwise we can't create it again. // (Perhaps it should not have been committed because we failed to // update the parent, but currently it is.) std::auto_ptr subdir_read(get_raid_file(subdirid)); BackupStoreDirectory subdir(static_cast >(subdir_read)); { BackupStoreDirectory::Iterator i(subdir); en = i.FindMatchingClearName( BackupStoreFilenameClear("lovely_directory")); } TEST_THAT_OR(en, return false); protocol.Reopen(); protocol.QueryDeleteDirectory(en->GetObjectID()); set_refcount(en->GetObjectID(), 0); // This should have fixed the error, so we should be able to add the // entry now. This should push the object size back up. int64_t dir2id = create_directory(protocol, subdirid); TEST_EQUAL(new_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); TEST_EQUAL(new_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); // Delete it again, which should reduce the object size again protocol.QueryDeleteDirectory(dir2id); // Reduce the limits, to remove it permanently from the store protocol.QueryFinished(); protocolReadOnly.QueryFinished(); TEST_THAT(change_account_limits("0B", "20000B")); TEST_THAT(run_housekeeping_and_check_account()); set_refcount(dir2id, 0); protocol.Reopen(); protocolReadOnly.Reopen(); // Check that the entry in the root directory was updated TEST_EQUAL(old_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); // Check that bbstoreaccounts check fix will detect and repair when // a directory's parent entry has the wrong size for the directory. protocol.QueryFinished(); root_read = get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID); root.ReadFromStream(*root_read, IOStream::TimeOutInfinite); en = root.FindEntryByID(subdirid); TEST_THAT_OR(en != 0, return false); en->SetSizeInBlocks(1234); TEST_THAT(write_dir(root)); TEST_EQUAL(1234, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); TEST_EQUAL(1, check_account_for_errors()); TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, BACKUPSTORE_ROOT_DIRECTORY_ID)); protocolReadOnly.QueryFinished(); tearDown(); return true; } bool test_cannot_open_multiple_writable_connections() { SETUP(); TEST_THAT_THROWONFAIL(StartServer()); // First try a local protocol. This works even on Windows. BackupProtocolLocal2 protocolWritable(0x01234567, "test", "backup/01234567/", 0, false); // Not read-only // Set the client store marker protocolWritable.QuerySetClientStoreMarker(0x8732523ab23aLL); // First try a local protocol. This works even on Windows. BackupProtocolLocal2 protocolWritable2(0x01234567, "test", "backup/01234567/", 0, false); // Not read-only assert_writable_connection_fails(protocolWritable2); BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", "backup/01234567/", 0, true); // Read-only TEST_EQUAL(0x8732523ab23aLL, assert_readonly_connection_succeeds(protocolReadOnly)); // Try network connections too. TLSContext context; BackupProtocolClient protocolWritable3(open_conn("localhost", context)); assert_writable_connection_fails(protocolWritable3); BackupProtocolClient protocolReadOnly2(open_conn("localhost", context)); TEST_EQUAL(0x8732523ab23aLL, assert_readonly_connection_succeeds(protocolReadOnly2)); tearDown(); return true; } bool test_encoding() { // Now test encoded files // TODO: This test needs to check failure situations as well as everything working, // but this will be saved for the full implementation. SETUP(); int encfile[ENCFILE_SIZE]; { for(int l = 0; l < ENCFILE_SIZE; ++l) { encfile[l] = l * 173; } // Encode and decode a small block (shouldn't be compressed) { #define SMALL_BLOCK_SIZE 251 int encBlockSize = BackupStoreFile::MaxBlockSizeForChunkSize(SMALL_BLOCK_SIZE); TEST_THAT(encBlockSize > SMALL_BLOCK_SIZE); BackupStoreFile::EncodingBuffer encoded; encoded.Allocate(encBlockSize / 8); // make sure reallocation happens // Encode! int encSize = BackupStoreFile::EncodeChunk(encfile, SMALL_BLOCK_SIZE, encoded); // Check the header says it's not been compressed TEST_THAT((encoded.mpBuffer[0] & 1) == 0); // Check the output size has been inflated (no compression) TEST_THAT(encSize > SMALL_BLOCK_SIZE); // Decode it int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(SMALL_BLOCK_SIZE); TEST_THAT(decBlockSize > SMALL_BLOCK_SIZE); uint8_t *decoded = (uint8_t*)malloc(decBlockSize); int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize); TEST_THAT(decSize < decBlockSize); TEST_THAT(decSize == SMALL_BLOCK_SIZE); // Check it came out of the wash the same TEST_THAT(::memcmp(encfile, decoded, SMALL_BLOCK_SIZE) == 0); free(decoded); } // Encode and decode a big block (should be compressed) { int encBlockSize = BackupStoreFile::MaxBlockSizeForChunkSize(ENCFILE_SIZE); TEST_THAT(encBlockSize > ENCFILE_SIZE); BackupStoreFile::EncodingBuffer encoded; encoded.Allocate(encBlockSize / 8); // make sure reallocation happens // Encode! int encSize = BackupStoreFile::EncodeChunk(encfile, ENCFILE_SIZE, encoded); // Check the header says it's compressed TEST_THAT((encoded.mpBuffer[0] & 1) == 1); // Check the output size make it likely that it's compressed (is very compressible data) TEST_THAT(encSize < ENCFILE_SIZE); // Decode it int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(ENCFILE_SIZE); TEST_THAT(decBlockSize > ENCFILE_SIZE); uint8_t *decoded = (uint8_t*)malloc(decBlockSize); int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize); TEST_THAT(decSize < decBlockSize); TEST_THAT(decSize == ENCFILE_SIZE); // Check it came out of the wash the same TEST_THAT(::memcmp(encfile, decoded, ENCFILE_SIZE) == 0); free(decoded); } // The test block to a file { FileStream f("testfiles/testenc1", O_WRONLY | O_CREAT); f.Write(encfile, sizeof(encfile)); } // Encode it { FileStream out("testfiles/testenc1_enc", O_WRONLY | O_CREAT); BackupStoreFilenameClear name("testfiles/testenc1"); std::auto_ptr encoded(BackupStoreFile::EncodeFile("testfiles/testenc1", 32, name)); encoded->CopyStreamTo(out); } // Verify it { FileStream enc("testfiles/testenc1_enc"); TEST_THAT(BackupStoreFile::VerifyEncodedFileFormat(enc) == true); } // Decode it { UNLINK_IF_EXISTS("testfiles/testenc1_orig"); FileStream enc("testfiles/testenc1_enc"); BackupStoreFile::DecodeFile(enc, "testfiles/testenc1_orig", IOStream::TimeOutInfinite); } // Read in rebuilt original, and compare contents { TEST_THAT(TestGetFileSize("testfiles/testenc1_orig") == sizeof(encfile)); FileStream in("testfiles/testenc1_orig"); int encfile_i[ENCFILE_SIZE]; in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } // Check how many blocks it had, and test the stream based interface { FileStream enc("testfiles/testenc1_enc"); std::auto_ptr decoded(BackupStoreFile::DecodeFileStream(enc, IOStream::TimeOutInfinite)); CollectInBufferStream d; decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 971 /* buffer block size */); d.SetForReading(); TEST_THAT(d.GetSize() == sizeof(encfile)); TEST_THAT(memcmp(encfile, d.GetBuffer(), sizeof(encfile)) == 0); TEST_THAT(decoded->GetNumBlocks() == 3); } // Test that the last block in a file, if less than 256 bytes, gets put into the last block { #define FILE_SIZE_JUST_OVER ((4096*2)+58) FileStream f("testfiles/testenc2", O_WRONLY | O_CREAT); f.Write(encfile + 2, FILE_SIZE_JUST_OVER); BackupStoreFilenameClear name("testenc2"); std::auto_ptr encoded(BackupStoreFile::EncodeFile("testfiles/testenc2", 32, name)); CollectInBufferStream e; encoded->CopyStreamTo(e); e.SetForReading(); std::auto_ptr decoded(BackupStoreFile::DecodeFileStream(e, IOStream::TimeOutInfinite)); CollectInBufferStream d; decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 879 /* buffer block size */); d.SetForReading(); TEST_THAT(d.GetSize() == FILE_SIZE_JUST_OVER); TEST_THAT(memcmp(encfile + 2, d.GetBuffer(), FILE_SIZE_JUST_OVER) == 0); TEST_THAT(decoded->GetNumBlocks() == 2); } // Test that reordered streams work too { FileStream enc("testfiles/testenc1_enc"); std::auto_ptr reordered(BackupStoreFile::ReorderFileToStreamOrder(&enc, false)); std::auto_ptr decoded(BackupStoreFile::DecodeFileStream(*reordered, IOStream::TimeOutInfinite)); CollectInBufferStream d; decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 971 /* buffer block size */); d.SetForReading(); TEST_THAT(d.GetSize() == sizeof(encfile)); TEST_THAT(memcmp(encfile, d.GetBuffer(), sizeof(encfile)) == 0); TEST_THAT(decoded->GetNumBlocks() == 3); } } tearDown(); return true; } bool test_symlinks() { #ifndef WIN32 // no symlinks on Win32 SETUP(); // TODO FIXME indentation // Try out doing this on a symlink { UNLINK_IF_EXISTS("testfiles/testsymlink"); TEST_THAT(::symlink("does/not/exist", "testfiles/testsymlink") == 0); BackupStoreFilenameClear name("testsymlink"); std::auto_ptr encoded(BackupStoreFile::EncodeFile("testfiles/testsymlink", 32, name)); // Can't decode it from the stream, because it's in file order, and doesn't have the // required properties to be able to reorder it. So buffer it... CollectInBufferStream b; encoded->CopyStreamTo(b); b.SetForReading(); // Decode it UNLINK_IF_EXISTS("testfiles/testsymlink_2"); BackupStoreFile::DecodeFile(b, "testfiles/testsymlink_2", IOStream::TimeOutInfinite); } tearDown(); #endif return true; } bool test_store_info() { SETUP(); { RaidFileWrite::CreateDirectory(0, "test-info"); BackupStoreInfo::CreateNew(76, "test-info/", 0, 3461231233455433LL, 2934852487LL); TEST_CHECK_THROWS(BackupStoreInfo::CreateNew(76, "test-info/", 0, 0, 0), RaidFileException, CannotOverwriteExistingFile); std::auto_ptr info(BackupStoreInfo::Load(76, "test-info/", 0, true)); TEST_CHECK_THROWS(info->Save(), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->ChangeBlocksUsed(1), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(1), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(1), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->RemovedDeletedDirectory(2), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->AddDeletedDirectory(2), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->SetAccountName("hello"), BackupStoreException, StoreInfoIsReadOnly); } { std::auto_ptr info(BackupStoreInfo::Load(76, "test-info/", 0, false)); info->ChangeBlocksUsed(8); info->ChangeBlocksInOldFiles(9); info->ChangeBlocksInDeletedFiles(10); info->ChangeBlocksUsed(-1); info->ChangeBlocksInOldFiles(-4); info->ChangeBlocksInDeletedFiles(-9); TEST_CHECK_THROWS(info->ChangeBlocksUsed(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); info->AddDeletedDirectory(2); info->AddDeletedDirectory(3); info->AddDeletedDirectory(4); info->RemovedDeletedDirectory(3); info->SetAccountName("whee"); TEST_CHECK_THROWS(info->RemovedDeletedDirectory(9), BackupStoreException, StoreInfoDirNotInList); info->Save(); } { std::auto_ptr info(BackupStoreInfo::Load(76, "test-info/", 0, true)); TEST_THAT(info->GetBlocksUsed() == 7); TEST_THAT(info->GetBlocksInOldFiles() == 5); TEST_THAT(info->GetBlocksInDeletedFiles() == 1); TEST_THAT(info->GetBlocksSoftLimit() == 3461231233455433LL); TEST_THAT(info->GetBlocksHardLimit() == 2934852487LL); TEST_THAT(info->GetAccountName() == "whee"); const std::vector &delfiles(info->GetDeletedDirectories()); TEST_THAT(delfiles.size() == 2); TEST_THAT(delfiles[0] == 2); TEST_THAT(delfiles[1] == 4); } tearDown(); return true; } TLSContext context; bool test_login_without_account() { // First, try logging in without an account having been created... just make sure login fails. SETUP(); delete_account(); TEST_THAT_THROWONFAIL(StartServer()); // BLOCK { // Open a connection to the server BackupProtocolClient protocol(open_conn("localhost", context)); // Check the version std::auto_ptr serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login TEST_CHECK_THROWS(std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, 0)), ConnectionException, Protocol_UnexpectedReply); // Finish the connection protocol.QueryFinished(); } // Recreate the account so that tearDown() doesn't freak out // TEST_THAT_THROWONFAIL(create_account(10000, 20000)); tearDown(); return true; } bool test_bbstoreaccounts_create() { SETUP(); // Delete the account, and create it again using bbstoreaccounts delete_account(); TEST_THAT_THROWONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf -Wwarning create 01234567 0 " "10000B 20000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); tearDown(); return true; } bool test_bbstoreaccounts_delete() { SETUP(); TEST_THAT_THROWONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf -Wwarning delete 01234567 yes") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Recreate the account so that tearDown() doesn't freak out TEST_THAT_THROWONFAIL(create_account(10000, 20000)); tearDown(); return true; } // Test that login fails on a disabled account bool test_login_with_disabled_account() { SETUP(); TEST_THAT_THROWONFAIL(StartServer()); TEST_THAT(TestDirExists("testfiles/0_0/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_1/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567")); TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8); // make sure something is written to it std::auto_ptr apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); std::auto_ptr apReferences( BackupStoreRefCountDatabase::Load( apAccounts->GetEntry(0x1234567), true)); TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, apReferences->GetLastObjectIDUsed()); TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) apReferences.reset(); // Test that login fails on a disabled account TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 no") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // BLOCK { // Open a connection to the server BackupProtocolClient protocol(open_conn("localhost", context)); // Check the version std::auto_ptr serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login TEST_CHECK_THROWS(std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, 0)), ConnectionException, Protocol_UnexpectedReply); int type, subType; TEST_EQUAL_LINE(true, protocol.GetLastError(type, subType), "expected a protocol error"); TEST_EQUAL_LINE(BackupProtocolError::ErrorType, type, "expected a BackupProtocolError"); TEST_EQUAL_LINE(BackupProtocolError::Err_DisabledAccount, subType, "expected an Err_DisabledAccount"); // Finish the connection protocol.QueryFinished(); } tearDown(); return true; } bool test_login_with_no_refcount_db() { SETUP(); TEST_THAT_THROWONFAIL(StartServer()); // It's easier to test this if we disable housekeeping, so run // without a server. // TEST_THAT_THROWONFAIL(StartServer()); TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 yes") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Delete the refcount database and try to log in again. Check that // we're locked out of the account until housekeeping has recreated // the refcount db. TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); TEST_CHECK_THROWS(test_server_login("localhost", context), ConnectionException, TLSReadFailed); TEST_THAT(ServerIsAlive(bbstored_pid)); std::auto_ptr apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); run_housekeeping(account); // Check that housekeeping fixed the ref counts std::auto_ptr apReferences = BackupStoreRefCountDatabase::Load(account, true); TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, apReferences->GetLastObjectIDUsed()); TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) apReferences.reset(); TEST_THAT(ServerIsAlive(bbstored_pid)); // test that all object reference counts have the // expected values apReferences = BackupStoreRefCountDatabase::Load(account, true); set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); TEST_EQUAL(ExpectedRefCounts.size() - 1, apReferences->GetLastObjectIDUsed()); for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID; i < ExpectedRefCounts.size(); i++) { TEST_EQUAL_LINE(ExpectedRefCounts[i], apReferences->GetRefCount(i), "object " << BOX_FORMAT_OBJECTID(i)); } // Delete the refcount database again, and let // housekeeping recreate it and fix the ref counts. // This could also happen after upgrade, if a housekeeping // runs before the user logs in. apReferences.reset(); TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); run_housekeeping(account); apReferences = BackupStoreRefCountDatabase::Load(account, true); tearDown(); return true; } bool test_housekeeping_deletes_files() { // Test the deletion of objects by the housekeeping system SETUP(); BackupProtocolLocal2 protocolLocal(0x01234567, "test", "backup/01234567/", 0, false); // Not read-only // Create some nice recursive directories write_test_file(1); int64_t dirtodelete = create_test_data_subdirs(protocolLocal, BackupProtocolListDirectory::RootDirectory, "test_delete", 6 /* depth */, NULL /* pRefCount */); TEST_EQUAL(dirtodelete, protocolLocal.QueryDeleteDirectory(dirtodelete)->GetObjectID()); test_everything_deleted(protocolLocal, dirtodelete); protocolLocal.QueryFinished(); // First, things as they are now. TEST_THAT_ABORTONFAIL(StartServer()); recursive_count_objects_results before = {0,0,0}; recursive_count_objects("localhost", BackupProtocolListDirectory::RootDirectory, before); TEST_THAT(before.objectsNotDel == 0); TEST_THAT(before.deleted != 0); TEST_THAT(before.old != 0); // Kill it TEST_THAT(StopServer()); // Set a new limit on the account -- leave the hard limit // high to make sure the target for freeing space is the // soft limit. TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf setlimit 01234567 " "10B 20000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Start things up TEST_THAT(StartServer()); // wait for housekeeping to happen printf("waiting for housekeeping:\n"); for(int l = 0; l < 30; ++l) { ::sleep(1); printf("."); fflush(stdout); } printf("\n"); // Count the objects again recursive_count_objects_results after = {0,0,0}; recursive_count_objects("localhost", BackupProtocolListDirectory::RootDirectory, after); // If these tests fail then try increasing the timeout above TEST_EQUAL(before.objectsNotDel, after.objectsNotDel); TEST_EQUAL(0, after.deleted); TEST_EQUAL(0, after.old); // Adjust reference counts on deleted files, so that the final checks in // tearDown() don't fail. ExpectedRefCounts.resize(2); // Delete the account to stop tearDown from checking it. TODO FIXME // I'm aware of the block count mismatch that tearDown catches and // will investigate. delete_account(); tearDown(); return true; } bool test_account_limits_respected() { SETUP(); TEST_THAT_THROWONFAIL(StartServer()); // Set a really small hard limit TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf setlimit 01234567 " "2B 2B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Try to upload a file and create a directory, and check an error is generated { // Open a connection to the server BackupProtocolClient protocol(open_conn("localhost", context)); int64_t modtime = 0; write_test_file(3); BackupStoreFilenameClear fnx("exceed-limit"); std::auto_ptr upload(BackupStoreFile::EncodeFile("testfiles/test3", BackupProtocolListDirectory::RootDirectory, fnx, &modtime)); TEST_THAT(modtime != 0); TEST_CHECK_THROWS(protocol.QueryStoreFile( BackupProtocolListDirectory::RootDirectory, modtime, modtime, /* use it for attr hash too */ 0, /* diff from ID */ fnx, upload), ConnectionException, Protocol_UnexpectedReply); // This currently causes a fatal error on the server, which // kills the connection. TODO FIXME return an error instead. std::auto_ptr attr(new MemBlockStream(&modtime, sizeof(modtime))); BackupStoreFilenameClear fnxd("exceed-limit-dir"); TEST_CHECK_THROWS(protocol.QueryCreateDirectory( BackupProtocolListDirectory::RootDirectory, FAKE_ATTR_MODIFICATION_TIME, fnxd, attr), ConnectionException, Protocol_UnexpectedReply); // Finish the connection. TODO FIXME reinstate this. protocol.QueryFinished(); } tearDown(); return true; } int multi_server() { printf("Starting server for connection from remote machines...\n"); // Create an account for the test client TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf create 01234567 0 " "30000B 40000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // First, try logging in without an account having been created... just make sure login fails. int pid = LaunchServer(BBSTORED " testfiles/bbstored_multi.conf", "testfiles/bbstored.pid"); TEST_THAT(pid != -1 && pid != 0); if(pid > 0) { ::sleep(1); TEST_THAT(ServerIsAlive(pid)); // Wait for a keypress printf("Press ENTER to terminate the server\n"); char line[512]; fgets(line, 512, stdin); printf("Terminating server...\n"); // Kill it TEST_THAT(StopServer()); } return 0; } bool test_open_files_with_limited_win32_permissions() { #ifdef WIN32 // this had better work, or bbstored will die when combining diffs char* file = "foo"; DWORD accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY | FILE_READ_EA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_DATA | FILE_WRITE_EA /*| FILE_ALL_ACCESS*/; DWORD shareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; HANDLE h1 = CreateFileA(file, accessRights, shareMode, NULL, OPEN_ALWAYS, FILE_FLAG_BACKUP_SEMANTICS, NULL); assert(h1 != INVALID_HANDLE_VALUE); TEST_THAT(h1 != INVALID_HANDLE_VALUE); accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY | FILE_READ_EA; HANDLE h2 = CreateFileA(file, accessRights, shareMode, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); assert(h2 != INVALID_HANDLE_VALUE); TEST_THAT(h2 != INVALID_HANDLE_VALUE); CloseHandle(h2); CloseHandle(h1); h1 = openfile(file, O_CREAT | O_RDWR, 0); TEST_THAT(h1 != INVALID_HANDLE_VALUE); h2 = openfile(file, O_RDWR, 0); TEST_THAT(h2 != INVALID_HANDLE_VALUE); CloseHandle(h2); CloseHandle(h1); #endif return true; } void compare_backupstoreinfo_values_to_expected ( const std::string& test_phase, const info_StreamFormat_1& expected, const BackupStoreInfo& actual, const std::string& expected_account_name, bool expected_account_enabled, const MemBlockStream& extra_data = MemBlockStream(/* empty */) ) { TEST_EQUAL_LINE(ntohl(expected.mAccountID), actual.GetAccountID(), test_phase << " AccountID"); #define TEST_INFO_EQUAL(property) \ TEST_EQUAL_LINE(box_ntoh64(expected.m ## property), \ actual.Get ## property (), test_phase << " " #property) TEST_INFO_EQUAL(ClientStoreMarker); TEST_INFO_EQUAL(LastObjectIDUsed); TEST_INFO_EQUAL(BlocksUsed); TEST_INFO_EQUAL(BlocksInOldFiles); TEST_INFO_EQUAL(BlocksInDeletedFiles); TEST_INFO_EQUAL(BlocksInDirectories); TEST_INFO_EQUAL(BlocksSoftLimit); TEST_INFO_EQUAL(BlocksHardLimit); #undef TEST_INFO_EQUAL // These attributes of the v2 structure are not supported by v1, // so they should all be initialised to 0 TEST_EQUAL_LINE(0, actual.GetBlocksInCurrentFiles(), test_phase << " BlocksInCurrentFiles"); TEST_EQUAL_LINE(0, actual.GetNumOldFiles(), test_phase << " NumOldFiles"); TEST_EQUAL_LINE(0, actual.GetNumDeletedFiles(), test_phase << " NumDeletedFiles"); TEST_EQUAL_LINE(0, actual.GetNumDirectories(), test_phase << " NumDirectories"); // These attributes of the old v1 structure are not actually loaded // or used: // TEST_INFO_EQUAL(CurrentMarkNumber); // TEST_INFO_EQUAL(OptionsPresent); TEST_EQUAL_LINE(box_ntoh64(expected.mNumberDeletedDirectories), actual.GetDeletedDirectories().size(), test_phase << " number of deleted directories"); for (size_t i = 0; i < box_ntoh64(expected.mNumberDeletedDirectories) && i < actual.GetDeletedDirectories().size(); i++) { TEST_EQUAL_LINE(13 + i, actual.GetDeletedDirectories()[i], test_phase << " deleted directory " << (i+1)); } TEST_EQUAL_LINE(expected_account_name, actual.GetAccountName(), test_phase << " AccountName"); TEST_EQUAL_LINE(expected_account_enabled, actual.IsAccountEnabled(), test_phase << " AccountEnabled"); TEST_EQUAL_LINE(extra_data.GetSize(), actual.GetExtraData().GetSize(), test_phase << " extra data has wrong size"); TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), actual.GetExtraData().GetBuffer(), extra_data.GetSize()), test_phase << " extra data has wrong contents"); } bool test_read_old_backupstoreinfo_files() { SETUP(); // Create an account for the test client std::auto_ptr apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), "'bbstoreaccounts create' should have set AccountEnabled flag"); info_StreamFormat_1 info_v1; info_v1.mMagicValue = htonl(INFO_MAGIC_VALUE_1); info_v1.mAccountID = htonl(0x01234567); info_v1.mClientStoreMarker = box_hton64(3); info_v1.mLastObjectIDUsed = box_hton64(4); info_v1.mBlocksUsed = box_hton64(5); info_v1.mBlocksInOldFiles = box_hton64(6); info_v1.mBlocksInDeletedFiles = box_hton64(7); info_v1.mBlocksInDirectories = box_hton64(8); info_v1.mBlocksSoftLimit = box_hton64(9); info_v1.mBlocksHardLimit = box_hton64(10); info_v1.mCurrentMarkNumber = htonl(11); info_v1.mOptionsPresent = htonl(12); info_v1.mNumberDeletedDirectories = box_hton64(2); // Then mNumberDeletedDirectories * int64_t IDs for the deleted directories // Generate the filename std::string info_filename("backup/01234567/" INFO_FILENAME); std::auto_ptr rfw(new RaidFileWrite(0, info_filename)); rfw->Open(/* AllowOverwrite = */ true); rfw->Write(&info_v1, sizeof(info_v1)); // Write mNumberDeletedDirectories * int64_t IDs for the deleted directories std::auto_ptr apArchive(new Archive(*rfw, IOStream::TimeOutInfinite)); apArchive->Write((int64_t) 13); apArchive->Write((int64_t) 14); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded from v1", info_v1, *apInfo, "" /* no name by default */, true /* enabled by default */); apInfo->SetAccountName("bonk"); // Save the info again apInfo->Save(/* allowOverwrite */ true); // Check that it was saved in the new Archive format std::auto_ptr rfr(RaidFileRead::Open(0, info_filename, 0)); int32_t magic; if(!rfr->ReadFullBuffer(&magic, sizeof(magic), 0)) { THROW_FILE_ERROR("Failed to read store info file: " "short read of magic number", info_filename, BackupStoreException, CouldNotLoadStoreInfo); } TEST_EQUAL_LINE(INFO_MAGIC_VALUE_2, ntohl(magic), "format version in newly saved BackupStoreInfo"); rfr.reset(); // load it, and check that all values are loaded properly apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded in v1, resaved in v2", info_v1, *apInfo, "bonk", true); // Check that the new AccountEnabled flag is saved properly apInfo->SetAccountEnabled(false); apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("saved in v2, loaded in v2", info_v1, *apInfo, "bonk", false /* as modified above */); apInfo->SetAccountEnabled(true); apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ true); compare_backupstoreinfo_values_to_expected("resaved in v2 with " "account enabled", info_v1, *apInfo, "bonk", true /* as modified above */); // Now save the info in v2 format without the AccountEnabled flag // (boxbackup 0.11 format) and check that the flag is set to true // for backwards compatibility rfw.reset(new RaidFileWrite(0, info_filename)); rfw->Open(/* allowOverwrite */ true); magic = htonl(INFO_MAGIC_VALUE_2); apArchive.reset(new Archive(*rfw, IOStream::TimeOutInfinite)); rfw->Write(&magic, sizeof(magic)); apArchive->Write(apInfo->GetAccountID()); apArchive->Write(std::string("test")); apArchive->Write(apInfo->GetClientStoreMarker()); apArchive->Write(apInfo->GetLastObjectIDUsed()); apArchive->Write(apInfo->GetBlocksUsed()); apArchive->Write(apInfo->GetBlocksInCurrentFiles()); apArchive->Write(apInfo->GetBlocksInOldFiles()); apArchive->Write(apInfo->GetBlocksInDeletedFiles()); apArchive->Write(apInfo->GetBlocksInDirectories()); apArchive->Write(apInfo->GetBlocksSoftLimit()); apArchive->Write(apInfo->GetBlocksHardLimit()); apArchive->Write(apInfo->GetNumCurrentFiles()); apArchive->Write(apInfo->GetNumOldFiles()); apArchive->Write(apInfo->GetNumDeletedFiles()); apArchive->Write(apInfo->GetNumDirectories()); apArchive->Write((int64_t) apInfo->GetDeletedDirectories().size()); apArchive->Write(apInfo->GetDeletedDirectories()[0]); apArchive->Write(apInfo->GetDeletedDirectories()[1]); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("saved in v2 without " "AccountEnabled", info_v1, *apInfo, "test", true); // Default for missing AccountEnabled should be true // Rewrite using full length, so that the first 4 bytes of extra data // doesn't get swallowed by "extra data". apInfo->Save(/* allowOverwrite */ true); // Append some extra data after the known account values, to simulate a // new addition to the store format. Check that this extra data is loaded // and resaved with the info file. We made the mistake of deleting it in // 0.11, let's not make the same mistake again. CollectInBufferStream info_data; rfr = RaidFileRead::Open(0, info_filename, 0); rfr->CopyStreamTo(info_data); rfr.reset(); info_data.SetForReading(); rfw.reset(new RaidFileWrite(0, info_filename)); rfw->Open(/* allowOverwrite */ true); info_data.CopyStreamTo(*rfw); char extra_string[] = "hello!"; MemBlockStream extra_data(extra_string, strlen(extra_string)); extra_data.CopyStreamTo(*rfw); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); TEST_EQUAL_LINE(extra_data.GetSize(), apInfo->GetExtraData().GetSize(), "wrong amount of extra data loaded from info file"); TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), apInfo->GetExtraData().GetBuffer(), extra_data.GetSize()), "extra data loaded from info file has wrong contents"); // Save the file and load again, check that the extra data is still there apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); compare_backupstoreinfo_values_to_expected("saved in future format " "with extra_data", info_v1, *apInfo, "test", true, extra_data); // Check that the new bbstoreaccounts command sets the flag properly TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 no") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); TEST_EQUAL_LINE(false, apInfo->IsAccountEnabled(), "'bbstoreaccounts disabled no' should have reset AccountEnabled flag"); TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 yes") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), "'bbstoreaccounts disabled yes' should have set AccountEnabled flag"); // Check that BackupStoreInfo::CreateForRegeneration saves all the // expected properties, including any extra data for forward // compatibility extra_data.Seek(0, IOStream::SeekType_Absolute); apInfo = BackupStoreInfo::CreateForRegeneration( apInfo->GetAccountID(), "spurtle" /* rAccountName */, "backup/01234567/" /* rRootDir */, 0 /* DiscSet */, apInfo->GetLastObjectIDUsed(), apInfo->GetBlocksUsed(), apInfo->GetBlocksInCurrentFiles(), apInfo->GetBlocksInOldFiles(), apInfo->GetBlocksInDeletedFiles(), apInfo->GetBlocksInDirectories(), apInfo->GetBlocksSoftLimit(), apInfo->GetBlocksHardLimit(), false /* AccountEnabled */, extra_data); // CreateForRegeneration always sets the ClientStoreMarker to 0 info_v1.mClientStoreMarker = 0; // CreateForRegeneration does not store any deleted directories info_v1.mNumberDeletedDirectories = 0; // check that the store info has the correct values in memory compare_backupstoreinfo_values_to_expected("stored by " "BackupStoreInfo::CreateForRegeneration", info_v1, *apInfo, "spurtle", false /* AccountEnabled */, extra_data); // Save the file and load again, check that the extra data is still there apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); compare_backupstoreinfo_values_to_expected("saved by " "BackupStoreInfo::CreateForRegeneration and reloaded", info_v1, *apInfo, "spurtle", false /* AccountEnabled */, extra_data); // Delete the account to stop tearDown checking it for errors. apInfo.reset(); TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf delete 01234567 yes") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); tearDown(); return true; } int test(int argc, const char *argv[]) { TEST_THAT(test_open_files_with_limited_win32_permissions()); // Initialise the raid file controller RaidFileController &rcontroller = RaidFileController::GetController(); rcontroller.Initialise("testfiles/raidfile.conf"); TEST_THAT(test_read_old_backupstoreinfo_files()); // SSL library SSLLib::Initialise(); // Use the setup crypto command to set up all these keys, so that the bbackupquery command can be used // for seeing what's going on. BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); // encode in some filenames -- can't do static initialisation // because the key won't be set up when these are initialised { MEMLEAKFINDER_NO_LEAKS for(unsigned int l = 0; l < sizeof(ens_filenames) / sizeof(ens_filenames[0]); ++l) { ens[l].fn = BackupStoreFilenameClear(ens_filenames[l]); } for(unsigned int l = 0; l < sizeof(uploads_filenames) / sizeof(uploads_filenames[0]); ++l) { uploads[l].name = BackupStoreFilenameClear(uploads_filenames[l]); } } // Trace errors out SET_DEBUG_SSLLIB_TRACE_ERRORS { R250 r(3465657); for(int l = 0; l < ATTR1_SIZE; ++l) {attr1[l] = r.next();} for(int l = 0; l < ATTR2_SIZE; ++l) {attr2[l] = r.next();} for(int l = 0; l < ATTR3_SIZE; ++l) {attr3[l] = r.next();} } TEST_THAT(test_filename_encoding()); TEST_THAT(test_bbstoreaccounts_create()); TEST_THAT(test_bbstoreaccounts_delete()); TEST_THAT(test_backupstore_directory()); TEST_THAT(test_directory_parent_entry_tracks_directory_size()); TEST_THAT(test_encoding()); TEST_THAT(test_symlinks()); TEST_THAT(test_store_info()); context.Initialise(false /* client */, "testfiles/clientCerts.pem", "testfiles/clientPrivKey.pem", "testfiles/clientTrustedCAs.pem"); TEST_THAT(test_login_without_account()); TEST_THAT(test_login_with_disabled_account()); TEST_THAT(test_login_with_no_refcount_db()); TEST_THAT(test_server_housekeeping()); TEST_THAT(test_account_limits_respected()); TEST_THAT(test_multiple_uploads()); TEST_THAT(test_housekeeping_deletes_files()); return 0; }