// -------------------------------------------------------------------------- // // File // Name: testbackupstore.cpp // Purpose: Test backup store server // Created: 2003/08/20 // // -------------------------------------------------------------------------- #include "Box.h" #include #include #include "Archive.h" #include "BackupClientCryptoKeys.h" #include "BackupClientFileAttributes.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreAccounts.h" #include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreInfo.h" #include "BackupStoreFilenameClear.h" #include "BackupStoreFileEncodeStream.h" #include "BackupStoreRefCountDatabase.h" #include "BackupStoreFile.h" #include "BoxPortsAndFiles.h" #include "CollectInBufferStream.h" #include "Configuration.h" #include "FileStream.h" #include "HousekeepStoreAccount.h" #include "MemBlockStream.h" #include "RaidFileController.h" #include "RaidFileException.h" #include "RaidFileRead.h" #include "RaidFileWrite.h" #include "SSLLib.h" #include "ServerControl.h" #include "Socket.h" #include "SocketStreamTLS.h" #include "StoreStructure.h" #include "TLSContext.h" #include "Test.h" #include "autogen_BackupProtocol.h" #include "MemLeakFindOn.h" #define ENCFILE_SIZE 2765 // Make some test attributes #define ATTR1_SIZE 245 #define ATTR2_SIZE 23 #define ATTR3_SIZE 122 int attr1[ATTR1_SIZE]; int attr2[ATTR2_SIZE]; int attr3[ATTR3_SIZE]; typedef struct { BackupStoreFilenameClear fn; box_time_t mod; int64_t id; int64_t size; int16_t flags; box_time_t attrmod; } dirtest; static dirtest ens[] = { {BackupStoreFilenameClear(), 324324, 3432, 324, BackupStoreDirectory::Entry::Flags_File, 458763243422LL}, {BackupStoreFilenameClear(), 3432, 32443245645LL, 78, BackupStoreDirectory::Entry::Flags_Dir, 3248972347LL}, {BackupStoreFilenameClear(), 544435, 234234, 23324, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_Deleted, 2348974782LL}, {BackupStoreFilenameClear(), 234, 235436, 6523, BackupStoreDirectory::Entry::Flags_File, 32458923175634LL}, {BackupStoreFilenameClear(), 0x3242343532144LL, 8978979789LL, 21345, BackupStoreDirectory::Entry::Flags_File, 329483243432LL}, {BackupStoreFilenameClear(), 324265765734LL, 12312312321LL, 324987324329874LL, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_Deleted, 32489747234LL}, {BackupStoreFilenameClear(), 3452134, 7868578768LL, 324243, BackupStoreDirectory::Entry::Flags_Dir, 34786457432LL}, {BackupStoreFilenameClear(), 43543543, 324234, 21432, BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, 3489723478327LL}, {BackupStoreFilenameClear(), 325654765874324LL, 4353543, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 32489734789237LL}, {BackupStoreFilenameClear(), 32144325, 436547657, 9, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 234897347234LL} }; static const char *ens_filenames[] = {"obj1ertewt", "obj2", "obj3", "obj4dfedfg43", "obj5", "obj6dfgs", "obj7", "obj8xcvbcx", "obj9", "obj10fgjhfg"}; #define DIR_NUM 10 #define DIR_DIRS 3 #define DIR_FILES 7 #define DIR_OLD 2 #define DIR_DELETED 3 typedef struct { const char *fnextra; BackupStoreFilenameClear name; int seed; int size; box_time_t mod_time; int64_t allocated_objid; bool should_be_old_version; bool delete_file; } uploadtest; #define TEST_FILE_FOR_PATCHING "testfiles/test2" // a few bytes will be inserted at this point: #define TEST_FILE_FOR_PATCHING_PATCH_AT ((64*1024)-128) #define TEST_FILE_FOR_PATCHING_SIZE ((128*1024)+2564) #define UPLOAD_PATCH_EN 2 uploadtest uploads[] = { {"0", BackupStoreFilenameClear(), 324, 455, 0, 0, false, false}, {"1", BackupStoreFilenameClear(), 3232432, 2674, 0, 0, true, false}, // old ver {"2", BackupStoreFilenameClear(), 234, TEST_FILE_FOR_PATCHING_SIZE, 0, 0, false, false}, {"3", BackupStoreFilenameClear(), 324324, 6763, 0, 0, false, false}, {"4", BackupStoreFilenameClear(), 23456, 124, 0, 0, true, false}, // old ver {"5", BackupStoreFilenameClear(), 675745, 1, 0, 0, false, false}, // will upload new attrs for this one! {"6", BackupStoreFilenameClear(), 345213, 0, 0, 0, false, false}, {"7", BackupStoreFilenameClear(), 12313, 3246, 0, 0, true, true}, // old ver, will get deleted {"8", BackupStoreFilenameClear(), 457, 3434, 0, 0, false, false}, // overwrites {"9", BackupStoreFilenameClear(), 12315, 446, 0, 0, false, false}, {"a", BackupStoreFilenameClear(), 3476, 2466, 0, 0, false, false}, {"b", BackupStoreFilenameClear(), 124334, 4562, 0, 0, false, false}, {"c", BackupStoreFilenameClear(), 45778, 234, 0, 0, false, false}, // overwrites {"d", BackupStoreFilenameClear(), 2423425, 435, 0, 0, false, true} // overwrites, will be deleted }; static const char *uploads_filenames[] = {"49587fds", "cvhjhj324", "sdfcscs324", "dsfdsvsdc3214", "XXsfdsdf2342", "dsfdsc232", "sfdsdce2345", "YYstfbdtrdf76", "cvhjhj324", "fbfd098.ycy", "dfs98732hj", "svd987kjsad", "XXsfdsdf2342", "YYstfbdtrdf76"}; #define UPLOAD_NUM 14 #define UPLOAD_LATEST_FILES 12 // file we'll upload some new attributes for #define UPLOAD_ATTRS_EN 5 #define UPLOAD_DELETE_EN 13 // file which will be moved (as well as it's old version) #define UPLOAD_FILE_TO_MOVE 8 #define UNLINK_IF_EXISTS(filename) \ if (FileExists(filename)) { TEST_THAT(unlink(filename) == 0); } // Nice random data for testing written files class R250 { public: // Set up internal state table with 32-bit random numbers. // The bizarre bit-twiddling is because rand() returns 16 bits of which // the bottom bit is always zero! Hence, I use only some of the bits. // You might want to do something better than this.... R250(int seed) : posn1(0), posn2(103) { // populate the state and incr tables srand(seed); for (int i = 0; i != stateLen; ++i) { state[i] = ((rand() >> 2) << 19) ^ ((rand() >> 2) << 11) ^ (rand() >> 2); incrTable[i] = i == stateLen - 1 ? 0 : i + 1; } // stir up the numbers to ensure they're random for (int j = 0; j != stateLen * 4; ++j) (void) next(); } // Returns the next random number. Xor together two elements separated // by 103 mod 250, replacing the first element with the result. Then // increment the two indices mod 250. inline int next() { int ret = (state[posn1] ^= state[posn2]); // xor and replace element posn1 = incrTable[posn1]; // increment indices using lookup table posn2 = incrTable[posn2]; return ret; } private: enum { stateLen = 250 }; // length of the state table int state[stateLen]; // holds the random number state int incrTable[stateLen]; // lookup table: maps i to (i+1) % stateLen int posn1, posn2; // indices into the state table }; int SkipEntries(int e, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { if(e >= DIR_NUM) return e; bool skip = false; do { skip = false; if(FlagsMustBeSet != BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING) { if((ens[e].flags & FlagsMustBeSet) != FlagsMustBeSet) { skip = true; } } if((ens[e].flags & FlagsNotToBeSet) != 0) { skip = true; } if(skip) { ++e; } } while(skip && e < DIR_NUM); return e; } void CheckEntries(BackupStoreDirectory &rDir, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { int e = 0; BackupStoreDirectory::Iterator i(rDir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { TEST_THAT(e < DIR_NUM); // Skip to entry in the ens array which matches e = SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet); // Does it match? TEST_THAT(en->GetName() == ens[e].fn && en->GetModificationTime() == ens[e].mod && en->GetObjectID() == ens[e].id && en->GetFlags() == ens[e].flags && en->GetSizeInBlocks() == ens[e].size); // next ++e; } // Got them all? TEST_THAT(en == 0); TEST_THAT(DIR_NUM == SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet)); } bool create_account(int soft, int hard) { std::string errs; std::auto_ptr config( Configuration::LoadAndVerify ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); BackupStoreAccountsControl control(*config); Logging::Guard guard(Log::WARNING); int result = control.CreateAccount(0x01234567, 0, soft, hard); TEST_EQUAL(0, result); return (result == 0); } std::vector ExpectedRefCounts; void set_refcount(int64_t ObjectID, uint32_t RefCount = 1); // test that all object reference counts have the expected values bool check_reference_counts() { std::auto_ptr apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); std::auto_ptr apReferences( BackupStoreRefCountDatabase::Load(account, true)); TEST_EQUAL(ExpectedRefCounts.size(), apReferences->GetLastObjectIDUsed() + 1); bool counts_ok = true; for (unsigned int i = BackupProtocolListDirectory::RootDirectory; i < ExpectedRefCounts.size(); i++) { TEST_EQUAL_LINE(ExpectedRefCounts[i], apReferences->GetRefCount(i), "object " << BOX_FORMAT_OBJECTID(i)); if (ExpectedRefCounts[i] != apReferences->GetRefCount(i)) { counts_ok = false; } } return counts_ok; } bool check_account(Log::Level log_level = Log::WARNING); int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount); bool run_housekeeping_and_check_account(); int bbstored_pid = 0; bool StartServer() { std::string cmd = BBSTORED " " + bbstored_args + " testfiles/bbstored.conf"; bbstored_pid = LaunchServer(cmd.c_str(), "testfiles/bbstored.pid"); TEST_THAT(bbstored_pid != -1 && bbstored_pid != 0); if(bbstored_pid <= 0) { return false; } ::sleep(1); TEST_THAT_THROWONFAIL(ServerIsAlive(bbstored_pid)); return true; } bool StopServer() { TEST_THAT_THROWONFAIL(ServerIsAlive(bbstored_pid)); TEST_THAT_THROWONFAIL(KillServer(bbstored_pid)); ::sleep(1); TEST_THAT_THROWONFAIL(!ServerIsAlive(bbstored_pid)); #ifdef WIN32 int unlink_result = unlink("testfiles/bbstored.pid"); TEST_EQUAL_LINE(0, unlink_result, "unlink testfiles/bbstored.pid"); if(unlink_result != 0) { return false; } #else TestRemoteProcessMemLeaks("bbstored.memleaks"); #endif return true; } #define SETUP() if (!setUp(__FUNCTION__)) return true; // skip this test bool setUp(const char* function_name) { if (!run_only_named_tests.empty()) { bool run_this_test = false; for (std::list::iterator i = run_only_named_tests.begin(); i != run_only_named_tests.end(); i++) { if (*i == function_name) { run_this_test = true; break; } } if (!run_this_test) { // not in the list, so don't run it. return false; } } printf("\n\n== %s ==\n", function_name); if (ServerIsAlive(bbstored_pid)) { StopServer(); } TEST_THAT_THROWONFAIL(system("rm -rf testfiles/0_0 testfiles/0_1 " "testfiles/0_2 testfiles/accounts.txt testfiles/test* " "testfiles/file*") == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/0_0", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/0_1", 0755) == 0); TEST_THAT_THROWONFAIL(mkdir("testfiles/0_2", 0755) == 0); TEST_THAT_THROWONFAIL(system("touch testfiles/accounts.txt") == 0); TEST_THAT_THROWONFAIL(create_account(10000, 20000)); set_refcount(BackupProtocolListDirectory::RootDirectory, 1); ExpectedRefCounts[1] = 1; return true; } void tearDown() { if (ServerIsAlive(bbstored_pid)) { StopServer(); } if (FileExists("testfiles/0_0/backup/01234567/info.rf")) { TEST_THAT(check_reference_counts()); TEST_THAT(check_account()); } } int test1(int argc, const char *argv[]) { // test some basics -- encoding and decoding filenames { // Make some filenames in various ways BackupStoreFilenameClear fn1; fn1.SetClearFilename(std::string("filenameXYZ")); BackupStoreFilenameClear fn2(std::string("filenameXYZ")); BackupStoreFilenameClear fn3(fn1); TEST_THAT(fn1 == fn2); TEST_THAT(fn1 == fn3); // Check that it's been encrypted std::string name(fn2.GetEncodedFilename()); TEST_THAT(name.find("name") == name.npos); // Bung it in a stream, get it out in a Clear filename { CollectInBufferStream stream; fn1.WriteToStream(stream); stream.SetForReading(); BackupStoreFilenameClear fn4; fn4.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn4.GetClearFilename() == "filenameXYZ"); TEST_THAT(fn4 == fn1); } // Bung it in a stream, get it out in a server non-Clear filename (two of them into the same var) { BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf "); CollectInBufferStream stream; fn1.WriteToStream(stream); fno.WriteToStream(stream); stream.SetForReading(); BackupStoreFilename fn5; fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5 == fn1); fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5 == fno); } // Same again with clear strings { BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf "); CollectInBufferStream stream; fn1.WriteToStream(stream); fno.WriteToStream(stream); stream.SetForReading(); BackupStoreFilenameClear fn5; fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5.GetClearFilename() == "filenameXYZ"); fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5.GetClearFilename() == "pinglet dksfnsf jksjdf "); } // Test a very big filename { const char *fnr = "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789"; BackupStoreFilenameClear fnLong(fnr); CollectInBufferStream stream; fnLong.WriteToStream(stream); stream.SetForReading(); BackupStoreFilenameClear fn9; fn9.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn9.GetClearFilename() == fnr); TEST_THAT(fn9 == fnLong); } // Test a filename which went wrong once { BackupStoreFilenameClear dodgy("content-negotiation.html"); } } return 0; } int test2(int argc, const char *argv[]) { { // Now play with directories // Fill in... BackupStoreDirectory dir1(12, 98); for(int e = 0; e < DIR_NUM; ++e) { dir1.AddEntry(ens[e].fn, ens[e].mod, ens[e].id, ens[e].size, ens[e].flags, ens[e].attrmod); } // Got the right number TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM); // Stick it into a stream and get it out again { CollectInBufferStream stream; dir1.WriteToStream(stream); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_NUM); TEST_THAT(dir2.GetObjectID() == 12); TEST_THAT(dir2.GetContainerID() == 98); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } // Then do selective writes and reads { CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } { CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_DIRS); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_Dir, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } { CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - DIR_OLD); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion); } // Finally test deleting items { dir1.DeleteEntry(12312312321LL); // Verify TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM - 1); CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - 1); } // Check attributes { int attrI[4] = {1, 2, 3, 4}; StreamableMemBlock attr(attrI, sizeof(attrI)); BackupStoreDirectory d1(16, 546); d1.SetAttributes(attr, 56234987324232LL); TEST_THAT(d1.GetAttributes() == attr); TEST_THAT(d1.GetAttributesModTime() == 56234987324232LL); CollectInBufferStream stream; d1.WriteToStream(stream); stream.SetForReading(); BackupStoreDirectory d2(stream); TEST_THAT(d2.GetAttributes() == attr); TEST_THAT(d2.GetAttributesModTime() == 56234987324232LL); } } return 0; } void write_test_file(int t) { std::string filename("testfiles/test"); filename += uploads[t].fnextra; printf("%s\n", filename.c_str()); FileStream write(filename.c_str(), O_WRONLY | O_CREAT); R250 r(uploads[t].seed); unsigned char *data = (unsigned char*)malloc(uploads[t].size); for(int l = 0; l < uploads[t].size; ++l) { data[l] = r.next() & 0xff; } write.Write(data, uploads[t].size); free(data); } void test_test_file(int t, IOStream &rStream) { // Decode to a file BackupStoreFile::DecodeFile(rStream, "testfiles/test_download", IOStream::TimeOutInfinite); // Compare... FileStream in("testfiles/test_download"); TEST_THAT(in.BytesLeftToRead() == uploads[t].size); R250 r(uploads[t].seed); unsigned char *data = (unsigned char*)malloc(uploads[t].size); TEST_THAT(in.ReadFullBuffer(data, uploads[t].size, 0 /* not interested in bytes read if this fails */)); for(int l = 0; l < uploads[t].size; ++l) { TEST_THAT(data[l] == (r.next() & 0xff)); } free(data); in.Close(); TEST_THAT(unlink("testfiles/test_download") == 0); } void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID) { printf("Test for del: %llx\n", (unsigned long long)DirID); // Command std::auto_ptr dirreply(protocol.QueryListDirectory( DirID, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(protocol.ReceiveStream()); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; int files = 0; int dirs = 0; while((en = i.Next()) != 0) { if(en->GetFlags() & BackupProtocolListDirectory::Flags_Dir) { dirs++; // Recurse test_everything_deleted(protocol, en->GetObjectID()); } else { files++; } // Check it's deleted TEST_THAT(en->GetFlags() & BackupProtocolListDirectory::Flags_Deleted); } // Check there were the right number of files and directories TEST_THAT(files == 3); TEST_THAT(dirs == 0 || dirs == 2); } std::vector ExpectedRefCounts; void set_refcount(int64_t ObjectID, uint32_t RefCount) { if ((int64_t)ExpectedRefCounts.size() <= ObjectID) { ExpectedRefCounts.resize(ObjectID + 1, 0); } ExpectedRefCounts[ObjectID] = RefCount; } void create_file_in_dir(std::string name, std::string source, int64_t parentId, BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount) { BackupStoreFilenameClear name_encoded("file_One"); std::auto_ptr upload(BackupStoreFile::EncodeFile( source.c_str(), parentId, name_encoded)); std::auto_ptr stored( protocol.QueryStoreFile( parentId, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ name_encoded, upload)); int64_t objectId = stored->GetObjectID(); TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed()); TEST_EQUAL(1, rRefCount.GetRefCount(objectId)) set_refcount(objectId, 1); } int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, const char *name, int depth, BackupStoreRefCountDatabase& rRefCount) { // Create a directory int64_t subdirid = 0; BackupStoreFilenameClear dirname(name); { // Create with dummy attributes int attrS = 0; std::auto_ptr attr(new MemBlockStream(&attrS, sizeof(attrS))); std::auto_ptr dirCreate(protocol.QueryCreateDirectory( indir, 9837429842987984LL, dirname, attr)); subdirid = dirCreate->GetObjectID(); } printf("Create subdirs, depth = %d, dirid = %llx\n", depth, (unsigned long long)subdirid); TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed()); TEST_EQUAL(1, rRefCount.GetRefCount(subdirid)) set_refcount(subdirid, 1); // Put more directories in it, if we haven't gone down too far if(depth > 0) { create_test_data_subdirs(protocol, subdirid, "dir_One", depth - 1, rRefCount); create_test_data_subdirs(protocol, subdirid, "dir_Two", depth - 1, rRefCount); } // Stick some files in it create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol, rRefCount); create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol, rRefCount); create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol, rRefCount); return subdirid; } void check_dir_after_uploads(BackupProtocolClient &protocol, const StreamableMemBlock &Attributes) { // Command std::auto_ptr dirreply(protocol.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); TEST_THAT(dirreply->GetObjectID() == BackupProtocolListDirectory::RootDirectory); // Stream BackupStoreDirectory dir(protocol.ReceiveStream()); TEST_THAT(dir.GetNumberOfEntries() == UPLOAD_NUM + 1 /* for the first test file */); TEST_THAT(!dir.HasAttributes()); // Check them! BackupStoreDirectory::Iterator i(dir); // Discard first BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); for(int t = 0; t < UPLOAD_NUM; ++t) { en = i.Next(); TEST_THAT(en != 0); TEST_THAT(en->GetName() == uploads[t].name); TEST_THAT(en->GetObjectID() == uploads[t].allocated_objid); TEST_THAT(en->GetModificationTime() == uploads[t].mod_time); int correct_flags = BackupProtocolListDirectory::Flags_File; if(uploads[t].should_be_old_version) correct_flags |= BackupProtocolListDirectory::Flags_OldVersion; if(uploads[t].delete_file) correct_flags |= BackupProtocolListDirectory::Flags_Deleted; TEST_THAT(en->GetFlags() == correct_flags); if(t == UPLOAD_ATTRS_EN) { TEST_THAT(en->HasAttributes()); TEST_THAT(en->GetAttributesHash() == 32498749832475LL); TEST_THAT(en->GetAttributes() == Attributes); } else { // No attributes on this one TEST_THAT(!en->HasAttributes()); } } en = i.Next(); TEST_THAT(en == 0); } typedef struct { int objectsNotDel; int deleted; int old; } recursive_count_objects_results; void recursive_count_objects_r(BackupProtocolClient &protocol, int64_t id, recursive_count_objects_results &results) { // Command std::auto_ptr dirreply(protocol.QueryListDirectory( id, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(protocol.ReceiveStream()); // Check them! BackupStoreDirectory::Iterator i(dir); // Discard first BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { if((en->GetFlags() & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) == 0) results.objectsNotDel++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) results.deleted++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) results.old++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) { recursive_count_objects_r(protocol, en->GetObjectID(), results); } } } void recursive_count_objects(const char *hostname, int64_t id, recursive_count_objects_results &results) { // Context TLSContext context; context.Initialise(false /* client */, "testfiles/clientCerts.pem", "testfiles/clientPrivKey.pem", "testfiles/clientTrustedCAs.pem"); // Get a connection SocketStreamTLS connReadOnly; connReadOnly.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED_TEST); BackupProtocolClient protocolReadOnly(connReadOnly); { std::auto_ptr serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); std::auto_ptr loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); } // Count objects recursive_count_objects_r(protocolReadOnly, id, results); // Close it protocolReadOnly.QueryFinished(); } bool check_block_index(const char *encoded_file, IOStream &rBlockIndex) { // Open file, and move to the right position FileStream enc(encoded_file); BackupStoreFile::MoveStreamPositionToBlockIndex(enc); bool same = true; // Now compare the two... while(enc.StreamDataLeft()) { char buffer1[2048]; char buffer2[2048]; int s = enc.Read(buffer1, sizeof(buffer1)); if(rBlockIndex.Read(buffer2, s) != s) { same = false; break; } if(::memcmp(buffer1, buffer2, s) != 0) { same = false; break; } } if(rBlockIndex.StreamDataLeft()) { same = false; // Absorb all this excess data so procotol is in the first state char buffer[2048]; while(rBlockIndex.StreamDataLeft()) { rBlockIndex.Read(buffer, sizeof(buffer)); } } return same; } bool check_files_same(const char *f1, const char *f2) { // Open file, and move to the right position FileStream f1s(f1); FileStream f2s(f2); bool same = true; // Now compare the two... while(f1s.StreamDataLeft()) { char buffer1[2048]; char buffer2[2048]; int s = f1s.Read(buffer1, sizeof(buffer1)); if(f2s.Read(buffer2, s) != s) { same = false; break; } if(::memcmp(buffer1, buffer2, s) != 0) { same = false; break; } } if(f2s.StreamDataLeft()) { same = false; } return same; } std::auto_ptr get_raid_file(int64_t ObjectID) { std::string filename; StoreStructure::MakeObjectFilename(ObjectID, "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, filename, false /* EnsureDirectoryExists */); return RaidFileRead::Open(0, filename); } bool check_num_files(int files, int old, int deleted, int dirs); bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, int Deleted, int Dirs, int Total); bool change_account_limits(const char* soft, const char* hard) { std::string errs; std::auto_ptr config( Configuration::LoadAndVerify ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); BackupStoreAccountsControl control(*config); int result = control.SetLimit(0x01234567, soft, hard); TEST_EQUAL(0, result); return (result == 0); } void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protocolReadOnly) { int encfile[ENCFILE_SIZE]; { for(int l = 0; l < ENCFILE_SIZE; ++l) { encfile[l] = l * 173; } // Write this to a file { FileStream f("testfiles/file1", O_WRONLY | O_CREAT); f.Write(encfile, sizeof(encfile)); } } // Read the root directory a few times (as it's cached, so make sure it doesn't hurt anything) for(int l = 0; l < 3; ++l) { // Command std::auto_ptr dirreply(protocol.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir; std::auto_ptr dirstream(protocol.ReceiveStream()); dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); TEST_THAT(dir.GetNumberOfEntries() == 0); } // Read the dir from the readonly connection (make sure it gets in the cache) { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir; std::auto_ptr dirstream(protocolReadOnly.ReceiveStream()); dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); TEST_THAT(dir.GetNumberOfEntries() == 0); } std::string root_dir_fn; StoreStructure::MakeObjectFilename( BackupProtocolListDirectory::RootDirectory, "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, root_dir_fn, false /* EnsureDirectoryExists */); std::auto_ptr storedFile(RaidFileRead::Open(0, root_dir_fn)); int root_dir_blocks = storedFile->GetDiscUsageInBlocks(); TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, root_dir_blocks)); // Store a file -- first make the encoded file BackupStoreFilenameClear store1name("file1"); { FileStream out("testfiles/file1_upload1", O_WRONLY | O_CREAT); std::auto_ptr encoded(BackupStoreFile::EncodeFile("testfiles/file1", BackupProtocolListDirectory::RootDirectory, store1name)); encoded->CopyStreamTo(out); } // printf("SKIPPING\n"); // goto skip; { // Then send it int64_t store1objid = 0; { std::auto_ptr upload(new FileStream("testfiles/file1_upload1")); std::auto_ptr stored(protocol.QueryStoreFile( BackupProtocolListDirectory::RootDirectory, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ store1name, upload)); store1objid = stored->GetObjectID(); TEST_THAT(store1objid == 2); } set_refcount(store1objid, 1); // And retrieve it { // Retrieve as object std::auto_ptr getfile(protocol.QueryGetObject(store1objid)); TEST_THAT(getfile->GetObjectID() == store1objid); // BLOCK { // Get stream std::auto_ptr filestream(protocol.ReceiveStream()); // Need to put it in another stream, because it's not in stream order CollectInBufferStream f; filestream->CopyStreamTo(f); f.SetForReading(); // Get and decode UNLINK_IF_EXISTS("testfiles/file1_upload_retrieved"); BackupStoreFile::DecodeFile(f, "testfiles/file1_upload_retrieved", IOStream::TimeOutInfinite); } // Retrieve as file std::auto_ptr getobj(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, store1objid)); TEST_THAT(getobj->GetObjectID() == store1objid); // BLOCK { UNLINK_IF_EXISTS("testfiles/file1_upload_retrieved_str"); // Get stream std::auto_ptr filestream(protocol.ReceiveStream()); // Get and decode BackupStoreFile::DecodeFile(*filestream, "testfiles/file1_upload_retrieved_str", IOStream::TimeOutInfinite); } // Read in rebuilt original, and compare contents { FileStream in("testfiles/file1_upload_retrieved"); int encfile_i[ENCFILE_SIZE]; in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } { FileStream in("testfiles/file1_upload_retrieved_str"); int encfile_i[ENCFILE_SIZE]; in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } // Retrieve the block index, by ID { std::auto_ptr getblockindex(protocol.QueryGetBlockIndexByID(store1objid)); TEST_THAT(getblockindex->GetObjectID() == store1objid); std::auto_ptr blockIndexStream(protocol.ReceiveStream()); // Check against uploaded file TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream)); } // and again, by name { std::auto_ptr getblockindex(protocol.QueryGetBlockIndexByName(BackupProtocolListDirectory::RootDirectory, store1name)); TEST_THAT(getblockindex->GetObjectID() == store1objid); std::auto_ptr blockIndexStream(protocol.ReceiveStream()); // Check against uploaded file TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream)); } } // Get the directory again, and see if the entry is in it { // Command std::auto_ptr dirreply(protocol.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(protocol.ReceiveStream()); TEST_THAT(dir.GetNumberOfEntries() == 1); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); TEST_THAT(i.Next() == 0); if(en != 0) { TEST_THAT(en->GetName() == store1name); TEST_THAT(en->GetModificationTime() == 0x123456789abcdefLL); TEST_THAT(en->GetAttributesHash() == 0x7362383249872dfLL); TEST_THAT(en->GetObjectID() == store1objid); TEST_THAT(en->GetSizeInBlocks() < ((ENCFILE_SIZE * 4 * 3) / 2 / 2048)+2); TEST_THAT(en->GetFlags() == BackupStoreDirectory::Entry::Flags_File); } } std::string file1_fn; StoreStructure::MakeObjectFilename(store1objid, "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, file1_fn, false /* EnsureDirectoryExists */); storedFile = RaidFileRead::Open(0, file1_fn); int file1_blocks = storedFile->GetDiscUsageInBlocks(); TEST_THAT(check_num_blocks(protocol, file1_blocks, 0, 0, root_dir_blocks, file1_blocks + root_dir_blocks)); // Try using GetFile on a directory { TEST_CHECK_THROWS(std::auto_ptr getFile(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::RootDirectory)), ConnectionException, Conn_Protocol_UnexpectedReply); } } void init_context(TLSContext& rContext) { rContext.Initialise(false /* client */, "testfiles/clientCerts.pem", "testfiles/clientPrivKey.pem", "testfiles/clientTrustedCAs.pem"); } std::auto_ptr open_conn(const char *hostname, TLSContext& rContext) { init_context(rContext); std::auto_ptr conn(new SocketStreamTLS); conn->Open(rContext, Socket::TypeINET, hostname, BOX_PORT_BBSTORED_TEST); return conn; } std::auto_ptr test_server_login(const char *hostname, TLSContext& rContext, std::auto_ptr& rapConn) { rapConn = open_conn(hostname, rContext); // Make a protocol std::auto_ptr protocol(new BackupProtocolClient(*rapConn)); // Check the version std::auto_ptr serverVersion( protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login std::auto_ptr loginConf( protocol->QueryLogin(0x01234567, 0)); return protocol; } bool check_num_files(int files, int old, int deleted, int dirs) { std::auto_ptr apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); TEST_EQUAL_LINE(files, apInfo->GetNumCurrentFiles(), "current files"); TEST_EQUAL_LINE(old, apInfo->GetNumOldFiles(), "old files"); TEST_EQUAL_LINE(deleted, apInfo->GetNumDeletedFiles(), "deleted files"); TEST_EQUAL_LINE(dirs, apInfo->GetNumDirectories(), "directories"); return (files == apInfo->GetNumCurrentFiles() && old == apInfo->GetNumOldFiles() && deleted == apInfo->GetNumDeletedFiles() && dirs == apInfo->GetNumDirectories()); } bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, int Deleted, int Dirs, int Total) { std::auto_ptr usage = Client.QueryGetAccountUsage2(); TEST_EQUAL_LINE(Total, usage->GetBlocksUsed(), "wrong BlocksUsed"); TEST_EQUAL_LINE(Current, usage->GetBlocksInCurrentFiles(), "wrong BlocksInCurrentFiles"); TEST_EQUAL_LINE(Old, usage->GetBlocksInOldFiles(), "wrong BlocksInOldFiles"); TEST_EQUAL_LINE(Deleted, usage->GetBlocksInDeletedFiles(), "wrong BlocksInDeletedFiles"); TEST_EQUAL_LINE(Dirs, usage->GetBlocksInDirectories(), "wrong BlocksInDirectories"); return (Total == usage->GetBlocksUsed() && Current == usage->GetBlocksInCurrentFiles() && Old == usage->GetBlocksInOldFiles() && Deleted == usage->GetBlocksInDeletedFiles() && Dirs == usage->GetBlocksInDirectories()); } bool check_account(Log::Level log_level) { int errors_fixed; { Logging::Guard guard(log_level); Logging::Tagger tag("check fix", true); Logging::ShowTagOnConsole show; std::string errs; std::auto_ptr config( Configuration::LoadAndVerify("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); BackupStoreAccountsControl control(*config); errors_fixed = control.CheckAccount(0x01234567, true, // FixErrors false); // Quiet } TEST_EQUAL_LINE(0, errors_fixed, "store errors found and fixed"); return (errors_fixed == 0); } int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount) { std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount); int discSet = rAccount.GetDiscSet(); // Do housekeeping on this account HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir, discSet, NULL); housekeeping.DoHousekeeping(true /* keep trying forever */); return housekeeping.GetErrorCount(); } // Run housekeeping (for which we need to disconnect ourselves) and check // that it doesn't change the numbers of files. // // Also check that bbstoreaccounts doesn't change anything bool run_housekeeping_and_check_account() { int error_count; { Logging::Tagger tag("", true); Logging::ShowTagOnConsole show; std::auto_ptr apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); error_count = run_housekeeping(account); } TEST_EQUAL_LINE(0, error_count, "housekeeping errors"); bool check_account_is_ok = check_account(); TEST_THAT(check_account_is_ok); return error_count == 0 && check_account_is_ok; } int64_t create_directory(BackupProtocolCallable& protocol) { // Create a directory BackupStoreFilenameClear dirname("lovely_directory"); // Attributes std::auto_ptr attr(new MemBlockStream(attr1, sizeof(attr1))); std::auto_ptr dirCreate(protocol.QueryCreateDirectory( BACKUPSTORE_ROOT_DIRECTORY_ID, 9837429842987984LL, dirname, attr)); int64_t subdirid = dirCreate->GetObjectID(); set_refcount(subdirid, 1); return subdirid; } int64_t create_file(BackupProtocolCallable& protocol, int64_t subdirid) { // Stick a file in it write_test_file(0); std::string filename("testfiles/test0"); int64_t modtime; std::auto_ptr upload(BackupStoreFile::EncodeFile(filename, subdirid, uploads[0].name, &modtime)); std::auto_ptr stored(protocol.QueryStoreFile( subdirid, modtime, modtime, /* use for attr hash too */ 0, /* diff from ID */ uploads[0].name, upload)); int64_t subdirfileid = stored->GetObjectID(); set_refcount(subdirfileid, 1); return subdirfileid; } int test_server(const char *hostname) { TLSContext context; std::auto_ptr conn; std::auto_ptr apProtocol = test_server_login(hostname, context, conn); { R250 r(3465657); for(int l = 0; l < ATTR1_SIZE; ++l) {attr1[l] = r.next();} for(int l = 0; l < ATTR2_SIZE; ++l) {attr2[l] = r.next();} for(int l = 0; l < ATTR3_SIZE; ++l) {attr3[l] = r.next();} } // BLOCK { // Get it logging FILE *protocolLog = ::fopen("testfiles/protocol.log", "w"); TEST_THAT(protocolLog != 0); apProtocol->SetLogToFile(protocolLog); #ifndef WIN32 // Check that we can't open a new connection which requests write permissions { SocketStreamTLS conn; conn.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED_TEST); BackupProtocolClient protocol(conn); std::auto_ptr serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); TEST_CHECK_THROWS(std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, 0)), ConnectionException, Conn_Protocol_UnexpectedReply); protocol.QueryFinished(); } #endif // Set the client store marker apProtocol->QuerySetClientStoreMarker(0x8732523ab23aLL); #ifndef WIN32 // Open a new connection which is read only SocketStreamTLS connReadOnly; connReadOnly.Open(context, Socket::TypeINET, hostname, BOX_PORT_BBSTORED_TEST); BackupProtocolClient protocolReadOnly(connReadOnly); // Get it logging FILE *protocolReadOnlyLog = ::fopen("testfiles/protocolReadOnly.log", "w"); TEST_THAT(protocolReadOnlyLog != 0); protocolReadOnly.SetLogToFile(protocolReadOnlyLog); { std::auto_ptr serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); std::auto_ptr loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); // Check client store marker TEST_THAT(loginConf->GetClientStoreMarker() == 0x8732523ab23aLL); } #else // WIN32 #define protocolReadOnly (*apProtocol) #endif test_server_1(*apProtocol, protocolReadOnly); apProtocol->QueryFinished(); TEST_THAT(check_num_files(1, 0, 0, 1)); TEST_THAT(run_housekeeping_and_check_account()); TEST_THAT(check_num_files(1, 0, 0, 1)); apProtocol = test_server_login("localhost", context, conn); // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); // Create and upload some test files int64_t maxID = 0; for(int t = 0; t < UPLOAD_NUM; ++t) { write_test_file(t); std::string filename("testfiles/test"); filename += uploads[t].fnextra; int64_t modtime = 0; std::auto_ptr upload(BackupStoreFile::EncodeFile(filename.c_str(), BackupProtocolListDirectory::RootDirectory, uploads[t].name, &modtime)); TEST_THAT(modtime != 0); std::auto_ptr stored(apProtocol->QueryStoreFile( BackupProtocolListDirectory::RootDirectory, modtime, modtime, /* use it for attr hash too */ 0, /* diff from ID */ uploads[t].name, upload)); uploads[t].allocated_objid = stored->GetObjectID(); uploads[t].mod_time = modtime; if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID(); set_refcount(stored->GetObjectID(), 1); BOX_TRACE("wrote file " << filename << " to server " "as object " << BOX_FORMAT_OBJECTID(stored->GetObjectID())); // Some of the uploaded files replace old ones, increasing // the old file count instead of the current file count. int expected_num_old_files = 0; if (t >= 8) expected_num_old_files++; if (t >= 12) expected_num_old_files++; if (t >= 13) expected_num_old_files++; int expected_num_current_files = t + 2 - expected_num_old_files; TEST_THAT(check_num_files(expected_num_current_files, expected_num_old_files, 0, 1)); apProtocol->QueryFinished(); TEST_THAT(run_housekeeping_and_check_account()); apProtocol = test_server_login("localhost", context, conn); TEST_THAT(check_num_files(expected_num_current_files, expected_num_old_files, 0, 1)); } // Add some attributes onto one of them { TEST_THAT(check_num_files(UPLOAD_NUM + 1, 0, 0, 1)); std::auto_ptr attrnew( new MemBlockStream(attr3, sizeof(attr3))); std::auto_ptr set(apProtocol->QuerySetReplacementFileAttributes( BackupProtocolListDirectory::RootDirectory, 32498749832475LL, uploads[UPLOAD_ATTRS_EN].name, attrnew)); TEST_THAT(set->GetObjectID() == uploads[UPLOAD_ATTRS_EN].allocated_objid); TEST_THAT(check_num_files(UPLOAD_NUM + 1, 0, 0, 1)); } // Delete one of them (will implicitly delete an old version) { std::auto_ptr del(apProtocol->QueryDeleteFile( BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_DELETE_EN].name)); TEST_THAT(del->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid); TEST_THAT(check_num_files(UPLOAD_NUM, 0, 1, 1)); } // Check that the block index can be obtained by name even though it's been deleted { // Fetch the raw object { FileStream out("testfiles/downloaddelobj", O_WRONLY | O_CREAT); std::auto_ptr getobj(apProtocol->QueryGetObject(uploads[UPLOAD_DELETE_EN].allocated_objid)); std::auto_ptr objstream(apProtocol->ReceiveStream()); objstream->CopyStreamTo(out); } // query index and test std::auto_ptr getblockindex(apProtocol->QueryGetBlockIndexByName( BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_DELETE_EN].name)); TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid); std::auto_ptr blockIndexStream(apProtocol->ReceiveStream()); TEST_THAT(check_block_index("testfiles/downloaddelobj", *blockIndexStream)); } // Download them all... (even deleted files) for(int t = 0; t < UPLOAD_NUM; ++t) { printf("%d\n", t); std::auto_ptr getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, uploads[t].allocated_objid)); TEST_THAT(getFile->GetObjectID() == uploads[t].allocated_objid); std::auto_ptr filestream(apProtocol->ReceiveStream()); test_test_file(t, *filestream); } { StreamableMemBlock attrtest(attr3, sizeof(attr3)); // Use the read only connection to verify that the directory is as we expect printf("\n\n==== Reading directory using read-only connection\n"); check_dir_after_uploads(protocolReadOnly, attrtest); printf("done.\n\n"); // And on the read/write one check_dir_after_uploads(*apProtocol, attrtest); } // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); // Check diffing and rsync like stuff... // Build a modified file { // Basically just insert a bit in the middle TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING) == TEST_FILE_FOR_PATCHING_SIZE); FileStream in(TEST_FILE_FOR_PATCHING); void *buf = ::malloc(TEST_FILE_FOR_PATCHING_SIZE); FileStream out(TEST_FILE_FOR_PATCHING ".mod", O_WRONLY | O_CREAT); TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_PATCH_AT); out.Write(buf, TEST_FILE_FOR_PATCHING_PATCH_AT); char insert[13] = "INSERTINSERT"; out.Write(insert, sizeof(insert)); TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT); out.Write(buf, TEST_FILE_FOR_PATCHING_SIZE - TEST_FILE_FOR_PATCHING_PATCH_AT); ::free(buf); } TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 1, 1)); // Run housekeeping (for which we need to disconnect // ourselves) and check that it doesn't change the numbers // of files apProtocol->QueryFinished(); std::auto_ptr apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); run_housekeeping(account); // Also check that bbstoreaccounts doesn't change anything TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf check 01234567 fix") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apProtocol = test_server_login(hostname, context, conn); TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 1, 1)); { // Fetch the block index for this one std::auto_ptr getblockindex(apProtocol->QueryGetBlockIndexByName( BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_PATCH_EN].name)); TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_PATCH_EN].allocated_objid); std::auto_ptr blockIndexStream(apProtocol->ReceiveStream()); // Do the patching bool isCompletelyDifferent = false; int64_t modtime; std::auto_ptr patchstream( BackupStoreFile::EncodeFileDiff( TEST_FILE_FOR_PATCHING ".mod", BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_PATCH_EN].name, uploads[UPLOAD_PATCH_EN].allocated_objid, *blockIndexStream, IOStream::TimeOutInfinite, NULL, // pointer to DiffTimer impl &modtime, &isCompletelyDifferent)); TEST_THAT(isCompletelyDifferent == false); // Sent this to a file, so we can check the size, rather than uploading it directly { FileStream patch(TEST_FILE_FOR_PATCHING ".patch", O_WRONLY | O_CREAT); patchstream->CopyStreamTo(patch); } // Make sure the stream is a plausible size for a patch containing only one new block TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING ".patch") < (8*1024)); // Upload it int64_t patchedID = 0; { std::auto_ptr uploadpatch(new FileStream(TEST_FILE_FOR_PATCHING ".patch")); std::auto_ptr stored(apProtocol->QueryStoreFile( BackupProtocolListDirectory::RootDirectory, modtime, modtime, /* use it for attr hash too */ uploads[UPLOAD_PATCH_EN].allocated_objid, /* diff from ID */ uploads[UPLOAD_PATCH_EN].name, uploadpatch)); TEST_THAT(stored->GetObjectID() > 0); if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID(); patchedID = stored->GetObjectID(); } set_refcount(patchedID, 1); // Then download it to check it's OK std::auto_ptr getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, patchedID)); TEST_THAT(getFile->GetObjectID() == patchedID); std::auto_ptr filestream(apProtocol->ReceiveStream()); BackupStoreFile::DecodeFile(*filestream, TEST_FILE_FOR_PATCHING ".downloaded", IOStream::TimeOutInfinite); // Check it's the same TEST_THAT(check_files_same(TEST_FILE_FOR_PATCHING ".downloaded", TEST_FILE_FOR_PATCHING ".mod")); TEST_THAT(check_num_files(UPLOAD_NUM - 3, 4, 1, 1)); } // Create a directory int64_t subdirid = create_directory(*apProtocol); TEST_THAT(check_num_files(UPLOAD_NUM - 3, 4, 1, 2)); // Stick a file in it int64_t subdirfileid = create_file(*apProtocol, subdirid); TEST_THAT(check_num_files(UPLOAD_NUM - 3, 4, 1, 2)); printf("\n==== Checking upload using read-only connection\n"); // Check the directories on the read only connection { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); // Stream BackupStoreDirectory dir(protocolReadOnly.ReceiveStream()); TEST_EQUAL(UPLOAD_NUM + 3, dir.GetNumberOfEntries()); // for the first test file, the patched upload, and this new dir. // Check the last one... BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; BackupStoreDirectory::Entry *t = 0; BackupStoreFilenameClear dirname("lovely_directory"); while((t = i.Next()) != 0) { if(en != 0) { // here for all but last object TEST_THAT(en->GetObjectID() != subdirid); TEST_THAT(en->GetName() != dirname); } en = t; } // Does it look right? TEST_THAT(en->GetName() == dirname); TEST_THAT(en->GetFlags() == BackupProtocolListDirectory::Flags_Dir); TEST_THAT(en->GetObjectID() == subdirid); TEST_THAT(en->GetModificationTime() == 0); // dirs don't have modification times. } { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( subdirid, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, true /* get attributes */)); TEST_THAT(dirreply->GetObjectID() == subdirid); BackupStoreDirectory dir(protocolReadOnly.ReceiveStream()); TEST_THAT(dir.GetNumberOfEntries() == 1); // Check the last one... BackupStoreDirectory::Iterator i(dir); // Discard first BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); // Does it look right? TEST_THAT(en->GetName() == uploads[0].name); TEST_THAT(en->GetFlags() == BackupProtocolListDirectory::Flags_File); TEST_THAT(en->GetObjectID() == subdirfileid); TEST_THAT(en->GetModificationTime() != 0); // Attributes TEST_THAT(dir.HasAttributes()); TEST_THAT(dir.GetAttributesModTime() == 9837429842987984LL); StreamableMemBlock attr(attr1, sizeof(attr1)); TEST_THAT(dir.GetAttributes() == attr); } printf("done.\n\n"); // Check that we don't get attributes if we don't ask for them { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( subdirid, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); // Stream BackupStoreDirectory dir(protocolReadOnly.ReceiveStream()); TEST_THAT(!dir.HasAttributes()); } // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); // Change attributes on the directory { std::auto_ptr attrnew( new MemBlockStream(attr2, sizeof(attr2))); std::auto_ptr changereply(apProtocol->QueryChangeDirAttributes( subdirid, 329483209443598LL, attrnew)); TEST_THAT(changereply->GetObjectID() == subdirid); } // Check the new attributes { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( subdirid, 0, // no flags BackupProtocolListDirectory::Flags_EXCLUDE_EVERYTHING, true /* get attributes */)); // Stream BackupStoreDirectory dir(protocolReadOnly.ReceiveStream()); TEST_THAT(dir.GetNumberOfEntries() == 0); // Attributes TEST_THAT(dir.HasAttributes()); TEST_THAT(dir.GetAttributesModTime() == 329483209443598LL); StreamableMemBlock attrtest(attr2, sizeof(attr2)); TEST_THAT(dir.GetAttributes() == attrtest); } // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); // Test moving a file { BackupStoreFilenameClear newName("moved-files"); std::auto_ptr rep(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, BackupProtocolListDirectory::RootDirectory, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName)); TEST_THAT(rep->GetObjectID() == uploads[UPLOAD_FILE_TO_MOVE].allocated_objid); } // Try some dodgy renames { BackupStoreFilenameClear newName("moved-files"); TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, BackupProtocolListDirectory::RootDirectory, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName), ConnectionException, Conn_Protocol_UnexpectedReply); TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, subdirid, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName), ConnectionException, Conn_Protocol_UnexpectedReply); } // Rename within a directory { BackupStoreFilenameClear newName("moved-files-x"); apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, subdirid, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName); } // Check it's all gone from the root directory... { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir; std::auto_ptr dirstream(protocolReadOnly.ReceiveStream()); dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); // Read all entries BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) { TEST_THAT(en->GetName() != uploads[UPLOAD_FILE_TO_MOVE].name); } } // Check the old and new versions are in the other directory { BackupStoreFilenameClear lookFor("moved-files-x"); // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( subdirid, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir; std::auto_ptr dirstream(protocolReadOnly.ReceiveStream()); dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); // Check entries BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; bool foundCurrent = false; bool foundOld = false; while((en = i.Next()) != 0) { if(en->GetName() == lookFor) { if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File)) foundCurrent = true; if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion)) foundOld = true; } } TEST_THAT(foundCurrent); TEST_THAT(foundOld); } // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); // make a little bit more of a thing to look at int64_t subsubdirid = 0; int64_t subsubfileid = 0; { BackupStoreFilenameClear nd("sub2"); // Attributes std::auto_ptr attr(new MemBlockStream(attr1, sizeof(attr1))); std::auto_ptr dirCreate(apProtocol->QueryCreateDirectory( subdirid, 9837429842987984LL, nd, attr)); subsubdirid = dirCreate->GetObjectID(); std::auto_ptr upload( new FileStream("testfiles/file1_upload1")); BackupStoreFilenameClear nf("file2"); std::auto_ptr stored(apProtocol->QueryStoreFile( subsubdirid, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ nf, upload)); subsubfileid = stored->GetObjectID(); } set_refcount(subsubdirid, 1); set_refcount(subsubfileid, 1); // Query names -- test that invalid stuff returns not found OK { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(3248972347823478927LL, subsubdirid)); TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(subsubfileid, 2342378424LL)); TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(38947234789LL, 2342378424LL)); TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, 2234342378424LL)); TEST_THAT(nameRep->GetNumNameElements() == 0); } // Query names... first, get info for the file { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(subsubfileid, subsubdirid)); std::auto_ptr namestream(apProtocol->ReceiveStream()); TEST_THAT(nameRep->GetNumNameElements() == 3); TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_File); TEST_THAT(nameRep->GetModificationTime() == 0x123456789abcdefLL); TEST_THAT(nameRep->GetAttributesHash() == 0x7362383249872dfLL); static const char *testnames[] = {"file2","sub2","lovely_directory"}; for(int l = 0; l < nameRep->GetNumNameElements(); ++l) { BackupStoreFilenameClear fn; fn.ReadFromStream(*namestream, 10000); TEST_THAT(fn.GetClearFilename() == testnames[l]); } } // Query names... secondly, for the directory { std::auto_ptr nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, subsubdirid)); std::auto_ptr namestream(apProtocol->ReceiveStream()); TEST_THAT(nameRep->GetNumNameElements() == 2); TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_Dir); static const char *testnames[] = {"sub2","lovely_directory"}; for(int l = 0; l < nameRep->GetNumNameElements(); ++l) { BackupStoreFilenameClear fn; fn.ReadFromStream(*namestream, 10000); TEST_THAT(fn.GetClearFilename() == testnames[l]); } } //} skip: std::auto_ptr apRefCount( BackupStoreRefCountDatabase::Load( apAccounts->GetEntry(0x1234567), true)); // Create some nice recursive directories int64_t dirtodelete = create_test_data_subdirs(*apProtocol, BackupProtocolListDirectory::RootDirectory, "test_delete", 6 /* depth */, *apRefCount); // And delete them { std::auto_ptr dirdel(apProtocol->QueryDeleteDirectory( dirtodelete)); TEST_THAT(dirdel->GetObjectID() == dirtodelete); } // Get the root dir, checking for deleted items { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::Flags_Dir | BackupProtocolListDirectory::Flags_Deleted, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream BackupStoreDirectory dir(protocolReadOnly.ReceiveStream()); // Check there's only that one entry TEST_THAT(dir.GetNumberOfEntries() == 1); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); if(en) { TEST_THAT(en->GetObjectID() == dirtodelete); BackupStoreFilenameClear n("test_delete"); TEST_THAT(en->GetName() == n); } // Then... check everything's deleted test_everything_deleted(protocolReadOnly, dirtodelete); } // Finish the connections #ifndef WIN32 protocolReadOnly.QueryFinished(); #endif apProtocol->QueryFinished(); // Close logs #ifndef WIN32 ::fclose(protocolReadOnlyLog); #endif ::fclose(protocolLog); } return 0; } int test3(int argc, const char *argv[]) { // Now test encoded files // TODO: This test needs to check failure situations as well as everything working, // but this will be saved for the full implementation. int encfile[ENCFILE_SIZE]; { for(int l = 0; l < ENCFILE_SIZE; ++l) { encfile[l] = l * 173; } // Encode and decode a small block (shouldn't be compressed) { #define SMALL_BLOCK_SIZE 251 int encBlockSize = BackupStoreFile::MaxBlockSizeForChunkSize(SMALL_BLOCK_SIZE); TEST_THAT(encBlockSize > SMALL_BLOCK_SIZE); BackupStoreFile::EncodingBuffer encoded; encoded.Allocate(encBlockSize / 8); // make sure reallocation happens // Encode! int encSize = BackupStoreFile::EncodeChunk(encfile, SMALL_BLOCK_SIZE, encoded); // Check the header says it's not been compressed TEST_THAT((encoded.mpBuffer[0] & 1) == 0); // Check the output size has been inflated (no compression) TEST_THAT(encSize > SMALL_BLOCK_SIZE); // Decode it int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(SMALL_BLOCK_SIZE); TEST_THAT(decBlockSize > SMALL_BLOCK_SIZE); uint8_t *decoded = (uint8_t*)malloc(decBlockSize); int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize); TEST_THAT(decSize < decBlockSize); TEST_THAT(decSize == SMALL_BLOCK_SIZE); // Check it came out of the wash the same TEST_THAT(::memcmp(encfile, decoded, SMALL_BLOCK_SIZE) == 0); free(decoded); } // Encode and decode a big block (should be compressed) { int encBlockSize = BackupStoreFile::MaxBlockSizeForChunkSize(ENCFILE_SIZE); TEST_THAT(encBlockSize > ENCFILE_SIZE); BackupStoreFile::EncodingBuffer encoded; encoded.Allocate(encBlockSize / 8); // make sure reallocation happens // Encode! int encSize = BackupStoreFile::EncodeChunk(encfile, ENCFILE_SIZE, encoded); // Check the header says it's compressed TEST_THAT((encoded.mpBuffer[0] & 1) == 1); // Check the output size make it likely that it's compressed (is very compressible data) TEST_THAT(encSize < ENCFILE_SIZE); // Decode it int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(ENCFILE_SIZE); TEST_THAT(decBlockSize > ENCFILE_SIZE); uint8_t *decoded = (uint8_t*)malloc(decBlockSize); int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize); TEST_THAT(decSize < decBlockSize); TEST_THAT(decSize == ENCFILE_SIZE); // Check it came out of the wash the same TEST_THAT(::memcmp(encfile, decoded, ENCFILE_SIZE) == 0); free(decoded); } // The test block to a file { FileStream f("testfiles/testenc1", O_WRONLY | O_CREAT); f.Write(encfile, sizeof(encfile)); } // Encode it { FileStream out("testfiles/testenc1_enc", O_WRONLY | O_CREAT); BackupStoreFilenameClear name("testfiles/testenc1"); std::auto_ptr encoded(BackupStoreFile::EncodeFile("testfiles/testenc1", 32, name)); encoded->CopyStreamTo(out); } // Verify it { FileStream enc("testfiles/testenc1_enc"); TEST_THAT(BackupStoreFile::VerifyEncodedFileFormat(enc) == true); } // Decode it { UNLINK_IF_EXISTS("testfiles/testenc1_orig"); FileStream enc("testfiles/testenc1_enc"); BackupStoreFile::DecodeFile(enc, "testfiles/testenc1_orig", IOStream::TimeOutInfinite); } // Read in rebuilt original, and compare contents { TEST_THAT(TestGetFileSize("testfiles/testenc1_orig") == sizeof(encfile)); FileStream in("testfiles/testenc1_orig"); int encfile_i[ENCFILE_SIZE]; in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } // Check how many blocks it had, and test the stream based interface { FileStream enc("testfiles/testenc1_enc"); std::auto_ptr decoded(BackupStoreFile::DecodeFileStream(enc, IOStream::TimeOutInfinite)); CollectInBufferStream d; decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 971 /* buffer block size */); d.SetForReading(); TEST_THAT(d.GetSize() == sizeof(encfile)); TEST_THAT(memcmp(encfile, d.GetBuffer(), sizeof(encfile)) == 0); TEST_THAT(decoded->GetNumBlocks() == 3); } // Test that the last block in a file, if less than 256 bytes, gets put into the last block { #define FILE_SIZE_JUST_OVER ((4096*2)+58) FileStream f("testfiles/testenc2", O_WRONLY | O_CREAT); f.Write(encfile + 2, FILE_SIZE_JUST_OVER); BackupStoreFilenameClear name("testenc2"); std::auto_ptr encoded(BackupStoreFile::EncodeFile("testfiles/testenc2", 32, name)); CollectInBufferStream e; encoded->CopyStreamTo(e); e.SetForReading(); std::auto_ptr decoded(BackupStoreFile::DecodeFileStream(e, IOStream::TimeOutInfinite)); CollectInBufferStream d; decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 879 /* buffer block size */); d.SetForReading(); TEST_THAT(d.GetSize() == FILE_SIZE_JUST_OVER); TEST_THAT(memcmp(encfile + 2, d.GetBuffer(), FILE_SIZE_JUST_OVER) == 0); TEST_THAT(decoded->GetNumBlocks() == 2); } // Test that reordered streams work too { FileStream enc("testfiles/testenc1_enc"); std::auto_ptr reordered(BackupStoreFile::ReorderFileToStreamOrder(&enc, false)); std::auto_ptr decoded(BackupStoreFile::DecodeFileStream(*reordered, IOStream::TimeOutInfinite)); CollectInBufferStream d; decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 971 /* buffer block size */); d.SetForReading(); TEST_THAT(d.GetSize() == sizeof(encfile)); TEST_THAT(memcmp(encfile, d.GetBuffer(), sizeof(encfile)) == 0); TEST_THAT(decoded->GetNumBlocks() == 3); } #ifndef WIN32 // no symlinks on Win32 // Try out doing this on a symlink { UNLINK_IF_EXISTS("testfiles/testsymlink"); TEST_THAT(::symlink("does/not/exist", "testfiles/testsymlink") == 0); BackupStoreFilenameClear name("testsymlink"); std::auto_ptr encoded(BackupStoreFile::EncodeFile("testfiles/testsymlink", 32, name)); // Can't decode it from the stream, because it's in file order, and doesn't have the // required properties to be able to reorder it. So buffer it... CollectInBufferStream b; encoded->CopyStreamTo(b); b.SetForReading(); // Decode it UNLINK_IF_EXISTS("testfiles/testsymlink_2"); BackupStoreFile::DecodeFile(b, "testfiles/testsymlink_2", IOStream::TimeOutInfinite); } #endif } // Store info { RaidFileWrite::CreateDirectory(0, "test-info"); BackupStoreInfo::CreateNew(76, "test-info/", 0, 3461231233455433LL, 2934852487LL); TEST_CHECK_THROWS(BackupStoreInfo::CreateNew(76, "test-info/", 0, 0, 0), RaidFileException, CannotOverwriteExistingFile); std::auto_ptr info(BackupStoreInfo::Load(76, "test-info/", 0, true)); TEST_CHECK_THROWS(info->Save(), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->ChangeBlocksUsed(1), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(1), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(1), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->RemovedDeletedDirectory(2), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->AddDeletedDirectory(2), BackupStoreException, StoreInfoIsReadOnly); TEST_CHECK_THROWS(info->SetAccountName("hello"), BackupStoreException, StoreInfoIsReadOnly); } { std::auto_ptr info(BackupStoreInfo::Load(76, "test-info/", 0, false)); info->ChangeBlocksUsed(8); info->ChangeBlocksInOldFiles(9); info->ChangeBlocksInDeletedFiles(10); info->ChangeBlocksUsed(-1); info->ChangeBlocksInOldFiles(-4); info->ChangeBlocksInDeletedFiles(-9); TEST_CHECK_THROWS(info->ChangeBlocksUsed(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); TEST_CHECK_THROWS(info->ChangeBlocksInOldFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); TEST_CHECK_THROWS(info->ChangeBlocksInDeletedFiles(-100), BackupStoreException, StoreInfoBlockDeltaMakesValueNegative); info->AddDeletedDirectory(2); info->AddDeletedDirectory(3); info->AddDeletedDirectory(4); info->RemovedDeletedDirectory(3); info->SetAccountName("whee"); TEST_CHECK_THROWS(info->RemovedDeletedDirectory(9), BackupStoreException, StoreInfoDirNotInList); info->Save(); } { std::auto_ptr info(BackupStoreInfo::Load(76, "test-info/", 0, true)); TEST_THAT(info->GetBlocksUsed() == 7); TEST_THAT(info->GetBlocksInOldFiles() == 5); TEST_THAT(info->GetBlocksInDeletedFiles() == 1); TEST_THAT(info->GetBlocksSoftLimit() == 3461231233455433LL); TEST_THAT(info->GetBlocksHardLimit() == 2934852487LL); TEST_THAT(info->GetAccountName() == "whee"); const std::vector &delfiles(info->GetDeletedDirectories()); TEST_THAT(delfiles.size() == 2); TEST_THAT(delfiles[0] == 2); TEST_THAT(delfiles[1] == 4); } //printf("SKIPPINGTESTS---------\n"); //return 0; // Context TLSContext context; context.Initialise(false /* client */, "testfiles/clientCerts.pem", "testfiles/clientPrivKey.pem", "testfiles/clientTrustedCAs.pem"); // First, try logging in without an account having been created... just make sure login fails. std::string cmd = BBSTORED " " + bbstored_args + " testfiles/bbstored.conf"; int pid = LaunchServer(cmd.c_str(), "testfiles/bbstored.pid"); TEST_THAT(pid != -1 && pid != 0); if(pid <= 0) { return 1; } ::sleep(1); TEST_THAT(ServerIsAlive(pid)); // BLOCK { // Open a connection to the server SocketStreamTLS conn; conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED_TEST); // Make a protocol BackupProtocolClient protocol(conn); // Check the version std::auto_ptr serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login TEST_CHECK_THROWS(std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, 0)), ConnectionException, Conn_Protocol_UnexpectedReply); // Finish the connection protocol.QueryFinished(); } // Create an account for the test client TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf create 01234567 0 " "10000B 20000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); TEST_THAT(TestDirExists("testfiles/0_0/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_1/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567")); TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8); // make sure something is written to it std::auto_ptr apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); std::auto_ptr apReferences( BackupStoreRefCountDatabase::Load( apAccounts->GetEntry(0x1234567), true)); TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, apReferences->GetLastObjectIDUsed()); TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) apReferences.reset(); // Test that login fails on a disabled account TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 no") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // BLOCK { // Open a connection to the server SocketStreamTLS conn; conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED_TEST); // Make a protocol BackupProtocolClient protocol(conn); // Check the version std::auto_ptr serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login TEST_CHECK_THROWS(std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, 0)), ConnectionException, Conn_Protocol_UnexpectedReply); int type, subType; TEST_EQUAL_LINE(true, protocol.GetLastError(type, subType), "expected a protocol error"); TEST_EQUAL_LINE(BackupProtocolError::ErrorType, type, "expected a BackupProtocolError"); TEST_EQUAL_LINE(BackupProtocolError::Err_DisabledAccount, subType, "expected an Err_DisabledAccount"); // Finish the connection protocol.QueryFinished(); } // Re-enable the account so that subsequent logins should succeed TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 yes") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Delete the refcount database and try to log in again. Check that // we're locked out of the account until housekeeping has recreated // the refcount db. TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); std::auto_ptr conn(new SocketStreamTLS); TEST_CHECK_THROWS(test_server_login("localhost", context, conn), ConnectionException, Conn_TLSReadFailed); TEST_THAT(ServerIsAlive(pid)); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); run_housekeeping(account); // Check that housekeeping fixed the ref counts apReferences = BackupStoreRefCountDatabase::Load(account, true); TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, apReferences->GetLastObjectIDUsed()); TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) apReferences.reset(); TEST_THAT(ServerIsAlive(pid)); TEST_THAT(test_server("localhost") == 0); // test that all object reference counts have the // expected values apReferences = BackupStoreRefCountDatabase::Load(account, true); set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); TEST_EQUAL(ExpectedRefCounts.size() - 1, apReferences->GetLastObjectIDUsed()); for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID; i < ExpectedRefCounts.size(); i++) { TEST_EQUAL_LINE(ExpectedRefCounts[i], apReferences->GetRefCount(i), "object " << BOX_FORMAT_OBJECTID(i)); } // Delete the refcount database again, and let // housekeeping recreate it and fix the ref counts. // This could also happen after upgrade, if a housekeeping // runs before the user logs in. apReferences.reset(); TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); run_housekeeping(account); apReferences = BackupStoreRefCountDatabase::Load(account, true); TEST_EQUAL(ExpectedRefCounts.size() - 1, apReferences->GetLastObjectIDUsed()); for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID; i < ExpectedRefCounts.size(); i++) { TEST_EQUAL_LINE(ExpectedRefCounts[i], apReferences->GetRefCount(i), "object " << BOX_FORMAT_OBJECTID(i)); } // Test the deletion of objects by the housekeeping system // First, things as they are now. recursive_count_objects_results before = {0,0,0}; recursive_count_objects("localhost", BackupProtocolListDirectory::RootDirectory, before); TEST_THAT(before.objectsNotDel != 0); TEST_THAT(before.deleted != 0); TEST_THAT(before.old != 0); // Kill it TEST_THAT(KillServer(pid)); ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); #ifdef WIN32 TEST_THAT(unlink("testfiles/bbstored.pid") == 0); #else TestRemoteProcessMemLeaks("bbstored.memleaks"); #endif // Set a new limit on the account -- leave the hard limit // high to make sure the target for freeing space is the // soft limit. TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf setlimit 01234567 " "10B 20000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Start things up pid = LaunchServer(BBSTORED " testfiles/bbstored.conf", "testfiles/bbstored.pid"); ::sleep(1); TEST_THAT(ServerIsAlive(pid)); // wait for housekeeping to happen printf("waiting for housekeeping:\n"); for(int l = 0; l < 30; ++l) { ::sleep(1); printf("."); fflush(stdout); } printf("\n"); // Count the objects again recursive_count_objects_results after = {0,0,0}; recursive_count_objects("localhost", BackupProtocolListDirectory::RootDirectory, after); // If these tests fail then try increasing the timeout above TEST_EQUAL(before.objectsNotDel, after.objectsNotDel); TEST_EQUAL(0, after.deleted); TEST_EQUAL(0, after.old); // Set a really small hard limit TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf setlimit 01234567 " "10B 20B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // Try to upload a file and create a directory, and check an error is generated { // Open a connection to the server SocketStreamTLS conn; conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED_TEST); // Make a protocol BackupProtocolClient protocol(conn); // Check the version std::auto_ptr serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, 0)); int64_t modtime = 0; BackupStoreFilenameClear fnx("exceed-limit"); std::auto_ptr upload(BackupStoreFile::EncodeFile("testfiles/test3", BackupProtocolListDirectory::RootDirectory, fnx, &modtime)); TEST_THAT(modtime != 0); TEST_CHECK_THROWS(std::auto_ptr stored(protocol.QueryStoreFile( BackupProtocolListDirectory::RootDirectory, modtime, modtime, /* use it for attr hash too */ 0, /* diff from ID */ fnx, upload)), ConnectionException, Conn_Protocol_UnexpectedReply); std::auto_ptr attr(new MemBlockStream(&modtime, sizeof(modtime))); BackupStoreFilenameClear fnxd("exceed-limit-dir"); TEST_CHECK_THROWS(std::auto_ptr dirCreate( protocol.QueryCreateDirectory( BackupProtocolListDirectory::RootDirectory, 9837429842987984LL, fnxd, attr)), ConnectionException, Conn_Protocol_UnexpectedReply); // Finish the connection protocol.QueryFinished(); } // Kill it again TEST_THAT(KillServer(pid)); ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); #ifdef WIN32 TEST_THAT(unlink("testfiles/bbstored.pid") == 0); #else TestRemoteProcessMemLeaks("bbstored.memleaks"); #endif return 0; } int multi_server() { printf("Starting server for connection from remote machines...\n"); // Create an account for the test client TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf create 01234567 0 " "30000B 40000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // First, try logging in without an account having been created... just make sure login fails. int pid = LaunchServer(BBSTORED " testfiles/bbstored_multi.conf", "testfiles/bbstored.pid"); TEST_THAT(pid != -1 && pid != 0); if(pid > 0) { ::sleep(1); TEST_THAT(ServerIsAlive(pid)); // Wait for a keypress printf("Press ENTER to terminate the server\n"); char line[512]; fgets(line, 512, stdin); printf("Terminating server...\n"); // Kill it TEST_THAT(KillServer(pid)); ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); #ifdef WIN32 TEST_THAT(unlink("testfiles/bbstored.pid") == 0); #else TestRemoteProcessMemLeaks("bbstored.memleaks"); #endif } return 0; } void test_open_files_with_limited_win32_permissions() { #ifdef WIN32 // this had better work, or bbstored will die when combining diffs char* file = "foo"; DWORD accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY | FILE_READ_EA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_DATA | FILE_WRITE_EA /*| FILE_ALL_ACCESS*/; DWORD shareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; HANDLE h1 = CreateFileA(file, accessRights, shareMode, NULL, OPEN_ALWAYS, FILE_FLAG_BACKUP_SEMANTICS, NULL); assert(h1 != INVALID_HANDLE_VALUE); TEST_THAT(h1 != INVALID_HANDLE_VALUE); accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY | FILE_READ_EA; HANDLE h2 = CreateFileA(file, accessRights, shareMode, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); assert(h2 != INVALID_HANDLE_VALUE); TEST_THAT(h2 != INVALID_HANDLE_VALUE); CloseHandle(h2); CloseHandle(h1); h1 = openfile(file, O_CREAT | O_RDWR, 0); TEST_THAT(h1 != INVALID_HANDLE_VALUE); h2 = openfile(file, O_RDWR, 0); TEST_THAT(h2 != INVALID_HANDLE_VALUE); CloseHandle(h2); CloseHandle(h1); #endif } void compare_backupstoreinfo_values_to_expected ( const std::string& test_phase, const info_StreamFormat_1& expected, const BackupStoreInfo& actual, const std::string& expected_account_name, bool expected_account_enabled, const MemBlockStream& extra_data = MemBlockStream(/* empty */) ) { TEST_EQUAL_LINE(ntohl(expected.mAccountID), actual.GetAccountID(), test_phase << " AccountID"); #define TEST_INFO_EQUAL(property) \ TEST_EQUAL_LINE(box_ntoh64(expected.m ## property), \ actual.Get ## property (), test_phase << " " #property) TEST_INFO_EQUAL(ClientStoreMarker); TEST_INFO_EQUAL(LastObjectIDUsed); TEST_INFO_EQUAL(BlocksUsed); TEST_INFO_EQUAL(BlocksInOldFiles); TEST_INFO_EQUAL(BlocksInDeletedFiles); TEST_INFO_EQUAL(BlocksInDirectories); TEST_INFO_EQUAL(BlocksSoftLimit); TEST_INFO_EQUAL(BlocksHardLimit); #undef TEST_INFO_EQUAL // These attributes of the v2 structure are not supported by v1, // so they should all be initialised to 0 TEST_EQUAL_LINE(0, actual.GetBlocksInCurrentFiles(), test_phase << " BlocksInCurrentFiles"); TEST_EQUAL_LINE(0, actual.GetNumOldFiles(), test_phase << " NumOldFiles"); TEST_EQUAL_LINE(0, actual.GetNumDeletedFiles(), test_phase << " NumDeletedFiles"); TEST_EQUAL_LINE(0, actual.GetNumDirectories(), test_phase << " NumDirectories"); // These attributes of the old v1 structure are not actually loaded // or used: // TEST_INFO_EQUAL(CurrentMarkNumber); // TEST_INFO_EQUAL(OptionsPresent); TEST_EQUAL_LINE(box_ntoh64(expected.mNumberDeletedDirectories), actual.GetDeletedDirectories().size(), test_phase << " number of deleted directories"); for (size_t i = 0; i < box_ntoh64(expected.mNumberDeletedDirectories) && i < actual.GetDeletedDirectories().size(); i++) { TEST_EQUAL_LINE(13 + i, actual.GetDeletedDirectories()[i], test_phase << " deleted directory " << (i+1)); } TEST_EQUAL_LINE(expected_account_name, actual.GetAccountName(), test_phase << " AccountName"); TEST_EQUAL_LINE(expected_account_enabled, actual.IsAccountEnabled(), test_phase << " AccountEnabled"); TEST_EQUAL_LINE(extra_data.GetSize(), actual.GetExtraData().GetSize(), test_phase << " extra data has wrong size"); TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), actual.GetExtraData().GetBuffer(), extra_data.GetSize()), test_phase << " extra data has wrong contents"); } int test_read_old_backupstoreinfo_files() { // Create an account for the test client TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf create 01234567 0 " "10000B 20000B") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); std::auto_ptr apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), "'bbstoreaccounts create' should have set AccountEnabled flag"); info_StreamFormat_1 info_v1; info_v1.mMagicValue = htonl(INFO_MAGIC_VALUE_1); info_v1.mAccountID = htonl(0x01234567); info_v1.mClientStoreMarker = box_hton64(3); info_v1.mLastObjectIDUsed = box_hton64(4); info_v1.mBlocksUsed = box_hton64(5); info_v1.mBlocksInOldFiles = box_hton64(6); info_v1.mBlocksInDeletedFiles = box_hton64(7); info_v1.mBlocksInDirectories = box_hton64(8); info_v1.mBlocksSoftLimit = box_hton64(9); info_v1.mBlocksHardLimit = box_hton64(10); info_v1.mCurrentMarkNumber = htonl(11); info_v1.mOptionsPresent = htonl(12); info_v1.mNumberDeletedDirectories = box_hton64(2); // Then mNumberDeletedDirectories * int64_t IDs for the deleted directories // Generate the filename std::string info_filename("backup/01234567/" INFO_FILENAME); std::auto_ptr rfw(new RaidFileWrite(0, info_filename)); rfw->Open(/* AllowOverwrite = */ true); rfw->Write(&info_v1, sizeof(info_v1)); // Write mNumberDeletedDirectories * int64_t IDs for the deleted directories std::auto_ptr apArchive(new Archive(*rfw, IOStream::TimeOutInfinite)); apArchive->Write((int64_t) 13); apArchive->Write((int64_t) 14); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded from v1", info_v1, *apInfo, "" /* no name by default */, true /* enabled by default */); apInfo->SetAccountName("bonk"); // Save the info again apInfo->Save(/* allowOverwrite */ true); // Check that it was saved in the new Archive format std::auto_ptr rfr(RaidFileRead::Open(0, info_filename, 0)); int32_t magic; if(!rfr->ReadFullBuffer(&magic, sizeof(magic), 0)) { THROW_FILE_ERROR("Failed to read store info file: " "short read of magic number", info_filename, BackupStoreException, CouldNotLoadStoreInfo); } TEST_EQUAL_LINE(INFO_MAGIC_VALUE_2, ntohl(magic), "format version in newly saved BackupStoreInfo"); rfr.reset(); // load it, and check that all values are loaded properly apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded in v1, resaved in v2", info_v1, *apInfo, "bonk", true); // Check that the new AccountEnabled flag is saved properly apInfo->SetAccountEnabled(false); apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("saved in v2, loaded in v2", info_v1, *apInfo, "bonk", false /* as modified above */); apInfo->SetAccountEnabled(true); apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ true); compare_backupstoreinfo_values_to_expected("resaved in v2 with " "account enabled", info_v1, *apInfo, "bonk", true /* as modified above */); // Now save the info in v2 format without the AccountEnabled flag // (boxbackup 0.11 format) and check that the flag is set to true // for backwards compatibility rfw.reset(new RaidFileWrite(0, info_filename)); rfw->Open(/* allowOverwrite */ true); magic = htonl(INFO_MAGIC_VALUE_2); apArchive.reset(new Archive(*rfw, IOStream::TimeOutInfinite)); rfw->Write(&magic, sizeof(magic)); apArchive->Write(apInfo->GetAccountID()); apArchive->Write(std::string("test")); apArchive->Write(apInfo->GetClientStoreMarker()); apArchive->Write(apInfo->GetLastObjectIDUsed()); apArchive->Write(apInfo->GetBlocksUsed()); apArchive->Write(apInfo->GetBlocksInCurrentFiles()); apArchive->Write(apInfo->GetBlocksInOldFiles()); apArchive->Write(apInfo->GetBlocksInDeletedFiles()); apArchive->Write(apInfo->GetBlocksInDirectories()); apArchive->Write(apInfo->GetBlocksSoftLimit()); apArchive->Write(apInfo->GetBlocksHardLimit()); apArchive->Write(apInfo->GetNumCurrentFiles()); apArchive->Write(apInfo->GetNumOldFiles()); apArchive->Write(apInfo->GetNumDeletedFiles()); apArchive->Write(apInfo->GetNumDirectories()); apArchive->Write((int64_t) apInfo->GetDeletedDirectories().size()); apArchive->Write(apInfo->GetDeletedDirectories()[0]); apArchive->Write(apInfo->GetDeletedDirectories()[1]); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("saved in v2 without " "AccountEnabled", info_v1, *apInfo, "test", true); // Default for missing AccountEnabled should be true // Rewrite using full length, so that the first 4 bytes of extra data // doesn't get swallowed by "extra data". apInfo->Save(/* allowOverwrite */ true); // Append some extra data after the known account values, to simulate a // new addition to the store format. Check that this extra data is loaded // and resaved with the info file. We made the mistake of deleting it in // 0.11, let's not make the same mistake again. CollectInBufferStream info_data; rfr = RaidFileRead::Open(0, info_filename, 0); rfr->CopyStreamTo(info_data); rfr.reset(); info_data.SetForReading(); rfw.reset(new RaidFileWrite(0, info_filename)); rfw->Open(/* allowOverwrite */ true); info_data.CopyStreamTo(*rfw); char extra_string[] = "hello!"; MemBlockStream extra_data(extra_string, strlen(extra_string)); extra_data.CopyStreamTo(*rfw); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); TEST_EQUAL_LINE(extra_data.GetSize(), apInfo->GetExtraData().GetSize(), "wrong amount of extra data loaded from info file"); TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), apInfo->GetExtraData().GetBuffer(), extra_data.GetSize()), "extra data loaded from info file has wrong contents"); // Save the file and load again, check that the extra data is still there apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); compare_backupstoreinfo_values_to_expected("saved in future format " "with extra_data", info_v1, *apInfo, "test", true, extra_data); // Check that the new bbstoreaccounts command sets the flag properly TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 no") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); TEST_EQUAL_LINE(false, apInfo->IsAccountEnabled(), "'bbstoreaccounts disabled no' should have reset AccountEnabled flag"); TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf enabled 01234567 yes") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), "'bbstoreaccounts disabled yes' should have set AccountEnabled flag"); // Check that BackupStoreInfo::CreateForRegeneration saves all the // expected properties, including any extra data for forward // compatibility extra_data.Seek(0, IOStream::SeekType_Absolute); apInfo = BackupStoreInfo::CreateForRegeneration( apInfo->GetAccountID(), "spurtle" /* rAccountName */, "backup/01234567/" /* rRootDir */, 0 /* DiscSet */, apInfo->GetLastObjectIDUsed(), apInfo->GetBlocksUsed(), apInfo->GetBlocksInCurrentFiles(), apInfo->GetBlocksInOldFiles(), apInfo->GetBlocksInDeletedFiles(), apInfo->GetBlocksInDirectories(), apInfo->GetBlocksSoftLimit(), apInfo->GetBlocksHardLimit(), false /* AccountEnabled */, extra_data); // CreateForRegeneration always sets the ClientStoreMarker to 0 info_v1.mClientStoreMarker = 0; // CreateForRegeneration does not store any deleted directories info_v1.mNumberDeletedDirectories = 0; // check that the store info has the correct values in memory compare_backupstoreinfo_values_to_expected("stored by " "BackupStoreInfo::CreateForRegeneration", info_v1, *apInfo, "spurtle", false /* AccountEnabled */, extra_data); // Save the file and load again, check that the extra data is still there apInfo->Save(/* allowOverwrite */ true); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); compare_backupstoreinfo_values_to_expected("saved by " "BackupStoreInfo::CreateForRegeneration and reloaded", info_v1, *apInfo, "spurtle", false /* AccountEnabled */, extra_data); // Delete the account to leave the store in the same state as before apInfo.reset(); TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf delete 01234567 yes") == 0); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); return 0; } int test(int argc, const char *argv[]) { test_open_files_with_limited_win32_permissions(); // Initialise the raid file controller RaidFileController &rcontroller = RaidFileController::GetController(); rcontroller.Initialise("testfiles/raidfile.conf"); int ret = test_read_old_backupstoreinfo_files(); if (ret != 0) { return ret; } // SSL library SSLLib::Initialise(); // Give a test key for the filenames // BackupStoreFilenameClear::SetBlowfishKey(FilenameEncodingKey, sizeof(FilenameEncodingKey)); // And set the encoding to blowfish // BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish); // And for directory attributes -- need to set it, as used in file encoding // BackupClientFileAttributes::SetBlowfishKey(AttributesEncodingKey, sizeof(AttributesEncodingKey)); // And finally for file encoding // BackupStoreFile::SetBlowfishKeys(FileEncodingKey, sizeof(FileEncodingKey), FileBlockEntryEncodingKey, sizeof(FileBlockEntryEncodingKey)); // Use the setup crypto command to set up all these keys, so that the bbackupquery command can be used // for seeing what's going on. BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); // encode in some filenames -- can't do static initialisation // because the key won't be set up when these are initialised { MEMLEAKFINDER_NO_LEAKS for(unsigned int l = 0; l < sizeof(ens_filenames) / sizeof(ens_filenames[0]); ++l) { ens[l].fn = BackupStoreFilenameClear(ens_filenames[l]); } for(unsigned int l = 0; l < sizeof(uploads_filenames) / sizeof(uploads_filenames[0]); ++l) { uploads[l].name = BackupStoreFilenameClear(uploads_filenames[l]); } } // Trace errors out SET_DEBUG_SSLLIB_TRACE_ERRORS if(argc == 2 && strcmp(argv[1], "server") == 0) { return multi_server(); } if(argc == 3 && strcmp(argv[1], "client") == 0) { return test_server(argv[2]); } // large file test /* { int64_t modtime = 0; std::auto_ptr upload(BackupStoreFile::EncodeFile("/Users/ben/temp/large.tar", BackupProtocolListDirectory::RootDirectory, uploads[0].name, &modtime)); TEST_THAT(modtime != 0); FileStream write("testfiles/large.enc", O_WRONLY | O_CREAT); upload->CopyStreamTo(write); } printf("SKIPPING TESTS ------------------------------------------------------\n"); return 0;*/ int r = 0; r = test1(argc, argv); if(r != 0) return r; r = test2(argc, argv); if(r != 0) return r; r = test3(argc, argv); if(r != 0) return r; return 0; }