diff options
| -rw-r--r-- | lib/backupclient/BackupDaemonConfigVerify.cpp | 10 | ||||
| -rw-r--r-- | lib/backupstore/BackupAccountControl.cpp | 127 | ||||
| -rw-r--r-- | lib/backupstore/BackupAccountControl.h | 48 | ||||
| -rw-r--r-- | lib/common/Test.h | 2 | ||||
| -rw-r--r-- | lib/httpserver/HTTPException.txt | 29 | ||||
| -rw-r--r-- | modules.txt | 1 |
6 files changed, 193 insertions, 24 deletions
diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index 8390f2f3..865ee413 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -48,11 +48,11 @@ static const ConfigurationVerifyKey verifyserverkeys[] = static const ConfigurationVerifyKey verifys3keys[] = { // These values are only required for Amazon S3-compatible stores - ConfigurationVerifyKey("BucketName", 0), - ConfigurationVerifyKey("Port", ConfigTest_IsInt, 80), - ConfigurationVerifyKey("BasePath", 0), - ConfigurationVerifyKey("AccessKey", 0), - ConfigurationVerifyKey("SecretKey", ConfigTest_LastEntry) + ConfigurationVerifyKey("HostName", ConfigTest_Exists), + ConfigurationVerifyKey("Port", ConfigTest_Exists | ConfigTest_IsInt, 80), + ConfigurationVerifyKey("BasePath", ConfigTest_Exists), + ConfigurationVerifyKey("AccessKey", ConfigTest_Exists), + ConfigurationVerifyKey("SecretKey", ConfigTest_Exists | ConfigTest_LastEntry) }; static const ConfigurationVerify verifyserver[] = diff --git a/lib/backupstore/BackupAccountControl.cpp b/lib/backupstore/BackupAccountControl.cpp index 6fbbc30e..edb25068 100644 --- a/lib/backupstore/BackupAccountControl.cpp +++ b/lib/backupstore/BackupAccountControl.cpp @@ -12,9 +12,13 @@ #include <climits> #include <iostream> +#include "autogen_CommonException.h" #include "BackupAccountControl.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" #include "BackupStoreInfo.h" #include "Configuration.h" +#include "HTTPResponse.h" #include "MemLeakFindOn.h" @@ -129,3 +133,126 @@ int BackupAccountControl::PrintAccountInfo(const BackupStoreInfo& info, return 0; } +S3BackupAccountControl::S3BackupAccountControl(const Configuration& config, + bool machineReadableOutput) +: BackupAccountControl(config, machineReadableOutput) +{ + if(!mConfig.SubConfigurationExists("S3Store")) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + InvalidConfiguration, + "The S3Store configuration subsection is required " + "when S3Store mode is enabled"); + } + const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); + + mBasePath = s3config.GetKeyValue("BasePath"); + if(mBasePath.size() == 0) + { + mBasePath = "/"; + } + else + { + if(mBasePath[0] != '/' || mBasePath[mBasePath.size() - 1] != '/') + { + THROW_EXCEPTION_MESSAGE(CommonException, + InvalidConfiguration, + "If S3Store.BasePath is not empty then it must start and " + "end with a slash, e.g. '/subdir/', but it currently does not."); + } + } + + mapS3Client.reset(new S3Client( + s3config.GetKeyValue("HostName"), + s3config.GetKeyValueInt("Port"), + s3config.GetKeyValue("AccessKey"), + s3config.GetKeyValue("SecretKey"))); + + mapFileSystem.reset(new S3BackupFileSystem(mConfig, mBasePath, *mapS3Client)); +} + +std::string S3BackupAccountControl::GetFullURL(const std::string ObjectPath) const +{ + const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); + return std::string("http://") + s3config.GetKeyValue("HostName") + ":" + + s3config.GetKeyValue("Port") + GetFullPath(ObjectPath); +} + +int S3BackupAccountControl::CreateAccount(const std::string& name, int32_t SoftLimit, + int32_t HardLimit) +{ + // Try getting the info file. If we get a 200 response then it already + // exists, and we should bail out. If we get a 404 then it's safe to + // continue. Otherwise something else is wrong and we should bail out. + std::string info_url = GetFullURL(S3_INFO_FILE_NAME); + + HTTPResponse response = GetObject(S3_INFO_FILE_NAME); + if(response.GetResponseCode() != HTTPResponse::Code_NotFound) + { + mapS3Client->CheckResponse(response, std::string("The BackupStoreInfo file already " + "exists at this URL: ") + info_url); + } + + BackupStoreInfo info(0, // fake AccountID for S3 stores + info_url, // FileName, + SoftLimit, HardLimit); + info.SetAccountName(name); + + // And an empty directory + BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + int64_t rootDirSize = mapFileSystem->PutDirectory(rootDir); + + // Update the store info to reflect the size of the root directory + info.ChangeBlocksUsed(rootDirSize); + info.ChangeBlocksInDirectories(rootDirSize); + info.AdjustNumDirectories(1); + int64_t id = info.AllocateObjectID(); + ASSERT(id == BACKUPSTORE_ROOT_DIRECTORY_ID); + + CollectInBufferStream out; + info.Save(out); + out.SetForReading(); + + response = PutObject(S3_INFO_FILE_NAME, out); + mapS3Client->CheckResponse(response, std::string("Failed to upload the new BackupStoreInfo " + "file to this URL: ") + info_url); + + // Now get the file again, to check that it really worked. + response = GetObject(S3_INFO_FILE_NAME); + mapS3Client->CheckResponse(response, std::string("Failed to download the new BackupStoreInfo " + "file that we just created: ") + info_url); + + return 0; +} + +std::string S3BackupFileSystem::GetDirectoryURI(int64_t ObjectID) +{ + std::ostringstream out; + out << mBasePath << "dirs/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir"; + return out.str(); +} + +std::auto_ptr<HTTPResponse> S3BackupFileSystem::GetDirectory(BackupStoreDirectory& rDir) +{ + std::string uri = GetDirectoryURI(rDir.GetObjectID()); + HTTPResponse response = mrClient.GetObject(uri); + mrClient.CheckResponse(response, + std::string("Failed to download directory: ") + uri); + return std::auto_ptr<HTTPResponse>(new HTTPResponse(response)); +} + +int S3BackupFileSystem::PutDirectory(BackupStoreDirectory& rDir) +{ + CollectInBufferStream out; + rDir.WriteToStream(out); + out.SetForReading(); + + std::string uri = GetDirectoryURI(rDir.GetObjectID()); + HTTPResponse response = mrClient.PutObject(uri, out); + mrClient.CheckResponse(response, + std::string("Failed to upload directory: ") + uri); + + int blocks = (out.GetSize() + S3_NOTIONAL_BLOCK_SIZE - 1) / S3_NOTIONAL_BLOCK_SIZE; + return blocks; +} + diff --git a/lib/backupstore/BackupAccountControl.h b/lib/backupstore/BackupAccountControl.h index 27075c99..bc041794 100644 --- a/lib/backupstore/BackupAccountControl.h +++ b/lib/backupstore/BackupAccountControl.h @@ -13,9 +13,11 @@ #include <string> #include "BackupStoreAccountDatabase.h" -#include "S3Client.h" +#include "HTTPResponse.h" #include "NamedLock.h" +#include "S3Client.h" +class BackupStoreDirectory; class BackupStoreInfo; class Configuration; @@ -37,20 +39,56 @@ public: int PrintAccountInfo(const BackupStoreInfo& info, int BlockSize); }; +class S3BackupFileSystem +{ +private: + const Configuration& mConfig; + std::string mBasePath; + S3Client& mrClient; +public: + S3BackupFileSystem(const Configuration& config, const std::string& BasePath, + S3Client& rClient) + : mConfig(config), + mBasePath(BasePath), + mrClient(rClient) + { } + std::string GetDirectoryURI(int64_t ObjectID); + std::auto_ptr<HTTPResponse> GetDirectory(BackupStoreDirectory& rDir); + int PutDirectory(BackupStoreDirectory& rDir); +}; + class S3BackupAccountControl : public BackupAccountControl { +private: + std::string mBasePath; + std::auto_ptr<S3Client> mapS3Client; + std::auto_ptr<S3BackupFileSystem> mapFileSystem; public: S3BackupAccountControl(const Configuration& config, - bool machineReadableOutput = false) - : BackupAccountControl(config, machineReadableOutput) - { } - std::string GetStoreRootURL(); + bool machineReadableOutput = false); + std::string GetFullPath(const std::string ObjectPath) const + { + return mBasePath + ObjectPath; + } + std::string GetFullURL(const std::string ObjectPath) const; int CreateAccount(const std::string& name, int32_t SoftLimit, int32_t HardLimit); int GetBlockSize() { return 4096; } + HTTPResponse GetObject(const std::string& name) + { + return mapS3Client->GetObject(GetFullPath(name)); + } + HTTPResponse PutObject(const std::string& name, IOStream& rStreamToSend, + const char* pContentType = NULL) + { + return mapS3Client->PutObject(GetFullPath(name), rStreamToSend, + pContentType); + } }; // max size of soft limit as percent of hard limit #define MAX_SOFT_LIMIT_SIZE 97 +#define S3_INFO_FILE_NAME "boxbackup.info" +#define S3_NOTIONAL_BLOCK_SIZE 1048576 #endif // BACKUPACCOUNTCONTROL__H diff --git a/lib/common/Test.h b/lib/common/Test.h index f6bbb49d..15a3db6f 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -34,6 +34,8 @@ extern int num_failures; extern int first_fail_line; +extern int num_tests_selected; +extern int old_failure_count; extern std::string first_fail_file; extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; extern std::list<std::string> run_only_named_tests; diff --git a/lib/httpserver/HTTPException.txt b/lib/httpserver/HTTPException.txt index 52630cda..c9b3f940 100644 --- a/lib/httpserver/HTTPException.txt +++ b/lib/httpserver/HTTPException.txt @@ -1,16 +1,17 @@ EXCEPTION HTTP 10 -Internal 0 -RequestReadFailed 1 -RequestAlreadyBeenRead 2 -BadRequest 3 -UnknownResponseCodeUsed 4 -NoContentTypeSet 5 -POSTContentTooLong 6 -CannotSetRedirectIfReponseHasData 7 -CannotSetNotFoundIfReponseHasData 8 -NotImplemented 9 -RequestNotInitialised 10 -BadResponse 11 -ResponseReadFailed 12 -NoStreamConfigured 13 +Internal 0 +RequestReadFailed 1 +RequestAlreadyBeenRead 2 +BadRequest 3 +UnknownResponseCodeUsed 4 +NoContentTypeSet 5 +POSTContentTooLong 6 +CannotSetRedirectIfReponseHasData 7 +CannotSetNotFoundIfReponseHasData 8 +NotImplemented 9 +RequestNotInitialised 10 +BadResponse 11 +ResponseReadFailed 12 +NoStreamConfigured 13 +RequestFailedUnexpectedly 14 The request was expected to succeed, but it failed. diff --git a/modules.txt b/modules.txt index 680dbd67..09cee59b 100644 --- a/modules.txt +++ b/modules.txt @@ -40,6 +40,7 @@ test/backupstorefix bin/bbstored bin/bbstoreaccounts lib/backupstore lib/raidfil test/backupstorepatch bin/bbstored bin/bbstoreaccounts lib/backupclient test/backupdiff lib/backupclient test/bbackupd bin/bbackupd bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupctl lib/server lib/backupstore lib/backupclient lib/intercept +test/s3store lib/backupclient lib/httpserver bin/s3simulator # HTTP server system lib/httpserver lib/server |