diff options
Diffstat (limited to 'lib')
49 files changed, 1410 insertions, 219 deletions
diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp index d896a363..af156a1d 100644 --- a/lib/backupclient/BackupClientFileAttributes.cpp +++ b/lib/backupclient/BackupClientFileAttributes.cpp @@ -78,6 +78,9 @@ typedef struct typedef struct { int32_t uid, gid, mode; + #ifdef WIN32 + int64_t fileCreationTime; + #endif } attributeHashData; // Use default packing @@ -221,12 +224,15 @@ bool BackupClientFileAttributes::operator==(const BackupClientFileAttributes &rA // // Function // Name: BackupClientFileAttributes::Compare(const BackupClientFileAttributes &, bool) -// Purpose: Compare, optionally ignoring the attribute modification time and/or modification time, and some data which is -// irrelevant in practise (eg file generation number) +// Purpose: Compare, optionally ignoring the attribute +// modification time and/or modification time, and some +// data which is irrelevant in practise (eg file +// generation number) // Created: 10/12/03 // // -------------------------------------------------------------------------- -bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, bool IgnoreAttrModTime, bool IgnoreModTime) const +bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr, + bool IgnoreAttrModTime, bool IgnoreModTime) const { EnsureClearAvailable(); rAttr.EnsureClearAvailable(); @@ -234,6 +240,10 @@ bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr // Check sizes are the same, as a first check if(mpClearAttributes->GetSize() != rAttr.mpClearAttributes->GetSize()) { + BOX_TRACE("Attribute Compare: Attributes objects are " + "different sizes, cannot compare them: local " << + mpClearAttributes->GetSize() << " bytes, remote " << + rAttr.mpClearAttributes->GetSize() << " bytes"); return false; } @@ -241,32 +251,51 @@ bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr // Bytes are checked in network order, but this doesn't matter as we're only checking for equality. attr_StreamFormat *a1 = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); attr_StreamFormat *a2 = (attr_StreamFormat*)rAttr.mpClearAttributes->GetBuffer(); - - if(a1->AttributeType != a2->AttributeType - || a1->UID != a2->UID - || a1->GID != a2->GID - || a1->UserDefinedFlags != a2->UserDefinedFlags - || a1->Mode != a2->Mode) - { - return false; + + #define COMPARE(attribute, message) \ + if (a1->attribute != a2->attribute) \ + { \ + BOX_TRACE("Attribute Compare: " << message << " differ: " \ + "local " << ntoh(a1->attribute) << ", " \ + "remote " << ntoh(a2->attribute)); \ + return false; \ } - + COMPARE(AttributeType, "Attribute types"); + COMPARE(UID, "UIDs"); + COMPARE(GID, "GIDs"); + COMPARE(UserDefinedFlags, "User-defined flags"); + COMPARE(Mode, "Modes"); + if(!IgnoreModTime) { - int t1 = a1->ModificationTime / 1000000; - int t2 = a2->ModificationTime / 1000000; - if(t1 != t2) + uint64_t t1 = box_ntoh64(a1->ModificationTime); + uint64_t t2 = box_ntoh64(a2->ModificationTime); + time_t s1 = BoxTimeToSeconds(t1); + time_t s2 = BoxTimeToSeconds(t2); + if(s1 != s2) { + BOX_TRACE("Attribute Compare: File modification " + "times differ: local " << + FormatTime(t1, true) << " (" << s1 << "), " + "remote " << + FormatTime(t2, true) << " (" << s2 << ")"); return false; } } - + if(!IgnoreAttrModTime) { - int t1 = a1->AttrModificationTime / 1000000; - int t2 = a2->AttrModificationTime / 1000000; - if(t1 != t2) + uint64_t t1 = box_ntoh64(a1->AttrModificationTime); + uint64_t t2 = box_ntoh64(a2->AttrModificationTime); + time_t s1 = BoxTimeToSeconds(t1); + time_t s2 = BoxTimeToSeconds(t2); + if(s1 != s2) { + BOX_TRACE("Attribute Compare: Attribute modification " + "times differ: local " << + FormatTime(t1, true) << " (" << s1 << "), " + "remote " << + FormatTime(t2, true) << " (" << s2 << ")"); return false; } } @@ -276,8 +305,16 @@ bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr if(size > sizeof(attr_StreamFormat)) { // Symlink strings don't match. This also compares xattrs - if(::memcmp(a1 + 1, a2 + 1, size - sizeof(attr_StreamFormat)) != 0) + int datalen = size - sizeof(attr_StreamFormat); + + if(::memcmp(a1 + 1, a2 + 1, datalen) != 0) { + std::string s1((char *)(a1 + 1), datalen); + std::string s2((char *)(a2 + 1), datalen); + BOX_TRACE("Attribute Compare: Symbolic link target " + "or extended attributes differ: " + "local " << PrintEscapedBinaryData(s1) << ", " + "remote " << PrintEscapedBinaryData(s2)); return false; } } @@ -292,14 +329,21 @@ bool BackupClientFileAttributes::Compare(const BackupClientFileAttributes &rAttr // -------------------------------------------------------------------------- // // Function -// Name: BackupClientFileAttributes::ReadAttributes(const char *) -// Purpose: Read the attributes of the file, and store them ready for streaming. -// Optionally retrieve the modification time and attribute modification time. +// Name: BackupClientFileAttributes::ReadAttributes( +// const char *Filename, bool ZeroModificationTimes, +// box_time_t *pModTime, box_time_t *pAttrModTime, +// int64_t *pFileSize, InodeRefType *pInodeNumber, +// bool *pHasMultipleLinks) +// Purpose: Read the attributes of the file, and store them +// ready for streaming. Optionally retrieve the +// modification time and attribute modification time. // Created: 2003/10/07 // // -------------------------------------------------------------------------- -void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroModificationTimes, box_time_t *pModTime, - box_time_t *pAttrModTime, int64_t *pFileSize, InodeRefType *pInodeNumber, bool *pHasMultipleLinks) +void BackupClientFileAttributes::ReadAttributes(const char *Filename, + bool ZeroModificationTimes, box_time_t *pModTime, + box_time_t *pAttrModTime, int64_t *pFileSize, + InodeRefType *pInodeNumber, bool *pHasMultipleLinks) { StreamableMemBlock *pnewAttr = 0; try @@ -603,6 +647,62 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc #endif } +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientFileAttributes::GetModificationTimes() +// Purpose: Returns the modification time embedded in the +// attributes. +// Created: 2010/02/24 +// +// -------------------------------------------------------------------------- +void BackupClientFileAttributes::GetModificationTimes( + box_time_t *pModificationTime, + box_time_t *pAttrModificationTime) const +{ + // Got something loaded + if(GetSize() <= 0) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Make sure there are clear attributes to use + EnsureClearAvailable(); + ASSERT(mpClearAttributes != 0); + + // Check if the decrypted attributes are small enough, and the type of attributes stored + if(mpClearAttributes->GetSize() < (int)sizeof(int32_t)) + { + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + int32_t *type = (int32_t*)mpClearAttributes->GetBuffer(); + ASSERT(type != 0); + if(ntohl(*type) != ATTRIBUTETYPE_GENERIC_UNIX) + { + // Don't know what to do with these + THROW_EXCEPTION(BackupStoreException, AttributesNotUnderstood); + } + + // Check there is enough space for an attributes block + if(mpClearAttributes->GetSize() < (int)sizeof(attr_StreamFormat)) + { + // Too small + THROW_EXCEPTION(BackupStoreException, AttributesNotLoaded); + } + + // Get pointer to structure + attr_StreamFormat *pattr = (attr_StreamFormat*)mpClearAttributes->GetBuffer(); + + if(pModificationTime) + { + *pModificationTime = box_ntoh64(pattr->ModificationTime); + } + + if(pAttrModificationTime) + { + *pAttrModificationTime = box_ntoh64(pattr->AttrModificationTime); + } +} // -------------------------------------------------------------------------- // @@ -1032,14 +1132,18 @@ void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int // -------------------------------------------------------------------------- // // Function -// Name: BackupClientFileAttributes::GenerateAttributeHash(struct stat &, const std::string &, const std::string &) -// Purpose: Generate a 64 bit hash from the attributes, used to detect changes. -// Include filename in the hash, so that it changes from one file to another, -// so don't reveal identical attributes. +// Name: BackupClientFileAttributes::GenerateAttributeHash( +// struct stat &, const std::string &, +// const std::string &) +// Purpose: Generate a 64 bit hash from the attributes, used to +// detect changes. Include filename in the hash, so +// that it changes from one file to another, so don't +// reveal identical attributes. // Created: 25/4/04 // // -------------------------------------------------------------------------- -uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname) +uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, + const std::string &filename, const std::string &leafname) { if(sAttributeHashSecretLength == 0) { @@ -1054,6 +1158,16 @@ uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, hashData.gid = htonl(st.st_gid); hashData.mode = htonl(st.st_mode); + #ifdef WIN32 + // On Windows, the "file attribute modification time" is the + // file creation time, and we want to back this up, restore + // it and compare it. + // + // On other platforms, it's not very important and can't + // reliably be set to anything other than the current time. + hashData.fileCreationTime = box_hton64(st.st_ctime); + #endif + StreamableMemBlock xattr; FillExtendedAttr(xattr, filename.c_str()); @@ -1062,7 +1176,7 @@ uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, digest.Add(&hashData, sizeof(hashData)); digest.Add(xattr.GetBuffer(), xattr.GetSize()); digest.Add(leafname.c_str(), leafname.size()); - digest.Add(sAttributeHashSecret, sAttributeHashSecretLength); + digest.Add(sAttributeHashSecret, sAttributeHashSecretLength); digest.Finish(); // Return the first 64 bits of the hash diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h index b32c14dd..f9a0d883 100644 --- a/lib/backupclient/BackupClientFileAttributes.h +++ b/lib/backupclient/BackupClientFileAttributes.h @@ -47,7 +47,9 @@ public: InodeRefType *pInodeNumber = 0, bool *pHasMultipleLinks = 0); void WriteAttributes(const char *Filename, bool MakeUserWritable = false) const; - + void GetModificationTimes(box_time_t *pModificationTime, + box_time_t *pAttrModificationTime) const; + bool IsSymLink() const; static void SetBlowfishKey(const void *pKey, int KeyLength); diff --git a/lib/backupclient/BackupStoreException.txt b/lib/backupclient/BackupStoreException.txt index 50c615b2..528a8c94 100644 --- a/lib/backupclient/BackupStoreException.txt +++ b/lib/backupclient/BackupStoreException.txt @@ -68,3 +68,4 @@ SignalReceived 64 A signal was received by the process, restart or terminate IncompatibleFromAndDiffFiles 65 Attempt to use a diff and a from file together, when they're not related DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file must be in the same directory PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. +UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. diff --git a/lib/backupclient/Makefile.extra b/lib/backupclient/Makefile.extra index 925c880e..df3319df 100644 --- a/lib/backupclient/Makefile.extra +++ b/lib/backupclient/Makefile.extra @@ -5,12 +5,12 @@ GEN_CMD_SRV = $(MAKEPROTOCOL) Client ../../bin/bbstored/backupprotocol.txt # AUTOGEN SEEDING autogen_BackupProtocolClient.cpp autogen_BackupProtocolClient.h: $(MAKEPROTOCOL) ../../bin/bbstored/backupprotocol.txt - $(PERL) $(GEN_CMD_SRV) + $(_PERL) $(GEN_CMD_SRV) MAKEEXCEPTION = ../../lib/common/makeexception.pl # AUTOGEN SEEDING autogen_BackupStoreException.h autogen_BackupStoreException.cpp: $(MAKEEXCEPTION) BackupStoreException.txt - $(PERL) $(MAKEEXCEPTION) BackupStoreException.txt + $(_PERL) $(MAKEEXCEPTION) BackupStoreException.txt diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp index 46cab68f..72a813d5 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.cpp +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -289,7 +289,8 @@ bool BackupStoreAccountDatabase::EntryExists(int32_t ID) const // Created: 2003/08/21 // // -------------------------------------------------------------------------- -const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(int32_t ID) const +BackupStoreAccountDatabase::Entry BackupStoreAccountDatabase::GetEntry( + int32_t ID) const { // Check that we're using the latest version of the database CheckUpToDate(); @@ -311,12 +312,14 @@ const BackupStoreAccountDatabase::Entry &BackupStoreAccountDatabase::GetEntry(in // Created: 2003/08/21 // // -------------------------------------------------------------------------- -void BackupStoreAccountDatabase::AddEntry(int32_t ID, int DiscSet) +BackupStoreAccountDatabase::Entry BackupStoreAccountDatabase::AddEntry( + int32_t ID, int DiscSet) { // Check that we're using the latest version of the database CheckUpToDate(); pImpl->mDatabase[ID] = Entry(ID, DiscSet); + return pImpl->mDatabase[ID]; } diff --git a/lib/backupstore/BackupStoreAccountDatabase.h b/lib/backupstore/BackupStoreAccountDatabase.h index 8d6e7ad8..79573242 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.h +++ b/lib/backupstore/BackupStoreAccountDatabase.h @@ -55,8 +55,8 @@ public: }; bool EntryExists(int32_t ID) const; - const Entry &GetEntry(int32_t ID) const; - void AddEntry(int32_t ID, int DiscSet); + Entry GetEntry(int32_t ID) const; + Entry AddEntry(int32_t ID, int DiscSet); void DeleteEntry(int32_t ID); // This interface should change in the future. But for now it'll do. diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp index eb10b385..5c7e4d38 100644 --- a/lib/backupstore/BackupStoreAccounts.cpp +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -14,6 +14,7 @@ #include "BoxPortsAndFiles.h" #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" +#include "BackupStoreRefCountDatabase.h" #include "RaidFileWrite.h" #include "BackupStoreInfo.h" #include "BackupStoreDirectory.h" @@ -61,6 +62,10 @@ BackupStoreAccounts::~BackupStoreAccounts() // -------------------------------------------------------------------------- void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, int64_t SizeHardLimit, const std::string &rAsUsername) { + // Create the entry in the database + BackupStoreAccountDatabase::Entry Entry(mrDatabase.AddEntry(ID, + DiscSet)); + { // Become the user specified in the config file? std::auto_ptr<UnixUser> user; @@ -100,15 +105,17 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, // Save it back info->Save(); + + // Create the refcount database + BackupStoreRefCountDatabase::CreateNew(Entry); + std::auto_ptr<BackupStoreRefCountDatabase> refcount( + BackupStoreRefCountDatabase::Load(Entry, false)); + refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID); } // As the original user... - - // Create the entry in the database - mrDatabase.AddEntry(ID, DiscSet); - // Write the database back - mrDatabase.Write(); + mrDatabase.Write(); } @@ -138,7 +145,7 @@ void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, i // Created: 2003/08/21 // // -------------------------------------------------------------------------- -std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) const +std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) { char accid[64]; // big enough! ::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID); diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h index 0c3dd103..224d7353 100644 --- a/lib/backupstore/BackupStoreAccounts.h +++ b/lib/backupstore/BackupStoreAccounts.h @@ -12,7 +12,7 @@ #include <string> -class BackupStoreAccountDatabase; +#include "BackupStoreAccountDatabase.h" // -------------------------------------------------------------------------- // @@ -35,9 +35,14 @@ public: bool AccountExists(int32_t ID); void GetAccountRoot(int32_t ID, std::string &rRootDirOut, int &rDiscSetOut) const; + static std::string GetAccountRoot(const + BackupStoreAccountDatabase::Entry &rEntry) + { + return MakeAccountRootDir(rEntry.GetID(), rEntry.GetDiscSet()); + } private: - std::string MakeAccountRootDir(int32_t ID, int DiscSet) const; + static std::string MakeAccountRootDir(int32_t ID, int DiscSet); private: BackupStoreAccountDatabase &mrDatabase; diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp new file mode 100644 index 00000000..f6db2ca4 --- /dev/null +++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp @@ -0,0 +1,321 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreRefCountDatabase.cpp +// Purpose: Backup store object reference count database storage +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <algorithm> + +#include "BackupStoreRefCountDatabase.h" +#include "BackupStoreException.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreAccounts.h" +#include "RaidFileController.h" +#include "RaidFileUtil.h" +#include "RaidFileException.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +#define REFCOUNT_MAGIC_VALUE 0x52656643 // RefC +#define REFCOUNT_FILENAME "refcount" + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::BackupStoreRefCountDatabase() +// Purpose: Default constructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const + BackupStoreAccountDatabase::Entry& rAccount) +: mAccount(rAccount), + mFilename(GetFilename(rAccount)), + mReadOnly(true), + mIsModified(false) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase +// Purpose: Destructor +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase() +{ +} + +std::string BackupStoreRefCountDatabase::GetFilename(const + BackupStoreAccountDatabase::Entry& rAccount) +{ + std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount); + ASSERT(RootDir[RootDir.size() - 1] == '/' || + RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); + + std::string fn(RootDir + "refcount.db"); + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); + return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Create(int32_t, +// const std::string &, int, bool) +// Purpose: Create a new database, overwriting an existing +// one only if AllowOverwrite is true. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +void BackupStoreRefCountDatabase::Create(const + BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite) +{ + // Initial header + refcount_StreamFormat hdr; + hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE); + hdr.mAccountID = htonl(rAccount.GetID()); + + // Generate the filename + std::string Filename = GetFilename(rAccount); + + // Open the file for writing + if (FileExists(Filename) && !AllowOverwrite) + { + BOX_ERROR("Attempted to overwrite refcount database file: " << + Filename); + THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile); + } + + int flags = O_CREAT | O_BINARY | O_RDWR; + if (!AllowOverwrite) + { + flags |= O_EXCL; + } + + std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags)); + + // Write header + DatabaseFile->Write(&hdr, sizeof(hdr)); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Load(int32_t AccountID, +// BackupStoreAccountDatabase& rAccountDatabase, +// bool ReadOnly); +// Purpose: Loads the info from disc, given the root +// information. Can be marked as read only. +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( + const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly) +{ + // Generate the filename + std::string filename = GetFilename(rAccount); + int flags = ReadOnly ? O_RDONLY : O_RDWR; + + // Open the file for read/write + std::auto_ptr<FileStream> dbfile(new FileStream(filename, + flags | O_BINARY)); + + // Read in a header + refcount_StreamFormat hdr; + if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) + { + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) + } + + // Check it + if(ntohl(hdr.mMagicValue) != REFCOUNT_MAGIC_VALUE || + (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) + { + THROW_EXCEPTION(BackupStoreException, BadStoreInfoOnLoad) + } + + // Make new object + std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount)); + + // Put in basic location info + refcount->mReadOnly = ReadOnly; + refcount->mapDatabaseFile = dbfile; + + // return it to caller + return refcount; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::Save() +// Purpose: Save modified info back to disc +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- +/* +void BackupStoreRefCountDatabase::Save() +{ + // Make sure we're initialised (although should never come to this) + if(mFilename.empty() || mAccount.GetID() == 0) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Can we do this? + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) + } + + // Then... open a write file + RaidFileWrite rf(mAccount.GetDiscSet(), mFilename); + rf.Open(true); // allow overwriting + + // Make header + info_StreamFormat hdr; + hdr.mMagicValue = htonl(INFO_MAGIC_VALUE); + hdr.mAccountID = htonl(mAccountID); + hdr.mClientStoreMarker = box_hton64(mClientStoreMarker); + hdr.mLastObjectIDUsed = box_hton64(mLastObjectIDUsed); + hdr.mBlocksUsed = box_hton64(mBlocksUsed); + hdr.mBlocksInOldFiles = box_hton64(mBlocksInOldFiles); + hdr.mBlocksInDeletedFiles = box_hton64(mBlocksInDeletedFiles); + hdr.mBlocksInDirectories = box_hton64(mBlocksInDirectories); + hdr.mBlocksSoftLimit = box_hton64(mBlocksSoftLimit); + hdr.mBlocksHardLimit = box_hton64(mBlocksHardLimit); + hdr.mCurrentMarkNumber = 0; + hdr.mOptionsPresent = 0; + hdr.mNumberDeletedDirectories = box_hton64(mDeletedDirectories.size()); + + // Write header + rf.Write(&hdr, sizeof(hdr)); + + // Write the deleted object list + if(mDeletedDirectories.size() > 0) + { + int64_t objs[NUM_DELETED_DIRS_BLOCK]; + + int tosave = mDeletedDirectories.size(); + std::vector<int64_t>::iterator i(mDeletedDirectories.begin()); + while(tosave > 0) + { + // How many in this one? + int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); + + // Add them + for(int t = 0; t < b; ++t) + { + ASSERT(i != mDeletedDirectories.end()); + objs[t] = box_hton64((*i)); + i++; + } + + // Write + rf.Write(objs, b * sizeof(int64_t)); + + // Number saved + tosave -= b; + } + } + + // Commit it to disc, converting it to RAID now + rf.Commit(true); + + // Mark is as not modified + mIsModified = false; +} +*/ + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreRefCountDatabase::GetRefCount(int64_t +// ObjectID) +// Purpose: Get the number of references to the specified object +// out of the database +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- +BackupStoreRefCountDatabase::refcount_t +BackupStoreRefCountDatabase::GetRefCount(int64_t ObjectID) const +{ + IOStream::pos_type offset = GetOffset(ObjectID); + + if (GetSize() < offset + GetEntrySize()) + { + BOX_ERROR("attempted read of unknown refcount for object " << + BOX_FORMAT_OBJECTID(ObjectID)); + THROW_EXCEPTION(BackupStoreException, + UnknownObjectRefCountRequested); + } + + mapDatabaseFile->Seek(offset, SEEK_SET); + + refcount_t refcount; + if (mapDatabaseFile->Read(&refcount, sizeof(refcount)) != + sizeof(refcount)) + { + BOX_LOG_SYS_ERROR("short read on refcount database: " << + mFilename); + THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo); + } + + return ntohl(refcount); +} + +int64_t BackupStoreRefCountDatabase::GetLastObjectIDUsed() const +{ + return (GetSize() - sizeof(refcount_StreamFormat)) / + sizeof(refcount_t); +} + +void BackupStoreRefCountDatabase::AddReference(int64_t ObjectID) +{ + refcount_t refcount; + + if (ObjectID > GetLastObjectIDUsed()) + { + // new object, assume no previous references + refcount = 0; + } + else + { + // read previous value from database + refcount = GetRefCount(ObjectID); + } + + refcount++; + + SetRefCount(ObjectID, refcount); +} + +void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID, + refcount_t NewRefCount) +{ + IOStream::pos_type offset = GetOffset(ObjectID); + mapDatabaseFile->Seek(offset, SEEK_SET); + refcount_t RefCountNetOrder = htonl(NewRefCount); + mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder)); +} + +bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) +{ + refcount_t refcount = GetRefCount(ObjectID); // must exist in database + ASSERT(refcount > 0); + refcount--; + SetRefCount(ObjectID, refcount); + return (refcount > 0); +} + diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h new file mode 100644 index 00000000..93c79afb --- /dev/null +++ b/lib/backupstore/BackupStoreRefCountDatabase.h @@ -0,0 +1,128 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreRefCountDatabase.h +// Purpose: Main backup store information storage +// Created: 2003/08/28 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPSTOREREFCOUNTDATABASE__H +#define BACKUPSTOREREFCOUNTDATABASE__H + +#include <memory> +#include <string> +#include <vector> + +#include "BackupStoreAccountDatabase.h" +#include "FileStream.h" + +class BackupStoreCheck; +class BackupStoreContext; + +// set packing to one byte +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "BeginStructPackForWire.h" +#else +BEGIN_STRUCTURE_PACKING_FOR_WIRE +#endif + +typedef struct +{ + uint32_t mMagicValue; // also the version number + uint32_t mAccountID; +} refcount_StreamFormat; + +// Use default packing +#ifdef STRUCTURE_PACKING_FOR_WIRE_USE_HEADERS +#include "EndStructPackForWire.h" +#else +END_STRUCTURE_PACKING_FOR_WIRE +#endif + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreRefCountDatabase +// Purpose: Backup store reference count database storage +// Created: 2009/06/01 +// +// -------------------------------------------------------------------------- +class BackupStoreRefCountDatabase +{ + friend class BackupStoreCheck; + friend class BackupStoreContext; + friend class HousekeepStoreAccount; + +public: + ~BackupStoreRefCountDatabase(); +private: + // Creation through static functions only + BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry& + rAccount); + // No copying allowed + BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &); + +public: + // Create a new database for a new account. This method will refuse + // to overwrite any existing file. + static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount) + { + Create(rAccount, false); + } + + // Load it from the store + static std::auto_ptr<BackupStoreRefCountDatabase> Load(const + BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); + + typedef uint32_t refcount_t; + + // Data access functions + refcount_t GetRefCount(int64_t ObjectID) const; + int64_t GetLastObjectIDUsed() const; + + // Data modification functions + void AddReference(int64_t ObjectID); + // RemoveReference returns false if refcount drops to zero + bool RemoveReference(int64_t ObjectID); + +private: + // Create a new database for an existing account. Used during + // account checking if opening the old database throws an exception. + // This method will overwrite any existing file. + static void CreateForRegeneration(const + BackupStoreAccountDatabase::Entry& rAccount) + { + Create(rAccount, true); + } + + static void Create(const BackupStoreAccountDatabase::Entry& rAccount, + bool AllowOverwrite); + + static std::string GetFilename(const BackupStoreAccountDatabase::Entry& + rAccount); + IOStream::pos_type GetSize() const + { + return mapDatabaseFile->GetPosition() + + mapDatabaseFile->BytesLeftToRead(); + } + IOStream::pos_type GetEntrySize() const + { + return sizeof(refcount_t); + } + IOStream::pos_type GetOffset(int64_t ObjectID) const + { + return ((ObjectID - 1) * GetEntrySize()) + + sizeof(refcount_StreamFormat); + } + void SetRefCount(int64_t ObjectID, refcount_t NewRefCount); + + // Location information + BackupStoreAccountDatabase::Entry mAccount; + std::string mFilename; + bool mReadOnly; + bool mIsModified; + std::auto_ptr<FileStream> mapDatabaseFile; +}; + +#endif // BACKUPSTOREREFCOUNTDATABASE__H diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h index c25ac433..e40224da 100644 --- a/lib/common/BannerText.h +++ b/lib/common/BannerText.h @@ -12,7 +12,7 @@ #define BANNER_TEXT(UtilityName) \ "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and " \ - "contributors 2003-2008" + "contributors 2003-2010" #endif // BANNERTEXT__H diff --git a/lib/common/Box.h b/lib/common/Box.h index 1124a062..158fab7b 100644 --- a/lib/common/Box.h +++ b/lib/common/Box.h @@ -103,19 +103,28 @@ #define THROW_EXCEPTION(type, subtype) \ { \ - OPTIONAL_DO_BACKTRACE \ - BOX_WARNING("Exception thrown: " #type "(" #subtype ") " \ - "at " __FILE__ "(" << __LINE__ << ")") \ + if(!HideExceptionMessageGuard::ExceptionsHidden()) \ + { \ + OPTIONAL_DO_BACKTRACE \ + BOX_WARNING("Exception thrown: " \ + #type "(" #subtype ") " \ + "at " __FILE__ "(" << __LINE__ << ")") \ + } \ throw type(type::subtype); \ } #define THROW_EXCEPTION_MESSAGE(type, subtype, message) \ { \ - OPTIONAL_DO_BACKTRACE \ - BOX_WARNING("Exception thrown: " #type "(" #subtype ") " \ - " (" message ") at " \ - __FILE__ "(" << __LINE__ << ")") \ - throw type(type::subtype, message); \ + std::ostringstream _box_throw_line; \ + _box_throw_line << message; \ + if(!HideExceptionMessageGuard::ExceptionsHidden()) \ + { \ + OPTIONAL_DO_BACKTRACE \ + BOX_WARNING("Exception thrown: " \ + #type "(" #subtype ") (" << message << \ + ") at " __FILE__ "(" << __LINE__ << ")") \ + } \ + throw type(type::subtype, _box_throw_line.str()); \ } // extra macros for converting to network byte order @@ -154,5 +163,23 @@ inline uint64_t box_swap64(uint64_t x) #define box_ntoh64(x) box_swap64(x) #endif +// overloaded auto-conversion functions +inline uint64_t hton(uint64_t in) { return box_hton64(in); } +inline uint32_t hton(uint32_t in) { return htonl(in); } +inline uint16_t hton(uint16_t in) { return htons(in); } +inline uint8_t hton(uint8_t in) { return in; } +inline int64_t hton(int64_t in) { return box_hton64(in); } +inline int32_t hton(int32_t in) { return htonl(in); } +inline int16_t hton(int16_t in) { return htons(in); } +inline int8_t hton(int8_t in) { return in; } +inline uint64_t ntoh(uint64_t in) { return box_ntoh64(in); } +inline uint32_t ntoh(uint32_t in) { return ntohl(in); } +inline uint16_t ntoh(uint16_t in) { return ntohs(in); } +inline uint8_t ntoh(uint8_t in) { return in; } +inline int64_t ntoh(int64_t in) { return box_ntoh64(in); } +inline int32_t ntoh(int32_t in) { return ntohl(in); } +inline int16_t ntoh(int16_t in) { return ntohs(in); } +inline int8_t ntoh(int8_t in) { return in; } + #endif // BOX__H diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h index c625a7c7..617aa031 100644 --- a/lib/common/BoxPlatform.h +++ b/lib/common/BoxPlatform.h @@ -176,6 +176,14 @@ #include "emu.h" +#ifdef WIN32 + #define INVALID_FILE INVALID_HANDLE_VALUE + typedef HANDLE tOSFileHandle; +#else + #define INVALID_FILE -1 + typedef int tOSFileHandle; +#endif + // Solaris has no dirfd(x) macro or function, and we need one for // intercept tests. We cannot define macros with arguments directly // using AC_DEFINE, so do it here instead of in configure.ac. diff --git a/lib/common/BoxPortsAndFiles.h.in b/lib/common/BoxPortsAndFiles.h.in index e30dcf90..41bad0ba 100644 --- a/lib/common/BoxPortsAndFiles.h.in +++ b/lib/common/BoxPortsAndFiles.h.in @@ -15,6 +15,7 @@ // Backup store daemon #define BOX_PORT_BBSTORED (BOX_PORT_BASE+1) +#define BOX_PORT_BBSTORED_TEST 22011 // directory within the RAIDFILE root for the backup store daemon #define BOX_RAIDFILE_ROOT_BBSTORED "backup" diff --git a/lib/common/FileModificationTime.cpp b/lib/common/FileModificationTime.cpp new file mode 100644 index 00000000..1109b15f --- /dev/null +++ b/lib/common/FileModificationTime.cpp @@ -0,0 +1,64 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: FileModificationTime.cpp +// Purpose: Function for getting file modification time. +// Created: 2010/02/15 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <sys/stat.h> + +#include "BoxTime.h" +#include "FileModificationTime.h" + +#include "MemLeakFindOn.h" + +box_time_t FileModificationTime(EMU_STRUCT_STAT &st) +{ +#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC + box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); +#else + box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#endif + + return datamodified; +} + +box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st) +{ + box_time_t statusmodified = +#ifdef HAVE_STRUCT_STAT_ST_MTIMESPEC + (((int64_t)st.st_ctimespec.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC + (((int64_t)st.st_ctim.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctim.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIMENSEC + (((int64_t)st.st_ctimensec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); +#else // no nanoseconds anywhere + (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); +#endif + + return statusmodified; +} + +box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st) +{ +#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC + box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); + box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL); +#else + box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); + box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#endif + + return (datamodified > statusmodified)?datamodified:statusmodified; +} + diff --git a/lib/common/FileModificationTime.h b/lib/common/FileModificationTime.h index 5f13c015..e6e6c172 100644 --- a/lib/common/FileModificationTime.h +++ b/lib/common/FileModificationTime.h @@ -14,51 +14,9 @@ #include "BoxTime.h" -inline box_time_t FileModificationTime(EMU_STRUCT_STAT &st) -{ -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC - box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); -#else - box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) - + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); -#endif - - return datamodified; -} - -inline box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st) -{ - box_time_t statusmodified = -#ifdef HAVE_STRUCT_STAT_ST_MTIMESPEC - (((int64_t)st.st_ctimespec.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + - (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); -#elif defined HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC - (((int64_t)st.st_ctim.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + - (((int64_t)st.st_ctim.tv_sec) * (MICRO_SEC_IN_SEC_LL)); -#elif defined HAVE_STRUCT_STAT_ST_ATIMENSEC - (((int64_t)st.st_ctimensec) / (NANO_SEC_IN_USEC_LL)) + - (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); -#else // no nanoseconds anywhere - (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); -#endif - - return statusmodified; -} - -inline box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st) -{ -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC - box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); - box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL); -#else - box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) - + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); - box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) - + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); -#endif - - return (datamodified > statusmodified)?datamodified:statusmodified; -} +box_time_t FileModificationTime(EMU_STRUCT_STAT &st); +box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st); +box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st); #endif // FILEMODIFICATIONTIME__H diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp index d6a3c5da..5be8237c 100644 --- a/lib/common/FileStream.cpp +++ b/lib/common/FileStream.cpp @@ -67,19 +67,20 @@ void FileStream::AfterOpen() { MEMLEAKFINDER_NOT_A_LEAK(this); + #ifdef WIN32 + BOX_LOG_WIN_WARNING_NUMBER("Failed to open file: " << + mFileName, winerrno); + #else + BOX_LOG_SYS_WARNING("Failed to open file: " << + mFileName); + #endif + if(errno == EACCES) { THROW_EXCEPTION(CommonException, AccessDenied) } else { - #ifdef WIN32 - BOX_LOG_WIN_WARNING_NUMBER("Failed to open file: " << - mFileName, winerrno); - #else - BOX_LOG_SYS_WARNING("Failed to open file: " << - mFileName); - #endif THROW_EXCEPTION(CommonException, OSFileOpenError) } } diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h index 7c4118cd..9101a968 100644 --- a/lib/common/FileStream.h +++ b/lib/common/FileStream.h @@ -20,14 +20,6 @@ #include <unistd.h> #endif -#ifdef WIN32 - #define INVALID_FILE INVALID_HANDLE_VALUE - typedef HANDLE tOSFileHandle; -#else - #define INVALID_FILE -1 - typedef int tOSFileHandle; -#endif - class FileStream : public IOStream { public: diff --git a/lib/common/InvisibleTempFileStream.cpp b/lib/common/InvisibleTempFileStream.cpp index a7e19ad3..abfcb5f6 100644 --- a/lib/common/InvisibleTempFileStream.cpp +++ b/lib/common/InvisibleTempFileStream.cpp @@ -24,9 +24,9 @@ // -------------------------------------------------------------------------- InvisibleTempFileStream::InvisibleTempFileStream(const char *Filename, int flags, int mode) #ifdef WIN32 - : FileStream(::openfile(Filename, flags | O_TEMPORARY, mode)) + : FileStream(Filename, flags | O_TEMPORARY, mode) #else - : FileStream(::open(Filename, flags, mode)) + : FileStream(Filename, flags, mode) #endif { #ifndef WIN32 diff --git a/lib/common/Logging.cpp b/lib/common/Logging.cpp index 1666b487..296443ea 100644 --- a/lib/common/Logging.cpp +++ b/lib/common/Logging.cpp @@ -33,6 +33,8 @@ bool Logging::sLogToSyslog = false; bool Logging::sLogToConsole = false; bool Logging::sContextSet = false; +bool HideExceptionMessageGuard::sHiddenState = false; + std::vector<Logger*> Logging::sLoggers; std::string Logging::sContext; Console* Logging::spConsole = NULL; @@ -493,3 +495,24 @@ bool FileLogger::Log(Log::Level Level, const std::string& rFile, Logging::Add(this); return true; } + +std::string PrintEscapedBinaryData(const std::string& rInput) +{ + std::ostringstream output; + + for (size_t i = 0; i < rInput.length(); i++) + { + if (isprint(rInput[i])) + { + output << rInput[i]; + } + else + { + output << "\\x" << std::hex << std::setw(2) << + std::setfill('0') << (int) rInput[i] << + std::dec; + } + } + + return output.str(); +} diff --git a/lib/common/Logging.h b/lib/common/Logging.h index 9bb2cf6c..24863d2c 100644 --- a/lib/common/Logging.h +++ b/lib/common/Logging.h @@ -18,17 +18,6 @@ #include "FileStream.h" -/* -#define BOX_LOG(level, stuff) \ -{ \ - if(Log::sMaxLoggingLevelForAnyOutput >= level) \ - std::ostringstream line; \ - line << stuff; \ - Log::Write(level, __FILE__, __LINE__, line.str()); \ - } \ -} -*/ - #define BOX_LOG(level, stuff) \ { \ std::ostringstream _box_log_line; \ @@ -52,12 +41,20 @@ if (Logging::IsEnabled(Log::TRACE)) \ { BOX_LOG(Log::TRACE, stuff) } +#define BOX_SYS_ERROR(stuff) \ + stuff << ": " << std::strerror(errno) << " (" << errno << ")" + #define BOX_LOG_SYS_WARNING(stuff) \ - BOX_WARNING(stuff << ": " << std::strerror(errno) << " (" << errno << ")") + BOX_WARNING(BOX_SYS_ERROR(stuff)) #define BOX_LOG_SYS_ERROR(stuff) \ - BOX_ERROR(stuff << ": " << std::strerror(errno) << " (" << errno << ")") + BOX_ERROR(BOX_SYS_ERROR(stuff)) #define BOX_LOG_SYS_FATAL(stuff) \ - BOX_FATAL(stuff << ": " << std::strerror(errno) << " (" << errno << ")") + BOX_FATAL(BOX_SYS_ERROR(stuff)) + +#define LOG_AND_THROW_ERROR(message, filename, exception, subtype) \ + BOX_LOG_SYS_ERROR(message << ": " << filename); \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_SYS_ERROR(message << ": " << filename)); inline std::string GetNativeErrorMessage() { @@ -320,4 +317,25 @@ class FileLogger : public Logger virtual void SetProgramName(const std::string& rProgramName) { } }; +class HideExceptionMessageGuard +{ + public: + HideExceptionMessageGuard() + { + mOldHiddenState = sHiddenState; + sHiddenState = true; + } + ~HideExceptionMessageGuard() + { + sHiddenState = mOldHiddenState; + } + static bool ExceptionsHidden() { return sHiddenState; } + + private: + static bool sHiddenState; + bool mOldHiddenState; +}; + +std::string PrintEscapedBinaryData(const std::string& rInput); + #endif // LOGGING__H diff --git a/lib/common/Makefile.extra b/lib/common/Makefile.extra index 04dd8e71..cc3f3a7a 100644 --- a/lib/common/Makefile.extra +++ b/lib/common/Makefile.extra @@ -3,9 +3,9 @@ MAKEEXCEPTION = ../../lib/common/makeexception.pl # AUTOGEN SEEDING autogen_CommonException.h autogen_CommonException.cpp: $(MAKEEXCEPTION) CommonException.txt - $(PERL) $(MAKEEXCEPTION) CommonException.txt + $(_PERL) $(MAKEEXCEPTION) CommonException.txt # AUTOGEN SEEDING autogen_ConversionException.h autogen_ConversionException.cpp: $(MAKEEXCEPTION) ConversionException.txt - $(PERL) $(MAKEEXCEPTION) ConversionException.txt + $(_PERL) $(MAKEEXCEPTION) ConversionException.txt diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp index b3d0009b..f96f80b5 100644 --- a/lib/common/NamedLock.cpp +++ b/lib/common/NamedLock.cpp @@ -65,7 +65,7 @@ NamedLock::~NamedLock() // Created: 2003/08/28 // // -------------------------------------------------------------------------- -bool NamedLock::TryAndGetLock(const char *Filename, int mode) +bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) { // Check if(mFileDescriptor != -1) @@ -75,7 +75,8 @@ bool NamedLock::TryAndGetLock(const char *Filename, int mode) // See if the lock can be got #if HAVE_DECL_O_EXLOCK - int fd = ::open(Filename, O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode); + int fd = ::open(rFilename.c_str(), + O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode); if(fd != -1) { // Got a lock, lovely @@ -92,10 +93,10 @@ bool NamedLock::TryAndGetLock(const char *Filename, int mode) return false; #else - int fd = ::open(Filename, O_WRONLY | O_CREAT | O_TRUNC, mode); + int fd = ::open(rFilename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, mode); if(fd == -1) { - BOX_WARNING("Failed to open lockfile: " << Filename); + BOX_WARNING("Failed to open lockfile: " << rFilename); THROW_EXCEPTION(CommonException, OSFileError) } diff --git a/lib/common/NamedLock.h b/lib/common/NamedLock.h index 5eac712a..534115db 100644 --- a/lib/common/NamedLock.h +++ b/lib/common/NamedLock.h @@ -28,7 +28,7 @@ private: NamedLock(const NamedLock &); public: - bool TryAndGetLock(const char *Filename, int mode = 0755); + bool TryAndGetLock(const std::string& rFilename, int mode = 0755); bool GotLock() {return mFileDescriptor != -1;} void ReleaseLock(); diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp index e903f41e..56638058 100644 --- a/lib/common/Test.cpp +++ b/lib/common/Test.cpp @@ -416,7 +416,7 @@ void terminate_bbackupd(int pid) // Wait a given number of seconds for something to complete -void wait_for_operation(int seconds, char* message) +void wait_for_operation(int seconds, const char* message) { if (Logging::GetGlobalLevel() >= Log::TRACE) { @@ -462,10 +462,24 @@ void safe_sleep(int seconds) ts.tv_nsec = 0; while (nanosleep(&ts, &ts) == -1 && errno == EINTR) { - BOX_TRACE("safe_sleep interrupted with " << - ts.tv_sec << "." << ts.tv_nsec << - " secs remaining, sleeping again"); - /* sleep again */ + // FIXME evil hack for OSX, where ts.tv_sec contains + // a negative number interpreted as unsigned 32-bit + // when nanosleep() returns later than expected. + + int32_t secs = (int32_t) ts.tv_sec; + int64_t remain_ns = (secs * 1000000000) + ts.tv_nsec; + + if (remain_ns < 0) + { + BOX_WARNING("nanosleep interrupted " << + ((float)(0 - remain_ns) / 1000000000) << + " secs late"); + return; + } + + BOX_TRACE("nanosleep interrupted with " << + (remain_ns / 1000000000) << " secs remaining, " + "sleeping again"); } #endif } diff --git a/lib/common/Test.h b/lib/common/Test.h index 362b43af..08ba4542 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -18,14 +18,14 @@ #define BBSTORED "..\\..\\bin\\bbstored\\bbstored.exe" #define BBACKUPQUERY "..\\..\\bin\\bbackupquery\\bbackupquery.exe" #define BBSTOREACCOUNTS "..\\..\\bin\\bbstoreaccounts\\bbstoreaccounts.exe" -#define TEST_RETURN(actual, expected) TEST_THAT(actual == expected); +#define TEST_RETURN(actual, expected) TEST_EQUAL(expected, actual); #else #define BBACKUPCTL "../../bin/bbackupctl/bbackupctl" #define BBACKUPD "../../bin/bbackupd/bbackupd" #define BBSTORED "../../bin/bbstored/bbstored" #define BBACKUPQUERY "../../bin/bbackupquery/bbackupquery" #define BBSTOREACCOUNTS "../../bin/bbstoreaccounts/bbstoreaccounts" -#define TEST_RETURN(actual, expected) TEST_THAT(actual == expected*256); +#define TEST_RETURN(actual, expected) TEST_EQUAL((expected << 8), actual); #endif extern int failures; @@ -54,6 +54,7 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; #define TEST_CHECK_THROWS(statement, excepttype, subtype) \ { \ bool didthrow = false; \ + HideExceptionMessageGuard hide; \ try \ { \ statement; \ @@ -90,8 +91,8 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; \ if(_exp_str != _found_str) \ { \ - printf("Expected <%s> but found <%s>\n", \ - _exp_str.c_str(), _found_str.c_str()); \ + BOX_WARNING("Expected <" << _exp_str << "> but found <" << \ + _found_str << ">"); \ \ std::ostringstream _oss3; \ _oss3 << #_found << " != " << #_expected; \ @@ -113,7 +114,9 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; \ if(_exp_str != _found_str) \ { \ - std::string _line_str = _line; \ + std::ostringstream _ossl; \ + _ossl << _line; \ + std::string _line_str = _ossl.str(); \ printf("Expected <%s> but found <%s> in <%s>\n", \ _exp_str.c_str(), _found_str.c_str(), _line_str.c_str()); \ \ @@ -158,7 +161,7 @@ void sync_and_wait(); void terminate_bbackupd(int pid); // Wait a given number of seconds for something to complete -void wait_for_operation(int seconds, char* message); +void wait_for_operation(int seconds, const char* message); void safe_sleep(int seconds); #endif // TEST__H diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp index f45ed26b..6f21330d 100644 --- a/lib/common/Utils.cpp +++ b/lib/common/Utils.cpp @@ -30,6 +30,11 @@ #include "MemLeakFindOn.h" +std::string GetBoxBackupVersion() +{ + return BOX_VERSION; +} + // -------------------------------------------------------------------------- // // Function @@ -156,15 +161,16 @@ void DumpStackBacktrace() // -------------------------------------------------------------------------- // // Function -// Name: FileExists(const char *) +// Name: FileExists(const std::string& rFilename) // Purpose: Does a file exist? // Created: 20/11/03 // // -------------------------------------------------------------------------- -bool FileExists(const char *Filename, int64_t *pFileSize, bool TreatLinksAsNotExisting) +bool FileExists(const std::string& rFilename, int64_t *pFileSize, + bool TreatLinksAsNotExisting) { EMU_STRUCT_STAT st; - if(EMU_LSTAT(Filename, &st) != 0) + if(EMU_LSTAT(rFilename.c_str(), &st) != 0) { if(errno == ENOENT) { diff --git a/lib/common/Utils.h b/lib/common/Utils.h index d6792077..8d98a520 100644 --- a/lib/common/Utils.h +++ b/lib/common/Utils.h @@ -15,13 +15,16 @@ #include "MemLeakFindOn.h" +std::string GetBoxBackupVersion(); + void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput); #ifdef SHOW_BACKTRACE_ON_EXCEPTION void DumpStackBacktrace(); #endif -bool FileExists(const char *Filename, int64_t *pFileSize = 0, bool TreatLinksAsNotExisting = false); +bool FileExists(const std::string& rFilename, int64_t *pFileSize = 0, + bool TreatLinksAsNotExisting = false); enum { diff --git a/lib/compress/Makefile.extra b/lib/compress/Makefile.extra index 3598f1c9..a29fcdb4 100644 --- a/lib/compress/Makefile.extra +++ b/lib/compress/Makefile.extra @@ -3,5 +3,5 @@ MAKEEXCEPTION = ../../lib/common/makeexception.pl # AUTOGEN SEEDING autogen_CompressException.h autogen_CompressException.cpp: $(MAKEEXCEPTION) CompressException.txt - $(PERL) $(MAKEEXCEPTION) CompressException.txt + $(_PERL) $(MAKEEXCEPTION) CompressException.txt diff --git a/lib/crypto/Makefile.extra b/lib/crypto/Makefile.extra index f9d63a42..3c94389f 100644 --- a/lib/crypto/Makefile.extra +++ b/lib/crypto/Makefile.extra @@ -3,5 +3,5 @@ MAKEEXCEPTION = ../../lib/common/makeexception.pl # AUTOGEN SEEDING autogen_CipherException.cpp autogen_CipherException.h: $(MAKEEXCEPTION) CipherException.txt - $(PERL) $(MAKEEXCEPTION) CipherException.txt + $(_PERL) $(MAKEEXCEPTION) CipherException.txt diff --git a/lib/httpserver/HTTPRequest.cpp b/lib/httpserver/HTTPRequest.cpp index e146a0a3..4c5dc149 100644 --- a/lib/httpserver/HTTPRequest.cpp +++ b/lib/httpserver/HTTPRequest.cpp @@ -535,9 +535,11 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) { ++dataStart; } - - if(p == sizeof("Content-Length")-1 - && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) + + std::string header_name(ToLowerCase(std::string(h, + p))); + + if (header_name == "content-length") { // Decode number long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK @@ -545,14 +547,12 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) // Store mContentLength = len; } - else if(p == sizeof("Content-Type")-1 - && ::strncasecmp(h, "Content-Type", sizeof("Content-Type")-1) == 0) + else if (header_name == "content-type") { // Store rest of string as content type mContentType = h + dataStart; } - else if(p == sizeof("Host")-1 - && ::strncasecmp(h, "Host", sizeof("Host")-1) == 0) + else if (header_name == "host") { // Store host header mHostName = h + dataStart; @@ -572,14 +572,12 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) "port = " << mHostPort); } } - else if(p == sizeof("Cookie")-1 - && ::strncasecmp(h, "Cookie", sizeof("Cookie")-1) == 0) + else if (header_name == "cookie") { // Parse cookies ParseCookies(header, dataStart); } - else if(p == sizeof("Connection")-1 - && ::strncasecmp(h, "Connection", sizeof("Connection")-1) == 0) + else if (header_name == "connection") { // Connection header, what is required? const char *v = h + dataStart; @@ -595,8 +593,7 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) } else { - std::string name = header.substr(0, p); - mExtraHeaders.push_back(Header(name, + mExtraHeaders.push_back(Header(header_name, h + dataStart)); } diff --git a/lib/httpserver/HTTPRequest.h b/lib/httpserver/HTTPRequest.h index 90215751..ca50e60f 100644 --- a/lib/httpserver/HTTPRequest.h +++ b/lib/httpserver/HTTPRequest.h @@ -97,16 +97,19 @@ public: const std::string &GetCookie(const char *CookieName) const; bool GetHeader(const std::string& rName, std::string* pValueOut) const { + std::string header = ToLowerCase(rName); + for (std::vector<Header>::const_iterator i = mExtraHeaders.begin(); i != mExtraHeaders.end(); i++) { - if (i->first == rName) + if (i->first == header) { *pValueOut = i->second; return true; } } + return false; } std::vector<Header> GetHeaders() { return mExtraHeaders; } @@ -128,7 +131,7 @@ public: void AddHeader(const std::string& rName, const std::string& rValue) { - mExtraHeaders.push_back(Header(rName, rValue)); + mExtraHeaders.push_back(Header(ToLowerCase(rName), rValue)); } bool IsExpectingContinue() const { return mExpectContinue; } const char* GetVerb() const @@ -168,6 +171,17 @@ private: bool mExpectContinue; IOStream* mpStreamToReadFrom; std::string mHttpVerb; + + std::string ToLowerCase(const std::string& rInput) const + { + std::string output = rInput; + for (std::string::iterator c = output.begin(); + c != output.end(); c++) + { + *c = tolower(*c); + } + return output; + } }; #endif // HTTPREQUEST__H diff --git a/lib/httpserver/HTTPServer.cpp b/lib/httpserver/HTTPServer.cpp index b8b02249..be1db687 100644 --- a/lib/httpserver/HTTPServer.cpp +++ b/lib/httpserver/HTTPServer.cpp @@ -153,21 +153,21 @@ void HTTPServer::Connection(SocketStream &rStream) // Generate a response HTTPResponse response(&rStream); + try { Handle(request, response); } catch(BoxException &e) { - char exceptionCode[64]; - ::sprintf(exceptionCode, "(%d/%d)", e.GetType(), e.GetSubType()); - SendInternalErrorResponse(exceptionCode, rStream); - return; + char exceptionCode[256]; + ::sprintf(exceptionCode, "%s (%d/%d)", e.what(), + e.GetType(), e.GetSubType()); + SendInternalErrorResponse(exceptionCode, response); } catch(...) { - SendInternalErrorResponse("unknown", rStream); - return; + SendInternalErrorResponse("unknown", response); } // Keep alive? @@ -186,7 +186,7 @@ void HTTPServer::Connection(SocketStream &rStream) response.Send(request.GetMethod() == HTTPRequest::Method_HEAD); } - // Notify derived claases + // Notify derived classes HTTPConnectionClosing(); } @@ -194,12 +194,14 @@ void HTTPServer::Connection(SocketStream &rStream) // -------------------------------------------------------------------------- // // Function -// Name: HTTPServer::SendInternalErrorResponse(const char *, SocketStream &) -// Purpose: Sends an error response to the remote side +// Name: HTTPServer::SendInternalErrorResponse(const char*, +// HTTPResponse&) +// Purpose: Generates an error message in the provided response // Created: 26/3/04 // // -------------------------------------------------------------------------- -void HTTPServer::SendInternalErrorResponse(const char *Error, SocketStream &rStream) +void HTTPServer::SendInternalErrorResponse(const std::string& rErrorMsg, + HTTPResponse& rResponse) { #define ERROR_HTML_1 "<html><head><title>Internal Server Error</title></head>\n" \ "<h1>Internal Server Error</h1>\n" \ @@ -209,15 +211,11 @@ void HTTPServer::SendInternalErrorResponse(const char *Error, SocketStream &rStr "</body>\n</html>\n" // Generate the error page - HTTPResponse response(&rStream); - response.SetResponseCode(HTTPResponse::Code_InternalServerError); - response.SetContentType("text/html"); - response.Write(ERROR_HTML_1, sizeof(ERROR_HTML_1) - 1); - response.Write(Error, ::strlen(Error)); - response.Write(ERROR_HTML_2, sizeof(ERROR_HTML_2) - 1); - - // Send the error response - response.Send(); + // rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); + rResponse.SetContentType("text/html"); + rResponse.Write(ERROR_HTML_1, sizeof(ERROR_HTML_1) - 1); + rResponse.IOStream::Write(rErrorMsg.c_str()); + rResponse.Write(ERROR_HTML_2, sizeof(ERROR_HTML_2) - 1); } diff --git a/lib/httpserver/HTTPServer.h b/lib/httpserver/HTTPServer.h index 8009438d..d9f74949 100644 --- a/lib/httpserver/HTTPServer.h +++ b/lib/httpserver/HTTPServer.h @@ -52,15 +52,17 @@ public: virtual void HTTPConnectionOpening(); virtual void HTTPConnectionClosing(); +protected: + void SendInternalErrorResponse(const std::string& rErrorMsg, + HTTPResponse& rResponse); + int GetTimeout() { return mTimeout; } + private: + int mTimeout; // Timeout for read operations const char *DaemonName() const; const ConfigurationVerify *GetConfigVerify() const; void Run(); void Connection(SocketStream &rStream); - void SendInternalErrorResponse(const char *Error, SocketStream &rStream); - -private: - int mTimeout; // Timeout for read operations }; // Root level diff --git a/lib/httpserver/Makefile.extra b/lib/httpserver/Makefile.extra index f0ca62be..ef47f398 100644 --- a/lib/httpserver/Makefile.extra +++ b/lib/httpserver/Makefile.extra @@ -3,5 +3,5 @@ MAKEEXCEPTION = ../../lib/common/makeexception.pl # AUTOGEN SEEDING autogen_HTTPException.h autogen_HTTPException.cpp: $(MAKEEXCEPTION) HTTPException.txt - perl $(MAKEEXCEPTION) HTTPException.txt + $(_PERL) $(MAKEEXCEPTION) HTTPException.txt diff --git a/lib/httpserver/S3Simulator.cpp b/lib/httpserver/S3Simulator.cpp new file mode 100644 index 00000000..4f6bb3e6 --- /dev/null +++ b/lib/httpserver/S3Simulator.cpp @@ -0,0 +1,309 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Client.cpp +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <algorithm> +#include <cstring> + +// #include <cstdio> +// #include <ctime> + +#include <openssl/hmac.h> + +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "autogen_HTTPException.h" +#include "IOStream.h" +#include "Logging.h" +#include "S3Simulator.h" +#include "decode.h" +#include "encode.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::GetConfigVerify() +// Purpose: Returns additional configuration options for the +// S3 simulator. Currently the access key, secret key +// and store directory can be configured. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify* S3Simulator::GetConfigVerify() const +{ + static ConfigurationVerifyKey verifyserverkeys[] = + { + HTTPSERVER_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default addresses + }; + + static ConfigurationVerify verifyserver[] = + { + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } + }; + + static ConfigurationVerifyKey verifyrootkeys[] = + { + ConfigurationVerifyKey("AccessKey", ConfigTest_Exists), + ConfigurationVerifyKey("SecretKey", ConfigTest_Exists), + ConfigurationVerifyKey("StoreDirectory", ConfigTest_Exists), + HTTPSERVER_VERIFY_ROOT_KEYS + }; + + static ConfigurationVerify verify = + { + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + }; + + return &verify; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::Handle(HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles any incoming S3 request, by checking +// authorization and then dispatching to one of the +// private Handle* methods. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + // if anything goes wrong, return a 500 error + rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); + rResponse.SetContentType("text/plain"); + + try + { + const Configuration& rConfig(GetConfiguration()); + std::string access_key = rConfig.GetKeyValue("AccessKey"); + std::string secret_key = rConfig.GetKeyValue("SecretKey"); + + std::string md5, date, bucket; + rRequest.GetHeader("content-md5", &md5); + rRequest.GetHeader("date", &date); + + std::string host = rRequest.GetHostName(); + std::string s3suffix = ".s3.amazonaws.com"; + if (host.size() > s3suffix.size()) + { + std::string suffix = host.substr(host.size() - + s3suffix.size(), s3suffix.size()); + if (suffix == s3suffix) + { + bucket = host.substr(0, host.size() - + s3suffix.size()); + } + } + + std::ostringstream data; + data << rRequest.GetVerb() << "\n"; + data << md5 << "\n"; + data << rRequest.GetContentType() << "\n"; + data << date << "\n"; + + // header names are already in lower case, i.e. canonical form + + std::vector<HTTPRequest::Header> headers = rRequest.GetHeaders(); + std::sort(headers.begin(), headers.end()); + + for (std::vector<HTTPRequest::Header>::iterator + i = headers.begin(); i != headers.end(); i++) + { + if (i->first.substr(0, 5) == "x-amz") + { + data << i->first << ":" << i->second << "\n"; + } + } + + if (! bucket.empty()) + { + data << "/" << bucket; + } + + data << rRequest.GetRequestURI(); + std::string data_string = data.str(); + + unsigned char digest_buffer[EVP_MAX_MD_SIZE]; + unsigned int digest_size = sizeof(digest_buffer); + /* unsigned char* mac = */ HMAC(EVP_sha1(), + secret_key.c_str(), secret_key.size(), + (const unsigned char*)data_string.c_str(), + data_string.size(), digest_buffer, &digest_size); + std::string digest((const char *)digest_buffer, digest_size); + + base64::encoder encoder; + std::string expectedAuth = "AWS " + access_key + ":" + + encoder.encode(digest); + + if (expectedAuth[expectedAuth.size() - 1] == '\n') + { + expectedAuth = expectedAuth.substr(0, + expectedAuth.size() - 1); + } + + std::string actualAuth; + if (!rRequest.GetHeader("authorization", &actualAuth) || + actualAuth != expectedAuth) + { + rResponse.SetResponseCode(HTTPResponse::Code_Unauthorized); + SendInternalErrorResponse("Authentication Failed", + rResponse); + } + else if (rRequest.GetMethod() == HTTPRequest::Method_GET) + { + HandleGet(rRequest, rResponse); + } + else if (rRequest.GetMethod() == HTTPRequest::Method_PUT) + { + HandlePut(rRequest, rResponse); + } + else + { + rResponse.SetResponseCode(HTTPResponse::Code_MethodNotAllowed); + SendInternalErrorResponse("Unsupported Method", + rResponse); + } + } + catch (CommonException &ce) + { + SendInternalErrorResponse(ce.what(), rResponse); + } + catch (std::exception &e) + { + SendInternalErrorResponse(e.what(), rResponse); + } + catch (...) + { + SendInternalErrorResponse("Unknown exception", rResponse); + } + + if (rResponse.GetResponseCode() != 200 && + rResponse.GetSize() == 0) + { + // no error message written, provide a default + std::ostringstream s; + s << rResponse.GetResponseCode(); + SendInternalErrorResponse(s.str().c_str(), rResponse); + } + + return; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::HandleGet(HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles an S3 GET request, i.e. downloading an +// existing object. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +void S3Simulator::HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + std::string path = GetConfiguration().GetKeyValue("StoreDirectory"); + path += rRequest.GetRequestURI(); + std::auto_ptr<FileStream> apFile; + + try + { + apFile.reset(new FileStream(path)); + } + catch (CommonException &ce) + { + if (ce.GetSubType() == CommonException::OSFileOpenError) + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + } + else if (ce.GetSubType() == CommonException::AccessDenied) + { + rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); + } + throw; + } + + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingRESTOperations.html + apFile->CopyStreamTo(rResponse); + rResponse.AddHeader("x-amz-id-2", "qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY"); + rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); + rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); + rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); + rResponse.AddHeader("Server", "AmazonS3"); + rResponse.SetResponseCode(HTTPResponse::Code_OK); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Simulator::HandlePut(HTTPRequest &rRequest, +// HTTPResponse &rResponse) +// Purpose: Handles an S3 PUT request, i.e. uploading data to +// create or replace an object. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +void S3Simulator::HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + std::string path = GetConfiguration().GetKeyValue("StoreDirectory"); + path += rRequest.GetRequestURI(); + std::auto_ptr<FileStream> apFile; + + try + { + apFile.reset(new FileStream(path, O_CREAT | O_WRONLY)); + } + catch (CommonException &ce) + { + if (ce.GetSubType() == CommonException::OSFileOpenError) + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + } + else if (ce.GetSubType() == CommonException::AccessDenied) + { + rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); + } + throw; + } + + if (rRequest.IsExpectingContinue()) + { + rResponse.SendContinue(); + } + + rRequest.ReadContent(*apFile); + + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectPUT.html + rResponse.AddHeader("x-amz-id-2", "LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7"); + rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); + rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); + rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); + rResponse.SetContentType(""); + rResponse.AddHeader("Server", "AmazonS3"); + rResponse.SetResponseCode(HTTPResponse::Code_OK); +} diff --git a/lib/httpserver/S3Simulator.h b/lib/httpserver/S3Simulator.h new file mode 100644 index 00000000..f80770ee --- /dev/null +++ b/lib/httpserver/S3Simulator.h @@ -0,0 +1,40 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Simulator.h +// Purpose: Amazon S3 simulation HTTP server for S3 testing +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#ifndef S3SIMULATOR__H +#define S3SIMULATOR__H + +#include "HTTPServer.h" + +class ConfigurationVerify; +class HTTPRequest; +class HTTPResponse; + +// -------------------------------------------------------------------------- +// +// Class +// Name: S3Simulator +// Purpose: Amazon S3 simulation HTTP server for S3 testing +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- +class S3Simulator : public HTTPServer +{ +public: + S3Simulator() { } + ~S3Simulator() { } + + const ConfigurationVerify* GetConfigVerify() const; + virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse); +}; + +#endif // S3SIMULATOR__H + diff --git a/lib/raidfile/Makefile.extra b/lib/raidfile/Makefile.extra index 4d904e8a..bf06ed2f 100644 --- a/lib/raidfile/Makefile.extra +++ b/lib/raidfile/Makefile.extra @@ -3,5 +3,5 @@ MAKEEXCEPTION = ../../lib/common/makeexception.pl # AUTOGEN SEEDING autogen_RaidFileException.h autogen_RaidFileException.cpp: $(MAKEEXCEPTION) RaidFileException.txt - $(PERL) $(MAKEEXCEPTION) RaidFileException.txt + $(_PERL) $(MAKEEXCEPTION) RaidFileException.txt diff --git a/lib/raidfile/RaidFileException.txt b/lib/raidfile/RaidFileException.txt index c69dc2a2..c7ddfcc5 100644 --- a/lib/raidfile/RaidFileException.txt +++ b/lib/raidfile/RaidFileException.txt @@ -23,3 +23,6 @@ CanOnlyGetUsageBeforeCommit 19 CanOnlyGetFileSizeBeforeCommit 20 ErrorOpeningWriteFileOnTruncate 21 FileIsCurrentlyOpenForWriting 22 +RequestedModifyUnreferencedFile 23 Internal error: the server attempted to modify a file which has no references. +RequestedModifyMultiplyReferencedFile 24 Internal error: the server attempted to modify a file which has multiple references. +RequestedDeleteReferencedFile 25 Internal error: the server attempted to delete a file which is still referenced. diff --git a/lib/raidfile/RaidFileUtil.h b/lib/raidfile/RaidFileUtil.h index 16670bf1..a581047c 100644 --- a/lib/raidfile/RaidFileUtil.h +++ b/lib/raidfile/RaidFileUtil.h @@ -37,7 +37,7 @@ public: AsRaidWithMissingNotRecoverable = 4 } ExistType; - typedef enum + enum { Stripe1Exists = 1, Stripe2Exists = 2, diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp index efec43a2..2d852f86 100644 --- a/lib/raidfile/RaidFileWrite.cpp +++ b/lib/raidfile/RaidFileWrite.cpp @@ -42,20 +42,50 @@ // // Function // Name: RaidFileWrite::RaidFileWrite(int, const std::string &) -// Purpose: Construtor, just stores requried details +// Purpose: Simple constructor, just stores required details // Created: 2003/07/10 // // -------------------------------------------------------------------------- RaidFileWrite::RaidFileWrite(int SetNumber, const std::string &Filename) : mSetNumber(SetNumber), mFilename(Filename), - mOSFileHandle(-1) // not valid file handle + mOSFileHandle(-1), // not valid file handle + mRefCount(-1) // unknown refcount { } // -------------------------------------------------------------------------- // // Function +// Name: RaidFileWrite::RaidFileWrite(int, +// const std::string &, int refcount) +// Purpose: Constructor with check for overwriting file +// with multiple references +// Created: 2009/07/05 +// +// -------------------------------------------------------------------------- +RaidFileWrite::RaidFileWrite(int SetNumber, const std::string &Filename, + int refcount) + : mSetNumber(SetNumber), + mFilename(Filename), + mOSFileHandle(-1), // not valid file handle + mRefCount(refcount) +{ + // Can't check for zero refcount here, because it's legal + // to create a RaidFileWrite to delete an object with zero refcount. + // Check in Commit() and Delete() instead. + if (refcount > 1) + { + BOX_ERROR("Attempted to modify object " << mFilename << + ", which has " << refcount << " references"); + THROW_EXCEPTION(RaidFileException, + RequestedModifyMultiplyReferencedFile); + } +} + +// -------------------------------------------------------------------------- +// +// Function // Name: RaidFileWrite::~RaidFileWrite() // Purpose: Destructor (will discard written file if not commited) // Created: 2003/07/10 @@ -113,6 +143,7 @@ void RaidFileWrite::Open(bool AllowOverwrite) S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); if(mOSFileHandle == -1) { + BOX_LOG_SYS_ERROR("Failed to open file: " << writeFilename); THROW_EXCEPTION(RaidFileException, ErrorOpeningWriteFile) } @@ -249,7 +280,15 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) { THROW_EXCEPTION(RaidFileException, NotOpen) } - + + if (mRefCount == 0) + { + BOX_ERROR("Attempted to modify object " << mFilename << + ", which has no references"); + THROW_EXCEPTION(RaidFileException, + RequestedModifyUnreferencedFile); + } + // Rename it into place -- BEFORE it's closed so lock remains #ifdef WIN32 @@ -274,13 +313,15 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) if(::unlink(renameTo.c_str()) != 0 && GetLastError() != ERROR_FILE_NOT_FOUND) { - BOX_LOG_WIN_ERROR("failed to delete file: " << renameTo); + BOX_LOG_WIN_ERROR("Failed to delete file: " << renameTo); THROW_EXCEPTION(RaidFileException, OSError) } #endif if(::rename(renameFrom.c_str(), renameTo.c_str()) != 0) { + BOX_LOG_SYS_ERROR("Failed to rename file: " << renameFrom << + " to " << renameTo); THROW_EXCEPTION(RaidFileException, OSError) } @@ -335,6 +376,7 @@ void RaidFileWrite::Discard() ::close(mOSFileHandle) != 0) #endif // !WIN32 { + BOX_LOG_SYS_ERROR("Failed to delete file: " << writeFilename); THROW_EXCEPTION(RaidFileException, OSError) } @@ -562,6 +604,8 @@ void RaidFileWrite::TransformToRaidStorage() ASSERT((::lseek(parity, 0, SEEK_CUR) % blockSize) == 0); if(::write(parity, &sw, sizeof(sw)) != sizeof(sw)) { + BOX_LOG_SYS_ERROR("Failed to write to file: " << + writeFilename); THROW_EXCEPTION(RaidFileException, OSError) } } @@ -600,6 +644,8 @@ void RaidFileWrite::TransformToRaidStorage() // Finally delete the write file if(::unlink(writeFilename.c_str()) != 0) { + BOX_LOG_SYS_ERROR("Failed to delete file: " << + writeFilename); THROW_EXCEPTION(RaidFileException, OSError) } } @@ -630,6 +676,14 @@ void RaidFileWrite::TransformToRaidStorage() // -------------------------------------------------------------------------- void RaidFileWrite::Delete() { + if (mRefCount != 0 && mRefCount != -1) + { + BOX_ERROR("Attempted to delete object " << mFilename << + " which has " << mRefCount << " references"); + THROW_EXCEPTION(RaidFileException, + RequestedDeleteReferencedFile); + } + // Get disc set RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mSetNumber)); diff --git a/lib/raidfile/RaidFileWrite.h b/lib/raidfile/RaidFileWrite.h index d7e51f21..418f90ee 100644 --- a/lib/raidfile/RaidFileWrite.h +++ b/lib/raidfile/RaidFileWrite.h @@ -28,6 +28,7 @@ class RaidFileWrite : public IOStream { public: RaidFileWrite(int SetNumber, const std::string &Filename); + RaidFileWrite(int SetNumber, const std::string &Filename, int refcount); ~RaidFileWrite(); private: RaidFileWrite(const RaidFileWrite &rToCopy); @@ -60,6 +61,7 @@ private: int mSetNumber; std::string mFilename; int mOSFileHandle; + int mRefCount; }; #endif // RAIDFILEWRITE__H diff --git a/lib/server/LocalProcessStream.cpp b/lib/server/LocalProcessStream.cpp index af24de1b..c331a135 100644 --- a/lib/server/LocalProcessStream.cpp +++ b/lib/server/LocalProcessStream.cpp @@ -43,13 +43,14 @@ // Created: 12/3/04 // // -------------------------------------------------------------------------- -std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidOut) +std::auto_ptr<IOStream> LocalProcessStream(const std::string& rCommandLine, + pid_t &rPidOut) { #ifndef WIN32 // Split up command std::vector<std::string> command; - SplitString(std::string(CommandLine), ' ', command); + SplitString(rCommandLine, ' ', command); // Build arguments char *args[MAX_ARGUMENTS + 4]; @@ -137,8 +138,8 @@ std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidO startupInfo.hStdInput = INVALID_HANDLE_VALUE; startupInfo.dwFlags |= STARTF_USESTDHANDLES; - CHAR* commandLineCopy = (CHAR*)malloc(strlen(CommandLine) + 1); - strcpy(commandLineCopy, CommandLine); + CHAR* commandLineCopy = (CHAR*)malloc(rCommandLine.size() + 1); + strcpy(commandLineCopy, rCommandLine.c_str()); BOOL result = CreateProcess(NULL, commandLineCopy, // command line @@ -155,7 +156,7 @@ std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidO if(!result) { - BOX_ERROR("Failed to CreateProcess: '" << CommandLine << + BOX_ERROR("Failed to CreateProcess: '" << rCommandLine << "': " << GetErrorMessage(GetLastError())); CloseHandle(writeInChild); CloseHandle(readFromChild); diff --git a/lib/server/LocalProcessStream.h b/lib/server/LocalProcessStream.h index 490c0f45..51e51f8a 100644 --- a/lib/server/LocalProcessStream.h +++ b/lib/server/LocalProcessStream.h @@ -13,7 +13,8 @@ #include <memory> #include "IOStream.h" -std::auto_ptr<IOStream> LocalProcessStream(const char *CommandLine, pid_t &rPidOut); +std::auto_ptr<IOStream> LocalProcessStream(const std::string& rCommandLine, + pid_t &rPidOut); #endif // LOCALPROCESSSTREAM__H diff --git a/lib/server/Makefile.extra b/lib/server/Makefile.extra index 33fac0a1..7fc6baf9 100644 --- a/lib/server/Makefile.extra +++ b/lib/server/Makefile.extra @@ -3,9 +3,9 @@ MAKEEXCEPTION = ../../lib/common/makeexception.pl # AUTOGEN SEEDING autogen_ServerException.h autogen_ServerException.cpp: $(MAKEEXCEPTION) ServerException.txt - $(PERL) $(MAKEEXCEPTION) ServerException.txt + $(_PERL) $(MAKEEXCEPTION) ServerException.txt # AUTOGEN SEEDING autogen_ConnectionException.h autogen_ConnectionException.cpp: $(MAKEEXCEPTION) ConnectionException.txt - $(PERL) $(MAKEEXCEPTION) ConnectionException.txt + $(_PERL) $(MAKEEXCEPTION) ConnectionException.txt diff --git a/lib/server/ServerControl.cpp b/lib/server/ServerControl.cpp index c4668b57..b9650cee 100644 --- a/lib/server/ServerControl.cpp +++ b/lib/server/ServerControl.cpp @@ -1,6 +1,7 @@ #include "Box.h" #include <errno.h> +#include <stdio.h> #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h index 34de7def..e49dbcbe 100644 --- a/lib/server/ServerStream.h +++ b/lib/server/ServerStream.h @@ -302,16 +302,7 @@ public: // Clean up child processes (if forking daemon) if(ForkToHandleRequests && !IsSingleProcess()) { - int status = 0; - int p = 0; - do - { - if((p = ::waitpid(0 /* any child in process group */, &status, WNOHANG)) == -1 - && errno != ECHILD && errno != EINTR) - { - THROW_EXCEPTION(ServerException, ServerWaitOnChildError) - } - } while(p > 0); + WaitForChildren(); } #endif // !WIN32 } @@ -326,6 +317,49 @@ public: DeleteSockets(); } + #ifndef WIN32 // no waitpid() on Windows + void WaitForChildren() + { + int p = 0; + do + { + int status = 0; + p = ::waitpid(0 /* any child in process group */, + &status, WNOHANG); + + if(p == -1 && errno != ECHILD && errno != EINTR) + { + THROW_EXCEPTION(ServerException, + ServerWaitOnChildError) + } + else if(p == 0) + { + // no children exited, will return from + // function + } + else if(WIFEXITED(status)) + { + BOX_INFO("child process " << p << " " + "terminated normally"); + } + else if(WIFSIGNALED(status)) + { + int sig = WTERMSIG(status); + BOX_ERROR("child process " << p << " " + "terminated abnormally with " + "signal " << sig); + } + else + { + BOX_WARNING("something unknown happened " + "to child process " << p << ": " + "status = " << status); + } + } + while(p > 0); + } + #endif + virtual void HandleConnection(StreamType &rStream) { Connection(rStream); diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index ad8c3041..3a56661a 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -417,6 +417,32 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName) } tmpStr += filename; + + // We are using direct filename access, which does not support .., + // so we need to implement it ourselves. + + for (std::string::size_type i = 1; i < tmpStr.size() - 3; i++) + { + if (tmpStr.substr(i, 3) == "\\..") + { + std::string::size_type lastSlash = + tmpStr.rfind('\\', i - 1); + + if (lastSlash == std::string::npos) + { + // no previous directory, ignore it, + // CreateFile will fail with error 123 + } + else + { + tmpStr.replace(lastSlash, i + 3 - lastSlash, + ""); + } + + i = lastSlash; + } + } + return tmpStr; } diff --git a/lib/win32/emu.h b/lib/win32/emu.h index 811e6495..f3389590 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -117,7 +117,7 @@ inline struct passwd * getpwnam(const char * name) #ifndef __MINGW32__ //not sure if these are correct //S_IWRITE - writing permitted - //_S_IREAD - reading permitted + //_S_IREAD - reading permitted //_S_IREAD | _S_IWRITE - #define S_IRUSR S_IWRITE #define S_IWUSR S_IREAD @@ -242,6 +242,15 @@ int closedir(DIR *dp); extern DWORD winerrno; /* used to report errors from openfile() */ HANDLE openfile(const char *filename, int flags, int mode); +inline int closefile(HANDLE handle) +{ + if (CloseHandle(handle) != TRUE) + { + errno = EINVAL; + return -1; + } + return 0; +} #define LOG_DEBUG LOG_INFO #define LOG_INFO 6 |