diff options
Diffstat (limited to 'lib')
112 files changed, 7526 insertions, 1962 deletions
diff --git a/lib/backupclient/BackupClientCryptoKeys.cpp b/lib/backupclient/BackupClientCryptoKeys.cpp index 46b77f0a..7a8da7ba 100644 --- a/lib/backupclient/BackupClientCryptoKeys.cpp +++ b/lib/backupclient/BackupClientCryptoKeys.cpp @@ -28,40 +28,58 @@ // Created: 1/12/03 // // -------------------------------------------------------------------------- -void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename) +void BackupClientCryptoKeys_Setup(const std::string& rKeyMaterialFilename) { // Read in the key material unsigned char KeyMaterial[BACKUPCRYPTOKEYS_FILE_SIZE]; // Open the file - FileStream file(KeyMaterialFilename); + FileStream file(rKeyMaterialFilename); + // Read in data if(!file.ReadFullBuffer(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE, 0)) { THROW_EXCEPTION(BackupStoreException, CouldntLoadClientKeyMaterial) } - // Tell the filename how to encrypt - BackupStoreFilenameClear::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH, - KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH); - BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish); + // Setup keys and encoding method for filename encryption + BackupStoreFilenameClear::SetBlowfishKey( + KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, + BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH); + BackupStoreFilenameClear::SetEncodingMethod( + BackupStoreFilename::Encoding_Blowfish); + + // Setup key for attributes encryption + BackupClientFileAttributes::SetBlowfishKey( + KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH); + + // Setup secret for attribute hashing + BackupClientFileAttributes::SetAttributeHashSecret( + KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH); - // Tell the attributes how to encrypt - BackupClientFileAttributes::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH); - // and the secret for hashing - BackupClientFileAttributes::SetAttributeHashSecret(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH); + // Setup keys for file data encryption + BackupStoreFile::SetBlowfishKeys( + KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH); - // Tell the files how to encrypt - BackupStoreFile::SetBlowfishKeys(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH, - KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH); #ifndef HAVE_OLD_SSL // Use AES where available - BackupStoreFile::SetAESKey(KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH); + BackupStoreFile::SetAESKey( + KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, + BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH); #endif // Wipe the key material from memory - ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE); + #ifdef _MSC_VER // not defined on MinGW + SecureZeroMemory(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE); + #else + ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE); + #endif } - - diff --git a/lib/backupclient/BackupClientCryptoKeys.h b/lib/backupclient/BackupClientCryptoKeys.h index 5e3a7df2..f40e2e03 100644 --- a/lib/backupclient/BackupClientCryptoKeys.h +++ b/lib/backupclient/BackupClientCryptoKeys.h @@ -49,7 +49,7 @@ #define BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH 32 -void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename); +void BackupClientCryptoKeys_Setup(const std::string& rKeyMaterialFilename); #endif // BACKUPCLIENTCRYTOKEYS__H diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp index 3ffeb189..bb17d41f 100644 --- a/lib/backupclient/BackupClientFileAttributes.cpp +++ b/lib/backupclient/BackupClientFileAttributes.cpp @@ -15,11 +15,14 @@ #include <sys/types.h> #include <sys/stat.h> -#include <string.h> +#include <errno.h> #include <limits.h> + #include <algorithm> +#include <cstring> #include <new> #include <vector> + #ifdef HAVE_SYS_XATTR_H #include <cerrno> #include <sys/xattr.h> @@ -299,9 +302,11 @@ void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroM StreamableMemBlock *pnewAttr = 0; try { - struct stat st; - if(::lstat(Filename, &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) != 0) { + BOX_LOG_SYS_ERROR("Failed to stat file: '" << + Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } @@ -390,7 +395,7 @@ void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroM // Created: 2003/10/07 // // -------------------------------------------------------------------------- -void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st, bool ZeroModificationTimes) +void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, EMU_STRUCT_STAT &st, bool ZeroModificationTimes) { outputBlock.ResizeBlock(sizeof(attr_StreamFormat)); attr_StreamFormat *pattr = (attr_StreamFormat*)outputBlock.GetBuffer(); @@ -439,6 +444,7 @@ void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBl int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX); if(linkedToSize == -1) { + BOX_LOG_SYS_ERROR("Failed to readlink '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError); } @@ -463,7 +469,7 @@ void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBl void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename) { #ifdef HAVE_SYS_XATTR_H - int listBufferSize = 1000; + int listBufferSize = 10000; char* list = new char[listBufferSize]; try @@ -529,6 +535,9 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc int valueSize = ::lgetxattr(Filename, attrKey.c_str(), 0, 0); if(valueSize<0) { + BOX_LOG_SYS_ERROR("Failed to get " + "extended attributes size " + "for '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError); } @@ -544,6 +553,9 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc valueSize = ::lgetxattr(Filename, attrKey.c_str(), buffer+xattrSize, xattrBufferSize-xattrSize); if(valueSize<0) { + BOX_LOG_SYS_ERROR("Failed to get " + "extended attributes for " + "'" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError); } xattrSize += valueSize; @@ -559,9 +571,25 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc outputBlock.ResizeBlock(xattrSize); } - else if(listSize<0 && errno!=EOPNOTSUPP && errno!=EACCES) + else if(listSize<0) { - THROW_EXCEPTION(CommonException, OSFileError); + if(errno == EOPNOTSUPP || errno == EACCES) + { + // fail silently + } + else if(errno == ERANGE) + { + BOX_ERROR("Failed to list extended " + "attributes of '" << Filename << "': " + "buffer too small, not backed up"); + } + else + { + BOX_LOG_SYS_ERROR("Failed to list extended " + "attributes of '" << Filename << "', " + "not backed up"); + THROW_EXCEPTION(CommonException, OSFileError); + } } } catch(...) @@ -638,6 +666,8 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename, ::unlink(Filename); if(::symlink((char*)(pattr + 1), Filename) != 0) { + BOX_LOG_SYS_ERROR("Failed to symlink '" << Filename << + "' to '" << (char*)(pattr + 1) << "'"); THROW_EXCEPTION(CommonException, OSFileError) } #endif @@ -655,12 +685,18 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename, // Not a link, use normal chown if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) { + BOX_LOG_SYS_ERROR("Failed to change " + "owner of file " + "'" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } } #else - if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) // use the version which sets things on symlinks + // use the version which sets things on symlinks + if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) { + BOX_LOG_SYS_ERROR("Failed to change owner of " + "symbolic link '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } #endif @@ -705,6 +741,8 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename, // Try to apply if(::utimes(Filename, times) != 0) { + BOX_LOG_SYS_ERROR("Failed to change times of " + "file '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } } @@ -715,8 +753,12 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename, } // Apply everything else... (allowable mode flags only) - if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX)) != 0) // mode must be done last (think setuid) + // Mode must be done last (think setuid) + if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID + | S_ISGID | S_ISVTX)) != 0) { + BOX_LOG_SYS_ERROR("Failed to change permissions of file " + "'" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } } @@ -831,6 +873,8 @@ void BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xat // FIXME: Warn on EOPNOTSUPP if(::lsetxattr(Filename, key, buffer+xattrOffset, valueSize, 0)!=0 && errno!=EOPNOTSUPP) { + BOX_LOG_SYS_ERROR("Failed to set extended attributes " + "on file '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError); } @@ -993,7 +1037,7 @@ void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int // Created: 25/4/04 // // -------------------------------------------------------------------------- -uint64_t BackupClientFileAttributes::GenerateAttributeHash(struct stat &st, const std::string &filename, const std::string &leafname) +uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname) { if(sAttributeHashSecretLength == 0) { diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h index fa56ff65..b32c14dd 100644 --- a/lib/backupclient/BackupClientFileAttributes.h +++ b/lib/backupclient/BackupClientFileAttributes.h @@ -15,7 +15,7 @@ #include "StreamableMemBlock.h" #include "BoxTime.h" -struct stat; +EMU_STRUCT_STAT; // declaration // -------------------------------------------------------------------------- // @@ -53,11 +53,13 @@ public: static void SetBlowfishKey(const void *pKey, int KeyLength); static void SetAttributeHashSecret(const void *pSecret, int SecretLength); - static uint64_t GenerateAttributeHash(struct stat &st, const std::string &filename, const std::string &leafname); + static uint64_t GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname); static void FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename); private: - static void FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st, bool ZeroModificationTimes); + static void FillAttributes(StreamableMemBlock &outputBlock, + const char *Filename, EMU_STRUCT_STAT &st, + bool ZeroModificationTimes); static void FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st); void WriteExtendedAttr(const char *Filename, int xattrOffset) const; diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp index b5a54964..b1c5cd0f 100644 --- a/lib/backupclient/BackupClientRestore.cpp +++ b/lib/backupclient/BackupClientRestore.cpp @@ -193,6 +193,8 @@ typedef struct { bool PrintDots; bool RestoreDeleted; + bool ContinueAfterErrors; + bool ContinuedAfterError; std::string mRestoreResumeInfoFilename; RestoreResumeInfo mResumeInfo; } RestoreParams; @@ -202,20 +204,26 @@ typedef struct // -------------------------------------------------------------------------- // // Function -// Name: BackupClientRestoreDir(BackupProtocolClient &, int64_t, const char *, bool) +// Name: BackupClientRestoreDir(BackupProtocolClient &, +// int64_t, const char *, bool) // Purpose: Restore a directory // Created: 23/11/03 // // -------------------------------------------------------------------------- -static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName, +static int BackupClientRestoreDir(BackupProtocolClient &rConnection, + int64_t DirectoryID, std::string &rLocalDirectoryName, RestoreParams &Params, RestoreResumeInfo &rLevel) { - // If we're resuming... check that we haven't got a next level to look at + // If we're resuming... check that we haven't got a next level to + // look at if(rLevel.mpNextLevel != 0) { // Recurse immediately - std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName); - BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, localDirname, Params, *rLevel.mpNextLevel); + std::string localDirname(rLocalDirectoryName + + DIRECTORY_SEPARATOR_ASCHAR + + rLevel.mNextLevelLocalName); + BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, + localDirname, Params, *rLevel.mpNextLevel); // Add it to the list of done itmes rLevel.mRestoredObjects.insert(rLevel.mNextLevelID); @@ -259,22 +267,23 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir break; case ObjectExists_File: { - // File exists with this name, which is fun. Get rid of it. + // File exists with this name, which is fun. + // Get rid of it. BOX_WARNING("File present with name '" << - rLocalDirectoryName << "', removing " << + rLocalDirectoryName << "', removing " "out of the way of restored directory. " "Use specific restore with ID to " "restore this object."); if(::unlink(rLocalDirectoryName.c_str()) != 0) { - BOX_ERROR("Failed to delete file " << - rLocalDirectoryName << ": " << - strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to delete " + "file '" << + rLocalDirectoryName << "'"); return Restore_UnknownError; } BOX_TRACE("In restore, directory name " - "collision with file " << - rLocalDirectoryName); + "collision with file '" << + rLocalDirectoryName << "'"); } break; case ObjectExists_NoObject: @@ -378,10 +387,17 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir exists == ObjectExists_File) && ::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0) { - BOX_ERROR("Failed to create directory '" << - rLocalDirectoryName << "': " << - strerror(errno)); - return Restore_UnknownError; + BOX_LOG_SYS_ERROR("Failed to create directory '" << + rLocalDirectoryName << "'"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } // Save the restore info, in case it's needed later @@ -394,23 +410,39 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir BOX_ERROR("Failed to save resume info file '" << Params.mRestoreResumeInfoFilename << "': " << e.what()); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to save resume info file '" << Params.mRestoreResumeInfoFilename << "': unknown error"); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } // Fetch the directory listing from the server -- getting a // list of files which is appropriate to the restore type rConnection.QueryListDirectory( - DirectoryID, - Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING), - BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)), - true /* want attributes */); + DirectoryID, + Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING), + BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)), + true /* want attributes */); // Retrieve the directory from the stream following BackupStoreDirectory dir; @@ -429,13 +461,29 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir { BOX_ERROR("Failed to restore attributes for '" << rLocalDirectoryName << "': " << e.what()); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to restore attributes for '" << rLocalDirectoryName << "': unknown error"); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } int64_t bytesWrittenSinceLastRestoreInfoSave = 0; @@ -444,35 +492,51 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir { BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; - while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0) + while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) + != 0) { // Check ID hasn't already been done - if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end()) + if(rLevel.mRestoredObjects.find(en->GetObjectID()) + == rLevel.mRestoredObjects.end()) { // Local name BackupStoreFilenameClear nm(en->GetName()); - std::string localFilename(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename()); + std::string localFilename(rLocalDirectoryName + + DIRECTORY_SEPARATOR_ASCHAR + + nm.GetClearFilename()); // Unlink anything which already exists: // For resuming restores, we can't overwrite // files already there. - if(ObjectExists(localFilename) != ObjectExists_NoObject && + if(ObjectExists(localFilename) + != ObjectExists_NoObject && ::unlink(localFilename.c_str()) != 0) { - BOX_ERROR("Failed to delete file '" << - localFilename << "': " << - strerror(errno)); - return Restore_UnknownError; + BOX_LOG_SYS_ERROR("Failed to delete " + "file '" << localFilename << + "'"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } // Request it from the store - rConnection.QueryGetFile(DirectoryID, en->GetObjectID()); + rConnection.QueryGetFile(DirectoryID, + en->GetObjectID()); // Stream containing encoded file - std::auto_ptr<IOStream> objectStream(rConnection.ReceiveStream()); + std::auto_ptr<IOStream> objectStream( + rConnection.ReceiveStream()); - // Decode the file -- need to do different things depending on whether - // the directory entry has additional attributes + // Decode the file -- need to do different + // things depending on whether the directory + // entry has additional attributes try { if(en->HasAttributes()) @@ -493,14 +557,30 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir BOX_ERROR("Failed to restore file '" << localFilename << "': " << e.what()); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to restore file '" << localFilename << "': unknown error"); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } // Progress display? @@ -515,7 +595,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir // Save restore info? int64_t fileSize; - int exists; + bool exists = false; try { @@ -531,7 +611,15 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir "whether file exists: '" << localFilename << "': " << e.what()); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { @@ -539,17 +627,27 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir "whether file exists: '" << localFilename << "': " "unknown error"); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } if(exists) { // File exists... - bytesWrittenSinceLastRestoreInfoSave += fileSize; + bytesWrittenSinceLastRestoreInfoSave + += fileSize; if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES) { - // Save the restore info, in case it's needed later + // Save the restore info, in + // case it's needed later try { Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); @@ -590,14 +688,28 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir BOX_ERROR("Failed to save resume info file '" << Params.mRestoreResumeInfoFilename << "': " << e.what()); - return Restore_UnknownError; + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to save resume info file '" << Params.mRestoreResumeInfoFilename << "': unknown error"); - return Restore_UnknownError; + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } bytesWrittenSinceLastRestoreInfoSave = 0; @@ -608,17 +720,23 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir { BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; - while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0) + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) + != 0) { // Check ID hasn't already been done - if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end()) + if(rLevel.mRestoredObjects.find(en->GetObjectID()) + == rLevel.mRestoredObjects.end()) { // Local name BackupStoreFilenameClear nm(en->GetName()); - std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename()); + std::string localDirname(rLocalDirectoryName + + DIRECTORY_SEPARATOR_ASCHAR + + nm.GetClearFilename()); // Add the level for the next entry - RestoreResumeInfo &rnextLevel(rLevel.AddLevel(en->GetObjectID(), nm.GetClearFilename())); + RestoreResumeInfo &rnextLevel( + rLevel.AddLevel(en->GetObjectID(), + nm.GetClearFilename())); // Recurse int result = BackupClientRestoreDir( @@ -648,13 +766,27 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir { BOX_ERROR("Failed to restore attributes for '" << rLocalDirectoryName << "': " << e.what()); - return Restore_UnknownError; + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to restore attributes for '" << rLocalDirectoryName << "': unknown error"); - return Restore_UnknownError; + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } return Restore_Complete; @@ -664,33 +796,45 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir // -------------------------------------------------------------------------- // // Function -// Name: BackupClientRestore(BackupProtocolClient &, int64_t, const char *, bool, bool) -// Purpose: Restore a directory on the server to a local directory on the disc. -// -// The local directory must not already exist. +// Name: BackupClientRestore(BackupProtocolClient &, int64_t, +// const char *, bool, bool, bool, bool, bool) +// Purpose: Restore a directory on the server to a local +// directory on the disc. The local directory must not +// already exist. // -// If a restore is aborted for any reason, then it may be resumed if -// Resume == true. If Resume == false and resumption is possible, then -// Restore_ResumePossible is returned. +// If a restore is aborted for any reason, then it may +// be resumed if Resume == true. If Resume == false +// and resumption is possible, then +// Restore_ResumePossible is returned. // -// Set RestoreDeleted to restore a deleted directory. This may not give the -// directory structure when it was deleted, because files may have been deleted -// within it before it was deleted. +// Set RestoreDeleted to restore a deleted directory. +// This may not give the directory structure when it +// was deleted, because files may have been deleted +// within it before it was deleted. // -// Returns Restore_TargetExists if the target directory exists, but -// there is no restore possible. (Won't attempt to overwrite things.) +// Returns Restore_TargetExists if the target +// directory exists, but there is no restore possible. +// (Won't attempt to overwrite things.) // -// Returns Restore_Complete on success. (Exceptions on error.) +// Returns Restore_Complete on success. (Exceptions +// on error, unless ContinueAfterError is true and +// the error is recoverable, in which case it returns +// Restore_CompleteWithErrors) // Created: 23/11/03 // // -------------------------------------------------------------------------- -int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName, - bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume) +int BackupClientRestore(BackupProtocolClient &rConnection, + int64_t DirectoryID, const char *LocalDirectoryName, + bool PrintDots, bool RestoreDeleted, + bool UndeleteAfterRestoreDeleted, bool Resume, + bool ContinueAfterErrors) { // Parameter block RestoreParams params; params.PrintDots = PrintDots; params.RestoreDeleted = RestoreDeleted; + params.ContinueAfterErrors = ContinueAfterErrors; + params.ContinuedAfterError = false; params.mRestoreResumeInfoFilename = LocalDirectoryName; params.mRestoreResumeInfoFilename += ".boxbackupresume"; @@ -699,12 +843,13 @@ int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, // Does any resumption information exist? bool doingResume = false; - if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && targetExistance == ObjectExists_Dir) + if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && + targetExistance == ObjectExists_Dir) { if(!Resume) { - // Caller didn't specify that resume should be done, so refuse to do it - // but say why. + // Caller didn't specify that resume should be done, + // so refuse to do it but say why. return Restore_ResumePossible; } @@ -752,9 +897,7 @@ int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, // Delete the resume information file ::unlink(params.mRestoreResumeInfoFilename.c_str()); - return Restore_Complete; + return params.ContinuedAfterError ? Restore_CompleteWithErrors + : Restore_Complete; } - - - diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h index f7724030..7e492238 100644 --- a/lib/backupclient/BackupClientRestore.h +++ b/lib/backupclient/BackupClientRestore.h @@ -15,14 +15,21 @@ class BackupProtocolClient; enum { Restore_Complete = 0, - Restore_ResumePossible = 1, - Restore_TargetExists = 2, - Restore_TargetPathNotFound = 3, - Restore_UnknownError = 4, + Restore_ResumePossible, + Restore_TargetExists, + Restore_TargetPathNotFound, + Restore_UnknownError, + Restore_CompleteWithErrors, }; -int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName, - bool PrintDots = false, bool RestoreDeleted = false, bool UndeleteAfterRestoreDeleted = false, bool Resume = false); +int BackupClientRestore(BackupProtocolClient &rConnection, + int64_t DirectoryID, + const char *LocalDirectoryName, + bool PrintDots = false, + bool RestoreDeleted = false, + bool UndeleteAfterRestoreDeleted = false, + bool Resume = false, + bool ContinueAfterErrors = false); #endif // BACKUPSCLIENTRESTORE__H diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index 61033b5b..e70ba865 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -17,15 +17,15 @@ static const ConfigurationVerifyKey backuplocationkeys[] = { - {"ExcludeFile", 0, ConfigTest_MultiValueAllowed, 0}, - {"ExcludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0}, - {"ExcludeDir", 0, ConfigTest_MultiValueAllowed, 0}, - {"ExcludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0}, - {"AlwaysIncludeFile", 0, ConfigTest_MultiValueAllowed, 0}, - {"AlwaysIncludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0}, - {"AlwaysIncludeDir", 0, ConfigTest_MultiValueAllowed, 0}, - {"AlwaysIncludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0}, - {"Path", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("ExcludeFile", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("ExcludeFilesRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("ExcludeDir", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("ExcludeDirsRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeFile", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeFilesRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeDir", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeDirsRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("Path", ConfigTest_Exists | ConfigTest_LastEntry) }; static const ConfigurationVerify backuplocations[] = @@ -64,39 +64,62 @@ static const ConfigurationVerify verifyserver[] = static const ConfigurationVerifyKey verifyrootkeys[] = { - {"AccountNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - - {"UpdateStoreInterval", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"MinimumFileAge", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"MaxUploadWait", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"MaxFileTimeInFuture", "172800", ConfigTest_IsInt, 0}, // file is uploaded if the file is this much in the future (2 days default) - - {"AutomaticBackup", "yes", ConfigTest_IsBool, 0}, + ConfigurationVerifyKey("AccountNumber", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("UpdateStoreInterval", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("MinimumFileAge", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("MaxUploadWait", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("MaxFileTimeInFuture", ConfigTest_IsInt, 172800), + // file is uploaded if the file is this much in the future + // (2 days default) + ConfigurationVerifyKey("AutomaticBackup", ConfigTest_IsBool, true), - {"SyncAllowScript", 0, 0, 0}, // optional script to run to see if the sync should be started now - // return "now" if it's allowed, or a number of seconds if it's not + ConfigurationVerifyKey("SyncAllowScript", 0), + // script that returns "now" if backup is allowed now, or a number + // of seconds to wait before trying again if not - {"MaximumDiffingTime", 0, ConfigTest_IsInt, 0}, - {"DeleteRedundantLocationsAfter", "172800", ConfigTest_IsInt, 0}, + ConfigurationVerifyKey("MaximumDiffingTime", ConfigTest_IsInt), + ConfigurationVerifyKey("DeleteRedundantLocationsAfter", + ConfigTest_IsInt, 172800), - {"FileTrackingSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"DiffingUploadSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"StoreHostname", 0, ConfigTest_Exists, 0}, - {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // extended log to syslog - {"ExtendedLogFile", NULL, 0, 0}, // extended log to a file - {"LogAllFileAccess", "no", ConfigTest_IsBool, 0}, + ConfigurationVerifyKey("FileTrackingSizeThreshold", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("DiffingUploadSizeThreshold", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("StoreHostname", ConfigTest_Exists), + ConfigurationVerifyKey("StorePort", ConfigTest_IsInt, + BOX_PORT_BBSTORED), + ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), + // extended log to syslog + ConfigurationVerifyKey("ExtendedLogFile", 0), + // extended log to a file + ConfigurationVerifyKey("LogAllFileAccess", ConfigTest_IsBool, false), + // enable logging reasons why each file is backed up or not + ConfigurationVerifyKey("LogFile", 0), + // enable logging to a file + ConfigurationVerifyKey("LogFileLevel", 0), + // set the level of verbosity of file logging + ConfigurationVerifyKey("CommandSocket", 0), + // not compulsory to have this + ConfigurationVerifyKey("KeepAliveTime", ConfigTest_IsInt), + ConfigurationVerifyKey("StoreObjectInfoFile", 0), + // optional - {"CommandSocket", 0, 0, 0}, // not compulsory to have this - {"KeepAliveTime", 0, ConfigTest_IsInt, 0}, // optional - {"StoreObjectInfoFile", 0, 0, 0}, // optional - - {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full + ConfigurationVerifyKey("NotifyScript", 0), + // optional script to run when backup needs attention, eg store full - {"CertificateFile", 0, ConfigTest_Exists, 0}, - {"PrivateKeyFile", 0, ConfigTest_Exists, 0}, - {"TrustedCAsFile", 0, ConfigTest_Exists, 0}, - {"KeysFile", 0, ConfigTest_Exists, 0}, - {"DataDirectory", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("NotifyAlways", ConfigTest_IsBool, false), + // option to disable the suppression of duplicate notifications + + ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), + ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), + ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), + ConfigurationVerifyKey("KeysFile", ConfigTest_Exists), + ConfigurationVerifyKey("DataDirectory", + ConfigTest_Exists | ConfigTest_LastEntry), }; const ConfigurationVerify BackupDaemonConfigVerify = diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp index 7e93d59d..27e12bc8 100644 --- a/lib/backupclient/BackupStoreFile.cpp +++ b/lib/backupclient/BackupStoreFile.cpp @@ -65,22 +65,27 @@ BackupStoreFileStats BackupStoreFile::msStats = {0,0,0}; // Function // Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &) // Purpose: Encode a file into something for storing on file server. -// Requires a real filename so full info can be stored. +// Requires a real filename so full info can be stored. // -// Returns a stream. Most of the work is done by the stream -// when data is actually requested -- the file will be held -// open until the stream is deleted or the file finished. +// Returns a stream. Most of the work is done by the stream +// when data is actually requested -- the file will be held +// open until the stream is deleted or the file finished. // Created: 2003/08/28 // // -------------------------------------------------------------------------- -std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime) +std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(const char *Filename, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) { // Create the stream std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream); // Do the initial setup - ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, 0 /* no recipe, just encode */, - ContainerID, rStoreFilename, pModificationTime); + ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, + 0 /* no recipe, just encode */, + ContainerID, rStoreFilename, pModificationTime, pLogger, + pRunStatusProvider); // Return the stream for the caller return stream; @@ -267,8 +272,8 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) { // Does file exist? - struct stat st; - if(::stat(DecodedFilename, &st) == 0) + EMU_STRUCT_STAT st; + if(EMU_STAT(DecodedFilename, &st) == 0) { THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) } @@ -1255,8 +1260,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, // is it a symlink? bool sourceIsSymlink = false; { - struct stat st; - if(::lstat(Filename, &st) == -1) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) == -1) { THROW_EXCEPTION(CommonException, OSFileError) } diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h index 3ee5ddb0..f38cd821 100644 --- a/lib/backupclient/BackupStoreFile.h +++ b/lib/backupclient/BackupStoreFile.h @@ -10,11 +10,14 @@ #ifndef BACKUPSTOREFILE__H #define BACKUPSTOREFILE__H -#include "IOStream.h" +#include <cstdlib> +#include <memory> + #include "BackupClientFileAttributes.h" #include "BackupStoreFilename.h" - -#include <memory> +#include "IOStream.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" typedef struct { @@ -114,7 +117,11 @@ public: // Main interface - static std::auto_ptr<IOStream> EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0); + static std::auto_ptr<IOStream> EncodeFile(const char *Filename, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime = 0, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); static std::auto_ptr<IOStream> EncodeFileDiff ( const char *Filename, int64_t ContainerID, @@ -207,7 +214,7 @@ public: static BackupStoreFileStats msStats; // For debug -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD static bool TraceDetailsOfDiffProcess; #endif diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp index f7842a0b..e9da1ee7 100644 --- a/lib/backupclient/BackupStoreFileDiff.cpp +++ b/lib/backupclient/BackupStoreFileDiff.cpp @@ -9,6 +9,8 @@ #include "Box.h" +#include <string.h> + #include <new> #include <map> @@ -38,7 +40,7 @@ using namespace BackupStoreFileCreation; // By default, don't trace out details of the diff as we go along -- would fill up logs significantly. // But it's useful for the test. -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD bool BackupStoreFile::TraceDetailsOfDiffProcess = false; #endif @@ -128,8 +130,8 @@ std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff { // Is it a symlink? { - struct stat st; - if(::lstat(Filename, &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) != 0) { THROW_EXCEPTION(CommonException, OSFileError) } @@ -149,7 +151,7 @@ std::auto_ptr<IOStream> BackupStoreFile::EncodeFileDiff int64_t blocksInIndex = 0; bool canDiffFromThis = false; LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis); - //TRACE1("Diff: Blocks in index: %lld\n", blocksInIndex); + // BOX_TRACE("Diff: Blocks in index: " << blocksInIndex); if(!canDiffFromThis) { @@ -434,12 +436,14 @@ static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, i } // trace the size table in debug builds -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD if(BackupStoreFile::TraceDetailsOfDiffProcess) { for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) { - TRACE3("Diff block size %d: %d (count = %lld)\n", t, Sizes[t], sizeCounts[t]); + BOX_TRACE("Diff block size " << t << ": " << + Sizes[t] << " (count = " << + sizeCounts[t] << ")"); } } #endif @@ -459,11 +463,12 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer) { - Timer maximumDiffingTime(0); + Timer maximumDiffingTime(0, "MaximumDiffingTime"); if(pDiffTimer && pDiffTimer->IsManaged()) { - maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime()); + maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime(), + "MaximumDiffingTime"); } std::map<int64_t, int32_t> goodnessOfFit; @@ -626,7 +631,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> // Block matched, roll the checksum forward to the next block without doing // any more comparisons, because these are pointless (as any more matches will be ignored when - // the receipe is generated) and just take up valuable processor time. Edge cases are + // the recipe is generated) and just take up valuable processor time. Edge cases are // especially nasty, using huge amounts of time and memory. int skip = Sizes[s]; if(offset < bytesInEndings && skip > 0) @@ -723,7 +728,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> throw; } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD if(BackupStoreFile::TraceDetailsOfDiffProcess) { // Trace out the found blocks in debug mode @@ -774,7 +779,7 @@ static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int3 // Already present in table? if(pHashTable[hash] != 0) { - //TRACE1("Another hash entry for %d found\n", hash); + //BOX_TRACE("Another hash entry for " << hash << " found"); // Yes -- need to set the pointer in this entry to the current entry to build the linked list pIndex[b].mpNextInHashList = pHashTable[hash]; } @@ -805,7 +810,7 @@ static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChec ASSERT(pFirstInHashList != 0); ASSERT(pIndex != 0); -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD uint16_t DEBUG_Hash = fastSum.GetComponentForHashing(); #endif uint32_t Checksum = fastSum.GetChecksum(); @@ -840,17 +845,19 @@ static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChec // Then go through the entries in the hash list, comparing with the strong digest calculated scan = pFirstInHashList; - //TRACE0("second stage match\n"); + //BOX_TRACE("second stage match"); while(scan != 0) { - //TRACE3("scan size %d, block size %d, hash %d\n", scan->mSize, BlockSize, Hash); + //BOX_TRACE("scan size " << scan->mSize << + // ", block size " << BlockSize << + // ", hash " << Hash); ASSERT(scan->mSize == BlockSize); ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == DEBUG_Hash); // Compare? if(strong.DigestMatches(scan->mStrongChecksum)) { - //TRACE0("Match!\n"); + //BOX_TRACE("Match!\n"); // Found! Add to list of found blocks... int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset; int64_t blockIndex = (scan - pIndex); // pointer arthmitic is frowned upon. But most efficient way of doing it here -- alternative is to use more memory @@ -909,10 +916,11 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA instruction.mSpaceBefore = SizeOfInputFile; rRecipe.push_back(instruction); - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD if(BackupStoreFile::TraceDetailsOfDiffProcess) { - TRACE1("Diff: Default recipe generated, %lld bytes of file\n", SizeOfInputFile); + BOX_TRACE("Diff: Default recipe generated, " << + SizeOfInputFile << " bytes of file"); } #endif @@ -928,7 +936,7 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA ASSERT(i != rFoundBlocks.end()); // check logic // Counting for debug tracing -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD int64_t debug_NewBytesFound = 0; int64_t debug_OldBlocksUsed = 0; #endif @@ -955,7 +963,7 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA instruction.mSpaceBefore = i->first - loc; // Move location forward to match loc += instruction.mSpaceBefore; -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD debug_NewBytesFound += instruction.mSpaceBefore; #endif } @@ -981,7 +989,7 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA instruction.mBlocks += 1; } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD debug_OldBlocksUsed++; #endif @@ -997,18 +1005,22 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA { RESET_INSTRUCTION instruction.mSpaceBefore = SizeOfInputFile - loc; -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD debug_NewBytesFound += instruction.mSpaceBefore; #endif rRecipe.push_back(instruction); } // dump out the recipe -#ifndef NDEBUG - TRACE2("Diff: %lld new bytes found, %lld old blocks used\n", debug_NewBytesFound, debug_OldBlocksUsed); +#ifndef BOX_RELEASE_BUILD + BOX_TRACE("Diff: " << + debug_NewBytesFound << " new bytes found, " << + debug_OldBlocksUsed << " old blocks used"); if(BackupStoreFile::TraceDetailsOfDiffProcess) { - TRACE1("Diff: Recipe generated (size %d)\n======== ========= ========\nSpace b4 FirstBlk NumBlks\n", rRecipe.size()); + BOX_TRACE("Diff: Recipe generated (size " << rRecipe.size()); + BOX_TRACE("======== ========= ========"); + BOX_TRACE("Space b4 FirstBlk NumBlks"); { for(unsigned int e = 0; e < rRecipe.size(); ++e) { @@ -1018,10 +1030,15 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA #else sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); #endif - TRACE3("%8lld %s %8lld\n", rRecipe[e].mSpaceBefore, (rRecipe[e].mpStartBlock == 0)?" -":b, (int64_t)rRecipe[e].mBlocks); + BOX_TRACE(std::setw(8) << + rRecipe[e].mSpaceBefore << + " " << + ((rRecipe[e].mpStartBlock == 0)?" -":b) << + " " << std::setw(8) << + rRecipe[e].mBlocks); } } - TRACE0("======== ========= ========\n"); + BOX_TRACE("======== ========= ========"); } #endif } diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp index 423c11a3..b2d44697 100644 --- a/lib/backupclient/BackupStoreFileEncodeStream.cpp +++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp @@ -9,18 +9,20 @@ #include "Box.h" -#include "BackupStoreFileEncodeStream.h" +#include <string.h> + +#include "BackupClientFileAttributes.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" #include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFileWire.h" #include "BackupStoreObjectMagic.h" -#include "BackupStoreException.h" -#include "BackupStoreConstants.h" #include "BoxTime.h" -#include "BackupClientFileAttributes.h" #include "FileStream.h" -#include "RollingChecksum.h" #include "Random.h" +#include "RollingChecksum.h" #include "MemLeakFindOn.h" @@ -39,6 +41,7 @@ BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() : mpRecipe(0), mpFile(0), mpLogging(0), + mpRunStatusProvider(NULL), mStatus(Status_Header), mSendData(true), mTotalBlocks(0), @@ -105,8 +108,11 @@ BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() // Created: 8/12/03 // // -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEncodeStream::Recipe *pRecipe, - int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime) +void BackupStoreFileEncodeStream::Setup(const char *Filename, + BackupStoreFileEncodeStream::Recipe *pRecipe, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) { // Pointer to a blank recipe which we might create BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; @@ -126,9 +132,9 @@ void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEnc pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); BackupStoreFileEncodeStream::RecipeInstruction instruction; - instruction.mSpaceBefore = fileSize; // whole file - instruction.mBlocks = 0; // no blocks - instruction.mpStartBlock = 0; // no block + instruction.mSpaceBefore = fileSize; // whole file + instruction.mBlocks = 0; // no blocks + instruction.mpStartBlock = 0; // no block pblankRecipe->push_back(instruction); pRecipe = pblankRecipe; @@ -208,8 +214,18 @@ void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEnc // Open the file mpFile = new FileStream(Filename); - // Create logging stream - mpLogging = new ReadLoggingStream(*mpFile); + if (pLogger) + { + // Create logging stream + mpLogging = new ReadLoggingStream(*mpFile, + *pLogger); + } + else + { + // re-use FileStream instead + mpLogging = mpFile; + mpFile = NULL; + } // Work out the largest possible block required for the encoded data mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); @@ -220,7 +236,7 @@ void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEnc { throw std::bad_alloc(); } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD // In debug builds, make sure that the reallocation code is exercised. mEncodedBuffer.Allocate(mAllocatedBufferSize / 4); #else @@ -257,6 +273,8 @@ void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEnc } throw; } + + mpRunStatusProvider = pRunStatusProvider; } @@ -314,6 +332,11 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { return 0; } + + if(mpRunStatusProvider && mpRunStatusProvider->StopRun()) + { + THROW_EXCEPTION(BackupStoreException, SignalReceived); + } int bytesToRead = NBytes; uint8_t *buffer = (uint8_t*)pBuffer; @@ -529,22 +552,25 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() ASSERT(blockRawSize < mAllocatedBufferSize); // Check file open - if(mpFile == 0 || mpLogging == 0) + if(mpLogging == 0) { // File should be open, but isn't. So logical error. THROW_EXCEPTION(BackupStoreException, Internal) } // Read the data in - if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */)) + if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, + 0 /* not interested in size if failure */)) { - // TODO: Do something more intelligent, and abort this upload because the file - // has changed - THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer) + // TODO: Do something more intelligent, and abort + // this upload because the file has changed. + THROW_EXCEPTION(BackupStoreException, + Temp_FileEncodeStreamDidntReadBuffer) } // Encode it - mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer); + mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, + blockRawSize, mEncodedBuffer); //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); @@ -555,7 +581,8 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() strongChecksum.Finish(); // Add entry to the index - StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); + StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, + weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); // Set vars to reading this block mPositionInCurrentBlock = 0; diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h index fb5d0851..c5fa780a 100644 --- a/lib/backupclient/BackupStoreFileEncodeStream.h +++ b/lib/backupclient/BackupStoreFileEncodeStream.h @@ -18,6 +18,7 @@ #include "MD5Digest.h" #include "BackupStoreFile.h" #include "ReadLoggingStream.h" +#include "RunStatusProvider.h" namespace BackupStoreFileCreation { @@ -74,7 +75,11 @@ public: int64_t mOtherFileID; }; - void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime); + void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); virtual int Read(void *pBuffer, int NBytes, int Timeout); virtual void Write(const void *pBuffer, int NBytes); @@ -101,7 +106,8 @@ private: Recipe *mpRecipe; IOStream *mpFile; // source file CollectInBufferStream mData; // buffer for header and index entries - ReadLoggingStream *mpLogging; + IOStream *mpLogging; + RunStatusProvider* mpRunStatusProvider; int mStatus; bool mSendData; // true if there's file data to send (ie not a symlink) int64_t mTotalBlocks; // Total number of blocks in the file diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp index fbfe3313..72cd1acd 100644 --- a/lib/backupclient/BackupStoreFilename.cpp +++ b/lib/backupclient/BackupStoreFilename.cpp @@ -37,7 +37,7 @@ BackupStoreFilename::BackupStoreFilename() // // -------------------------------------------------------------------------- BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy) - : BackupStoreFilename_base(rToCopy) + : mEncryptedName(rToCopy.mEncryptedName) { } @@ -65,7 +65,7 @@ bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const { bool ok = true; - if(size() < 2) + if(mEncryptedName.size() < 2) { // Isn't long enough to have a header ok = false; @@ -73,14 +73,14 @@ bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const else { // Check size is consistent - unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(*this); - if(dsize != size()) + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(this->mEncryptedName); + if(dsize != mEncryptedName.size()) { ok = false; } // And encoding is an accepted value - unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this); + unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName); if(encoding < Encoding_Min || encoding > Encoding_Max) { ok = false; @@ -119,8 +119,8 @@ void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol) rProtocol.Read(data, dsize - 2); // assign to this string, storing the header and the extra data - assign(hdr, 2); - append(data.c_str(), data.size()); + mEncryptedName.assign(hdr, 2); + mEncryptedName.append(data.c_str(), data.size()); // Check it CheckValid(); @@ -141,7 +141,7 @@ void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const { CheckValid(); - rProtocol.Write(c_str(), size()); + rProtocol.Write(mEncryptedName.c_str(), mEncryptedName.size()); } // -------------------------------------------------------------------------- @@ -177,7 +177,7 @@ void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) buf[0] = hdr[0]; buf[1] = hdr[1]; // assign to this string, storing the header and the extra data - assign(buf, dsize); + mEncryptedName.assign(buf, dsize); } else { @@ -194,7 +194,7 @@ void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) data[0] = hdr[0]; data[1] = hdr[1]; // assign to this string, storing the header and the extra data - assign(data, dsize); + mEncryptedName.assign(data, dsize); } // Check it @@ -216,7 +216,7 @@ void BackupStoreFilename::WriteToStream(IOStream &rStream) const { CheckValid(); - rStream.Write(c_str(), size()); + rStream.Write(mEncryptedName.c_str(), mEncryptedName.size()); } // -------------------------------------------------------------------------- @@ -242,7 +242,8 @@ void BackupStoreFilename::EncodedFilenameChanged() // -------------------------------------------------------------------------- bool BackupStoreFilename::IsEncrypted() const { - return BACKUPSTOREFILENAME_GET_ENCODING(*this) != Encoding_Clear; + return BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName) != + Encoding_Clear; } @@ -250,8 +251,9 @@ bool BackupStoreFilename::IsEncrypted() const // // Function // Name: BackupStoreFilename::SetAsClearFilename(const char *) -// Purpose: Sets this object to be a valid filename, but with a filename in the clear. -// Used on the server to create filenames when there's no way of encrypting it. +// Purpose: Sets this object to be a valid filename, but with a +// filename in the clear. Used on the server to create +// filenames when there's no way of encrypting it. // Created: 22/4/04 // // -------------------------------------------------------------------------- @@ -268,7 +270,7 @@ void BackupStoreFilename::SetAsClearFilename(const char *Clear) ASSERT(encoded.size() == toEncode.size() + 2); // Store the encoded string - assign(encoded); + mEncryptedName.assign(encoded); // Stuff which must be done EncodedFilenameChanged(); diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h index a7b6c437..80db9516 100644 --- a/lib/backupclient/BackupStoreFilename.h +++ b/lib/backupclient/BackupStoreFilename.h @@ -40,8 +40,11 @@ class IOStream; // Created: 2003/08/26 // // -------------------------------------------------------------------------- -class BackupStoreFilename : public BackupStoreFilename_base +class BackupStoreFilename /* : public BackupStoreFilename_base */ { +private: + std::string mEncryptedName; + public: BackupStoreFilename(); BackupStoreFilename(const BackupStoreFilename &rToCopy); @@ -71,8 +74,27 @@ public: Encoding_Max = 2 }; + const std::string& GetEncodedFilename() const + { + return mEncryptedName; + } + + bool operator==(const BackupStoreFilename& rOther) const + { + return mEncryptedName == rOther.mEncryptedName; + } + + bool operator!=(const BackupStoreFilename& rOther) const + { + return mEncryptedName != rOther.mEncryptedName; + } + protected: virtual void EncodedFilenameChanged(); + void SetEncodedFilename(const std::string &rEncoded) + { + mEncryptedName = rEncoded; + } }; // On the wire utilities for class and derived class diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp index 9114fdd1..e529d8d3 100644 --- a/lib/backupclient/BackupStoreFilenameClear.cpp +++ b/lib/backupclient/BackupStoreFilenameClear.cpp @@ -160,15 +160,17 @@ void BackupStoreFilenameClear::MakeClearAvailable() const CheckValid(); // Decode the header - int size = BACKUPSTOREFILENAME_GET_SIZE(*this); - int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this); + int size = BACKUPSTOREFILENAME_GET_SIZE(GetEncodedFilename()); + int encoding = BACKUPSTOREFILENAME_GET_ENCODING(GetEncodedFilename()); // Decode based on encoding given in the header switch(encoding) { case Encoding_Clear: - TRACE0("**** BackupStoreFilename encoded with Clear encoding ****\n"); - mClearFilename.assign(c_str() + 2, size - 2); + BOX_TRACE("**** BackupStoreFilename encoded with " + "Clear encoding ****"); + mClearFilename.assign(GetEncodedFilename().c_str() + 2, + size - 2); break; case Encoding_Blowfish: @@ -193,7 +195,8 @@ static void EnsureEncDecBufferSize(int BufSize) if(spEncDecBuffer == 0) { #ifndef WIN32 - TRACE1("Allocating filename encoding/decoding buffer with size %d\n", BufSize); + BOX_TRACE("Allocating filename encoding/decoding buffer " + "with size " << BufSize); #endif spEncDecBuffer = new MemoryBlockGuard<uint8_t *>(BufSize); MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer); @@ -242,7 +245,7 @@ void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, Cipher BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding); // Store the encoded string - assign((char*)buffer, encSize); + SetEncodedFilename(std::string((char*)buffer, encSize)); } @@ -256,8 +259,10 @@ void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, Cipher // -------------------------------------------------------------------------- void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const { + const std::string& rEncoded = GetEncodedFilename(); + // Work out max size - int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(size()) + 4; + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rEncoded.size()) + 4; // Make sure encode/decode buffer has enough space EnsureEncDecBufferSize(maxOutSize); @@ -266,8 +271,8 @@ void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) con uint8_t *buffer = *spEncDecBuffer; // Decrypt - const char *str = c_str() + 2; - int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, size() - 2); + const char *str = rEncoded.c_str() + 2; + int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, rEncoded.size() - 2); // Assign to this mClearFilename.assign((char*)buffer, sizeOut); diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupclient/BackupStoreObjectDump.cpp index d3d9cc17..654317c1 100644 --- a/lib/backupclient/BackupStoreObjectDump.cpp +++ b/lib/backupclient/BackupStoreObjectDump.cpp @@ -47,7 +47,7 @@ static void OutputLine(FILE *file, bool ToTrace, const char *format, ...) } if(ToTrace) { - TRACE1("%s", text); + BOX_TRACE(text); } } @@ -70,7 +70,7 @@ void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) mAttributesModTime, mAttributes.GetSize()); // So repeated filenames can be illustrated, even though they can't be decoded - std::map<BackupStoreFilename, int> nameNum; + std::map<std::string, int> nameNum; int nameNumI = 0; // Dump items @@ -78,7 +78,7 @@ void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i) { // Choose file name index number for this file - std::map<BackupStoreFilename, int>::iterator nn(nameNum.find((*i)->GetName())); + std::map<std::string, int>::iterator nn(nameNum.find((*i)->GetName().GetEncodedFilename())); int ni = nameNumI; if(nn != nameNum.end()) { @@ -86,7 +86,7 @@ void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) } else { - nameNum[(*i)->GetName()] = nameNumI; + nameNum[(*i)->GetName().GetEncodedFilename()] = nameNumI; ++nameNumI; } @@ -124,7 +124,7 @@ void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) (*i)->GetSizeInBlocks(), (*i)->GetAttributesHash(), (*i)->GetAttributes().GetSize(), - (*i)->GetName().size(), + (*i)->GetName().GetEncodedFilename().size(), ni, ((f & BackupStoreDirectory::Entry::Flags_File)?" file":""), ((f & BackupStoreDirectory::Entry::Flags_Dir)?" dir":""), @@ -173,7 +173,8 @@ void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFi // Read the next two objects BackupStoreFilename fn; fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); - OutputLine(file, ToTrace, "Filename size: %d\n", fn.size()); + OutputLine(file, ToTrace, "Filename size: %d\n", + fn.GetEncodedFilename().size()); BackupClientFileAttributes attr; attr.ReadFromStream(rFile, IOStream::TimeOutInfinite); @@ -211,14 +212,16 @@ void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFi if(s > 0) { nnew++; - TRACE2("%8lld this s=%8lld\n", b, s); + BOX_TRACE(std::setw(8) << b << " this s=" << + std::setw(8) << s); } else { nold++; - TRACE2("%8lld other i=%8lld\n", b, 0 - s); + BOX_TRACE(std::setw(8) << b << " other i=" << + std::setw(8) << 0 - s); } } - TRACE0("======== ===== ==========\n"); + BOX_TRACE("======== ===== =========="); } diff --git a/lib/backupclient/RunStatusProvider.h b/lib/backupclient/RunStatusProvider.h new file mode 100644 index 00000000..89f361ca --- /dev/null +++ b/lib/backupclient/RunStatusProvider.h @@ -0,0 +1,29 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RunStatusProvider.h +// Purpose: Declares the RunStatusProvider interface. +// Created: 2008/08/14 +// +// -------------------------------------------------------------------------- + +#ifndef RUNSTATUSPROVIDER__H +#define RUNSTATUSPROVIDER__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: RunStatusProvider +// Purpose: Provides a StopRun() method which returns true if +// the current backup should be halted. +// Created: 2005/11/15 +// +// -------------------------------------------------------------------------- +class RunStatusProvider +{ + public: + virtual ~RunStatusProvider() { } + virtual bool StopRun() = 0; +}; + +#endif // RUNSTATUSPROVIDER__H diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp index 91a6b758..46cab68f 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.cpp +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -208,8 +208,8 @@ void BackupStoreAccountDatabase::CheckUpToDate() const // -------------------------------------------------------------------------- box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const { - struct stat st; - if(::stat(pImpl->mFilename.c_str(), &st) == -1) + EMU_STRUCT_STAT st; + if(EMU_STAT(pImpl->mFilename.c_str(), &st) == -1) { THROW_EXCEPTION(CommonException, OSFileError) } diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index 176ece8f..7598094e 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -268,7 +268,8 @@ void BackupStoreCheck::CheckObjects() } maxDir = CheckObjectsScanDir(0, 1, mStoreRoot); - TRACE1("Max dir starting ID is %llx\n", maxDir); + BOX_TRACE("Max dir starting ID is " << + BOX_FORMAT_OBJECTID(maxDir)); } // Then go through and scan all the objects within those directories diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h index 3f48312a..1d5c1b1e 100644 --- a/lib/backupstore/BackupStoreCheck.h +++ b/lib/backupstore/BackupStoreCheck.h @@ -48,7 +48,7 @@ The following problems can be fixed: // Size of blocks in the list of IDs -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define BACKUPSTORECHECK_BLOCK_SIZE (64*1024) #else #define BACKUPSTORECHECK_BLOCK_SIZE 8 @@ -150,7 +150,7 @@ private: return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK; } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD void DumpObjectInfo(); #define DUMP_OBJECT_INFO DumpObjectInfo(); #else diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 9c6f2452..bcb5c5e9 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -95,6 +95,21 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain mBlocksInDirectories += size; } +class BackupStoreDirectoryFixer +{ + private: + BackupStoreDirectory mDirectory; + std::string mFilename; + std::string mStoreRoot; + int mDiscSetNumber; + + public: + BackupStoreDirectoryFixer(std::string storeRoot, int discSetNumber, + int64_t ID); + void InsertObject(int64_t ObjectID, bool IsDirectory, + int32_t lostDirNameSerial); + ~BackupStoreDirectoryFixer(); +}; // -------------------------------------------------------------------------- // @@ -106,6 +121,10 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain // -------------------------------------------------------------------------- void BackupStoreCheck::CheckUnattachedObjects() { + typedef std::map<int64_t, BackupStoreDirectoryFixer*> fixers_t; + typedef std::pair<int64_t, BackupStoreDirectoryFixer*> fixer_pair_t; + fixers_t fixers; + // Scan all objects, finding ones which have no container for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) { @@ -118,7 +137,9 @@ void BackupStoreCheck::CheckUnattachedObjects() if((flags & Flags_IsContained) == 0) { // Unattached object... - BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached."); + BOX_WARNING("Object " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " is unattached."); ++mNumberErrorsFound; // What's to be done? @@ -196,14 +217,50 @@ void BackupStoreCheck::CheckUnattachedObjects() } ASSERT(putIntoDirectoryID != 0); + if (!mFixErrors) + { + continue; + } + + BackupStoreDirectoryFixer* pFixer; + fixers_t::iterator fi = + fixers.find(putIntoDirectoryID); + if (fi == fixers.end()) + { + // no match, create a new one + pFixer = new BackupStoreDirectoryFixer( + mStoreRoot, mDiscSetNumber, + putIntoDirectoryID); + fixers.insert(fixer_pair_t( + putIntoDirectoryID, pFixer)); + } + else + { + pFixer = fi->second; + } + + int32_t lostDirNameSerial = 0; + + if(flags & Flags_IsDir) + { + lostDirNameSerial = mLostDirNameSerial++; + } + // Add it to the directory - InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID, - ((flags & Flags_IsDir) == Flags_IsDir)); + pFixer->InsertObject(pblock->mID[e], + ((flags & Flags_IsDir) == Flags_IsDir), + lostDirNameSerial); } } } -} + // clean up all the fixers. Deleting them commits them automatically. + for (fixers_t::iterator i = fixers.begin(); i != fixers.end(); i++) + { + BackupStoreDirectoryFixer* pFixer = i->second; + delete pFixer; + } +} // -------------------------------------------------------------------------- // @@ -261,6 +318,86 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) return true; } +BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(std::string storeRoot, + int discSetNumber, int64_t ID) +: mStoreRoot(storeRoot), + mDiscSetNumber(discSetNumber) +{ + // Generate filename + StoreStructure::MakeObjectFilename(ID, mStoreRoot, mDiscSetNumber, + mFilename, false /* don't make sure the dir exists */); + + // Read it in + std::auto_ptr<RaidFileRead> file( + RaidFileRead::Open(mDiscSetNumber, mFilename)); + mDirectory.ReadFromStream(*file, IOStream::TimeOutInfinite); +} + +void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, + int32_t lostDirNameSerial) +{ + // Data for the object + BackupStoreFilename objectStoreFilename; + int64_t modTime = 100; // something which isn't zero or a special time + int32_t sizeInBlocks = 0; // suitable for directories + + if(IsDirectory) + { + // Directory -- simply generate a name for it. + char name[32]; + ::sprintf(name, "dir%08x", lostDirNameSerial); + objectStoreFilename.SetAsClearFilename(name); + } + else + { + // Files require a little more work... + // Open file + std::string fileFilename; + StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, + mDiscSetNumber, fileFilename, + false /* don't make sure the dir exists */); + std::auto_ptr<RaidFileRead> file( + RaidFileRead::Open(mDiscSetNumber, fileFilename)); + + // Fill in size information + sizeInBlocks = file->GetDiscUsageInBlocks(); + + // Read in header + file_StreamFormat hdr; + if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || + (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + )) + { + // This should never happen, everything has been + // checked before. + THROW_EXCEPTION(BackupStoreException, Internal) + } + // This tells us nice things + modTime = box_ntoh64(hdr.mModificationTime); + // And the filename comes next + objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Add a new entry in an appropriate place + mDirectory.AddUnattactedObject(objectStoreFilename, modTime, + ObjectID, sizeInBlocks, + IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); +} + +BackupStoreDirectoryFixer::~BackupStoreDirectoryFixer() +{ + // Fix any flags which have been broken, which there's a good chance of doing + mDirectory.CheckAndFix(); + + // Write it out + RaidFileWrite root(mDiscSetNumber, mFilename); + root.Open(true /* allow overwriting */); + mDirectory.WriteToStream(root); + root.Commit(true /* convert to raid now */); +} // -------------------------------------------------------------------------- // @@ -335,91 +472,6 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool) -// Purpose: -// Created: 22/4/04 -// -// -------------------------------------------------------------------------- -void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory) -{ - if(!mFixErrors) - { - // Don't do anything if we're not supposed to fix errors - return; - } - - // Data for the object - BackupStoreFilename objectStoreFilename; - int64_t modTime = 100; // something which isn't zero or a special time - int32_t sizeInBlocks = 0; // suitable for directories - - if(IsDirectory) - { - // Directory -- simply generate a name for it. - char name[32]; - ::sprintf(name, "dir%08x", mLostDirNameSerial++); - objectStoreFilename.SetAsClearFilename(name); - } - else - { - // Files require a little more work... - // Open file - std::string fileFilename; - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */); - std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, fileFilename)); - // Fill in size information - sizeInBlocks = file->GetDiscUsageInBlocks(); - // Read in header - file_StreamFormat hdr; - if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - )) - { - // This should never happen, everything has been checked before. - THROW_EXCEPTION(BackupStoreException, Internal) - } - // This tells us nice things - modTime = box_ntoh64(hdr.mModificationTime); - // And the filename comes next - objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite); - } - - // Directory object - BackupStoreDirectory dir; - - // Generate filename - std::string filename; - StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); - - // Read it in - { - std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename)); - dir.ReadFromStream(*file, IOStream::TimeOutInfinite); - } - - // Add a new entry in an appropraite place - dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks, - IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); - - // Fix any flags which have been broken, which there's a good change of going - dir.CheckAndFix(); - - // Write it out - if(mFixErrors) - { - RaidFileWrite root(mDiscSetNumber, filename); - root.Open(true /* allow overwriting */); - dir.WriteToStream(root); - root.Commit(true /* convert to raid now */); - } -} - - -// -------------------------------------------------------------------------- -// -// Function // Name: BackupStoreCheck::FixDirsWithWrongContainerID() // Purpose: Rewrites container IDs where required // Created: 22/4/04 @@ -594,6 +646,8 @@ void BackupStoreCheck::WriteNewStoreInfo() } } +#define FMT_OID(x) BOX_FORMAT_OBJECTID(x) +#define FMT_i BOX_FORMAT_OBJECTID((*i)->GetObjectID()) // -------------------------------------------------------------------------- // @@ -620,7 +674,11 @@ bool BackupStoreDirectory::CheckAndFix() if(newerEn == 0) { // Depends on something, but it isn't there. - TRACE2("Entry id %llx removed because depends on newer version %llx which doesn't exist\n", (*i)->GetObjectID(), dependsNewer); + BOX_TRACE("Entry id " << FMT_i << + " removed because depends " + "on newer version " << + FMT_OID(dependsNewer) << + " which doesn't exist"); // Remove delete *i; @@ -638,7 +696,12 @@ bool BackupStoreDirectory::CheckAndFix() if(newerEn->GetDependsOlder() != (*i)->GetObjectID()) { // Wrong entry - TRACE3("Entry id %llx, correcting DependsOlder to %llx, was %llx\n", dependsNewer, (*i)->GetObjectID(), newerEn->GetDependsOlder()); + BOX_TRACE("Entry id " << + FMT_OID(dependsNewer) << + ", correcting DependsOlder to " << + FMT_i << + ", was " << + FMT_OID(newerEn->GetDependsOlder())); newerEn->SetDependsOlder((*i)->GetObjectID()); // Mark as changed changed = true; @@ -657,7 +720,11 @@ bool BackupStoreDirectory::CheckAndFix() if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0) { // Has an older version marked, but this doesn't exist. Remove this mark - TRACE2("Entry id %llx was marked that %llx depended on it, which doesn't exist, dependency info cleared\n", (*i)->GetObjectID(), dependsOlder); + BOX_TRACE("Entry id " << FMT_i << + " was marked as depended on by " << + FMT_OID(dependsOlder) << ", " + "which doesn't exist, dependency " + "info cleared"); (*i)->SetDependsOlder(0); @@ -683,7 +750,7 @@ bool BackupStoreDirectory::CheckAndFix() // Records of things seen std::set<int64_t> idsEncountered; - std::set<BackupStoreFilename> filenamesEncountered; + std::set<std::string> filenamesEncountered; do { @@ -693,7 +760,7 @@ bool BackupStoreDirectory::CheckAndFix() bool removeEntry = false; if((*i) == 0) { - TRACE0("Remove because null pointer found\n"); + BOX_TRACE("Remove because null pointer found"); removeEntry = true; } else @@ -704,7 +771,8 @@ bool BackupStoreDirectory::CheckAndFix() if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File)) { // Bad! Unset the file flag - TRACE1("Entry %llx: File flag set when dir flag set\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": File flag and dir flag both set"); (*i)->RemoveFlags(Entry::Flags_File); changed = true; } @@ -713,7 +781,8 @@ bool BackupStoreDirectory::CheckAndFix() if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end()) { // ID already seen, or type doesn't match - TRACE1("Entry %llx: Remove because ID already seen\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": Remove because ID already seen"); removeEntry = true; } else @@ -723,14 +792,15 @@ bool BackupStoreDirectory::CheckAndFix() // Check to see if the name has already been encountered -- if not, then it // needs to have the old version flag set - if(filenamesEncountered.find((*i)->GetName()) != filenamesEncountered.end()) + if(filenamesEncountered.find((*i)->GetName().GetEncodedFilename()) != filenamesEncountered.end()) { // Seen before -- check old version flag set if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion && ((*i)->GetFlags() & Entry::Flags_Deleted) == 0) { // Not set, set it - TRACE1("Entry %llx: Set old flag\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": Set old flag"); (*i)->AddFlags(Entry::Flags_OldVersion); changed = true; } @@ -741,13 +811,14 @@ bool BackupStoreDirectory::CheckAndFix() if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion) { // Set, unset it - TRACE1("Entry %llx: Old flag unset\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": Old flag unset"); (*i)->RemoveFlags(Entry::Flags_OldVersion); changed = true; } // Remember filename - filenamesEncountered.insert((*i)->GetName()); + filenamesEncountered.insert((*i)->GetName().GetEncodedFilename()); } } } diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp index f22c8339..fed0c3f1 100644 --- a/lib/backupstore/BackupStoreCheckData.cpp +++ b/lib/backupstore/BackupStoreCheckData.cpp @@ -174,7 +174,7 @@ BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD // -------------------------------------------------------------------------- // // Function @@ -189,15 +189,18 @@ void BackupStoreCheck::DumpObjectInfo() { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; - TRACE2("BLOCK @ 0x%08x, %d entries\n", pblock, bentries); + BOX_TRACE("BLOCK @ " << BOX_FORMAT_HEX32(pblock) << + ", " << bentries << " entries"); for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); - TRACE4("id %llx, c %llx, %s, %s\n", - pblock->mID[e], pblock->mContainer[e], - (flags & Flags_IsDir)?"dir":"file", - (flags & Flags_IsContained)?"contained":"unattached"); + BOX_TRACE(std::hex << + "id " << pblock->mID[e] << + ", c " << pblock->mContainer[e] << + ", " << ((flags & Flags_IsDir)?"dir":"file") << + ", " << ((flags & Flags_IsContained) ? + "contained":"unattached")); } } } diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp index 784adfb8..cc6efcf5 100644 --- a/lib/backupstore/BackupStoreConfigVerify.cpp +++ b/lib/backupstore/BackupStoreConfigVerify.cpp @@ -16,7 +16,8 @@ static const ConfigurationVerifyKey verifyserverkeys[] = { - SERVERTLS_VERIFY_SERVER_KEYS(0) // no default listen addresses + SERVERTLS_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) + // no default listen addresses }; static const ConfigurationVerify verifyserver[] = @@ -32,16 +33,18 @@ static const ConfigurationVerify verifyserver[] = static const ConfigurationVerifyKey verifyrootkeys[] = { - {"AccountDatabase", 0, ConfigTest_Exists, 0}, - {"TimeBetweenHousekeeping", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file + ConfigurationVerifyKey("AccountDatabase", ConfigTest_Exists), + ConfigurationVerifyKey("TimeBetweenHousekeeping", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), + // make value "yes" to enable in config file #ifdef WIN32 - {"RaidFileConf", "", ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry) #else - {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry, + BOX_FILE_RAIDFILE_DEFAULT_CONFIG) #endif - }; const ConfigurationVerify BackupConfigFileVerify = diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index 3588cc00..1d55fdf0 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -55,7 +55,7 @@ typedef struct END_STRUCTURE_PACKING_FOR_WIRE #endif -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define NUM_DELETED_DIRS_BLOCK 256 #else #define NUM_DELETED_DIRS_BLOCK 2 @@ -182,7 +182,7 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, const st // Insert info from file info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker); info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed); - info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); + info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); info->mBlocksInOldFiles = box_ntoh64(hdr.mBlocksInOldFiles); info->mBlocksInDeletedFiles = box_ntoh64(hdr.mBlocksInDeletedFiles); info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories); diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h index 094c0deb..ffbe83dd 100644 --- a/lib/backupstore/StoreStructure.h +++ b/lib/backupstore/StoreStructure.h @@ -12,7 +12,7 @@ #include <string> -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define STORE_ID_SEGMENT_LENGTH 8 #define STORE_ID_SEGMENT_MASK 0xff #else diff --git a/lib/common/Box.h b/lib/common/Box.h index d0e7ab1e..1124a062 100644 --- a/lib/common/Box.h +++ b/lib/common/Box.h @@ -17,13 +17,14 @@ #include "BoxPlatform.h" -// uncomment this line to enable full memory leak finding on all malloc-ed blocks (at least, ones used by the STL) +// uncomment this line to enable full memory leak finding on all +// malloc-ed blocks (at least, ones used by the STL) //#define MEMLEAKFINDER_FULL_MALLOC_MONITORING -#ifndef NDEBUG - #ifdef HAVE_EXECINFO_H - #define SHOW_BACKTRACE_ON_EXCEPTION - #endif +// Show backtraces on exceptions in release builds until further notice +// (they are only logged at TRACE level anyway) +#ifdef HAVE_EXECINFO_H + #define SHOW_BACKTRACE_ON_EXCEPTION #endif #ifdef SHOW_BACKTRACE_ON_EXCEPTION @@ -36,14 +37,23 @@ #include "CommonException.h" #include "Logging.h" -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD extern bool AssertFailuresToSyslog; #define ASSERT_FAILS_TO_SYSLOG_ON {AssertFailuresToSyslog = true;} void BoxDebugAssertFailed(const char *cond, const char *file, int line); - #define ASSERT(cond) {if(!(cond)) {BoxDebugAssertFailed(#cond, __FILE__, __LINE__); THROW_EXCEPTION(CommonException, AssertFailed)}} + #define ASSERT(cond) \ + { \ + if(!(cond)) \ + { \ + BoxDebugAssertFailed(#cond, __FILE__, __LINE__); \ + THROW_EXCEPTION_MESSAGE(CommonException, \ + AssertFailed, #cond); \ + } \ + } - // Note that syslog tracing is independent of BoxDebugTraceOn, but stdout tracing is not + // Note that syslog tracing is independent of BoxDebugTraceOn, + // but stdout tracing is not extern bool BoxDebugTraceToSyslog; #define TRACE_TO_SYSLOG(x) {BoxDebugTraceToSyslog = x;} extern bool BoxDebugTraceToStdout; @@ -52,15 +62,6 @@ extern bool BoxDebugTraceOn; int BoxDebug_printf(const char *format, ...); int BoxDebugTrace(const char *format, ...); - #define TRACE0(msg) {BoxDebugTrace("%s", msg);} - #define TRACE1(msg, a0) {BoxDebugTrace(msg, a0);} - #define TRACE2(msg, a0, a1) {BoxDebugTrace(msg, a0, a1);} - #define TRACE3(msg, a0, a1, a2) {BoxDebugTrace(msg, a0, a1, a2);} - #define TRACE4(msg, a0, a1, a2, a3) {BoxDebugTrace(msg, a0, a1, a2, a3);} - #define TRACE5(msg, a0, a1, a2, a3, a4) {BoxDebugTrace(msg, a0, a1, a2, a3, a4);} - #define TRACE6(msg, a0, a1, a2, a3, a4, a5) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5);} - #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6);} - #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6, a7);} #ifndef PLATFORM_DISABLE_MEM_LEAK_TESTING #define BOX_MEMORY_LEAK_TESTING @@ -76,16 +77,6 @@ #define TRACE_TO_SYSLOG(x) {} #define TRACE_TO_STDOUT(x) {} - #define TRACE0(msg) - #define TRACE1(msg, a0) - #define TRACE2(msg, a0, a1) - #define TRACE3(msg, a0, a1, a2) - #define TRACE4(msg, a0, a1, a2, a3) - #define TRACE5(msg, a0, a1, a2, a3, a4) - #define TRACE6(msg, a0, a1, a2, a3, a4, a5) - #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6) - #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7) - // Box Backup builds release get extra information for exception logging #define EXCEPTION_CODENAMES_EXTENDED #define EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION @@ -113,13 +104,21 @@ #define THROW_EXCEPTION(type, subtype) \ { \ OPTIONAL_DO_BACKTRACE \ - BOX_WARNING("Exception thrown: " #type "(" #subtype ") at " \ - __FILE__ "(" << __LINE__ << ")") \ + BOX_WARNING("Exception thrown: " #type "(" #subtype ") " \ + "at " __FILE__ "(" << __LINE__ << ")") \ throw type(type::subtype); \ } -// extra macros for converting to network byte order +#define THROW_EXCEPTION_MESSAGE(type, subtype, message) \ + { \ + OPTIONAL_DO_BACKTRACE \ + BOX_WARNING("Exception thrown: " #type "(" #subtype ") " \ + " (" message ") at " \ + __FILE__ "(" << __LINE__ << ")") \ + throw type(type::subtype, message); \ + } +// extra macros for converting to network byte order #ifdef HAVE_NETINET_IN_H #include <netinet/in.h> #endif diff --git a/lib/common/BoxException.h b/lib/common/BoxException.h index eb992f57..a8f5d7a6 100644 --- a/lib/common/BoxException.h +++ b/lib/common/BoxException.h @@ -11,6 +11,7 @@ #define BOXEXCEPTION__H #include <exception> +#include <string> // -------------------------------------------------------------------------- // diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h index 2f0096aa..c625a7c7 100644 --- a/lib/common/BoxPlatform.h +++ b/lib/common/BoxPlatform.h @@ -29,9 +29,15 @@ #endif #ifdef WIN32 - // need msvcrt version 6.1 or higher for _gmtime64() - // must define this before importing <sys/types.h> - #define __MSVCRT_VERSION__ 0x0601 + #ifdef __MSVCRT_VERSION__ + #if __MSVCRT_VERSION__ < 0x0601 + #error Must include Box.h before sys/types.h + #endif + #else + // need msvcrt version 6.1 or higher for _gmtime64() + // must define this before importing <sys/types.h> + #define __MSVCRT_VERSION__ 0x0601 + #endif #endif #ifdef HAVE_SYS_TYPES_H @@ -66,10 +72,14 @@ #endif // Find out if credentials on UNIX sockets can be obtained -#ifndef HAVE_GETPEEREID - #if !HAVE_DECL_SO_PEERCRED - #define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET - #endif +#ifdef HAVE_GETPEEREID + // +#elif HAVE_DECL_SO_PEERCRED + // +#elif defined HAVE_UCRED_H && HAVE_GETPEERUCRED + // +#else + #define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET #endif #ifdef HAVE_DEFINE_PRAGMA @@ -150,7 +160,7 @@ #endif // for Unix compatibility with Windows :-) -#if !HAVE_DECL_O_BINARY +#ifndef O_BINARY #define O_BINARY 0 #endif @@ -162,14 +172,15 @@ #ifdef WIN32 #define WIN32_LEAN_AND_MEAN - #include "emu.h" #endif -// Solaris has no dirfd(x) macro or function, and we need one. -// We cannot define macros with arguments directly using AC_DEFINE, -// so do it here instead of in configure.ac. +#include "emu.h" + +// Solaris has no dirfd(x) macro or function, and we need one for +// intercept tests. We cannot define macros with arguments directly +// using AC_DEFINE, so do it here instead of in configure.ac. -#if ! HAVE_DECL_DIRFD +#if ! defined PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE && ! HAVE_DECL_DIRFD #ifdef HAVE_DIR_D_FD #define dirfd(x) (x)->d_fd #elif defined HAVE_DIR_DD_FD diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp index 1ddcffd4..d05c0a6c 100644 --- a/lib/common/BoxTime.cpp +++ b/lib/common/BoxTime.cpp @@ -39,8 +39,8 @@ box_time_t GetCurrentBoxTime() struct timeval tv; if (gettimeofday(&tv, NULL) != 0) { - BOX_ERROR("Failed to gettimeofday(), dropping " - "precision: " << strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to gettimeofday(), " + "dropping precision"); } else { @@ -52,3 +52,45 @@ box_time_t GetCurrentBoxTime() return SecondsToBoxTime(time(0)); } + +std::string FormatTime(box_time_t time, bool includeDate, bool showMicros) +{ + std::ostringstream buf; + + time_t seconds = BoxTimeToSeconds(time); + int micros = BoxTimeToMicroSeconds(time) % MICRO_SEC_IN_SEC; + + struct tm tm_now, *tm_ptr = &tm_now; + + #ifdef WIN32 + if ((tm_ptr = localtime(&seconds)) != NULL) + #else + if (localtime_r(&seconds, &tm_now) != NULL) + #endif + { + buf << std::setfill('0'); + + if (includeDate) + { + buf << std::setw(4) << (tm_ptr->tm_year + 1900) << "-" << + std::setw(2) << (tm_ptr->tm_mon + 1) << "-" << + std::setw(2) << (tm_ptr->tm_mday) << " "; + } + + buf << std::setw(2) << tm_ptr->tm_hour << ":" << + std::setw(2) << tm_ptr->tm_min << ":" << + std::setw(2) << tm_ptr->tm_sec; + + if (showMicros) + { + buf << "." << std::setw(6) << micros; + } + } + else + { + buf << strerror(errno); + } + + return buf.str(); +} + diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h index e62a77ab..6681bbbd 100644 --- a/lib/common/BoxTime.h +++ b/lib/common/BoxTime.h @@ -40,4 +40,7 @@ inline uint64_t BoxTimeToMicroSeconds(box_time_t Time) return Time; } +std::string FormatTime(box_time_t time, bool includeDate, + bool showMicros = false); + #endif // BOXTIME__H diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp index 4d76e0e0..7d2e0bac 100644 --- a/lib/common/Configuration.cpp +++ b/lib/common/Configuration.cpp @@ -9,8 +9,11 @@ #include "Box.h" -#include <stdlib.h> #include <limits.h> +#include <stdlib.h> +#include <string.h> + +#include <sstream> #include "Configuration.h" #include "CommonException.h" @@ -29,7 +32,105 @@ inline bool iw(int c) static const char *sValueBooleanStrings[] = {"yes", "true", "no", "false", 0}; static const bool sValueBooleanValue[] = {true, true, false, false}; +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + void *testFunction +) +: mName(name), + mHasDefaultValue(false), + mFlags(flags), + mTestFunction(testFunction) +{ } + +// to allow passing NULL for default ListenAddresses + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + NoDefaultValue_t t, + void *testFunction +) +: mName(name), + mHasDefaultValue(false), + mFlags(flags), + mTestFunction(testFunction) +{ } + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + std::string defaultValue, + void *testFunction +) +: mName(name), + mDefaultValue(defaultValue), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ } + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + const char *defaultValue, + void *testFunction +) +: mName(name), + mDefaultValue(defaultValue), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ } + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + int defaultValue, + void *testFunction +) +: mName(name), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ + ASSERT(flags & ConfigTest_IsInt); + std::ostringstream val; + val << defaultValue; + mDefaultValue = val.str(); +} +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + bool defaultValue, + void *testFunction +) +: mName(name), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ + ASSERT(flags & ConfigTest_IsBool); + mDefaultValue = defaultValue ? "yes" : "no"; +} + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + const ConfigurationVerifyKey& rToCopy +) +: mName(rToCopy.mName), + mDefaultValue(rToCopy.mDefaultValue), + mHasDefaultValue(rToCopy.mHasDefaultValue), + mFlags(rToCopy.mFlags), + mTestFunction(rToCopy.mTestFunction) +{ } // -------------------------------------------------------------------------- // @@ -55,8 +156,8 @@ Configuration::Configuration(const std::string &rName) // -------------------------------------------------------------------------- Configuration::Configuration(const Configuration &rToCopy) : mName(rToCopy.mName), - mSubConfigurations(rToCopy.mSubConfigurations), - mKeys(rToCopy.mKeys) + mKeys(rToCopy.mKeys), + mSubConfigurations(rToCopy.mSubConfigurations) { } @@ -98,32 +199,29 @@ std::auto_ptr<Configuration> Configuration::LoadAndVerify( FdGetLine getline(file); // Object to create - Configuration *pconfig = new Configuration(std::string("<root>")); + std::auto_ptr<Configuration> apConfig( + new Configuration(std::string("<root>"))); try { // Load - LoadInto(*pconfig, getline, rErrorMsg, true); + LoadInto(*apConfig, getline, rErrorMsg, true); if(!rErrorMsg.empty()) { // An error occured, return now - //TRACE1("Error message from LoadInto: %s", rErrorMsg.c_str()); - TRACE0("Error at Configuration::LoadInfo\n"); - delete pconfig; - pconfig = 0; + BOX_ERROR("Error in Configuration::LoadInto: " << + rErrorMsg); return std::auto_ptr<Configuration>(0); } // Verify? if(pVerify) { - if(!Verify(*pconfig, *pVerify, std::string(), rErrorMsg)) + if(!apConfig->Verify(*pVerify, std::string(), rErrorMsg)) { - //TRACE1("Error message from Verify: %s", rErrorMsg.c_str()); - TRACE0("Error at Configuration::Verify\n"); - delete pconfig; - pconfig = 0; + BOX_ERROR("Error verifying configuration: " << + rErrorMsg); return std::auto_ptr<Configuration>(0); } } @@ -131,13 +229,11 @@ std::auto_ptr<Configuration> Configuration::LoadAndVerify( catch(...) { // Clean up - delete pconfig; - pconfig = 0; throw; } // Success. Return result. - return std::auto_ptr<Configuration>(pconfig); + return apConfig; } @@ -173,10 +269,10 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s if(startBlockExpected) { // New config object - Configuration config(blockName); + Configuration subConfig(blockName); // Continue processing into this block - if(!LoadInto(config, rGetLine, rErrorMsg, false)) + if(!LoadInto(subConfig, rGetLine, rErrorMsg, false)) { // Abort error return false; @@ -185,11 +281,12 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s startBlockExpected = false; // Store... - rConfig.mSubConfigurations.push_back(std::pair<std::string, Configuration>(blockName, config)); + rConfig.AddSubConfig(blockName, subConfig); } else { - rErrorMsg += "Unexpected start block in " + rConfig.mName + "\n"; + rErrorMsg += "Unexpected start block in " + + rConfig.mName + "\n"; } } else @@ -200,7 +297,7 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s if(RootLevel) { // error -- root level doesn't have a close - rErrorMsg += "Root level has close block -- forget to terminate subblock?\n"; + rErrorMsg += "Root level has close block -- forgot to terminate subblock?\n"; // but otherwise ignore } else @@ -246,24 +343,11 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s { std::string key(line.substr(0, keyend)); std::string value(line.substr(valuestart)); - //TRACE2("KEY: |%s|=|%s|\n", key.c_str(), value.c_str()); - - // Check for duplicate values - if(rConfig.mKeys.find(key) != rConfig.mKeys.end()) - { - // Multi-values allowed here, but checked later on - rConfig.mKeys[key] += MultiValueSeparator; - rConfig.mKeys[key] += value; - } - else - { - // Store - rConfig.mKeys[key] = value; - } + rConfig.AddKeyValue(key, value); } else { - rErrorMsg += "Invalid key in block "+rConfig.mName+"\n"; + rErrorMsg += "Invalid configuration key: " + line + "\n"; } } else @@ -286,40 +370,60 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s return true; } +void Configuration::AddKeyValue(const std::string& rKey, + const std::string& rValue) +{ + // Check for duplicate values + if(mKeys.find(rKey) != mKeys.end()) + { + // Multi-values allowed here, but checked later on + mKeys[rKey] += MultiValueSeparator; + mKeys[rKey] += rValue; + } + else + { + // Store + mKeys[rKey] = rValue; + } +} + +void Configuration::AddSubConfig(const std::string& rName, + const Configuration& rSubConfig) +{ + mSubConfigurations.push_back( + std::pair<std::string, Configuration>(rName, rSubConfig)); +} + // -------------------------------------------------------------------------- // // Function -// Name: Configuration::KeyExists(const char *) +// Name: Configuration::KeyExists(const std::string&) // Purpose: Checks to see if a key exists // Created: 2003/07/23 // // -------------------------------------------------------------------------- -bool Configuration::KeyExists(const char *pKeyName) const +bool Configuration::KeyExists(const std::string& rKeyName) const { - if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - - return mKeys.find(pKeyName) != mKeys.end(); + return mKeys.find(rKeyName) != mKeys.end(); } // -------------------------------------------------------------------------- // // Function -// Name: Configuration::GetKeyValue(const char *) +// Name: Configuration::GetKeyValue(const std::string&) // Purpose: Returns the value of a configuration variable // Created: 2003/07/23 // // -------------------------------------------------------------------------- -const std::string &Configuration::GetKeyValue(const char *pKeyName) const +const std::string &Configuration::GetKeyValue(const std::string& rKeyName) const { - if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - - std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName)); + std::map<std::string, std::string>::const_iterator i(mKeys.find(rKeyName)); if(i == mKeys.end()) { - BOX_ERROR("Missing configuration key: " << pKeyName); + BOX_ERROR("Missing configuration key: " << rKeyName); THROW_EXCEPTION(CommonException, ConfigNoKey) } else @@ -332,16 +436,14 @@ const std::string &Configuration::GetKeyValue(const char *pKeyName) const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::GetKeyValueInt(const char *) +// Name: Configuration::GetKeyValueInt(const std::string& rKeyName) // Purpose: Gets a key value as an integer // Created: 2003/07/23 // // -------------------------------------------------------------------------- -int Configuration::GetKeyValueInt(const char *pKeyName) const +int Configuration::GetKeyValueInt(const std::string& rKeyName) const { - if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - - std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName)); + std::map<std::string, std::string>::const_iterator i(mKeys.find(rKeyName)); if(i == mKeys.end()) { @@ -362,16 +464,14 @@ int Configuration::GetKeyValueInt(const char *pKeyName) const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::GetKeyValueBool(const char *) const +// Name: Configuration::GetKeyValueBool(const std::string&) // Purpose: Gets a key value as a boolean // Created: 17/2/04 // // -------------------------------------------------------------------------- -bool Configuration::GetKeyValueBool(const char *pKeyName) const +bool Configuration::GetKeyValueBool(const std::string& rKeyName) const { - if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - - std::map<std::string, std::string>::const_iterator i(mKeys.find(pKeyName)); + std::map<std::string, std::string>::const_iterator i(mKeys.find(rKeyName)); if(i == mKeys.end()) { @@ -428,22 +528,21 @@ std::vector<std::string> Configuration::GetKeyNames() const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::SubConfigurationExists(const char *) +// Name: Configuration::SubConfigurationExists(const +// std::string&) // Purpose: Checks to see if a sub configuration exists // Created: 2003/07/23 // // -------------------------------------------------------------------------- -bool Configuration::SubConfigurationExists(const char *pSubName) const +bool Configuration::SubConfigurationExists(const std::string& rSubName) const { - if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - // Attempt to find it... std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin()); for(; i != mSubConfigurations.end(); ++i) { // This the one? - if(i->first == pSubName) + if(i->first == rSubName) { // Yes. return true; @@ -458,22 +557,52 @@ bool Configuration::SubConfigurationExists(const char *pSubName) const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::GetSubConfiguration(const char *) +// Name: Configuration::GetSubConfiguration(const +// std::string&) // Purpose: Gets a sub configuration // Created: 2003/07/23 // // -------------------------------------------------------------------------- -const Configuration &Configuration::GetSubConfiguration(const char *pSubName) const +const Configuration &Configuration::GetSubConfiguration(const std::string& + rSubName) const { - if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - // Attempt to find it... std::list<std::pair<std::string, Configuration> >::const_iterator i(mSubConfigurations.begin()); for(; i != mSubConfigurations.end(); ++i) { // This the one? - if(i->first == pSubName) + if(i->first == rSubName) + { + // Yes. + return i->second; + } + } + + THROW_EXCEPTION(CommonException, ConfigNoSubConfig) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetSubConfiguration(const +// std::string&) +// Purpose: Gets a sub configuration for editing +// Created: 2008/08/12 +// +// -------------------------------------------------------------------------- +Configuration &Configuration::GetSubConfigurationEditable(const std::string& + rSubName) +{ + // Attempt to find it... + + for(SubConfigListType::iterator + i = mSubConfigurations.begin(); + i != mSubConfigurations.end(); ++i) + { + // This the one? + if(i->first == rSubName) { // Yes. return i->second; @@ -510,12 +639,14 @@ std::vector<std::string> Configuration::GetSubConfigurationNames() const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::Verify(const Configuration &, const ConfigurationVerify &, const std::string &, std::string &) -// Purpose: Return list of sub configuration names +// Name: Configuration::Verify(const ConfigurationVerify &, const std::string &, std::string &) +// Purpose: Checks that the configuration is valid according to the +// supplied verifier // Created: 2003/07/24 // // -------------------------------------------------------------------------- -bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg) +bool Configuration::Verify(const ConfigurationVerify &rVerify, + const std::string &rLevel, std::string &rErrorMsg) { bool ok = true; @@ -528,15 +659,14 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV do { // Can the key be found? - ASSERT(pvkey->mpName); - if(rConfig.KeyExists(pvkey->mpName)) + if(KeyExists(pvkey->Name())) { // Get value - const std::string &rval = rConfig.GetKeyValue(pvkey->mpName); + const std::string &rval = GetKeyValue(pvkey->Name()); const char *val = rval.c_str(); // Check it's a number? - if((pvkey->Tests & ConfigTest_IsInt) == ConfigTest_IsInt) + if((pvkey->Flags() & ConfigTest_IsInt) == ConfigTest_IsInt) { // Test it... char *end; @@ -545,12 +675,12 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV { // not a good value ok = false; - rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid integer.\n"; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is not a valid integer.\n"; } } // Check it's a bool? - if((pvkey->Tests & ConfigTest_IsBool) == ConfigTest_IsBool) + if((pvkey->Flags() & ConfigTest_IsBool) == ConfigTest_IsBool) { // See if it's one of the allowed strings. bool found = false; @@ -568,37 +698,38 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV if(!found) { ok = false; - rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid boolean value.\n"; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is not a valid boolean value.\n"; } } // Check for multi valued statments where they're not allowed - if((pvkey->Tests & ConfigTest_MultiValueAllowed) == 0) + if((pvkey->Flags() & ConfigTest_MultiValueAllowed) == 0) { // Check to see if this key is a multi-value -- it shouldn't be if(rval.find(MultiValueSeparator) != rval.npos) { ok = false; - rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) multi value not allowed (duplicated key?).\n"; + rErrorMsg += rLevel + mName +"." + pvkey->Name() + " (key) multi value not allowed (duplicated key?).\n"; } } } else { // Is it required to exist? - if((pvkey->Tests & ConfigTest_Exists) == ConfigTest_Exists) + if((pvkey->Flags() & ConfigTest_Exists) == ConfigTest_Exists) { // Should exist, but doesn't. ok = false; - rErrorMsg += rLevel + rConfig.mName + "." + pvkey->mpName + " (key) is missing.\n"; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is missing.\n"; } - else if(pvkey->mpDefaultValue) + else if(pvkey->HasDefaultValue()) { - rConfig.mKeys[std::string(pvkey->mpName)] = std::string(pvkey->mpDefaultValue); + mKeys[pvkey->Name()] = + pvkey->DefaultValue(); } } - if((pvkey->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + if((pvkey->Flags() & ConfigTest_LastEntry) == ConfigTest_LastEntry) { // No more! todo = false; @@ -610,22 +741,22 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV } while(todo); // Check for additional keys - for(std::map<std::string, std::string>::const_iterator i = rConfig.mKeys.begin(); - i != rConfig.mKeys.end(); ++i) + for(std::map<std::string, std::string>::const_iterator i = mKeys.begin(); + i != mKeys.end(); ++i) { // Is the name in the list? const ConfigurationVerifyKey *scan = rVerify.mpKeys; bool found = false; while(scan) { - if(scan->mpName == i->first) + if(scan->Name() == i->first) { found = true; break; } // Next? - if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + if((scan->Flags() & ConfigTest_LastEntry) == ConfigTest_LastEntry) { break; } @@ -636,7 +767,7 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV { // Shouldn't exist, but does. ok = false; - rErrorMsg += rLevel + rConfig.mName + "." + i->first + " (key) is not a known key. Check spelling and placement.\n"; + rErrorMsg += rLevel + mName + "." + i->first + " (key) is not a known key. Check spelling and placement.\n"; } } } @@ -650,8 +781,7 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV const ConfigurationVerify *scan = rVerify.mpSubConfigurations; while(scan) { - ASSERT(scan->mpName); - if(scan->mpName[0] == '*') + if(scan->mName.length() > 0 && scan->mName[0] == '*') { wildcardverify = scan; } @@ -659,24 +789,25 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV // Required? if((scan->Tests & ConfigTest_Exists) == ConfigTest_Exists) { - if(scan->mpName[0] == '*') + if(scan->mName.length() > 0 && + scan->mName[0] == '*') { // Check something exists - if(rConfig.mSubConfigurations.size() < 1) + if(mSubConfigurations.size() < 1) { // A sub config should exist, but doesn't. ok = false; - rErrorMsg += rLevel + rConfig.mName + ".* (block) is missing (a block must be present).\n"; + rErrorMsg += rLevel + mName + ".* (block) is missing (a block must be present).\n"; } } else { // Check real thing exists - if(!rConfig.SubConfigurationExists(scan->mpName)) + if(!SubConfigurationExists(scan->mName)) { // Should exist, but doesn't. ok = false; - rErrorMsg += rLevel + rConfig.mName + "." + scan->mpName + " (block) is missing.\n"; + rErrorMsg += rLevel + mName + "." + scan->mName + " (block) is missing.\n"; } } } @@ -690,8 +821,9 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV } // Go through the sub configurations, one by one - for(std::list<std::pair<std::string, Configuration> >::const_iterator i(rConfig.mSubConfigurations.begin()); - i != rConfig.mSubConfigurations.end(); ++i) + for(SubConfigListType::iterator + i = mSubConfigurations.begin(); + i != mSubConfigurations.end(); ++i) { // Can this be found? const ConfigurationVerify *subverify = 0; @@ -701,7 +833,7 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV ASSERT(name); while(scan) { - if(strcmp(scan->mpName, name) == 0) + if(scan->mName == name) { // found it! subverify = scan; @@ -725,7 +857,8 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV if(subverify) { // override const-ness here... - if(!Verify((Configuration&)i->second, *subverify, rConfig.mName + '.', rErrorMsg)) + if(!i->second.Verify(*subverify, mName + '.', + rErrorMsg)) { ok = false; } diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h index 64e7568e..2babd753 100644 --- a/lib/common/Configuration.h +++ b/lib/common/Configuration.h @@ -29,20 +29,51 @@ enum class ConfigurationVerifyKey { public: - const char *mpName; // "*" for all other keys (not implemented yet) - const char *mpDefaultValue; // default for when it's not present - int Tests; - void *TestFunction; // set to zero for now, will implement later + typedef enum + { + NoDefaultValue = 1 + } NoDefaultValue_t; + + ConfigurationVerifyKey(std::string name, int flags, + void *testFunction = NULL); + // to allow passing ConfigurationVerifyKey::NoDefaultValue + // for default ListenAddresses + ConfigurationVerifyKey(std::string name, int flags, + NoDefaultValue_t t, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + std::string defaultValue, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + const char* defaultValue, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + int defaultValue, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + bool defaultValue, void *testFunction = NULL); + const std::string& Name() const { return mName; } + const std::string& DefaultValue() const { return mDefaultValue; } + const bool HasDefaultValue() const { return mHasDefaultValue; } + const int Flags() const { return mFlags; } + const void* TestFunction() const { return mTestFunction; } + ConfigurationVerifyKey(const ConfigurationVerifyKey& rToCopy); + +private: + ConfigurationVerifyKey& operator=(const ConfigurationVerifyKey& + noAssign); + + std::string mName; // "*" for all other keys (not implemented yet) + std::string mDefaultValue; // default for when it's not present + bool mHasDefaultValue; + int mFlags; + void *mTestFunction; // set to zero for now, will implement later }; class ConfigurationVerify { public: - const char *mpName; // "*" for all other sub config names + std::string mName; // "*" for all other sub config names const ConfigurationVerify *mpSubConfigurations; const ConfigurationVerifyKey *mpKeys; int Tests; - void *TestFunction; // set to zero for now, will implement later + void *TestFunction; // set to zero for now, will implement later }; class FdGetLine; @@ -57,9 +88,8 @@ class FdGetLine; // -------------------------------------------------------------------------- class Configuration { -private: - Configuration(const std::string &rName); public: + Configuration(const std::string &rName); Configuration(const Configuration &rToCopy); ~Configuration(); @@ -79,26 +109,36 @@ public: std::string &rErrorMsg) { return LoadAndVerify(rFilename, 0, rErrorMsg); } - bool KeyExists(const char *pKeyName) const; - const std::string &GetKeyValue(const char *pKeyName) const; - int GetKeyValueInt(const char *pKeyName) const; - bool GetKeyValueBool(const char *pKeyName) const; + bool KeyExists(const std::string& rKeyName) const; + const std::string &GetKeyValue(const std::string& rKeyName) const; + int GetKeyValueInt(const std::string& rKeyName) const; + bool GetKeyValueBool(const std::string& rKeyName) const; std::vector<std::string> GetKeyNames() const; - bool SubConfigurationExists(const char *pSubName) const; - const Configuration &GetSubConfiguration(const char *pSubName) const; + bool SubConfigurationExists(const std::string& rSubName) const; + const Configuration &GetSubConfiguration(const std::string& rSubName) const; + Configuration &GetSubConfigurationEditable(const std::string& rSubName); std::vector<std::string> GetSubConfigurationNames() const; + void AddKeyValue(const std::string& rKey, const std::string& rValue); + void AddSubConfig(const std::string& rName, const Configuration& rSubConfig); + + bool Verify(const ConfigurationVerify &rVerify, std::string &rErrorMsg) + { + return Verify(rVerify, std::string(), rErrorMsg); + } + +private: std::string mName; + // Order of keys not preserved + std::map<std::string, std::string> mKeys; // Order of sub blocks preserved typedef std::list<std::pair<std::string, Configuration> > SubConfigListType; SubConfigListType mSubConfigurations; - // Order of keys, not preserved - std::map<std::string, std::string> mKeys; -private: static bool LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel); - static bool Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg); + bool Verify(const ConfigurationVerify &rVerify, const std::string &rLevel, + std::string &rErrorMsg); }; #endif // CONFIGURATION__H diff --git a/lib/common/DebugAssertFailed.cpp b/lib/common/DebugAssertFailed.cpp index cceab0ef..e498d641 100644 --- a/lib/common/DebugAssertFailed.cpp +++ b/lib/common/DebugAssertFailed.cpp @@ -7,7 +7,7 @@ // // -------------------------------------------------------------------------- -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD #include "Box.h" @@ -33,5 +33,5 @@ void BoxDebugAssertFailed(const char *cond, const char *file, int line) } -#endif // NDEBUG +#endif // BOX_RELEASE_BUILD diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp index 87cdf00d..230d7163 100644 --- a/lib/common/DebugMemLeakFinder.cpp +++ b/lib/common/DebugMemLeakFinder.cpp @@ -8,7 +8,7 @@ // -------------------------------------------------------------------------- -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD #include "Box.h" @@ -75,6 +75,13 @@ namespace void memleakfinder_init() { ASSERT(!memleakfinder_initialised); + + { + // allocates a permanent buffer on Solaris. + // not a leak? + std::ostringstream oss; + } + memleakfinder_initialised = true; } @@ -146,7 +153,9 @@ void *memleakfinder_realloc(void *ptr, size_t size) std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); if(ptr && i == sMallocBlocks.end()) { - TRACE1("Block %x realloc(), but not in list. Error? Or allocated in startup static objects?\n", ptr); + BOX_WARNING("Block " << ptr << " realloc()ated, but not " + "in list. Error? Or allocated in startup static " + "objects?"); } void *b = ::realloc(ptr, size); @@ -193,7 +202,9 @@ void memleakfinder_free(void *ptr) } else { - TRACE1("Block %p freed, but not known. Error? Or allocated in startup static allocation?\n", ptr); + BOX_WARNING("Block " << ptr << " freed, but not " + "known. Error? Or allocated in startup " + "static allocation?"); } if(sTrackMallocInSection) @@ -293,16 +304,21 @@ void memleakfinder_traceblocksinsection() std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.find(*s)); if(i == sMallocBlocks.end()) { - TRACE0("Logical error in section block finding\n"); + BOX_WARNING("Logical error in section block finding"); } else { - TRACE4("Block %p size %d allocated at %s:%d\n", i->first, i->second.size, i->second.file, i->second.line); + BOX_TRACE("Block " << i->first << " size " << + i->second.size << " allocated at " << + i->second.file << ":" << i->second.line); } } for(std::map<void *, ObjectInfo>::const_iterator i(sSectionObjectBlocks.begin()); i != sSectionObjectBlocks.end(); ++i) { - TRACE5("Object%s %p size %d allocated at %s:%d\n", i->second.array?" []":"", i->first, i->second.size, i->second.file, i->second.line); + BOX_TRACE("Object" << (i->second.array?" []":"") << " " << + i->first << " size " << i->second.size << + " allocated at " << i->second.file << + ":" << i->second.line); } } @@ -335,13 +351,27 @@ void memleakfinder_reportleaks_file(FILE *file) ASSERT(!sTrackingDataDestroyed); - for(std::map<void *, MallocBlockInfo>::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) + for(std::map<void *, MallocBlockInfo>::const_iterator + i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) { - if(is_leak(i->first)) ::fprintf(file, "Block 0x%p size %d allocated at %s:%d\n", i->first, i->second.size, i->second.file, i->second.line); + if(is_leak(i->first)) + { + ::fprintf(file, "Block %p size %d allocated at " + "%s:%d\n", i->first, i->second.size, + i->second.file, i->second.line); + } } - for(std::map<void *, ObjectInfo>::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) + + for(std::map<void *, ObjectInfo>::const_iterator + i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) { - if(is_leak(i->first)) ::fprintf(file, "Object%s 0x%p size %d allocated at %s:%d\n", i->second.array?" []":"", i->first, i->second.size, i->second.file, i->second.line); + if(is_leak(i->first)) + { + ::fprintf(file, "Object%s %p size %d allocated at " + "%s:%d\n", i->second.array?" []":"", + i->first, i->second.size, i->second.file, + i->second.line); + } } } @@ -390,8 +420,10 @@ extern "C" void memleakfinder_atexit() void memleakfinder_setup_exit_report(const char *filename, const char *markertext) { - ::strcpy(atexit_filename, filename); - ::strcpy(atexit_markertext, markertext); + ::strncpy(atexit_filename, filename, sizeof(atexit_filename)-1); + ::strncpy(atexit_markertext, markertext, sizeof(atexit_markertext)-1); + atexit_filename[sizeof(atexit_filename)-1] = 0; + atexit_markertext[sizeof(atexit_markertext)-1] = 0; if(!atexit_registered) { atexit(memleakfinder_atexit); @@ -516,4 +548,4 @@ void operator delete(void *ptr) throw () internal_delete(ptr); } -#endif // NDEBUG +#endif // BOX_RELEASE_BUILD diff --git a/lib/common/DebugPrintf.cpp b/lib/common/DebugPrintf.cpp index 8d75f458..1335d473 100644 --- a/lib/common/DebugPrintf.cpp +++ b/lib/common/DebugPrintf.cpp @@ -7,7 +7,7 @@ // // -------------------------------------------------------------------------- -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD #include "Box.h" @@ -80,4 +80,4 @@ int BoxDebugTrace(const char *format, ...) } -#endif // NDEBUG +#endif // BOX_RELEASE_BUILD diff --git a/lib/common/EventWatchFilesystemObject.cpp b/lib/common/EventWatchFilesystemObject.cpp index 84781113..43533fc8 100644 --- a/lib/common/EventWatchFilesystemObject.cpp +++ b/lib/common/EventWatchFilesystemObject.cpp @@ -26,9 +26,10 @@ // -------------------------------------------------------------------------- // // Function -// Name: EventWatchFilesystemObject::EventWatchFilesystemObject(const char *) -// Purpose: Constructor -- opens the file object -// Created: 12/3/04 +// Name: EventWatchFilesystemObject::EventWatchFilesystemObject +// (const char *) +// Purpose: Constructor -- opens the file object +// Created: 12/3/04 // // -------------------------------------------------------------------------- EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) @@ -39,9 +40,8 @@ EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) #ifdef HAVE_KQUEUE if(mDescriptor == -1) { - BOX_ERROR("EventWatchFilesystemObject: " - "Failed to open file '" << Filename << "': " << - strerror(errno)); + BOX_LOG_SYS_ERROR("EventWatchFilesystemObject: " + "Failed to open file '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileOpenError) } #else @@ -53,9 +53,9 @@ EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) // -------------------------------------------------------------------------- // // Function -// Name: EventWatchFilesystemObject::~EventWatchFilesystemObject() -// Purpose: Destructor -// Created: 12/3/04 +// Name: EventWatchFilesystemObject::~EventWatchFilesystemObject() +// Purpose: Destructor +// Created: 12/3/04 // // -------------------------------------------------------------------------- EventWatchFilesystemObject::~EventWatchFilesystemObject() @@ -70,12 +70,14 @@ EventWatchFilesystemObject::~EventWatchFilesystemObject() // -------------------------------------------------------------------------- // // Function -// Name: EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &) -// Purpose: Copy constructor -// Created: 12/3/04 +// Name: EventWatchFilesystemObject::EventWatchFilesystemObject +// (const EventWatchFilesystemObject &) +// Purpose: Copy constructor +// Created: 12/3/04 // // -------------------------------------------------------------------------- -EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy) +EventWatchFilesystemObject::EventWatchFilesystemObject( + const EventWatchFilesystemObject &rToCopy) : mDescriptor(::dup(rToCopy.mDescriptor)) { if(mDescriptor == -1) @@ -89,17 +91,20 @@ EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesyste // -------------------------------------------------------------------------- // // Function -// Name: EventWatchFilesystemObject::FillInKEvent(struct kevent &, int) -// Purpose: For WaitForEvent -// Created: 12/3/04 +// Name: EventWatchFilesystemObject::FillInKEvent(struct kevent &, int) +// Purpose: For WaitForEvent +// Created: 12/3/04 // // -------------------------------------------------------------------------- -void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, int Flags) const +void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, + int Flags) const { - EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, NOTE_DELETE | NOTE_WRITE, 0, (void*)this); + EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, + NOTE_DELETE | NOTE_WRITE, 0, (void*)this); } #else -void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, int Flags) const +void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, + int Flags) const { THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) } diff --git a/lib/common/FdGetLine.h b/lib/common/FdGetLine.h index a18007a3..df43c3c9 100644 --- a/lib/common/FdGetLine.h +++ b/lib/common/FdGetLine.h @@ -12,7 +12,7 @@ #include <string> -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define FDGETLINE_BUFFER_SIZE 1024 #elif defined WIN32 // need enough space for at least one unicode character diff --git a/lib/common/FileModificationTime.h b/lib/common/FileModificationTime.h index a84df579..5f13c015 100644 --- a/lib/common/FileModificationTime.h +++ b/lib/common/FileModificationTime.h @@ -14,7 +14,7 @@ #include "BoxTime.h" -inline box_time_t FileModificationTime(struct stat &st) +inline box_time_t FileModificationTime(EMU_STRUCT_STAT &st) { #ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); @@ -26,19 +26,26 @@ inline box_time_t FileModificationTime(struct stat &st) return datamodified; } -inline box_time_t FileAttrModificationTime(struct stat &st) +inline box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st) { -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC - box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL); -#else - box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) - + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); + box_time_t statusmodified = +#ifdef HAVE_STRUCT_STAT_ST_MTIMESPEC + (((int64_t)st.st_ctimespec.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC + (((int64_t)st.st_ctim.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctim.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIMENSEC + (((int64_t)st.st_ctimensec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); +#else // no nanoseconds anywhere + (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); #endif return statusmodified; } -inline box_time_t FileModificationTimeMaxModAndAttr(struct stat &st) +inline box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st) { #ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp index e0806e10..d6a3c5da 100644 --- a/lib/common/FileStream.cpp +++ b/lib/common/FileStream.cpp @@ -24,13 +24,40 @@ // Created: 2003/07/31 // // -------------------------------------------------------------------------- -FileStream::FileStream(const char *Filename, int flags, int mode) +FileStream::FileStream(const std::string& rFilename, int flags, int mode) #ifdef WIN32 - : mOSFileHandle(::openfile(Filename, flags, mode)), + : mOSFileHandle(::openfile(rFilename.c_str(), flags, mode)), #else - : mOSFileHandle(::open(Filename, flags, mode)), + : mOSFileHandle(::open(rFilename.c_str(), flags, mode)), #endif - mIsEOF(false) + mIsEOF(false), + mFileName(rFilename) +{ + AfterOpen(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::FileStream(const char *, int, int) +// Purpose: Alternative constructor, takes a const char *, +// avoids const strings being interpreted as handles! +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +FileStream::FileStream(const char *pFilename, int flags, int mode) +#ifdef WIN32 + : mOSFileHandle(::openfile(pFilename, flags, mode)), +#else + : mOSFileHandle(::open(pFilename, flags, mode)), +#endif + mIsEOF(false), + mFileName(pFilename) +{ + AfterOpen(); +} + +void FileStream::AfterOpen() { #ifdef WIN32 if(mOSFileHandle == INVALID_HANDLE_VALUE) @@ -46,12 +73,16 @@ FileStream::FileStream(const char *Filename, int flags, int mode) } else { + #ifdef WIN32 + BOX_LOG_WIN_WARNING_NUMBER("Failed to open file: " << + mFileName, winerrno); + #else + BOX_LOG_SYS_WARNING("Failed to open file: " << + mFileName); + #endif THROW_EXCEPTION(CommonException, OSFileOpenError) } } -#ifdef WIN32 - this->fileName = Filename; -#endif } @@ -65,7 +96,8 @@ FileStream::FileStream(const char *Filename, int flags, int mode) // -------------------------------------------------------------------------- FileStream::FileStream(tOSFileHandle FileDescriptor) : mOSFileHandle(FileDescriptor), - mIsEOF(false) + mIsEOF(false), + mFileName("HANDLE") { #ifdef WIN32 if(mOSFileHandle == INVALID_HANDLE_VALUE) @@ -77,9 +109,6 @@ FileStream::FileStream(tOSFileHandle FileDescriptor) BOX_ERROR("FileStream: called with invalid file handle"); THROW_EXCEPTION(CommonException, OSFileOpenError) } -#ifdef WIN32 - this->fileName = "HANDLE"; -#endif } #if 0 @@ -150,27 +179,32 @@ int FileStream::Read(void *pBuffer, int NBytes, int Timeout) NULL ); - if ( valid ) + if(valid) { r = numBytesRead; } - else if (GetLastError() == ERROR_BROKEN_PIPE) + else if(GetLastError() == ERROR_BROKEN_PIPE) { r = 0; } else { - BOX_ERROR("Failed to read from file: " << - GetErrorMessage(GetLastError())); + BOX_LOG_WIN_ERROR("Failed to read from file: " << mFileName); r = -1; } #else int r = ::read(mOSFileHandle, pBuffer, NBytes); + if(r == -1) + { + BOX_LOG_SYS_ERROR("Failed to read from file: " << mFileName); + } #endif + if(r == -1) { THROW_EXCEPTION(CommonException, OSFileReadError) } + if(r == 0) { mIsEOF = true; @@ -190,8 +224,8 @@ int FileStream::Read(void *pBuffer, int NBytes, int Timeout) // -------------------------------------------------------------------------- IOStream::pos_type FileStream::BytesLeftToRead() { - struct stat st; - if(::fstat(mOSFileHandle, &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_FSTAT(mOSFileHandle, &st) != 0) { THROW_EXCEPTION(CommonException, OSFileError) } @@ -233,6 +267,7 @@ void FileStream::Write(const void *pBuffer, int NBytes) #else if(::write(mOSFileHandle, pBuffer, NBytes) != NBytes) { + BOX_LOG_SYS_ERROR("Failed to write to file: " << mFileName); THROW_EXCEPTION(CommonException, OSFileWriteError) } #endif @@ -368,3 +403,44 @@ bool FileStream::StreamClosed() return mIsEOF; } +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::CompareWith(IOStream&, int) +// Purpose: Compare bytes in this file with other stream's data +// Created: 2009/01/03 +// +// -------------------------------------------------------------------------- +bool FileStream::CompareWith(IOStream& rOther, int Timeout) +{ + // Size + IOStream::pos_type mySize = BytesLeftToRead(); + IOStream::pos_type otherSize = 0; + + // Test the contents + char buf1[2048]; + char buf2[2048]; + while(StreamDataLeft() && rOther.StreamDataLeft()) + { + int readSize = rOther.Read(buf1, sizeof(buf1), Timeout); + otherSize += readSize; + + if(Read(buf2, readSize) != readSize || + ::memcmp(buf1, buf2, readSize) != 0) + { + return false; + } + } + + // Check read all the data from the server and file -- can't be + // equal if local and remote aren't the same length. Can't use + // StreamDataLeft() test on local file, because if it's the same + // size, it won't know it's EOF yet. + + if(rOther.StreamDataLeft() || otherSize != mySize) + { + return false; + } + + return true; +} diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h index 721bf3dd..7c4118cd 100644 --- a/lib/common/FileStream.h +++ b/lib/common/FileStream.h @@ -31,13 +31,17 @@ class FileStream : public IOStream { public: - FileStream(const char *Filename, -#ifdef WIN32 + FileStream(const std::string& rFilename, int flags = (O_RDONLY | O_BINARY), -#else - int flags = O_RDONLY, -#endif int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + + // Ensure that const char * name doesn't end up as a handle + // on Windows! + + FileStream(const char *pFilename, + int flags = (O_RDONLY | O_BINARY), + int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + FileStream(tOSFileHandle FileDescriptor); virtual ~FileStream(); @@ -52,15 +56,16 @@ public: virtual bool StreamDataLeft(); virtual bool StreamClosed(); + bool CompareWith(IOStream& rOther, int Timeout = IOStream::TimeOutInfinite); + private: tOSFileHandle mOSFileHandle; bool mIsEOF; FileStream(const FileStream &rToCopy) { /* do not call */ } + void AfterOpen(); -#ifdef WIN32 // for debugging.. - std::string fileName; -#endif + std::string mFileName; }; diff --git a/lib/common/Guards.h b/lib/common/Guards.h index d2fb84e0..cd2e4628 100644 --- a/lib/common/Guards.h +++ b/lib/common/Guards.h @@ -37,8 +37,8 @@ public: { if(mOSFileHandle < 0) { - BOX_ERROR("FileHandleGuard: failed to open file '" << - rFilename << "': " << strerror(errno)); + BOX_LOG_SYS_ERROR("FileHandleGuard: failed to open " + "file '" << rFilename << "'"); THROW_EXCEPTION(CommonException, OSFileOpenError) } } diff --git a/lib/common/IOStream.cpp b/lib/common/IOStream.cpp index 3c7be561..fc9d0bc3 100644 --- a/lib/common/IOStream.cpp +++ b/lib/common/IOStream.cpp @@ -29,19 +29,6 @@ IOStream::IOStream() // -------------------------------------------------------------------------- // // Function -// Name: IOStream::IOStream(const IOStream &) -// Purpose: Copy constructor (exceptions) -// Created: 2003/07/31 -// -// -------------------------------------------------------------------------- -IOStream::IOStream(const IOStream &rToCopy) -{ - THROW_EXCEPTION(CommonException, NotSupported) -} - -// -------------------------------------------------------------------------- -// -// Function // Name: IOStream::~IOStream() // Purpose: Destructor // Created: 2003/07/31 @@ -238,4 +225,27 @@ bool IOStream::CopyStreamTo(IOStream &rCopyTo, int Timeout, int BufferSize) return true; // completed } +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::Flush(int Timeout) +// Purpose: Read and discard all remaining data in stream. +// Useful for protocol streams which must be flushed +// to avoid breaking the protocol. +// Created: 2008/08/20 +// +// -------------------------------------------------------------------------- +void IOStream::Flush(int Timeout) +{ + char buffer[4096]; + while(StreamDataLeft()) + { + Read(buffer, sizeof(buffer), Timeout); + } +} + +void IOStream::Write(const char *pBuffer) +{ + Write(pBuffer, strlen(pBuffer)); +} diff --git a/lib/common/IOStream.h b/lib/common/IOStream.h index 042ccca4..0b1cedd3 100644 --- a/lib/common/IOStream.h +++ b/lib/common/IOStream.h @@ -22,9 +22,13 @@ class IOStream { public: IOStream(); - IOStream(const IOStream &rToCopy); virtual ~IOStream(); - + +private: + IOStream(const IOStream &rToCopy); /* forbidden */ + IOStream& operator=(const IOStream &rToCopy); /* forbidden */ + +public: enum { TimeOutInfinite = -1, @@ -44,6 +48,7 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite) = 0; virtual pos_type BytesLeftToRead(); // may return IOStream::SizeOfStreamUnknown (and will for most stream types) virtual void Write(const void *pBuffer, int NBytes) = 0; + virtual void Write(const char *pBuffer); virtual void WriteAllBuffered(); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); @@ -57,6 +62,7 @@ public: // Utility functions bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout = IOStream::TimeOutInfinite); bool CopyStreamTo(IOStream &rCopyTo, int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024); + void Flush(int Timeout = IOStream::TimeOutInfinite); static int ConvertSeekTypeToOSWhence(int SeekType); }; diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h index cf152e5a..9a5d1818 100644 --- a/lib/common/IOStreamGetLine.h +++ b/lib/common/IOStreamGetLine.h @@ -14,7 +14,7 @@ #include "IOStream.h" -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define IOSTREAMGETLINE_BUFFER_SIZE 1024 #else #define IOSTREAMGETLINE_BUFFER_SIZE 4 diff --git a/lib/common/Logging.cpp b/lib/common/Logging.cpp index 2b81b52b..1f872d93 100644 --- a/lib/common/Logging.cpp +++ b/lib/common/Logging.cpp @@ -15,12 +15,15 @@ #ifdef HAVE_SYSLOG_H #include <syslog.h> #endif +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif -#include "Logging.h" - +#include <cstring> #include <iomanip> #include "BoxTime.h" +#include "Logging.h" bool Logging::sLogToSyslog = false; bool Logging::sLogToConsole = false; @@ -32,6 +35,7 @@ Console* Logging::spConsole = NULL; Syslog* Logging::spSyslog = NULL; Log::Level Logging::sGlobalLevel = Log::EVERYTHING; Logging Logging::sGlobalLogging; //automatic initialisation +std::string Logging::sProgramName; Logging::Logging() { @@ -148,12 +152,54 @@ void Logging::Log(Log::Level level, const std::string& rFile, } } +void Logging::LogToSyslog(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage) +{ + if (!sLogToSyslog) + { + return; + } + + if (level > sGlobalLevel) + { + return; + } + + std::string newMessage; + + if (sContextSet) + { + newMessage += "[" + sContext + "] "; + } + + newMessage += rMessage; + + spSyslog->Log(level, rFile, line, newMessage); +} + void Logging::SetContext(std::string context) { sContext = context; sContextSet = true; } +Log::Level Logging::GetNamedLevel(const std::string& rName) +{ + if (rName == "nothing") { return Log::NOTHING; } + else if (rName == "fatal") { return Log::FATAL; } + else if (rName == "error") { return Log::ERROR; } + else if (rName == "warning") { return Log::WARNING; } + else if (rName == "notice") { return Log::NOTICE; } + else if (rName == "info") { return Log::INFO; } + else if (rName == "trace") { return Log::TRACE; } + else if (rName == "everything") { return Log::EVERYTHING; } + else + { + BOX_ERROR("Unknown verbosity level: " << rName); + return Log::INVALID; + } +} + void Logging::ClearContext() { sContextSet = false; @@ -161,6 +207,8 @@ void Logging::ClearContext() void Logging::SetProgramName(const std::string& rProgramName) { + sProgramName = rProgramName; + for (std::vector<Logger*>::iterator i = sLoggers.begin(); i != sLoggers.end(); i++) { @@ -168,12 +216,23 @@ void Logging::SetProgramName(const std::string& rProgramName) } } +void Logging::SetFacility(int facility) +{ + spSyslog->SetFacility(facility); +} + Logger::Logger() : mCurrentLevel(Log::EVERYTHING) { Logging::Add(this); } +Logger::Logger(Log::Level Level) +: mCurrentLevel(Level) +{ + Logging::Add(this); +} + Logger::~Logger() { Logging::Remove(this); @@ -182,12 +241,17 @@ Logger::~Logger() bool Console::sShowTime = false; bool Console::sShowTimeMicros = false; bool Console::sShowTag = false; +bool Console::sShowPID = false; std::string Console::sTag; -void Console::SetTag(const std::string& rTag) +void Console::SetProgramName(const std::string& rProgramName) +{ + sTag = rProgramName; +} + +void Console::SetShowTag(bool enabled) { - sTag = rTag; - sShowTag = true; + sShowTag = enabled; } void Console::SetShowTime(bool enabled) @@ -200,6 +264,11 @@ void Console::SetShowTimeMicros(bool enabled) sShowTimeMicros = enabled; } +void Console::SetShowPID(bool enabled) +{ + sShowPID = enabled; +} + bool Console::Log(Log::Level level, const std::string& rFile, int line, std::string& rMessage) { @@ -215,69 +284,64 @@ bool Console::Log(Log::Level level, const std::string& rFile, target = stderr; } - std::string msg; + std::ostringstream buf; if (sShowTime) { - box_time_t time_now = GetCurrentBoxTime(); - time_t seconds = BoxTimeToSeconds(time_now); - int micros = BoxTimeToMicroSeconds(time_now) % MICRO_SEC_IN_SEC; - - struct tm tm_now, *tm_ptr = &tm_now; + buf << FormatTime(GetCurrentBoxTime(), false, sShowTimeMicros); + buf << " "; + } - #ifdef WIN32 - if ((tm_ptr = localtime(&seconds)) != NULL) - #else - if (localtime_r(&seconds, &tm_now) != NULL) - #endif + if (sShowTag) + { + if (sShowPID) { - std::ostringstream buf; - - buf << std::setfill('0') << - std::setw(2) << tm_ptr->tm_hour << ":" << - std::setw(2) << tm_ptr->tm_min << ":" << - std::setw(2) << tm_ptr->tm_sec; - - if (sShowTimeMicros) - { - buf << "." << std::setw(6) << micros; - } - - buf << " "; - msg += buf.str(); + buf << "[" << sTag << " " << getpid() << "] "; } else { - msg += strerror(errno); - msg += " "; + buf << "[" << sTag << "] "; } } - - if (sShowTag) + else if (sShowPID) { - msg += "[" + sTag + "] "; + buf << "[" << getpid() << "] "; } if (level <= Log::FATAL) { - msg += "FATAL: "; + buf << "FATAL: "; } else if (level <= Log::ERROR) { - msg += "ERROR: "; + buf << "ERROR: "; } else if (level <= Log::WARNING) { - msg += "WARNING: "; + buf << "WARNING: "; } else if (level <= Log::NOTICE) { - msg += "NOTICE: "; + buf << "NOTICE: "; } - - msg += rMessage; + else if (level <= Log::INFO) + { + buf << "INFO: "; + } + else if (level <= Log::TRACE) + { + buf << "TRACE: "; + } + + buf << rMessage; - fprintf(target, "%s\n", msg.c_str()); + #ifdef WIN32 + std::string output = buf.str(); + ConvertUtf8ToConsole(output.c_str(), output); + fprintf(target, "%s\n", output.c_str()); + #else + fprintf(target, "%s\n", buf.str().c_str()); + #endif return true; } @@ -295,6 +359,7 @@ bool Syslog::Log(Log::Level level, const std::string& rFile, switch(level) { case Log::NOTHING: /* fall through */ + case Log::INVALID: /* fall through */ case Log::FATAL: syslogLevel = LOG_CRIT; break; case Log::ERROR: syslogLevel = LOG_ERR; break; case Log::WARNING: syslogLevel = LOG_WARNING; break; @@ -330,9 +395,9 @@ bool Syslog::Log(Log::Level level, const std::string& rFile, return true; } -Syslog::Syslog() +Syslog::Syslog() : mFacility(LOG_LOCAL6) { - ::openlog("Box Backup", LOG_PID, LOG_LOCAL6); + ::openlog("Box Backup", LOG_PID, mFacility); } Syslog::~Syslog() @@ -344,5 +409,83 @@ void Syslog::SetProgramName(const std::string& rProgramName) { mName = rProgramName; ::closelog(); - ::openlog(mName.c_str(), LOG_PID, LOG_LOCAL6); + ::openlog(mName.c_str(), LOG_PID, mFacility); +} + +void Syslog::SetFacility(int facility) +{ + mFacility = facility; + ::closelog(); + ::openlog(mName.c_str(), LOG_PID, mFacility); +} + +int Syslog::GetNamedFacility(const std::string& rFacility) +{ + #define CASE_RETURN(x) if (rFacility == #x) { return LOG_ ## x; } + CASE_RETURN(LOCAL0) + CASE_RETURN(LOCAL1) + CASE_RETURN(LOCAL2) + CASE_RETURN(LOCAL3) + CASE_RETURN(LOCAL4) + CASE_RETURN(LOCAL5) + CASE_RETURN(LOCAL6) + CASE_RETURN(DAEMON) + #undef CASE_RETURN + + BOX_ERROR("Unknown log facility '" << rFacility << "', " + "using default LOCAL6"); + return LOG_LOCAL6; +} + +bool FileLogger::Log(Log::Level Level, const std::string& rFile, + int line, std::string& rMessage) +{ + if (Level > GetLevel()) + { + return true; + } + + /* avoid infinite loop if this throws an exception */ + Logging::Remove(this); + + std::ostringstream buf; + buf << FormatTime(GetCurrentBoxTime(), true, false); + buf << " "; + + if (Level <= Log::FATAL) + { + buf << "[FATAL] "; + } + else if (Level <= Log::ERROR) + { + buf << "[ERROR] "; + } + else if (Level <= Log::WARNING) + { + buf << "[WARNING] "; + } + else if (Level <= Log::NOTICE) + { + buf << "[NOTICE] "; + } + else if (Level <= Log::INFO) + { + buf << "[INFO] "; + } + else if (Level <= Log::TRACE) + { + buf << "[TRACE] "; + } + + buf << rMessage << "\n"; + std::string output = buf.str(); + + #ifdef WIN32 + ConvertUtf8ToConsole(output.c_str(), output); + #endif + + mLogFile.Write(output.c_str(), output.length()); + + Logging::Add(this); + return true; } diff --git a/lib/common/Logging.h b/lib/common/Logging.h index 78db2bea..9bb2cf6c 100644 --- a/lib/common/Logging.h +++ b/lib/common/Logging.h @@ -10,10 +10,14 @@ #ifndef LOGGING__H #define LOGGING__H +#include <cerrno> +#include <cstring> #include <iomanip> #include <sstream> #include <vector> +#include "FileStream.h" + /* #define BOX_LOG(level, stuff) \ { \ @@ -27,9 +31,16 @@ #define BOX_LOG(level, stuff) \ { \ - std::ostringstream line; \ - line << stuff; \ - Logging::Log(level, __FILE__, __LINE__, line.str()); \ + std::ostringstream _box_log_line; \ + _box_log_line << stuff; \ + Logging::Log(level, __FILE__, __LINE__, _box_log_line.str()); \ +} + +#define BOX_SYSLOG(level, stuff) \ +{ \ + std::ostringstream _box_log_line; \ + _box_log_line << stuff; \ + Logging::LogToSyslog(level, __FILE__, __LINE__, _box_log_line.str()); \ } #define BOX_FATAL(stuff) BOX_LOG(Log::FATAL, stuff) @@ -41,15 +52,56 @@ if (Logging::IsEnabled(Log::TRACE)) \ { BOX_LOG(Log::TRACE, stuff) } -#define BOX_FORMAT_ACCOUNT(accno) \ +#define BOX_LOG_SYS_WARNING(stuff) \ + BOX_WARNING(stuff << ": " << std::strerror(errno) << " (" << errno << ")") +#define BOX_LOG_SYS_ERROR(stuff) \ + BOX_ERROR(stuff << ": " << std::strerror(errno) << " (" << errno << ")") +#define BOX_LOG_SYS_FATAL(stuff) \ + BOX_FATAL(stuff << ": " << std::strerror(errno) << " (" << errno << ")") + +inline std::string GetNativeErrorMessage() +{ +#ifdef WIN32 + return GetErrorMessage(GetLastError()); +#else + std::ostringstream _box_log_line; + _box_log_line << std::strerror(errno) << " (" << errno << ")"; + return _box_log_line.str(); +#endif +} + +#ifdef WIN32 + #define BOX_LOG_WIN_ERROR(stuff) \ + BOX_ERROR(stuff << ": " << GetErrorMessage(GetLastError())) + #define BOX_LOG_WIN_WARNING(stuff) \ + BOX_WARNING(stuff << ": " << GetErrorMessage(GetLastError())) + #define BOX_LOG_WIN_ERROR_NUMBER(stuff, number) \ + BOX_ERROR(stuff << ": " << GetErrorMessage(number)) + #define BOX_LOG_WIN_WARNING_NUMBER(stuff, number) \ + BOX_WARNING(stuff << ": " << GetErrorMessage(number)) + #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_WIN_ERROR(stuff) + #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_WIN_WARNING(stuff) +#else + #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_SYS_ERROR(stuff) + #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_SYS_WARNING(stuff) +#endif + +#define BOX_LOG_SOCKET_ERROR(_type, _name, _port, stuff) \ + BOX_LOG_NATIVE_ERROR(stuff << " (type " << _type << ", name " << \ + _name << ", port " << _port << ")") + +#define BOX_FORMAT_HEX32(number) \ std::hex << \ std::showbase << \ std::internal << \ std::setw(10) << \ std::setfill('0') << \ - (accno) << \ + (number) << \ std::dec +#define BOX_FORMAT_ACCOUNT(accno) \ + BOX_FORMAT_HEX32(accno) + #define BOX_FORMAT_OBJECTID(objectid) \ std::hex << \ std::showbase << \ @@ -69,7 +121,8 @@ namespace Log NOTICE, INFO, TRACE, - EVERYTHING + EVERYTHING, + INVALID = -1 }; } @@ -89,6 +142,7 @@ class Logger public: Logger(); + Logger(Log::Level level); virtual ~Logger(); virtual bool Log(Log::Level level, const std::string& rFile, @@ -117,20 +171,22 @@ class Logger class Console : public Logger { private: + static bool sShowTag; static bool sShowTime; static bool sShowTimeMicros; - static bool sShowTag; + static bool sShowPID; static std::string sTag; public: virtual bool Log(Log::Level level, const std::string& rFile, int line, std::string& rMessage); virtual const char* GetType() { return "Console"; } - virtual void SetProgramName(const std::string& rProgramName) { } + virtual void SetProgramName(const std::string& rProgramName); - static void SetTag(const std::string& rTag); + static void SetShowTag(bool enabled); static void SetShowTime(bool enabled); static void SetShowTimeMicros(bool enabled); + static void SetShowPID(bool enabled); }; // -------------------------------------------------------------------------- @@ -146,6 +202,7 @@ class Syslog : public Logger { private: std::string mName; + int mFacility; public: Syslog(); @@ -155,6 +212,8 @@ class Syslog : public Logger int line, std::string& rMessage); virtual const char* GetType() { return "Syslog"; } virtual void SetProgramName(const std::string& rProgramName); + virtual void SetFacility(int facility); + static int GetNamedFacility(const std::string& rFacility); }; // -------------------------------------------------------------------------- @@ -178,6 +237,7 @@ class Logging static Syslog* spSyslog; static Log::Level sGlobalLevel; static Logging sGlobalLogging; + static std::string sProgramName; public: Logging (); @@ -190,14 +250,74 @@ class Logging static void Remove (Logger* pOldLogger); static void Log(Log::Level level, const std::string& rFile, int line, const std::string& rMessage); + static void LogToSyslog(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage); static void SetContext(std::string context); static void ClearContext(); static void SetGlobalLevel(Log::Level level) { sGlobalLevel = level; } + static Log::Level GetGlobalLevel() { return sGlobalLevel; } + static Log::Level GetNamedLevel(const std::string& rName); static bool IsEnabled(Log::Level level) { return (int)sGlobalLevel >= (int)level; } static void SetProgramName(const std::string& rProgramName); + static std::string GetProgramName() { return sProgramName; } + static void SetFacility(int facility); + + class Guard + { + private: + Log::Level mOldLevel; + + public: + Guard(Log::Level newLevel) + { + mOldLevel = Logging::GetGlobalLevel(); + Logging::SetGlobalLevel(newLevel); + } + ~Guard() + { + Logging::SetGlobalLevel(mOldLevel); + } + }; + + class Tagger + { + private: + std::string mOldTag; + + public: + Tagger(const std::string& rTempTag) + { + mOldTag = Logging::GetProgramName(); + Logging::SetProgramName(mOldTag + " " + rTempTag); + } + ~Tagger() + { + Logging::SetProgramName(mOldTag); + } + }; +}; + +class FileLogger : public Logger +{ + private: + FileStream mLogFile; + FileLogger(const FileLogger& forbidden) + : mLogFile("") { /* do not call */ } + + public: + FileLogger(const std::string& rFileName, Log::Level Level) + : Logger(Level), + mLogFile(rFileName, O_WRONLY | O_CREAT | O_APPEND) + { } + + virtual bool Log(Log::Level Level, const std::string& rFile, + int Line, std::string& rMessage); + + virtual const char* GetType() { return "FileLogger"; } + virtual void SetProgramName(const std::string& rProgramName) { } }; #endif // LOGGING__H diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp index 16c580ba..b3d0009b 100644 --- a/lib/common/NamedLock.cpp +++ b/lib/common/NamedLock.cpp @@ -2,7 +2,8 @@ // // File // Name: NamedLock.cpp -// Purpose: A global named lock, implemented as a lock file in file system +// Purpose: A global named lock, implemented as a lock file in +// file system // Created: 2003/08/28 // // -------------------------------------------------------------------------- @@ -58,8 +59,9 @@ NamedLock::~NamedLock() // // Function // Name: NamedLock::TryAndGetLock(const char *, int) -// Purpose: Trys to get a lock on the name in the file system. -// IMPORTANT NOTE: If a file exists with this name, it will be deleted. +// Purpose: Tries to get a lock on the name in the file system. +// IMPORTANT NOTE: If a file exists with this name, it +// will be deleted. // Created: 2003/08/28 // // -------------------------------------------------------------------------- diff --git a/lib/common/PartialReadStream.cpp b/lib/common/PartialReadStream.cpp index 76096738..f2f79715 100644 --- a/lib/common/PartialReadStream.cpp +++ b/lib/common/PartialReadStream.cpp @@ -44,7 +44,8 @@ PartialReadStream::~PartialReadStream() // Warn in debug mode if(mBytesLeft != 0) { - TRACE1("PartialReadStream::~PartialReadStream when mBytesLeft = %d\n", mBytesLeft); + BOX_TRACE("PartialReadStream destroyed with " << mBytesLeft << + " bytes remaining"); } } diff --git a/lib/common/ReadLoggingStream.cpp b/lib/common/ReadLoggingStream.cpp index 9023f827..54c99c95 100644 --- a/lib/common/ReadLoggingStream.cpp +++ b/lib/common/ReadLoggingStream.cpp @@ -25,12 +25,13 @@ // Created: 2007/01/16 // // -------------------------------------------------------------------------- -ReadLoggingStream::ReadLoggingStream(IOStream& rSource) +ReadLoggingStream::ReadLoggingStream(IOStream& rSource, Logger& rLogger) : mrSource(rSource), mOffset(0), mLength(mrSource.BytesLeftToRead()), mTotalRead(0), - mStartTime(GetCurrentBoxTime()) + mStartTime(GetCurrentBoxTime()), + mrLogger(rLogger) { } @@ -52,26 +53,21 @@ int ReadLoggingStream::Read(void *pBuffer, int NBytes, int Timeout) mOffset += numBytesRead; } - if (mLength >= 0 && mTotalRead > 0) + if (mLength == 0) { - box_time_t timeNow = GetCurrentBoxTime(); - box_time_t elapsed = timeNow - mStartTime; - box_time_t finish = (elapsed * mLength) / mTotalRead; - box_time_t remain = finish - elapsed; - - BOX_TRACE("Read " << numBytesRead << " bytes at " << mOffset << - ", " << (mLength - mOffset) << " remain, eta " << - BoxTimeToSeconds(remain) << "s"); + mrLogger.Log(numBytesRead, mOffset); } - else if (mLength >= 0 && mTotalRead == 0) + else if (mTotalRead == 0) { - BOX_TRACE("Read " << numBytesRead << " bytes at " << mOffset << - ", " << (mLength - mOffset) << " remain"); + mrLogger.Log(numBytesRead, mOffset, mLength); } else { - BOX_TRACE("Read " << numBytesRead << " bytes at " << mOffset << - ", unknown bytes remaining"); + box_time_t timeNow = GetCurrentBoxTime(); + box_time_t elapsed = timeNow - mStartTime; + box_time_t finish = (elapsed * mLength) / mTotalRead; + // box_time_t remain = finish - elapsed; + mrLogger.Log(numBytesRead, mOffset, mLength, elapsed, finish); } return numBytesRead; diff --git a/lib/common/ReadLoggingStream.h b/lib/common/ReadLoggingStream.h index 15c3ef48..b23b542c 100644 --- a/lib/common/ReadLoggingStream.h +++ b/lib/common/ReadLoggingStream.h @@ -15,13 +15,27 @@ class ReadLoggingStream : public IOStream { +public: + class Logger + { + public: + virtual ~Logger() { } + virtual void Log(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, + box_time_t finish) = 0; + virtual void Log(int64_t readSize, int64_t offset, + int64_t length) = 0; + virtual void Log(int64_t readSize, int64_t offset) = 0; + }; + private: IOStream& mrSource; IOStream::pos_type mOffset, mLength, mTotalRead; box_time_t mStartTime; + Logger& mrLogger; public: - ReadLoggingStream(IOStream& rSource); + ReadLoggingStream(IOStream& rSource, Logger& rLogger); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); @@ -35,7 +49,8 @@ public: private: ReadLoggingStream(const ReadLoggingStream &rToCopy) - : mrSource(rToCopy.mrSource) { /* do not call */ } + : mrSource(rToCopy.mrSource), mrLogger(rToCopy.mrLogger) + { /* do not call */ } }; #endif // READLOGGINGSTREAM__H diff --git a/lib/common/SelfFlushingStream.h b/lib/common/SelfFlushingStream.h new file mode 100644 index 00000000..36e9a4d3 --- /dev/null +++ b/lib/common/SelfFlushingStream.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SelfFlushingStream.h +// Purpose: A stream wrapper that always flushes the underlying +// stream, to ensure protocol safety. +// Created: 2008/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef SELFFLUSHINGSTREAM__H +#define SELFFLUSHINGSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: SelfFlushingStream +// Purpose: A stream wrapper that always flushes the underlying +// stream, to ensure protocol safety. +// Created: 2008/08/20 +// +// -------------------------------------------------------------------------- +class SelfFlushingStream : public IOStream +{ +public: + SelfFlushingStream(IOStream &rSource) + : mrSource(rSource) { } + + SelfFlushingStream(const SelfFlushingStream &rToCopy) + : mrSource(rToCopy.mrSource) { } + + ~SelfFlushingStream() + { + Flush(); + } + +private: + // no copying from IOStream allowed + SelfFlushingStream(const IOStream& rToCopy); + +public: + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + return mrSource.Read(pBuffer, NBytes, Timeout); + } + virtual pos_type BytesLeftToRead() + { + return mrSource.BytesLeftToRead(); + } + virtual void Write(const void *pBuffer, int NBytes) + { + mrSource.Write(pBuffer, NBytes); + } + virtual bool StreamDataLeft() + { + return mrSource.StreamDataLeft(); + } + virtual bool StreamClosed() + { + return mrSource.StreamClosed(); + } + +private: + IOStream &mrSource; +}; + +#endif // SELFFLUSHINGSTREAM__H + diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp new file mode 100644 index 00000000..04d778c1 --- /dev/null +++ b/lib/common/Test.cpp @@ -0,0 +1,418 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Test.cpp +// Purpose: Useful stuff for tests +// Created: 2008/04/05 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <errno.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> + +#include <sys/stat.h> +#include <sys/types.h> + +#ifdef HAVE_UNISTD_H + #include <unistd.h> +#endif + +#include "Test.h" + +bool TestFileExists(const char *Filename) +{ + EMU_STRUCT_STAT st; + return EMU_STAT(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0; +} + +bool TestFileNotEmpty(const char *Filename) +{ + EMU_STRUCT_STAT st; + return EMU_STAT(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0 && + st.st_size > 0; +} + +bool TestDirExists(const char *Filename) +{ + EMU_STRUCT_STAT st; + return EMU_STAT(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == S_IFDIR; +} + +// -1 if doesn't exist +int TestGetFileSize(const char *Filename) +{ + EMU_STRUCT_STAT st; + if(EMU_STAT(Filename, &st) == 0) + { + return st.st_size; + } + return -1; +} + +std::string ConvertPaths(const std::string& rOriginal) +{ +#ifdef WIN32 + // convert UNIX paths to native + + std::string converted; + for (size_t i = 0; i < rOriginal.size(); i++) + { + if (rOriginal[i] == '/') + { + converted += '\\'; + } + else + { + converted += rOriginal[i]; + } + } + return converted; + +#else // !WIN32 + return rOriginal; +#endif +} + +int RunCommand(const std::string& rCommandLine) +{ + return ::system(ConvertPaths(rCommandLine).c_str()); +} + +#ifdef WIN32 +#include <windows.h> +#endif + +bool ServerIsAlive(int pid) +{ + #ifdef WIN32 + + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, + false, pid); + if (hProcess == NULL) + { + if (GetLastError() != ERROR_INVALID_PARAMETER) + { + BOX_ERROR("Failed to open process " << pid << + ": " << + GetErrorMessage(GetLastError())); + } + return false; + } + + DWORD exitCode; + BOOL result = GetExitCodeProcess(hProcess, &exitCode); + CloseHandle(hProcess); + + if (result == 0) + { + BOX_ERROR("Failed to get exit code for process " << + pid << ": " << + GetErrorMessage(GetLastError())) + return false; + } + + if (exitCode == STILL_ACTIVE) + { + return true; + } + + return false; + + #else // !WIN32 + + if(pid == 0) return false; + return ::kill(pid, 0) != -1; + + #endif // WIN32 +} + +int ReadPidFile(const char *pidFile) +{ + if(!TestFileNotEmpty(pidFile)) + { + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file " + "(perhaps one was already running?)"); + return -1; + } + + int pid = -1; + + FILE *f = fopen(pidFile, "r"); + if(f == NULL || fscanf(f, "%d", &pid) != 1) + { + TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); + return -1; + } + fclose(f); + + return pid; +} + +int LaunchServer(const std::string& rCommandLine, const char *pidFile) +{ + ::fprintf(stdout, "Starting server: %s\n", rCommandLine.c_str()); + +#ifdef WIN32 + + PROCESS_INFORMATION procInfo; + + STARTUPINFO startInfo; + startInfo.cb = sizeof(startInfo); + startInfo.lpReserved = NULL; + startInfo.lpDesktop = NULL; + startInfo.lpTitle = NULL; + startInfo.dwFlags = 0; + startInfo.cbReserved2 = 0; + startInfo.lpReserved2 = NULL; + + std::string cmd = ConvertPaths(rCommandLine); + CHAR* tempCmd = strdup(cmd.c_str()); + + DWORD result = CreateProcess + ( + NULL, // lpApplicationName, naughty! + tempCmd, // lpCommandLine + NULL, // lpProcessAttributes + NULL, // lpThreadAttributes + false, // bInheritHandles + 0, // dwCreationFlags + NULL, // lpEnvironment + NULL, // lpCurrentDirectory + &startInfo, // lpStartupInfo + &procInfo // lpProcessInformation + ); + + free(tempCmd); + + if (result == 0) + { + DWORD err = GetLastError(); + printf("Launch failed: %s: error %d\n", rCommandLine.c_str(), + (int)err); + TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + return -1; + } + + CloseHandle(procInfo.hProcess); + CloseHandle(procInfo.hThread); + + return WaitForServerStartup(pidFile, (int)procInfo.dwProcessId); + +#else // !WIN32 + + if(RunCommand(rCommandLine) != 0) + { + TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + return -1; + } + + return WaitForServerStartup(pidFile, 0); + +#endif // WIN32 +} + +int WaitForServerStartup(const char *pidFile, int pidIfKnown) +{ + #ifdef WIN32 + if (pidFile == NULL) + { + return pidIfKnown; + } + #else + // on other platforms there is no other way to get + // the PID, so a NULL pidFile doesn't make sense. + ASSERT(pidFile != NULL); + #endif + + // time for it to start up + ::fprintf(stdout, "Waiting for server to start: "); + + for (int i = 0; i < 15; i++) + { + if (TestFileNotEmpty(pidFile)) + { + break; + } + + if (pidIfKnown && !ServerIsAlive(pidIfKnown)) + { + break; + } + + ::fprintf(stdout, "."); + ::fflush(stdout); + ::sleep(1); + } + + // on Win32 we can check whether the process is alive + // without even checking the PID file + + if (pidIfKnown && !ServerIsAlive(pidIfKnown)) + { + ::fprintf(stdout, " server died!\n"); + TEST_FAIL_WITH_MESSAGE("Server died!"); + return -1; + } + + if (!TestFileNotEmpty(pidFile)) + { + ::fprintf(stdout, " timed out!\n"); + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); + return -1; + } + + ::fprintf(stdout, " done.\n"); + + // wait a second for the pid to be written to the file + ::sleep(1); + + // read pid file + int pid = ReadPidFile(pidFile); + + // On Win32 we can check whether the PID in the pidFile matches + // the one returned by the system, which it always should. + + if (pidIfKnown && pid != pidIfKnown) + { + printf("Server wrote wrong pid to file (%s): expected %d " + "but found %d\n", pidFile, pidIfKnown, pid); + TEST_FAIL_WITH_MESSAGE("Server wrote wrong pid to file"); + return -1; + } + + return pid; +} + +void TestRemoteProcessMemLeaksFunc(const char *filename, + const char* file, int line) +{ +#ifdef BOX_MEMORY_LEAK_TESTING + // Does the file exist? + if(!TestFileExists(filename)) + { + if (failures == 0) + { + first_fail_file = file; + first_fail_line = line; + } + ++failures; + printf("FAILURE: MemLeak report not available (file %s) " + "at %s:%d\n", filename, file, line); + } + else + { + // Is it empty? + if(TestGetFileSize(filename) > 0) + { + if (failures == 0) + { + first_fail_file = file; + first_fail_line = line; + } + ++failures; + printf("FAILURE: Memory leaks found in other process " + "(file %s) at %s:%d\n==========\n", + filename, file, line); + FILE *f = fopen(filename, "r"); + char linebuf[512]; + while(::fgets(linebuf, sizeof(linebuf), f) != 0) + { + printf("%s", linebuf); + } + fclose(f); + printf("==========\n"); + } + + // Delete it + ::unlink(filename); + } +#endif +} + +void force_sync() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "force-sync") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void wait_for_sync_start() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "wait-for-sync") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void wait_for_sync_end() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "wait-for-end") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void sync_and_wait() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "sync-and-wait") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void terminate_bbackupd(int pid) +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "terminate") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + + for (int i = 0; i < 20; i++) + { + if (!ServerIsAlive(pid)) break; + fprintf(stdout, "."); + fflush(stdout); + sleep(1); + } + + TEST_THAT(!ServerIsAlive(pid)); + TestRemoteProcessMemLeaks("bbackupd.memleaks"); +} + + +// Wait a given number of seconds for something to complete +void wait_for_operation(int seconds) +{ + printf("Waiting: "); + fflush(stdout); + for(int l = 0; l < seconds; ++l) + { + sleep(1); + printf("."); + fflush(stdout); + } + printf(" done.\n"); + fflush(stdout); +} + +void safe_sleep(int seconds) +{ +#ifdef WIN32 + Sleep(seconds * 1000); +#else + struct timespec ts; + memset(&ts, 0, sizeof(ts)); + ts.tv_sec = seconds; + ts.tv_nsec = 0; + BOX_TRACE("sleeping for " << seconds << " seconds"); + while (nanosleep(&ts, &ts) == -1 && errno == EINTR) + { + BOX_TRACE("safe_sleep interrupted with " << + ts.tv_sec << "." << ts.tv_nsec << + " secs remaining, sleeping again"); + /* sleep again */ + } +#endif +} + + diff --git a/lib/common/Test.h b/lib/common/Test.h index 49b0ac66..f4766ddc 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -10,18 +10,6 @@ #ifndef TEST__H #define TEST__H -#include <errno.h> -#include <signal.h> -#include <stdio.h> -#include <stdlib.h> - -#include <sys/stat.h> -#include <sys/types.h> - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - #include <string> #ifdef WIN32 @@ -53,7 +41,8 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; first_fail_line = __LINE__; \ } \ failures++; \ - printf("FAILURE: " msg " at " __FILE__ "(%d)\n", __LINE__); \ + BOX_ERROR("**** TEST FAILURE: " << msg << " at " << __FILE__ << \ + ":" << __LINE__); \ } #define TEST_ABORT_WITH_MESSAGE(msg) {TEST_FAIL_WITH_MESSAGE(msg); return 1;} @@ -88,367 +77,88 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; } \ } -inline bool TestFileExists(const char *Filename) -{ - struct stat st; - return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0; -} - -inline bool TestDirExists(const char *Filename) -{ - struct stat st; - return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == S_IFDIR; -} - -// -1 if doesn't exist -inline int TestGetFileSize(const char *Filename) -{ - struct stat st; - if(::stat(Filename, &st) == 0) - { - return st.st_size; - } - return -1; -} - -inline std::string ConvertPaths(const std::string& rOriginal) -{ -#ifdef WIN32 - // convert UNIX paths to native - - std::string converted; - for (size_t i = 0; i < rOriginal.size(); i++) - { - if (rOriginal[i] == '/') - { - converted += '\\'; - } - else - { - converted += rOriginal[i]; - } - } - return converted; - -#else // !WIN32 - return rOriginal; -#endif -} - -inline int RunCommand(const std::string& rCommandLine) -{ - return ::system(ConvertPaths(rCommandLine).c_str()); -} - -#ifdef WIN32 -#include <windows.h> -#endif - -inline bool ServerIsAlive(int pid) -{ -#ifdef WIN32 - HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, false, pid); - if (hProcess == NULL) - { - if (GetLastError() != ERROR_INVALID_PARAMETER) - { - printf("Failed to open process %d: error %d\n", - pid, (int)GetLastError()); - } - return false; - } - CloseHandle(hProcess); - return true; -#else // !WIN32 - if(pid == 0) return false; - return ::kill(pid, 0) != -1; -#endif // WIN32 +// utility macro for comparing two strings in a line +#define TEST_EQUAL(_expected, _found) \ +{ \ + std::ostringstream _oss1; \ + _oss1 << _expected; \ + std::string _exp_str = _oss1.str(); \ + \ + std::ostringstream _oss2; \ + _oss2 << _found; \ + std::string _found_str = _oss2.str(); \ + \ + if(_exp_str != _found_str) \ + { \ + printf("Expected <%s> but found <%s>\n", \ + _exp_str.c_str(), _found_str.c_str()); \ + \ + std::ostringstream _oss3; \ + _oss3 << #_found << " != " << #_expected; \ + \ + TEST_FAIL_WITH_MESSAGE(_oss3.str().c_str()); \ + } \ } -inline int ReadPidFile(const char *pidFile) -{ - if(!TestFileExists(pidFile)) - { - TEST_FAIL_WITH_MESSAGE("Server didn't save PID file " - "(perhaps one was already running?)"); - return -1; - } - - int pid = -1; - - FILE *f = fopen(pidFile, "r"); - if(f == NULL || fscanf(f, "%d", &pid) != 1) - { - TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); - return -1; - } - fclose(f); - - return pid; +// utility macro for comparing two strings in a line +#define TEST_EQUAL_LINE(_expected, _found, _line) \ +{ \ + std::ostringstream _oss1; \ + _oss1 << _expected; \ + std::string _exp_str = _oss1.str(); \ + \ + std::ostringstream _oss2; \ + _oss2 << _found; \ + std::string _found_str = _oss2.str(); \ + \ + if(_exp_str != _found_str) \ + { \ + std::string _line_str = _line; \ + printf("Expected <%s> but found <%s> in <%s>\n", \ + _exp_str.c_str(), _found_str.c_str(), _line_str.c_str()); \ + \ + std::ostringstream _oss3; \ + _oss3 << #_found << " != " << #_expected << " in " << _line; \ + \ + TEST_FAIL_WITH_MESSAGE(_oss3.str().c_str()); \ + } \ } -inline int LaunchServer(const std::string& rCommandLine, const char *pidFile) -{ -#ifdef WIN32 - - PROCESS_INFORMATION procInfo; - - STARTUPINFO startInfo; - startInfo.cb = sizeof(startInfo); - startInfo.lpReserved = NULL; - startInfo.lpDesktop = NULL; - startInfo.lpTitle = NULL; - startInfo.dwFlags = 0; - startInfo.cbReserved2 = 0; - startInfo.lpReserved2 = NULL; - - std::string cmd = ConvertPaths(rCommandLine); - CHAR* tempCmd = strdup(cmd.c_str()); - - DWORD result = CreateProcess - ( - NULL, // lpApplicationName, naughty! - tempCmd, // lpCommandLine - NULL, // lpProcessAttributes - NULL, // lpThreadAttributes - false, // bInheritHandles - 0, // dwCreationFlags - NULL, // lpEnvironment - NULL, // lpCurrentDirectory - &startInfo, // lpStartupInfo - &procInfo // lpProcessInformation - ); - - free(tempCmd); - - if (result == 0) - { - DWORD err = GetLastError(); - printf("Launch failed: %s: error %d\n", rCommandLine.c_str(), - (int)err); - return -1; - } - - CloseHandle(procInfo.hProcess); - CloseHandle(procInfo.hThread); - -#else // !WIN32 - - if(RunCommand(rCommandLine) != 0) - { - printf("Server: %s\n", rCommandLine.c_str()); - TEST_FAIL_WITH_MESSAGE("Couldn't start server"); - return -1; - } - -#endif // WIN32 - - #ifdef WIN32 - // on other platforms there is no other way to get - // the PID, so a NULL pidFile doesn't make sense. - - if (pidFile == NULL) - { - return (int)procInfo.dwProcessId; - } - #endif - - // time for it to start up - ::fprintf(stdout, "Starting server: %s\n", rCommandLine.c_str()); - ::fprintf(stdout, "Waiting for server to start: "); - for (int i = 0; i < 15; i++) - { - if (TestFileExists(pidFile)) - { - break; - } - - #ifdef WIN32 - if (!ServerIsAlive((int)procInfo.dwProcessId)) - { - break; - } - #endif - - ::fprintf(stdout, "."); - ::fflush(stdout); - ::sleep(1); - } - - #ifdef WIN32 - // on Win32 we can check whether the process is alive - // without even checking the PID file - - if (!ServerIsAlive((int)procInfo.dwProcessId)) - { - ::fprintf(stdout, "server died!\n"); - TEST_FAIL_WITH_MESSAGE("Server died!"); - return -1; - } - #endif - - if (!TestFileExists(pidFile)) - { - ::fprintf(stdout, "timed out!\n"); - TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); - return -1; +// utility macro for testing a line +#define TEST_LINE(_condition, _line) \ + TEST_THAT(_condition); \ + if (!(_condition)) \ + { \ + printf("Test failed on <%s>\n", _line.c_str()); \ } - ::fprintf(stdout, "done.\n"); - - // wait a second for the pid to be written to the file - ::sleep(1); +bool TestFileExists(const char *Filename); +bool TestDirExists(const char *Filename); - // read pid file - int pid = ReadPidFile(pidFile); - - #ifdef WIN32 - // On Win32 we can check whether the PID in the pidFile matches - // the one returned by the system, which it always should. - - if (pid != (int)procInfo.dwProcessId) - { - printf("Server wrote wrong pid to file (%s): expected %d " - "but found %d\n", pidFile, - (int)procInfo.dwProcessId, pid); - TEST_FAIL_WITH_MESSAGE("Server wrote wrong pid to file"); - return -1; - } - #endif - - return pid; -} +// -1 if doesn't exist +int TestGetFileSize(const char *Filename); +std::string ConvertPaths(const std::string& rOriginal); +int RunCommand(const std::string& rCommandLine); +bool ServerIsAlive(int pid); +int ReadPidFile(const char *pidFile); +int LaunchServer(const std::string& rCommandLine, const char *pidFile); +int WaitForServerStartup(const char *pidFile, int pidIfKnown); #define TestRemoteProcessMemLeaks(filename) \ TestRemoteProcessMemLeaksFunc(filename, __FILE__, __LINE__) -inline void TestRemoteProcessMemLeaksFunc(const char *filename, - const char* file, int line) -{ -#ifdef BOX_MEMORY_LEAK_TESTING - // Does the file exist? - if(!TestFileExists(filename)) - { - if (failures == 0) - { - first_fail_file = file; - first_fail_line = line; - } - ++failures; - printf("FAILURE: MemLeak report not available (file %s) " - "at %s:%d\n", filename, file, line); - } - else - { - // Is it empty? - if(TestGetFileSize(filename) > 0) - { - if (failures == 0) - { - first_fail_file = file; - first_fail_line = line; - } - ++failures; - printf("FAILURE: Memory leaks found in other process " - "(file %s) at %s:%d\n==========\n", - filename, file, line); - FILE *f = fopen(filename, "r"); - char linebuf[512]; - while(::fgets(linebuf, sizeof(linebuf), f) != 0) - { - printf("%s", linebuf); - } - fclose(f); - printf("==========\n"); - } - - // Delete it - ::unlink(filename); - } -#endif -} - -inline void force_sync() -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "force-sync") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); -} - -inline void wait_for_sync_start() -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "wait-for-sync") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); -} - -inline void wait_for_sync_end() -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "wait-for-end") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); -} - -inline void sync_and_wait() -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "sync-and-wait") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); -} - -inline void terminate_bbackupd(int pid) -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "terminate") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); - - for (int i = 0; i < 20; i++) - { - if (!ServerIsAlive(pid)) break; - fprintf(stdout, "."); - fflush(stdout); - sleep(1); - } - - TEST_THAT(!ServerIsAlive(pid)); - TestRemoteProcessMemLeaks("bbackupd.memleaks"); -} +void TestRemoteProcessMemLeaksFunc(const char *filename, + const char* file, int line); +void force_sync(); +void wait_for_sync_start(); +void wait_for_sync_end(); +void sync_and_wait(); +void terminate_bbackupd(int pid); // Wait a given number of seconds for something to complete -inline void wait_for_operation(int seconds) -{ - printf("Waiting: "); - fflush(stdout); - for(int l = 0; l < seconds; ++l) - { - sleep(1); - printf("."); - fflush(stdout); - } - printf(" done.\n"); - fflush(stdout); -} - -inline void safe_sleep(int seconds) -{ -#ifdef WIN32 - Sleep(seconds * 1000); -#else - struct timespec ts; - memset(&ts, 0, sizeof(ts)); - ts.tv_sec = seconds; - ts.tv_nsec = 0; - BOX_TRACE("sleeping for " << seconds << " seconds"); - while (nanosleep(&ts, &ts) == -1 && errno == EINTR) - { - BOX_TRACE("safe_sleep interrupted with " << - ts.tv_sec << "." << ts.tv_nsec << - " secs remaining, sleeping again"); - /* sleep again */ - } -#endif -} +void wait_for_operation(int seconds); +void safe_sleep(int seconds); #endif // TEST__H diff --git a/lib/common/Timer.cpp b/lib/common/Timer.cpp index 16133ecc..137ad45f 100644 --- a/lib/common/Timer.cpp +++ b/lib/common/Timer.cpp @@ -8,9 +8,14 @@ // // -------------------------------------------------------------------------- +#ifdef WIN32 + #define _WIN32_WINNT 0x0500 +#endif + #include "Box.h" #include <signal.h> +#include <cstring> #include "Timer.h" #include "Logging.h" @@ -20,6 +25,9 @@ std::vector<Timer*>* Timers::spTimers = NULL; bool Timers::sRescheduleNeeded = false; +#define TIMER_ID "timer " << mName << " (" << this << ") " +#define TIMER_ID_OF(t) "timer " << (t).GetName() << " (" << &(t) << ") " + typedef void (*sighandler_t)(int); // -------------------------------------------------------------------------- @@ -35,9 +43,7 @@ void Timers::Init() ASSERT(!spTimers); #if defined WIN32 && ! defined PLATFORM_CYGWIN - // no support for signals at all - InitTimer(); - SetTimerHandler(Timers::SignalHandler); + // no init needed #else struct sigaction newact, oldact; newact.sa_handler = Timers::SignalHandler; @@ -72,9 +78,7 @@ void Timers::Cleanup() } #if defined WIN32 && ! defined PLATFORM_CYGWIN - // no support for signals at all - FiniTimer(); - SetTimerHandler(NULL); + // no cleanup needed #else struct itimerval timeout; memset(&timeout, 0, sizeof(timeout)); @@ -149,9 +153,22 @@ void Timers::Remove(Timer& rTimer) Reschedule(); } +void Timers::RequestReschedule() +{ + sRescheduleNeeded = true; +} + +void Timers::RescheduleIfNeeded() +{ + if (sRescheduleNeeded) + { + Reschedule(); + } +} + #define FORMAT_MICROSECONDS(t) \ (int)(t / 1000000) << "." << \ - (int)(t % 1000000) + (int)(t % 1000000) << " seconds" // -------------------------------------------------------------------------- // @@ -195,6 +212,9 @@ void Timers::Reschedule() // we will do it anyway. sRescheduleNeeded = false; +#ifdef WIN32 + // win32 timers need no management +#else box_time_t timeNow = GetCurrentBoxTime(); // scan for, trigger and remove expired timers. Removal requires @@ -212,8 +232,14 @@ void Timers::Reschedule() if (timeToExpiry <= 0) { + /* BOX_TRACE("timer " << *i << " has expired, " "triggering it"); + */ + BOX_TRACE(TIMER_ID_OF(**i) "has expired, " + "triggering " << + FORMAT_MICROSECONDS(-timeToExpiry) << + " late"); rTimer.OnExpire(); spTimers->erase(i); restart = true; @@ -221,10 +247,12 @@ void Timers::Reschedule() } else { + /* BOX_TRACE("timer " << *i << " has not " "expired, triggering in " << FORMAT_MICROSECONDS(timeToExpiry) << " seconds"); + */ } } } @@ -233,6 +261,7 @@ void Timers::Reschedule() // Scan to find the next one to fire (earliest deadline). int64_t timeToNextEvent = 0; + std::string nameOfNextEvent; for (std::vector<Timer*>::iterator i = spTimers->begin(); i != spTimers->end(); i++) @@ -240,6 +269,7 @@ void Timers::Reschedule() Timer& rTimer = **i; int64_t timeToExpiry = rTimer.GetExpiryTime() - timeNow; + ASSERT(timeToExpiry > 0) if (timeToExpiry <= 0) { timeToExpiry = 1; @@ -248,23 +278,36 @@ void Timers::Reschedule() if (timeToNextEvent == 0 || timeToNextEvent > timeToExpiry) { timeToNextEvent = timeToExpiry; + nameOfNextEvent = rTimer.GetName(); } } ASSERT(timeToNextEvent >= 0); - + + if (timeToNextEvent == 0) + { + BOX_TRACE("timer: no more events, going to sleep."); + } + else + { + BOX_TRACE("timer: next event: " << nameOfNextEvent << + " expires in " << FORMAT_MICROSECONDS(timeToNextEvent)); + } + struct itimerval timeout; memset(&timeout, 0, sizeof(timeout)); timeout.it_value.tv_sec = BoxTimeToSeconds(timeToNextEvent); timeout.it_value.tv_usec = (int) - (BoxTimeToMicroSeconds(timeToNextEvent) % MICRO_SEC_IN_SEC); + (BoxTimeToMicroSeconds(timeToNextEvent) + % MICRO_SEC_IN_SEC); if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) { - BOX_ERROR("Failed to initialise timer\n"); + BOX_ERROR("Failed to initialise system timer\n"); THROW_EXCEPTION(CommonException, Internal) } +#endif } // -------------------------------------------------------------------------- @@ -279,27 +322,42 @@ void Timers::Reschedule() // Created: 5/11/2006 // // -------------------------------------------------------------------------- -void Timers::SignalHandler(int iUnused) +void Timers::SignalHandler(int unused) { // ASSERT(spTimers); Timers::RequestReschedule(); } -Timer::Timer(size_t timeoutSecs) +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Timer(size_t timeoutSecs, +// const std::string& rName) +// Purpose: Standard timer constructor, takes a timeout in +// seconds from now, and an optional name for +// logging purposes. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +Timer::Timer(size_t timeoutSecs, const std::string& rName) : mExpires(GetCurrentBoxTime() + SecondsToBoxTime(timeoutSecs)), - mExpired(false) + mExpired(false), + mName(rName) +#ifdef WIN32 +, mTimerHandle(INVALID_HANDLE_VALUE) +#endif { - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD if (timeoutSecs == 0) { - BOX_TRACE("timer " << this << " initialised for " << - timeoutSecs << " secs, will not fire"); + BOX_TRACE(TIMER_ID "initialised for " << timeoutSecs << + " secs, will not fire"); } else { - BOX_TRACE("timer " << this << " initialised for " << - timeoutSecs << " secs, to fire at " << - FORMAT_MICROSECONDS(mExpires)); + BOX_TRACE(TIMER_ID "initialised for " << timeoutSecs << + " secs, to fire at " << FormatTime(mExpires, false, true)); } #endif @@ -310,37 +368,157 @@ Timer::Timer(size_t timeoutSecs) else { Timers::Add(*this); + Start(timeoutSecs * MICRO_SEC_IN_SEC_LL); } } +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Start() +// Purpose: This internal function initialises an OS TimerQueue +// timer on Windows, while on Unixes there is only a +// single global timer, managed by the Timers class, +// so this method does nothing. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::Start() +{ +#ifdef WIN32 + box_time_t timeNow = GetCurrentBoxTime(); + int64_t timeToExpiry = mExpires - timeNow; + + if (timeToExpiry <= 0) + { + BOX_WARNING(TIMER_ID << "fudging expiry from -" << + FORMAT_MICROSECONDS(-timeToExpiry)) + timeToExpiry = 1; + } + + Start(timeToExpiry); +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Start(int64_t delayInMicros) +// Purpose: This internal function initialises an OS TimerQueue +// timer on Windows, with a specified delay already +// calculated to save us doing it again. Like +// Timer::Start(), on Unixes it does nothing. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::Start(int64_t delayInMicros) +{ +#ifdef WIN32 + // only call me once! + ASSERT(mTimerHandle == INVALID_HANDLE_VALUE); + + int64_t delayInMillis = delayInMicros / 1000; + + // Windows XP always seems to fire timers up to 20 ms late, + // at least on my test laptop. Not critical in practice, but our + // tests are precise enough that they will fail if we don't + // correct for it. + delayInMillis -= 20; + + // Set a system timer to call our timer routine + if (CreateTimerQueueTimer(&mTimerHandle, NULL, TimerRoutine, + (PVOID)this, delayInMillis, 0, WT_EXECUTEINTIMERTHREAD) + == FALSE) + { + BOX_ERROR(TIMER_ID "failed to create timer: " << + GetErrorMessage(GetLastError())); + mTimerHandle = INVALID_HANDLE_VALUE; + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Stop() +// Purpose: This internal function deletes the associated OS +// TimerQueue timer on Windows, and on Unixes does +// nothing. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::Stop() +{ +#ifdef WIN32 + if (mTimerHandle != INVALID_HANDLE_VALUE) + { + if (DeleteTimerQueueTimer(NULL, mTimerHandle, + INVALID_HANDLE_VALUE) == FALSE) + { + BOX_ERROR(TIMER_ID "failed to delete timer: " << + GetErrorMessage(GetLastError())); + } + mTimerHandle = INVALID_HANDLE_VALUE; + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::~Timer() +// Purpose: Destructor for Timer objects. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + Timer::~Timer() { - #ifndef NDEBUG - BOX_TRACE("timer " << this << " destroyed"); + #ifndef BOX_RELEASE_BUILD + BOX_TRACE(TIMER_ID "destroyed"); #endif Timers::Remove(*this); + Stop(); } +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Timer(Timer& rToCopy) +// Purpose: Copy constructor for Timer objects. Creates a new +// timer that will trigger at the same time as the +// original. The original will usually be discarded. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + Timer::Timer(const Timer& rToCopy) : mExpires(rToCopy.mExpires), - mExpired(rToCopy.mExpired) + mExpired(rToCopy.mExpired), + mName(rToCopy.mName) +#ifdef WIN32 +, mTimerHandle(INVALID_HANDLE_VALUE) +#endif { - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD if (mExpired) { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << ", already expired, will not fire"); + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "already expired, will not fire"); } else if (mExpires == 0) { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << ", no expiry, will not fire"); + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "no expiry, will not fire"); } else { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << " to fire at " << + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "to fire at " << (int)(mExpires / 1000000) << "." << (int)(mExpires % 1000000)); } @@ -349,46 +527,100 @@ Timer::Timer(const Timer& rToCopy) if (!mExpired && mExpires != 0) { Timers::Add(*this); + Start(); } } +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::operator=(const Timer& rToCopy) +// Purpose: Assignment operator for Timer objects. Works +// exactly the same as the copy constructor, except +// that if the receiving timer is already running, +// it is stopped first. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + Timer& Timer::operator=(const Timer& rToCopy) { - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD if (rToCopy.mExpired) { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << ", already expired, will not fire"); + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "already expired, will not fire"); } else if (rToCopy.mExpires == 0) { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << ", no expiry, will not fire"); + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "no expiry, will not fire"); } else { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << " to fire at " << + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "to fire at " << (int)(rToCopy.mExpires / 1000000) << "." << (int)(rToCopy.mExpires % 1000000)); } #endif Timers::Remove(*this); + Stop(); + mExpires = rToCopy.mExpires; mExpired = rToCopy.mExpired; + mName = rToCopy.mName; + if (!mExpired && mExpires != 0) { Timers::Add(*this); + Start(); } + return *this; } +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::OnExpire() +// Purpose: Method called by Timers::Reschedule (on Unixes) +// on next poll after timer expires, or from +// Timer::TimerRoutine (on Windows) from a separate +// thread managed by the OS. Marks the timer as +// expired for future reference. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + void Timer::OnExpire() { - #ifndef NDEBUG - BOX_TRACE("timer " << this << " fired"); + #ifndef BOX_RELEASE_BUILD + BOX_TRACE(TIMER_ID "fired"); #endif mExpired = true; } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::TimerRoutine(PVOID lpParam, +// BOOLEAN TimerOrWaitFired) +// Purpose: Static method called by the Windows OS when a +// TimerQueue timer expires. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +#ifdef WIN32 +VOID CALLBACK Timer::TimerRoutine(PVOID lpParam, + BOOLEAN TimerOrWaitFired) +{ + Timer* pTimer = (Timer*)lpParam; + pTimer->OnExpire(); + // is it safe to write to write debug output from a timer? + // e.g. to write to the Event Log? +} +#endif diff --git a/lib/common/Timer.h b/lib/common/Timer.h index ba6d71f4..42b2e00f 100644 --- a/lib/common/Timer.h +++ b/lib/common/Timer.h @@ -40,35 +40,24 @@ class Timers static bool sRescheduleNeeded; static void SignalHandler(int iUnused); - + public: static void Init(); static void Cleanup(); static void Add (Timer& rTimer); static void Remove(Timer& rTimer); - static void RequestReschedule() - { - sRescheduleNeeded = true; - } - - static void RescheduleIfNeeded() - { - if (sRescheduleNeeded) - { - Reschedule(); - } - } + static void RequestReschedule(); + static void RescheduleIfNeeded(); }; class Timer { public: - Timer(size_t timeoutSecs); + Timer(size_t timeoutSecs, const std::string& rName = ""); virtual ~Timer(); Timer(const Timer &); Timer &operator=(const Timer &); -public: box_time_t GetExpiryTime() { return mExpires; } virtual void OnExpire(); bool HasExpired() @@ -76,10 +65,23 @@ public: Timers::RescheduleIfNeeded(); return mExpired; } + + const std::string& GetName() const { return mName; } private: - box_time_t mExpires; - bool mExpired; + box_time_t mExpires; + bool mExpired; + std::string mName; + + void Start(); + void Start(int64_t delayInMicros); + void Stop(); + + #ifdef WIN32 + HANDLE mTimerHandle; + static VOID CALLBACK TimerRoutine(PVOID lpParam, + BOOLEAN TimerOrWaitFired); + #endif }; #include "MemLeakFindOff.h" diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp index 83a12ccf..f45ed26b 100644 --- a/lib/common/Utils.cpp +++ b/lib/common/Utils.cpp @@ -13,11 +13,17 @@ #include <sys/stat.h> #include <errno.h> +#include <cstdlib> + #ifdef SHOW_BACKTRACE_ON_EXCEPTION #include <execinfo.h> #include <stdlib.h> #endif +#ifdef HAVE_CXXABI_H + #include <cxxabi.h> +#endif + #include "Utils.h" #include "CommonException.h" #include "Logging.h" @@ -52,11 +58,11 @@ void SplitString(const std::string &String, char SplitOn, std::vector<std::strin { rOutput.push_back(String.substr(b)); } -/*#ifndef NDEBUG - TRACE2("Splitting string '%s' on %c\n", String.c_str(), SplitOn); +/*#ifndef BOX_RELEASE_BUILD + BOX_TRACE("Splitting string '" << String << " on " << (char)SplitOn); for(unsigned int l = 0; l < rOutput.size(); ++l) { - TRACE2("%d = '%s'\n", l, rOutput[l].c_str()); + BOX_TRACE(l << " = '" << rOutput[l] << "'"); } #endif*/ } @@ -64,23 +70,83 @@ void SplitString(const std::string &String, char SplitOn, std::vector<std::strin #ifdef SHOW_BACKTRACE_ON_EXCEPTION void DumpStackBacktrace() { - void *array[10]; - size_t size; - char **strings; - size_t i; - - size = backtrace (array, 10); - strings = backtrace_symbols (array, size); + void *array[10]; + size_t size = backtrace (array, 10); + char **strings = backtrace_symbols (array, size); BOX_TRACE("Obtained " << size << " stack frames."); - for(i = 0; i < size; i++) + for(size_t i = 0; i < size; i++) { - BOX_TRACE(strings[i]); + // Demangling code copied from + // cctbx_sources/boost_adaptbx/meta_ext.cpp, BSD license + + std::string mangled_frame = strings[i]; + std::string output_frame = strings[i]; // default + + #ifdef HAVE_CXXABI_H + int start = mangled_frame.find('('); + int end = mangled_frame.find('+', start); + std::string mangled_func = mangled_frame.substr(start + 1, + end - start - 1); + + int status; + +#include "MemLeakFindOff.h" + char* result = abi::__cxa_demangle(mangled_func.c_str(), + NULL, NULL, &status); +#include "MemLeakFindOn.h" + + if (result == NULL) + { + if (status == 0) + { + BOX_WARNING("Demangle failed but no error: " << + mangled_func); + } + else if (status == -1) + { + BOX_WARNING("Demangle failed with " + "memory allocation error: " << + mangled_func); + } + else if (status == -2) + { + // Probably non-C++ name, don't demangle + /* + BOX_WARNING("Demangle failed with " + "with invalid name: " << + mangled_func); + */ + } + else if (status == -3) + { + BOX_WARNING("Demangle failed with " + "with invalid argument: " << + mangled_func); + } + else + { + BOX_WARNING("Demangle failed with " + "with unknown error " << status << + ": " << mangled_func); + } + } + else + { + output_frame = mangled_frame.substr(0, start + 1) + + result + mangled_frame.substr(end); +#include "MemLeakFindOff.h" + std::free(result); +#include "MemLeakFindOn.h" + } + #endif // HAVE_CXXABI_H + + BOX_TRACE("Stack frame " << i << ": " << output_frame); } #include "MemLeakFindOff.h" - free (strings); + std::free (strings); #include "MemLeakFindOn.h" } #endif @@ -97,8 +163,8 @@ void DumpStackBacktrace() // -------------------------------------------------------------------------- bool FileExists(const char *Filename, int64_t *pFileSize, bool TreatLinksAsNotExisting) { - struct stat st; - if(::lstat(Filename, &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) != 0) { if(errno == ENOENT) { @@ -142,8 +208,8 @@ bool FileExists(const char *Filename, int64_t *pFileSize, bool TreatLinksAsNotEx // -------------------------------------------------------------------------- int ObjectExists(const std::string& rFilename) { - struct stat st; - if(::stat(rFilename.c_str(), &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_STAT(rFilename.c_str(), &st) != 0) { if(errno == ENOENT) { @@ -159,6 +225,85 @@ int ObjectExists(const std::string& rFilename) return ((st.st_mode & S_IFDIR) == 0)?ObjectExists_File:ObjectExists_Dir; } +std::string HumanReadableSize(int64_t Bytes) +{ + double readableValue = Bytes; + std::string units = " B"; + + if (readableValue > 1024) + { + readableValue /= 1024; + units = "kB"; + } + + if (readableValue > 1024) + { + readableValue /= 1024; + units = "MB"; + } + + if (readableValue > 1024) + { + readableValue /= 1024; + units = "GB"; + } + + std::ostringstream result; + result << std::fixed << std::setprecision(2) << readableValue << + " " << units; + return result.str(); +} + +std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, + bool MachineReadable) +{ + std::ostringstream result; + + + if (MachineReadable) + { + result << (Bytes >> 10) << " kB, " << + std::setprecision(0) << ((Bytes*100)/Max) << "%"; + } + else + { + // Bar graph + char bar[17]; + unsigned int b = (int)((Bytes * (sizeof(bar)-1)) / Max); + if(b > sizeof(bar)-1) {b = sizeof(bar)-1;} + for(unsigned int l = 0; l < b; l++) + { + bar[l] = '*'; + } + for(unsigned int l = b; l < sizeof(bar) - 1; l++) + { + bar[l] = ' '; + } + bar[sizeof(bar)-1] = '\0'; + + result << std::fixed << + std::setw(10) << Blocks << " blocks, " << + std::setw(10) << HumanReadableSize(Bytes) << ", " << + std::setw(3) << std::setprecision(0) << + ((Bytes*100)/Max) << "% |" << bar << "|"; + } + + return result.str(); +} +std::string FormatUsageLineStart(const std::string& rName, + bool MachineReadable) +{ + std::ostringstream result; + if (MachineReadable) + { + result << rName << ": "; + } + else + { + result << std::setw(20) << std::right << rName << ": "; + } + return result.str(); +} diff --git a/lib/common/Utils.h b/lib/common/Utils.h index d0842b51..d6792077 100644 --- a/lib/common/Utils.h +++ b/lib/common/Utils.h @@ -30,6 +30,11 @@ enum ObjectExists_Dir = 2 }; int ObjectExists(const std::string& rFilename); +std::string HumanReadableSize(int64_t Bytes); +std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, + bool MachineReadable); +std::string FormatUsageLineStart(const std::string& rName, + bool MachineReadable); #include "MemLeakFindOff.h" diff --git a/lib/common/WaitForEvent.h b/lib/common/WaitForEvent.h index 52a073e9..045d6d67 100644 --- a/lib/common/WaitForEvent.h +++ b/lib/common/WaitForEvent.h @@ -10,6 +10,8 @@ #ifndef WAITFOREVENT__H #define WAITFOREVENT__H +#include <cstdlib> + #ifdef HAVE_KQUEUE #include <sys/event.h> #include <sys/time.h> diff --git a/lib/common/makeexception.pl.in b/lib/common/makeexception.pl.in index 1564b75b..76b9b02b 100755 --- a/lib/common/makeexception.pl.in +++ b/lib/common/makeexception.pl.in @@ -71,13 +71,14 @@ print H <<__E; class ${class}Exception : public BoxException { public: - ${class}Exception(unsigned int SubType) - : mSubType(SubType) + ${class}Exception(unsigned int SubType, + const std::string& rMessage = "") + : mSubType(SubType), mMessage(rMessage) { } ${class}Exception(const ${class}Exception &rToCopy) - : mSubType(rToCopy.mSubType) + : mSubType(rToCopy.mSubType), mMessage(rToCopy.mMessage) { } @@ -108,9 +109,14 @@ print H <<__E; virtual unsigned int GetType() const throw(); virtual unsigned int GetSubType() const throw(); virtual const char *what() const throw(); + virtual const std::string& GetMessage() const + { + return mMessage; + } private: unsigned int mSubType; + std::string mMessage; }; #endif // $guardname diff --git a/lib/compress/Compress.h b/lib/compress/Compress.h index 4a98a59e..de38af2c 100644 --- a/lib/compress/Compress.h +++ b/lib/compress/Compress.h @@ -52,10 +52,12 @@ public: if((r = ((Compressing)?(deflateEnd(&mStream)) :(inflateEnd(&mStream)))) != Z_OK) { - TRACE1("zlib error code = %d\n", r); + BOX_WARNING("zlib error code = " << r); if(r == Z_DATA_ERROR) { - TRACE0("WARNING: End of compress/decompress without all input being consumed -- possible corruption?\n"); + BOX_WARNING("End of compress/decompress " + "without all input being consumed, " + "possible corruption?"); } else { @@ -148,7 +150,7 @@ public: // Check errors if(ret < 0) { - TRACE1("zlib error code = %d\n", ret); + BOX_WARNING("zlib error code = " << ret); THROW_EXCEPTION(CompressException, TransformFailed) } diff --git a/lib/compress/CompressStream.cpp b/lib/compress/CompressStream.cpp index 2839b647..9bb73e3d 100644 --- a/lib/compress/CompressStream.cpp +++ b/lib/compress/CompressStream.cpp @@ -19,7 +19,7 @@ #include "MemLeakFindOn.h" // How big a buffer to use -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD // debug! #define BUFFER_SIZE 256 #else @@ -414,7 +414,7 @@ void CompressStream::CheckBuffer() { size *= 2; } - TRACE1("Allocating CompressStream buffer, size %d\n", size); + BOX_TRACE("Allocating CompressStream buffer, size " << size); mpBuffer = ::malloc(size); if(mpBuffer == 0) { diff --git a/lib/crypto/CipherContext.cpp b/lib/crypto/CipherContext.cpp index 8c3b25b2..e5cd9b0e 100644 --- a/lib/crypto/CipherContext.cpp +++ b/lib/crypto/CipherContext.cpp @@ -106,7 +106,8 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes #else // With the old version, a copy needs to be taken first. mpDescription = rDescription.Clone(); - // Mark it as not a leak, otherwise static cipher contexts cause supriously memory leaks to be reported + // Mark it as not a leak, otherwise static cipher contexts + // cause spurious memory leaks to be reported MEMLEAKFINDER_NOT_A_LEAK(mpDescription); mpDescription->SetupParameters(&ctx); #endif @@ -166,7 +167,8 @@ void CipherContext::Begin() // Warn if in a transformation (not an error, because a context might not have been finalised if an exception occured) if(mWithinTransform) { - TRACE0("CipherContext::Begin called when context flagged as within a transform\n"); + BOX_WARNING("CipherContext::Begin called when context " + "flagged as within a transform"); } // Initialise the cipher context again @@ -423,7 +425,8 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p // Warn if in a transformation if(mWithinTransform) { - TRACE0("CipherContext::TransformBlock called when context flagged as within a transform\n"); + BOX_WARNING("CipherContext::TransformBlock called when " + "context flagged as within a transform"); } // Check output buffer size @@ -521,7 +524,8 @@ void CipherContext::SetIV(const void *pIV) // Warn if in a transformation if(mWithinTransform) { - TRACE0("CipherContext::SetIV called when context flagged as within a transform\n"); + BOX_WARNING("CipherContext::SetIV called when context " + "flagged as within a transform"); } // Set IV @@ -559,7 +563,8 @@ const void *CipherContext::SetRandomIV(int &rLengthOut) // Warn if in a transformation if(mWithinTransform) { - TRACE0("CipherContext::SetRandomIV called when context flagged as within a transform\n"); + BOX_WARNING("CipherContext::SetRandomIV called when " + "context flagged as within a transform"); } // Get length of IV diff --git a/lib/httpserver/HTTPException.txt b/lib/httpserver/HTTPException.txt new file mode 100644 index 00000000..52630cda --- /dev/null +++ b/lib/httpserver/HTTPException.txt @@ -0,0 +1,16 @@ +EXCEPTION HTTP 10 + +Internal 0 +RequestReadFailed 1 +RequestAlreadyBeenRead 2 +BadRequest 3 +UnknownResponseCodeUsed 4 +NoContentTypeSet 5 +POSTContentTooLong 6 +CannotSetRedirectIfReponseHasData 7 +CannotSetNotFoundIfReponseHasData 8 +NotImplemented 9 +RequestNotInitialised 10 +BadResponse 11 +ResponseReadFailed 12 +NoStreamConfigured 13 diff --git a/lib/httpserver/HTTPQueryDecoder.cpp b/lib/httpserver/HTTPQueryDecoder.cpp new file mode 100644 index 00000000..c49ac2ce --- /dev/null +++ b/lib/httpserver/HTTPQueryDecoder.cpp @@ -0,0 +1,159 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPQueryDecoder.cpp +// Purpose: Utility class to decode HTTP query strings +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdlib.h> + +#include "HTTPQueryDecoder.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::HTTPQueryDecoder( +// HTTPRequest::Query_t &) +// Purpose: Constructor. Pass in the query contents you want +// to decode the query string into. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPQueryDecoder::HTTPQueryDecoder(HTTPRequest::Query_t &rDecodeInto) + : mrDecodeInto(rDecodeInto), + mInKey(true), + mEscapedState(0) +{ + // Insert the terminator for escaped characters + mEscaped[2] = '\0'; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::~HTTPQueryDecoder() +// Purpose: Destructor. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPQueryDecoder::~HTTPQueryDecoder() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::Decode(const char *, int) +// Purpose: Decode a chunk of query string -- call several times with +// the bits as they are received, and then call Finish() +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPQueryDecoder::DecodeChunk(const char *pQueryString, int QueryStringSize) +{ + for(int l = 0; l < QueryStringSize; ++l) + { + char c = pQueryString[l]; + + // BEFORE unescaping, check to see if we need to flip key / value + if(mEscapedState == 0) + { + if(mInKey && c == '=') + { + // Set to store characters in the value + mInKey = false; + continue; + } + else if(!mInKey && c == '&') + { + // Need to store the current key/value pair + mrDecodeInto.insert(HTTPRequest::QueryEn_t(mCurrentKey, mCurrentValue)); + // Blank the strings + mCurrentKey.erase(); + mCurrentValue.erase(); + + // Set to store characters in the key + mInKey = true; + continue; + } + } + + // Decode an escaped value? + if(mEscapedState == 1) + { + // Waiting for char one of the escaped hex value + mEscaped[0] = c; + mEscapedState = 2; + continue; + } + else if(mEscapedState == 2) + { + // Escaped value, decode it + mEscaped[1] = c; // str terminated in constructor + mEscapedState = 0; // stop being in escaped mode + long ch = ::strtol(mEscaped, NULL, 16); + if(ch <= 0 || ch > 255) + { + // Bad character, just ignore + continue; + } + + // Use this instead + c = (char)ch; + } + else if(c == '+') + { + c = ' '; + } + else if(c == '%') + { + mEscapedState = 1; + continue; + } + + // Store decoded value into the appropriate string + if(mInKey) + { + mCurrentKey += c; + } + else + { + mCurrentValue += c; + } + } + + // Don't do anything here with left over values, DecodeChunk might be called + // again. Let Finish() clean up. +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::Finish() +// Purpose: Finish the decoding. Necessary to get the last item! +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPQueryDecoder::Finish() +{ + // Insert any remaining value. + if(!mCurrentKey.empty()) + { + mrDecodeInto.insert(HTTPRequest::QueryEn_t(mCurrentKey, mCurrentValue)); + // Blank values, just in case + mCurrentKey.erase(); + mCurrentValue.erase(); + } +} + + diff --git a/lib/httpserver/HTTPQueryDecoder.h b/lib/httpserver/HTTPQueryDecoder.h new file mode 100644 index 00000000..ca5afe7e --- /dev/null +++ b/lib/httpserver/HTTPQueryDecoder.h @@ -0,0 +1,47 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPQueryDecoder.h +// Purpose: Utility class to decode HTTP query strings +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPQUERYDECODER__H +#define HTTPQUERYDECODER__H + +#include "HTTPRequest.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPQueryDecoder +// Purpose: Utility class to decode HTTP query strings +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPQueryDecoder +{ +public: + HTTPQueryDecoder(HTTPRequest::Query_t &rDecodeInto); + ~HTTPQueryDecoder(); +private: + // no copying + HTTPQueryDecoder(const HTTPQueryDecoder &); + HTTPQueryDecoder &operator=(const HTTPQueryDecoder &); +public: + + void DecodeChunk(const char *pQueryString, int QueryStringSize); + void Finish(); + +private: + HTTPRequest::Query_t &mrDecodeInto; + std::string mCurrentKey; + std::string mCurrentValue; + bool mInKey; + char mEscaped[4]; + int mEscapedState; +}; + +#endif // HTTPQUERYDECODER__H + diff --git a/lib/httpserver/HTTPRequest.cpp b/lib/httpserver/HTTPRequest.cpp new file mode 100644 index 00000000..e146a0a3 --- /dev/null +++ b/lib/httpserver/HTTPRequest.cpp @@ -0,0 +1,783 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPRequest.cpp +// Purpose: Request object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <string.h> +#include <strings.h> +#include <stdlib.h> +#include <stdio.h> + +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "HTTPQueryDecoder.h" +#include "autogen_HTTPException.h" +#include "IOStream.h" +#include "IOStreamGetLine.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +#define MAX_CONTENT_SIZE (128*1024) + +#define ENSURE_COOKIE_JAR_ALLOCATED \ + if(mpCookies == 0) {mpCookies = new CookieJar_t;} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::HTTPRequest() +// Purpose: Constructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPRequest::HTTPRequest() + : mMethod(Method_UNINITIALISED), + mHostPort(80), // default if not specified + mHTTPVersion(0), + mContentLength(-1), + mpCookies(0), + mClientKeepAliveRequested(false), + mExpectContinue(false), + mpStreamToReadFrom(NULL) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::HTTPRequest(enum Method, +// const std::string&) +// Purpose: Alternate constructor for hand-crafted requests +// Created: 03/01/09 +// +// -------------------------------------------------------------------------- +HTTPRequest::HTTPRequest(enum Method method, const std::string& rURI) + : mMethod(method), + mRequestURI(rURI), + mHostPort(80), // default if not specified + mHTTPVersion(HTTPVersion_1_1), + mContentLength(-1), + mpCookies(0), + mClientKeepAliveRequested(false), + mExpectContinue(false), + mpStreamToReadFrom(NULL) +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::~HTTPRequest() +// Purpose: Destructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPRequest::~HTTPRequest() +{ + // Clean up any cookies + if(mpCookies != 0) + { + delete mpCookies; + mpCookies = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::Receive(IOStreamGetLine &, int) +// Purpose: Read the request from an IOStreamGetLine (and +// attached stream). +// Returns false if there was no valid request, +// probably due to a kept-alive connection closing. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) +{ + // Check caller's logic + if(mMethod != Method_UNINITIALISED) + { + THROW_EXCEPTION(HTTPException, RequestAlreadyBeenRead) + } + + // Read the first line, which is of a different format to the rest of the lines + std::string requestLine; + if(!rGetLine.GetLine(requestLine, false /* no preprocessing */, Timeout)) + { + // Didn't get the request line, probably end of connection which had been kept alive + return false; + } + BOX_TRACE("Request line: " << requestLine); + + // Check the method + size_t p = 0; // current position in string + p = requestLine.find(' '); // end of first word + + if (p == std::string::npos) + { + // No terminating space, looks bad + p = requestLine.size(); + } + else + { + mHttpVerb = requestLine.substr(0, p); + if (mHttpVerb == "GET") + { + mMethod = Method_GET; + } + else if (mHttpVerb == "HEAD") + { + mMethod = Method_HEAD; + } + else if (mHttpVerb == "POST") + { + mMethod = Method_POST; + } + else if (mHttpVerb == "PUT") + { + mMethod = Method_PUT; + } + else + { + mMethod = Method_UNKNOWN; + } + } + + // Skip spaces to find URI + const char *requestLinePtr = requestLine.c_str(); + while(requestLinePtr[p] != '\0' && requestLinePtr[p] == ' ') + { + ++p; + } + + // Check there's a URI following... + if(requestLinePtr[p] == '\0') + { + // Didn't get the request line, probably end of connection which had been kept alive + return false; + } + + // Read the URI, unescaping any %XX hex codes + while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0') + { + // End of URI, on to query string? + if(requestLinePtr[p] == '?') + { + // Put the rest into the query string, without escaping anything + ++p; + while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0') + { + mQueryString += requestLinePtr[p]; + ++p; + } + break; + } + // Needs unescaping? + else if(requestLinePtr[p] == '+') + { + mRequestURI += ' '; + } + else if(requestLinePtr[p] == '%') + { + // Be tolerant about this... bad things are silently accepted, + // rather than throwing an error. + char code[4] = {0,0,0,0}; + code[0] = requestLinePtr[++p]; + if(code[0] != '\0') + { + code[1] = requestLinePtr[++p]; + } + + // Convert into a char code + long c = ::strtol(code, NULL, 16); + + // Accept it? + if(c > 0 && c <= 255) + { + mRequestURI += (char)c; + } + } + else + { + // Simple copy of character + mRequestURI += requestLinePtr[p]; + } + + ++p; + } + + // End of URL? + if(requestLinePtr[p] == '\0') + { + // Assume HTTP 0.9 + mHTTPVersion = HTTPVersion_0_9; + } + else + { + // Skip any more spaces + while(requestLinePtr[p] != '\0' && requestLinePtr[p] == ' ') + { + ++p; + } + + // Check to see if there's the right string next... + if(::strncmp(requestLinePtr + p, "HTTP/", 5) == 0) + { + // Find the version numbers + int major, minor; + if(::sscanf(requestLinePtr + p + 5, "%d.%d", &major, &minor) != 2) + { + THROW_EXCEPTION(HTTPException, BadRequest) + } + + // Store version + mHTTPVersion = (major * HTTPVersion__MajorMultiplier) + minor; + } + else + { + // Not good -- wrong string found + THROW_EXCEPTION(HTTPException, BadRequest) + } + } + + BOX_TRACE("HTTPRequest: method=" << mMethod << ", uri=" << + mRequestURI << ", version=" << mHTTPVersion); + + // If HTTP 1.1 or greater, assume keep-alive + if(mHTTPVersion >= HTTPVersion_1_1) + { + mClientKeepAliveRequested = true; + } + + // Decode query string? + if((mMethod == Method_GET || mMethod == Method_HEAD) && !mQueryString.empty()) + { + HTTPQueryDecoder decoder(mQuery); + decoder.DecodeChunk(mQueryString.c_str(), mQueryString.size()); + decoder.Finish(); + } + + // Now parse the headers + ParseHeaders(rGetLine, Timeout); + + std::string expected; + if (GetHeader("Expect", &expected)) + { + if (expected == "100-continue") + { + mExpectContinue = true; + } + } + + // Parse form data? + if(mMethod == Method_POST && mContentLength >= 0) + { + // Too long? Don't allow people to be nasty by sending lots of data + if(mContentLength > MAX_CONTENT_SIZE) + { + THROW_EXCEPTION(HTTPException, POSTContentTooLong) + } + + // Some data in the request to follow, parsing it bit by bit + HTTPQueryDecoder decoder(mQuery); + // Don't forget any data left in the GetLine object + int fromBuffer = rGetLine.GetSizeOfBufferedData(); + if(fromBuffer > mContentLength) fromBuffer = mContentLength; + if(fromBuffer > 0) + { + BOX_TRACE("Decoding " << fromBuffer << " bytes of " + "data from getline buffer"); + decoder.DecodeChunk((const char *)rGetLine.GetBufferedData(), fromBuffer); + // And tell the getline object to ignore the data we just used + rGetLine.IgnoreBufferedData(fromBuffer); + } + // Then read any more data, as required + int bytesToGo = mContentLength - fromBuffer; + while(bytesToGo > 0) + { + char buf[4096]; + int toRead = sizeof(buf); + if(toRead > bytesToGo) toRead = bytesToGo; + IOStream &rstream(rGetLine.GetUnderlyingStream()); + int r = rstream.Read(buf, toRead, Timeout); + if(r == 0) + { + // Timeout, just error + THROW_EXCEPTION(HTTPException, RequestReadFailed) + } + decoder.DecodeChunk(buf, r); + bytesToGo -= r; + } + // Finish off + decoder.Finish(); + } + else if (mContentLength > 0) + { + IOStream::pos_type bytesToCopy = rGetLine.GetSizeOfBufferedData(); + if (bytesToCopy > mContentLength) + { + bytesToCopy = mContentLength; + } + Write(rGetLine.GetBufferedData(), bytesToCopy); + SetForReading(); + mpStreamToReadFrom = &(rGetLine.GetUnderlyingStream()); + } + + return true; +} + +void HTTPRequest::ReadContent(IOStream& rStreamToWriteTo) +{ + Seek(0, SeekType_Absolute); + + CopyStreamTo(rStreamToWriteTo); + IOStream::pos_type bytesCopied = GetSize(); + + while (bytesCopied < mContentLength) + { + char buffer[1024]; + IOStream::pos_type bytesToCopy = sizeof(buffer); + if (bytesToCopy > mContentLength - bytesCopied) + { + bytesToCopy = mContentLength - bytesCopied; + } + bytesToCopy = mpStreamToReadFrom->Read(buffer, bytesToCopy); + rStreamToWriteTo.Write(buffer, bytesToCopy); + bytesCopied += bytesToCopy; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::Send(IOStream &, int) +// Purpose: Write the request to an IOStream using HTTP. +// Created: 03/01/09 +// +// -------------------------------------------------------------------------- +bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) +{ + switch (mMethod) + { + case Method_UNINITIALISED: + THROW_EXCEPTION(HTTPException, RequestNotInitialised); break; + case Method_UNKNOWN: + THROW_EXCEPTION(HTTPException, BadRequest); break; + case Method_GET: + rStream.Write("GET"); break; + case Method_HEAD: + rStream.Write("HEAD"); break; + case Method_POST: + rStream.Write("POST"); break; + case Method_PUT: + rStream.Write("PUT"); break; + } + + rStream.Write(" "); + rStream.Write(mRequestURI.c_str()); + rStream.Write(" "); + + switch (mHTTPVersion) + { + case HTTPVersion_0_9: rStream.Write("HTTP/0.9"); break; + case HTTPVersion_1_0: rStream.Write("HTTP/1.0"); break; + case HTTPVersion_1_1: rStream.Write("HTTP/1.1"); break; + default: + THROW_EXCEPTION(HTTPException, NotImplemented); + } + + rStream.Write("\n"); + std::ostringstream oss; + + if (mContentLength != -1) + { + oss << "Content-Length: " << mContentLength << "\n"; + } + + if (mContentType != "") + { + oss << "Content-Type: " << mContentType << "\n"; + } + + if (mHostName != "") + { + if (mHostPort != 80) + { + oss << "Host: " << mHostName << ":" << mHostPort << + "\n"; + } + else + { + oss << "Host: " << mHostName << "\n"; + } + } + + if (mpCookies) + { + THROW_EXCEPTION(HTTPException, NotImplemented); + } + + if (mClientKeepAliveRequested) + { + oss << "Connection: keep-alive\n"; + } + else + { + oss << "Connection: close\n"; + } + + for (std::vector<Header>::iterator i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + oss << i->first << ": " << i->second << "\n"; + } + + if (ExpectContinue) + { + oss << "Expect: 100-continue\n"; + } + + rStream.Write(oss.str().c_str()); + rStream.Write("\n"); + + return true; +} + +void HTTPRequest::SendWithStream(IOStream &rStreamToSendTo, int Timeout, + IOStream* pStreamToSend, HTTPResponse& rResponse) +{ + IOStream::pos_type size = pStreamToSend->BytesLeftToRead(); + + if (size != IOStream::SizeOfStreamUnknown) + { + mContentLength = size; + } + + Send(rStreamToSendTo, Timeout, true); + + rResponse.Receive(rStreamToSendTo, Timeout); + if (rResponse.GetResponseCode() != 100) + { + // bad response, abort now + return; + } + + pStreamToSend->CopyStreamTo(rStreamToSendTo, Timeout); + + // receive the final response + rResponse.Receive(rStreamToSendTo, Timeout); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::ParseHeaders(IOStreamGetLine &, int) +// Purpose: Private. Parse the headers of the request +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) +{ + std::string header; + bool haveHeader = false; + while(true) + { + if(rGetLine.IsEOF()) + { + // Header terminates unexpectedly + THROW_EXCEPTION(HTTPException, BadRequest) + } + + std::string currentLine; + if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) + { + // Timeout + THROW_EXCEPTION(HTTPException, RequestReadFailed) + } + + // Is this a continuation of the previous line? + bool processHeader = haveHeader; + if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) + { + // A continuation, don't process anything yet + processHeader = false; + } + //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); + + // Parse the header -- this will actually process the header + // from the previous run around the loop. + if(processHeader) + { + // Find where the : is in the line + const char *h = header.c_str(); + int p = 0; + while(h[p] != '\0' && h[p] != ':') + { + ++p; + } + // Skip white space + int dataStart = p + 1; + while(h[dataStart] == ' ' || h[dataStart] == '\t') + { + ++dataStart; + } + + if(p == sizeof("Content-Length")-1 + && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) + { + // Decode number + long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK + if(len < 0) len = 0; + // Store + mContentLength = len; + } + else if(p == sizeof("Content-Type")-1 + && ::strncasecmp(h, "Content-Type", sizeof("Content-Type")-1) == 0) + { + // Store rest of string as content type + mContentType = h + dataStart; + } + else if(p == sizeof("Host")-1 + && ::strncasecmp(h, "Host", sizeof("Host")-1) == 0) + { + // Store host header + mHostName = h + dataStart; + + // Is there a port number to split off? + std::string::size_type colon = mHostName.find_first_of(':'); + if(colon != std::string::npos) + { + // There's a port in the string... attempt to turn it into an int + mHostPort = ::strtol(mHostName.c_str() + colon + 1, 0, 10); + + // Truncate the string to just the hostname + mHostName = mHostName.substr(0, colon); + + BOX_TRACE("Host: header, hostname = " << + "'" << mHostName << "', host " + "port = " << mHostPort); + } + } + else if(p == sizeof("Cookie")-1 + && ::strncasecmp(h, "Cookie", sizeof("Cookie")-1) == 0) + { + // Parse cookies + ParseCookies(header, dataStart); + } + else if(p == sizeof("Connection")-1 + && ::strncasecmp(h, "Connection", sizeof("Connection")-1) == 0) + { + // Connection header, what is required? + const char *v = h + dataStart; + if(::strcasecmp(v, "close") == 0) + { + mClientKeepAliveRequested = false; + } + else if(::strcasecmp(v, "keep-alive") == 0) + { + mClientKeepAliveRequested = true; + } + // else don't understand, just assume default for protocol version + } + else + { + std::string name = header.substr(0, p); + mExtraHeaders.push_back(Header(name, + h + dataStart)); + } + + // Unset have header flag, as it's now been processed + haveHeader = false; + } + + // Store the chunk of header the for next time round + if(haveHeader) + { + header += currentLine; + } + else + { + header = currentLine; + haveHeader = true; + } + + // End of headers? + if(currentLine.empty()) + { + // All done! + break; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::ParseCookies(const std::string &, int) +// Purpose: Parse the cookie header +// Created: 20/8/04 +// +// -------------------------------------------------------------------------- +void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) +{ + const char *data = rHeader.c_str() + DataStarts; + const char *pos = data; + const char *itemStart = pos; + std::string name; + + enum + { + s_NAME, s_VALUE, s_VALUE_QUOTED, s_FIND_NEXT_NAME + } state = s_NAME; + + do + { + switch(state) + { + case s_NAME: + { + if(*pos == '=') + { + // Found the name. Store + name.assign(itemStart, pos - itemStart); + // Looking at values now + state = s_VALUE; + if((*(pos + 1)) == '"') + { + // Actually it's a quoted value, skip over that + ++pos; + state = s_VALUE_QUOTED; + } + // Record starting point for this item + itemStart = pos + 1; + } + } + break; + + case s_VALUE: + { + if(*pos == ';' || *pos == ',' || *pos == '\0') + { + // Name ends + ENSURE_COOKIE_JAR_ALLOCATED + std::string value(itemStart, pos - itemStart); + (*mpCookies)[name] = value; + // And move to the waiting stage + state = s_FIND_NEXT_NAME; + } + } + break; + + case s_VALUE_QUOTED: + { + if(*pos == '"') + { + // That'll do nicely, save it + ENSURE_COOKIE_JAR_ALLOCATED + std::string value(itemStart, pos - itemStart); + (*mpCookies)[name] = value; + // And move to the waiting stage + state = s_FIND_NEXT_NAME; + } + } + break; + + case s_FIND_NEXT_NAME: + { + // Skip over terminators and white space to get to the next name + if(*pos != ';' && *pos != ',' && *pos != ' ' && *pos != '\t') + { + // Name starts here + itemStart = pos; + state = s_NAME; + } + } + break; + + default: + // Ooops + THROW_EXCEPTION(HTTPException, Internal) + break; + } + } + while(*(pos++) != 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::GetCookie(const char *, std::string &) const +// Purpose: Fetch a cookie's value. If cookie not present, returns false +// and string is unaltered. +// Created: 20/8/04 +// +// -------------------------------------------------------------------------- +bool HTTPRequest::GetCookie(const char *CookieName, std::string &rValueOut) const +{ + // Got any cookies? + if(mpCookies == 0) + { + return false; + } + + // See if it's there + CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); + if(v != mpCookies->end()) + { + // Return the value + rValueOut = v->second; + return true; + } + + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::GetCookie(const char *) +// Purpose: Return a string for the given cookie, or the null string if the +// cookie has not been recieved. +// Created: 22/8/04 +// +// -------------------------------------------------------------------------- +const std::string &HTTPRequest::GetCookie(const char *CookieName) const +{ + static const std::string noCookie; + + // Got any cookies? + if(mpCookies == 0) + { + return noCookie; + } + + // See if it's there + CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); + if(v != mpCookies->end()) + { + // Return the value + return v->second; + } + + return noCookie; +} + + + diff --git a/lib/httpserver/HTTPRequest.h b/lib/httpserver/HTTPRequest.h new file mode 100644 index 00000000..90215751 --- /dev/null +++ b/lib/httpserver/HTTPRequest.h @@ -0,0 +1,174 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPRequest.h +// Purpose: Request object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPREQUEST__H +#define HTTPREQUEST__H + +#include <string> +#include <map> + +#include "CollectInBufferStream.h" + +class HTTPResponse; +class IOStream; +class IOStreamGetLine; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPRequest +// Purpose: Request object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPRequest : public CollectInBufferStream +{ +public: + enum Method + { + Method_UNINITIALISED = -1, + Method_UNKNOWN = 0, + Method_GET = 1, + Method_HEAD = 2, + Method_POST = 3, + Method_PUT = 4 + }; + + HTTPRequest(); + HTTPRequest(enum Method method, const std::string& rURI); + ~HTTPRequest(); +private: + // no copying + HTTPRequest(const HTTPRequest &); + HTTPRequest &operator=(const HTTPRequest &); +public: + typedef std::multimap<std::string, std::string> Query_t; + typedef std::pair<std::string, std::string> QueryEn_t, Header; + + enum + { + HTTPVersion__MajorMultiplier = 1000, + HTTPVersion_0_9 = 9, + HTTPVersion_1_0 = 1000, + HTTPVersion_1_1 = 1001 + }; + + bool Receive(IOStreamGetLine &rGetLine, int Timeout); + bool Send(IOStream &rStream, int Timeout, bool ExpectContinue = false); + void SendWithStream(IOStream &rStreamToSendTo, int Timeout, + IOStream* pStreamToSend, HTTPResponse& rResponse); + void ReadContent(IOStream& rStreamToWriteTo); + + typedef std::map<std::string, std::string> CookieJar_t; + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPResponse::Get*() + // Purpose: Various Get accessors + // Created: 26/3/04 + // + // -------------------------------------------------------------------------- + enum Method GetMethod() const {return mMethod;} + const std::string &GetRequestURI() const {return mRequestURI;} + + // Note: the HTTPRequest generates and parses the Host: header + // Do not attempt to set one yourself with AddHeader(). + const std::string &GetHostName() const {return mHostName;} + void SetHostName(const std::string& rHostName) + { + mHostName = rHostName; + } + + const int GetHostPort() const {return mHostPort;} + const std::string &GetQueryString() const {return mQueryString;} + int GetHTTPVersion() const {return mHTTPVersion;} + const Query_t &GetQuery() const {return mQuery;} + int GetContentLength() const {return mContentLength;} + const std::string &GetContentType() const {return mContentType;} + const CookieJar_t *GetCookies() const {return mpCookies;} // WARNING: May return NULL + bool GetCookie(const char *CookieName, std::string &rValueOut) const; + const std::string &GetCookie(const char *CookieName) const; + bool GetHeader(const std::string& rName, std::string* pValueOut) const + { + for (std::vector<Header>::const_iterator + i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + if (i->first == rName) + { + *pValueOut = i->second; + return true; + } + } + return false; + } + std::vector<Header> GetHeaders() { return mExtraHeaders; } + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPRequest::GetClientKeepAliveRequested() + // Purpose: Returns true if the client requested that the connection + // should be kept open for further requests. + // Created: 22/12/04 + // + // -------------------------------------------------------------------------- + bool GetClientKeepAliveRequested() const {return mClientKeepAliveRequested;} + void SetClientKeepAliveRequested(bool keepAlive) + { + mClientKeepAliveRequested = keepAlive; + } + + void AddHeader(const std::string& rName, const std::string& rValue) + { + mExtraHeaders.push_back(Header(rName, rValue)); + } + bool IsExpectingContinue() const { return mExpectContinue; } + const char* GetVerb() const + { + if (!mHttpVerb.empty()) + { + return mHttpVerb.c_str(); + } + switch (mMethod) + { + case Method_UNINITIALISED: return "Uninitialized"; + case Method_UNKNOWN: return "Unknown"; + case Method_GET: return "GET"; + case Method_HEAD: return "HEAD"; + case Method_POST: return "POST"; + case Method_PUT: return "PUT"; + } + return "Bad"; + } + +private: + void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); + void ParseCookies(const std::string &rHeader, int DataStarts); + + enum Method mMethod; + std::string mRequestURI; + std::string mHostName; + int mHostPort; + std::string mQueryString; + int mHTTPVersion; + Query_t mQuery; + int mContentLength; + std::string mContentType; + CookieJar_t *mpCookies; + bool mClientKeepAliveRequested; + std::vector<Header> mExtraHeaders; + bool mExpectContinue; + IOStream* mpStreamToReadFrom; + std::string mHttpVerb; +}; + +#endif // HTTPREQUEST__H + diff --git a/lib/httpserver/HTTPResponse.cpp b/lib/httpserver/HTTPResponse.cpp new file mode 100644 index 00000000..1a8c8447 --- /dev/null +++ b/lib/httpserver/HTTPResponse.cpp @@ -0,0 +1,648 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPResponse.cpp +// Purpose: Response object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> +#include <string.h> + +#include "HTTPResponse.h" +#include "IOStreamGetLine.h" +#include "autogen_HTTPException.h" + +#include "MemLeakFindOn.h" + +// Static variables +std::string HTTPResponse::msDefaultURIPrefix; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::HTTPResponse(IOStream*) +// Purpose: Constructor for response to be sent to a stream +// Created: 04/01/09 +// +// -------------------------------------------------------------------------- +HTTPResponse::HTTPResponse(IOStream* pStreamToSendTo) + : mResponseCode(HTTPResponse::Code_NoContent), + mResponseIsDynamicContent(true), + mKeepAlive(false), + mContentLength(-1), + mpStreamToSendTo(pStreamToSendTo) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::HTTPResponse() +// Purpose: Constructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPResponse::HTTPResponse() + : mResponseCode(HTTPResponse::Code_NoContent), + mResponseIsDynamicContent(true), + mKeepAlive(false), + mContentLength(-1), + mpStreamToSendTo(NULL) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::~HTTPResponse() +// Purpose: Destructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPResponse::~HTTPResponse() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::ResponseCodeToString(int) +// Purpose: Return string equivalent of the response code, +// suitable for Status: headers +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +const char *HTTPResponse::ResponseCodeToString(int ResponseCode) +{ + switch(ResponseCode) + { + case Code_OK: return "200 OK"; break; + case Code_NoContent: return "204 No Content"; break; + case Code_MovedPermanently: return "301 Moved Permanently"; break; + case Code_Found: return "302 Found"; break; + case Code_NotModified: return "304 Not Modified"; break; + case Code_TemporaryRedirect: return "307 Temporary Redirect"; break; + case Code_MethodNotAllowed: return "400 Method Not Allowed"; break; + case Code_Unauthorized: return "401 Unauthorized"; break; + case Code_Forbidden: return "403 Forbidden"; break; + case Code_NotFound: return "404 Not Found"; break; + case Code_InternalServerError: return "500 Internal Server Error"; break; + case Code_NotImplemented: return "501 Not Implemented"; break; + default: + { + THROW_EXCEPTION(HTTPException, UnknownResponseCodeUsed) + } + } + return "500 Internal Server Error"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetResponseCode(int) +// Purpose: Set the response code to be returned +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetResponseCode(int Code) +{ + mResponseCode = Code; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetContentType(const char *) +// Purpose: Set content type +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetContentType(const char *ContentType) +{ + mContentType = ContentType; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::Send(IOStream &, bool) +// Purpose: Build the response, and send via the stream. +// Optionally omitting the content. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::Send(bool OmitContent) +{ + if (!mpStreamToSendTo) + { + THROW_EXCEPTION(HTTPException, NoStreamConfigured); + } + + if (GetSize() != 0 && mContentType.empty()) + { + THROW_EXCEPTION(HTTPException, NoContentTypeSet); + } + + // Build and send header + { + std::string header("HTTP/1.1 "); + header += ResponseCodeToString(mResponseCode); + header += "\r\nContent-Type: "; + header += mContentType; + header += "\r\nContent-Length: "; + { + char len[32]; + ::sprintf(len, "%d", OmitContent?(0):(GetSize())); + header += len; + } + // Extra headers... + for(std::vector<std::pair<std::string, std::string> >::const_iterator i(mExtraHeaders.begin()); i != mExtraHeaders.end(); ++i) + { + header += "\r\n"; + header += i->first + ": " + i->second; + } + // NOTE: a line ending must be included here in all cases + // Control whether the response is cached + if(mResponseIsDynamicContent) + { + // dynamic is private and can't be cached + header += "\r\nCache-Control: no-cache, private"; + } + else + { + // static is allowed to be cached for a day + header += "\r\nCache-Control: max-age=86400"; + } + if(mKeepAlive) + { + header += "\r\nConnection: keep-alive\r\n\r\n"; + } + else + { + header += "\r\nConnection: close\r\n\r\n"; + } + // NOTE: header ends with blank line in all cases + + // Write to stream + mpStreamToSendTo->Write(header.c_str(), header.size()); + } + + // Send content + if(!OmitContent) + { + mpStreamToSendTo->Write(GetBuffer(), GetSize()); + } +} + +void HTTPResponse::SendContinue() +{ + mpStreamToSendTo->Write("HTTP/1.1 100 Continue\r\n"); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::ParseHeaders(IOStreamGetLine &, int) +// Purpose: Private. Parse the headers of the response +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) +{ + std::string header; + bool haveHeader = false; + while(true) + { + if(rGetLine.IsEOF()) + { + // Header terminates unexpectedly + THROW_EXCEPTION(HTTPException, BadRequest) + } + + std::string currentLine; + if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) + { + // Timeout + THROW_EXCEPTION(HTTPException, RequestReadFailed) + } + + // Is this a continuation of the previous line? + bool processHeader = haveHeader; + if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) + { + // A continuation, don't process anything yet + processHeader = false; + } + //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); + + // Parse the header -- this will actually process the header + // from the previous run around the loop. + if(processHeader) + { + // Find where the : is in the line + const char *h = header.c_str(); + int p = 0; + while(h[p] != '\0' && h[p] != ':') + { + ++p; + } + // Skip white space + int dataStart = p + 1; + while(h[dataStart] == ' ' || h[dataStart] == '\t') + { + ++dataStart; + } + + if(p == sizeof("Content-Length")-1 + && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) + { + // Decode number + long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK + if(len < 0) len = 0; + // Store + mContentLength = len; + } + else if(p == sizeof("Content-Type")-1 + && ::strncasecmp(h, "Content-Type", sizeof("Content-Type")-1) == 0) + { + // Store rest of string as content type + mContentType = h + dataStart; + } + else if(p == sizeof("Cookie")-1 + && ::strncasecmp(h, "Cookie", sizeof("Cookie")-1) == 0) + { + THROW_EXCEPTION(HTTPException, NotImplemented); + /* + // Parse cookies + ParseCookies(header, dataStart); + */ + } + else if(p == sizeof("Connection")-1 + && ::strncasecmp(h, "Connection", sizeof("Connection")-1) == 0) + { + // Connection header, what is required? + const char *v = h + dataStart; + if(::strcasecmp(v, "close") == 0) + { + mKeepAlive = false; + } + else if(::strcasecmp(v, "keep-alive") == 0) + { + mKeepAlive = true; + } + // else don't understand, just assume default for protocol version + } + else + { + std::string headerName = header.substr(0, p); + AddHeader(headerName, h + dataStart); + } + + // Unset have header flag, as it's now been processed + haveHeader = false; + } + + // Store the chunk of header the for next time round + if(haveHeader) + { + header += currentLine; + } + else + { + header = currentLine; + haveHeader = true; + } + + // End of headers? + if(currentLine.empty()) + { + // All done! + break; + } + } +} + +void HTTPResponse::Receive(IOStream& rStream, int Timeout) +{ + IOStreamGetLine rGetLine(rStream); + + if(rGetLine.IsEOF()) + { + // Connection terminated unexpectedly + THROW_EXCEPTION(HTTPException, BadResponse) + } + + std::string statusLine; + if(!rGetLine.GetLine(statusLine, false /* no preprocess */, Timeout)) + { + // Timeout + THROW_EXCEPTION(HTTPException, ResponseReadFailed) + } + + if (statusLine.substr(0, 7) != "HTTP/1." || + statusLine[8] != ' ') + { + // Status line terminated unexpectedly + BOX_ERROR("Bad response status line: " << statusLine); + THROW_EXCEPTION(HTTPException, BadResponse) + } + + if (statusLine[5] == '1' && statusLine[7] == '1') + { + // HTTP/1.1 default is to keep alive + mKeepAlive = true; + } + + // Decode the status code + long status = ::strtol(statusLine.substr(9, 3).c_str(), NULL, 10); + // returns zero in error case, this is OK + if (status < 0) status = 0; + // Store + mResponseCode = status; + + // 100 Continue responses have no headers, terminating newline, or body + if (status == 100) + { + return; + } + + ParseHeaders(rGetLine, Timeout); + + // push back whatever bytes we have left + // rGetLine.DetachFile(); + if (mContentLength > 0) + { + if (mContentLength < rGetLine.GetSizeOfBufferedData()) + { + // very small response, not good! + THROW_EXCEPTION(HTTPException, NotImplemented); + } + + mContentLength -= rGetLine.GetSizeOfBufferedData(); + + Write(rGetLine.GetBufferedData(), + rGetLine.GetSizeOfBufferedData()); + } + + while (mContentLength != 0) // could be -1 as well + { + char buffer[4096]; + int readSize = sizeof(buffer); + if (mContentLength > 0 && mContentLength < readSize) + { + readSize = mContentLength; + } + readSize = rStream.Read(buffer, readSize, Timeout); + if (readSize == 0) + { + break; + } + mContentLength -= readSize; + Write(buffer, readSize); + } + + SetForReading(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const char *) +// Purpose: Add header, given entire line +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +/* +void HTTPResponse::AddHeader(const char *EntireHeaderLine) +{ + mExtraHeaders.push_back(std::string(EntireHeaderLine)); +} +*/ + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const std::string &) +// Purpose: Add header, given entire line +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +/* +void HTTPResponse::AddHeader(const std::string &rEntireHeaderLine) +{ + mExtraHeaders.push_back(rEntireHeaderLine); +} +*/ + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const char *, const char *) +// Purpose: Add header, given header name and it's value +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::AddHeader(const char *pHeader, const char *pValue) +{ + mExtraHeaders.push_back(Header(pHeader, pValue)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const char *, const std::string &) +// Purpose: Add header, given header name and it's value +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::AddHeader(const char *pHeader, const std::string &rValue) +{ + mExtraHeaders.push_back(Header(pHeader, rValue)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const std::string &, const std::string &) +// Purpose: Add header, given header name and it's value +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::AddHeader(const std::string &rHeader, const std::string &rValue) +{ + mExtraHeaders.push_back(Header(rHeader, rValue)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetCookie(const char *, const char *, const char *, int) +// Purpose: Sets a cookie, using name, value, path and expiry time. +// Created: 20/8/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetCookie(const char *Name, const char *Value, const char *Path, int ExpiresAt) +{ + if(ExpiresAt != 0) + { + THROW_EXCEPTION(HTTPException, NotImplemented) + } + + // Appears you shouldn't use quotes when you generate set-cookie headers. + // Oh well. It was fun finding that out. +/* std::string h("Set-Cookie: "); + h += Name; + h += "=\""; + h += Value; + h += "\"; Version=\"1\"; Path=\""; + h += Path; + h += "\""; +*/ + std::string h; + h += Name; + h += "="; + h += Value; + h += "; Version=1; Path="; + h += Path; + + mExtraHeaders.push_back(Header("Set-Cookie", h)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetAsRedirect(const char *, bool) +// Purpose: Sets the response objects to be a redirect to another page. +// If IsLocalURL == true, the default prefix will be added. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetAsRedirect(const char *RedirectTo, bool IsLocalURI) +{ + if(mResponseCode != HTTPResponse::Code_NoContent + || !mContentType.empty() + || GetSize() != 0) + { + THROW_EXCEPTION(HTTPException, CannotSetRedirectIfReponseHasData) + } + + // Set response code + mResponseCode = Code_Found; + + // Set location to redirect to + std::string header; + if(IsLocalURI) header += msDefaultURIPrefix; + header += RedirectTo; + mExtraHeaders.push_back(Header("Location", header)); + + // Set up some default content + mContentType = "text/html"; + #define REDIRECT_HTML_1 "<html><head><title>Redirection</title></head>\n<body><p><a href=\"" + #define REDIRECT_HTML_2 "\">Redirect to content</a></p></body></html>\n" + Write(REDIRECT_HTML_1, sizeof(REDIRECT_HTML_1) - 1); + if(IsLocalURI) Write(msDefaultURIPrefix.c_str(), msDefaultURIPrefix.size()); + Write(RedirectTo, ::strlen(RedirectTo)); + Write(REDIRECT_HTML_2, sizeof(REDIRECT_HTML_2) - 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetAsNotFound(const char *) +// Purpose: Set the response object to be a standard page not found 404 response. +// Created: 7/4/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetAsNotFound(const char *URI) +{ + if(mResponseCode != HTTPResponse::Code_NoContent + || mExtraHeaders.size() != 0 + || !mContentType.empty() + || GetSize() != 0) + { + THROW_EXCEPTION(HTTPException, CannotSetNotFoundIfReponseHasData) + } + + // Set response code + mResponseCode = Code_NotFound; + + // Set data + mContentType = "text/html"; + #define NOT_FOUND_HTML_1 "<html><head><title>404 Not Found</title></head>\n<body><h1>404 Not Found</h1>\n<p>The URI <i>" + #define NOT_FOUND_HTML_2 "</i> was not found on this server.</p></body></html>\n" + Write(NOT_FOUND_HTML_1, sizeof(NOT_FOUND_HTML_1) - 1); + WriteStringDefang(std::string(URI)); + Write(NOT_FOUND_HTML_2, sizeof(NOT_FOUND_HTML_2) - 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::WriteStringDefang(const char *, unsigned int) +// Purpose: Writes a string 'defanged', ie has HTML special characters escaped +// so that people can't output arbitary HTML by playing with +// URLs and form parameters, and it's safe to write strings into +// HTML element attribute values. +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::WriteStringDefang(const char *String, unsigned int StringLen) +{ + while(StringLen > 0) + { + unsigned int toWrite = 0; + while(toWrite < StringLen + && String[toWrite] != '<' + && String[toWrite] != '>' + && String[toWrite] != '&' + && String[toWrite] != '"') + { + ++toWrite; + } + if(toWrite > 0) + { + Write(String, toWrite); + StringLen -= toWrite; + String += toWrite; + } + + // Is it a bad character next? + while(StringLen > 0) + { + bool notSpecial = false; + switch(*String) + { + case '<': Write("<", 4); break; + case '>': Write(">", 4); break; + case '&': Write("&", 5); break; + case '"': Write(""", 6); break; + default: + // Stop this loop + notSpecial = true; + break; + } + if(notSpecial) break; + ++String; + --StringLen; + } + } +} + + diff --git a/lib/httpserver/HTTPResponse.h b/lib/httpserver/HTTPResponse.h new file mode 100644 index 00000000..04051958 --- /dev/null +++ b/lib/httpserver/HTTPResponse.h @@ -0,0 +1,175 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPResponse.h +// Purpose: Response object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPRESPONSE__H +#define HTTPRESPONSE__H + +#include <string> +#include <vector> + +#include "CollectInBufferStream.h" + +class IOStreamGetLine; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPResponse +// Purpose: Response object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPResponse : public CollectInBufferStream +{ +public: + HTTPResponse(IOStream* pStreamToSendTo); + HTTPResponse(); + ~HTTPResponse(); + + // allow copying, but be very careful with the response stream, + // you can only read it once! (this class doesn't police it). + HTTPResponse(const HTTPResponse& rOther) + : mResponseCode(rOther.mResponseCode), + mResponseIsDynamicContent(rOther.mResponseIsDynamicContent), + mKeepAlive(rOther.mKeepAlive), + mContentType(rOther.mContentType), + mExtraHeaders(rOther.mExtraHeaders), + mContentLength(rOther.mContentLength), + mpStreamToSendTo(rOther.mpStreamToSendTo) + { + Write(rOther.GetBuffer(), rOther.GetSize()); + } + + HTTPResponse &operator=(const HTTPResponse &rOther) + { + Reset(); + Write(rOther.GetBuffer(), rOther.GetSize()); + mResponseCode = rOther.mResponseCode; + mResponseIsDynamicContent = rOther.mResponseIsDynamicContent; + mKeepAlive = rOther.mKeepAlive; + mContentType = rOther.mContentType; + mExtraHeaders = rOther.mExtraHeaders; + mContentLength = rOther.mContentLength; + mpStreamToSendTo = rOther.mpStreamToSendTo; + return *this; + } + + typedef std::pair<std::string, std::string> Header; + + void SetResponseCode(int Code); + int GetResponseCode() { return mResponseCode; } + void SetContentType(const char *ContentType); + const std::string& GetContentType() { return mContentType; } + + void SetAsRedirect(const char *RedirectTo, bool IsLocalURI = true); + void SetAsNotFound(const char *URI); + + void Send(bool OmitContent = false); + void SendContinue(); + void Receive(IOStream& rStream, int Timeout = IOStream::TimeOutInfinite); + + // void AddHeader(const char *EntireHeaderLine); + // void AddHeader(const std::string &rEntireHeaderLine); + void AddHeader(const char *Header, const char *Value); + void AddHeader(const char *Header, const std::string &rValue); + void AddHeader(const std::string &rHeader, const std::string &rValue); + bool GetHeader(const std::string& rName, std::string* pValueOut) const + { + for (std::vector<Header>::const_iterator + i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + if (i->first == rName) + { + *pValueOut = i->second; + return true; + } + } + return false; + } + std::string GetHeaderValue(const std::string& rName) + { + std::string value; + if (!GetHeader(rName, &value)) + { + THROW_EXCEPTION(CommonException, ConfigNoKey); + } + return value; + } + + // Set dynamic content flag, default is content is dynamic + void SetResponseIsDynamicContent(bool IsDynamic) {mResponseIsDynamicContent = IsDynamic;} + // Set keep alive control, default is to mark as to be closed + void SetKeepAlive(bool KeepAlive) {mKeepAlive = KeepAlive;} + + void SetCookie(const char *Name, const char *Value, const char *Path = "/", int ExpiresAt = 0); + + enum + { + Code_OK = 200, + Code_NoContent = 204, + Code_MovedPermanently = 301, + Code_Found = 302, // redirection + Code_NotModified = 304, + Code_TemporaryRedirect = 307, + Code_MethodNotAllowed = 400, + Code_Unauthorized = 401, + Code_Forbidden = 403, + Code_NotFound = 404, + Code_InternalServerError = 500, + Code_NotImplemented = 501 + }; + + static const char *ResponseCodeToString(int ResponseCode); + + void WriteStringDefang(const char *String, unsigned int StringLen); + void WriteStringDefang(const std::string &rString) {WriteStringDefang(rString.c_str(), rString.size());} + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPResponse::WriteString(const std::string &) + // Purpose: Write a string to the response (simple sugar function) + // Created: 9/4/04 + // + // -------------------------------------------------------------------------- + void WriteString(const std::string &rString) + { + Write(rString.c_str(), rString.size()); + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPResponse::SetDefaultURIPrefix(const std::string &) + // Purpose: Set default prefix used to local redirections + // Created: 26/3/04 + // + // -------------------------------------------------------------------------- + static void SetDefaultURIPrefix(const std::string &rPrefix) + { + msDefaultURIPrefix = rPrefix; + } + +private: + int mResponseCode; + bool mResponseIsDynamicContent; + bool mKeepAlive; + std::string mContentType; + std::vector<Header> mExtraHeaders; + int mContentLength; // only used when reading response from stream + IOStream* mpStreamToSendTo; // nonzero only when constructed with a stream + + static std::string msDefaultURIPrefix; + + void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); +}; + +#endif // HTTPRESPONSE__H + diff --git a/lib/httpserver/HTTPServer.cpp b/lib/httpserver/HTTPServer.cpp new file mode 100644 index 00000000..b8b02249 --- /dev/null +++ b/lib/httpserver/HTTPServer.cpp @@ -0,0 +1,249 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPServer.cpp +// Purpose: HTTP server class +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <stdio.h> + +#include "HTTPServer.h" +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "IOStreamGetLine.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::HTTPServer() +// Purpose: Constructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPServer::HTTPServer() + : mTimeout(20000) // default timeout leaves a little while for clients to get the second request in. +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::~HTTPServer() +// Purpose: Destructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPServer::~HTTPServer() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::DaemonName() +// Purpose: As interface, generic name for daemon +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +const char *HTTPServer::DaemonName() const +{ + return "generic-httpserver"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::GetConfigVerify() +// Purpose: As interface -- return most basic config so it's only necessary to +// provide this if you want to add extra directives. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify *HTTPServer::GetConfigVerify() const +{ + static ConfigurationVerifyKey verifyserverkeys[] = + { + HTTPSERVER_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default addresses + }; + + static ConfigurationVerify verifyserver[] = + { + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } + }; + + static ConfigurationVerifyKey verifyrootkeys[] = + { + HTTPSERVER_VERIFY_ROOT_KEYS + }; + + static ConfigurationVerify verify = + { + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + }; + + return &verify; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::Run() +// Purpose: As interface. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::Run() +{ + // Do some configuration stuff + const Configuration &conf(GetConfiguration()); + HTTPResponse::SetDefaultURIPrefix(conf.GetKeyValue("AddressPrefix")); + + // Let the base class do the work + ServerStream<SocketStream, 80>::Run(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::Connection(SocketStream &) +// Purpose: As interface, handle connection +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::Connection(SocketStream &rStream) +{ + // Create a get line object to use + IOStreamGetLine getLine(rStream); + + // Notify dervived claases + HTTPConnectionOpening(); + + bool handleRequests = true; + while(handleRequests) + { + // Parse the request + HTTPRequest request; + if(!request.Receive(getLine, mTimeout)) + { + // Didn't get request, connection probably closed. + break; + } + + // Generate a response + HTTPResponse response(&rStream); + try + { + Handle(request, response); + } + catch(BoxException &e) + { + char exceptionCode[64]; + ::sprintf(exceptionCode, "(%d/%d)", e.GetType(), e.GetSubType()); + SendInternalErrorResponse(exceptionCode, rStream); + return; + } + catch(...) + { + SendInternalErrorResponse("unknown", rStream); + return; + } + + // Keep alive? + if(request.GetClientKeepAliveRequested()) + { + // Mark the response to the client as supporting keepalive + response.SetKeepAlive(true); + } + else + { + // Stop now + handleRequests = false; + } + + // Send the response (omit any content if this is a HEAD method request) + response.Send(request.GetMethod() == HTTPRequest::Method_HEAD); + } + + // Notify derived claases + HTTPConnectionClosing(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::SendInternalErrorResponse(const char *, SocketStream &) +// Purpose: Sends an error response to the remote side +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::SendInternalErrorResponse(const char *Error, SocketStream &rStream) +{ + #define ERROR_HTML_1 "<html><head><title>Internal Server Error</title></head>\n" \ + "<h1>Internal Server Error</h1>\n" \ + "<p>An error, type " + #define ERROR_HTML_2 " occured when processing the request.</p>" \ + "<p>Please try again later.</p>" \ + "</body>\n</html>\n" + + // Generate the error page + HTTPResponse response(&rStream); + response.SetResponseCode(HTTPResponse::Code_InternalServerError); + response.SetContentType("text/html"); + response.Write(ERROR_HTML_1, sizeof(ERROR_HTML_1) - 1); + response.Write(Error, ::strlen(Error)); + response.Write(ERROR_HTML_2, sizeof(ERROR_HTML_2) - 1); + + // Send the error response + response.Send(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::HTTPConnectionOpening() +// Purpose: Override to get notifications of connections opening +// Created: 22/12/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::HTTPConnectionOpening() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::HTTPConnectionClosing() +// Purpose: Override to get notifications of connections closing +// Created: 22/12/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::HTTPConnectionClosing() +{ +} + + diff --git a/lib/httpserver/HTTPServer.h b/lib/httpserver/HTTPServer.h new file mode 100644 index 00000000..8009438d --- /dev/null +++ b/lib/httpserver/HTTPServer.h @@ -0,0 +1,79 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPServer.h +// Purpose: HTTP server class +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPSERVER__H +#define HTTPSERVER__H + +#include "ServerStream.h" +#include "SocketStream.h" + +class HTTPRequest; +class HTTPResponse; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPServer +// Purpose: HTTP server +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPServer : public ServerStream<SocketStream, 80> +{ +public: + HTTPServer(); + ~HTTPServer(); +private: + // no copying + HTTPServer(const HTTPServer &); + HTTPServer &operator=(const HTTPServer &); +public: + + int GetTimeout() const {return mTimeout;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPServer::Handle(const HTTPRequest &, HTTPResponse &) + // Purpose: Response to a request, filling in the response object for sending + // at some point in the future. + // Created: 26/3/04 + // + // -------------------------------------------------------------------------- + virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) = 0; + + // For notifications to derived classes + virtual void HTTPConnectionOpening(); + virtual void HTTPConnectionClosing(); + +private: + const char *DaemonName() const; + const ConfigurationVerify *GetConfigVerify() const; + void Run(); + void Connection(SocketStream &rStream); + void SendInternalErrorResponse(const char *Error, SocketStream &rStream); + +private: + int mTimeout; // Timeout for read operations +}; + +// Root level +#define HTTPSERVER_VERIFY_ROOT_KEYS \ + ConfigurationVerifyKey("AddressPrefix", \ + ConfigTest_Exists | ConfigTest_LastEntry) + +// AddressPrefix is, for example, http://localhost:1080 -- ie the beginning of the URI +// This is used for handling redirections. + +// Server level +#define HTTPSERVER_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ + SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) + +#endif // HTTPSERVER__H + diff --git a/lib/httpserver/Makefile.extra b/lib/httpserver/Makefile.extra new file mode 100644 index 00000000..f0ca62be --- /dev/null +++ b/lib/httpserver/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_HTTPException.h autogen_HTTPException.cpp: $(MAKEEXCEPTION) HTTPException.txt + perl $(MAKEEXCEPTION) HTTPException.txt + diff --git a/lib/httpserver/S3Client.cpp b/lib/httpserver/S3Client.cpp new file mode 100644 index 00000000..cd5988d5 --- /dev/null +++ b/lib/httpserver/S3Client.cpp @@ -0,0 +1,243 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Client.cpp +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <cstring> + +// #include <cstdio> +// #include <ctime> + +#include <openssl/hmac.h> + +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "HTTPServer.h" +#include "autogen_HTTPException.h" +#include "IOStream.h" +#include "Logging.h" +#include "S3Client.h" +#include "decode.h" +#include "encode.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::GetObject(const std::string& rObjectURI) +// Purpose: Retrieve the object with the specified URI (key) +// from your S3 bucket. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::GetObject(const std::string& rObjectURI) +{ + return FinishAndSendRequest(HTTPRequest::Method_GET, rObjectURI); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::PutObject(const std::string& rObjectURI, +// IOStream& rStreamToSend, const char* pContentType) +// Purpose: Upload the stream to S3, creating or overwriting the +// object with the specified URI (key) in your S3 +// bucket. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::PutObject(const std::string& rObjectURI, + IOStream& rStreamToSend, const char* pContentType) +{ + return FinishAndSendRequest(HTTPRequest::Method_PUT, rObjectURI, + &rStreamToSend, pContentType); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::FinishAndSendRequest( +// HTTPRequest::Method Method, +// const std::string& rRequestURI, +// IOStream* pStreamToSend, +// const char* pStreamContentType) +// Purpose: Internal method which creates an HTTP request to S3, +// populates the date and authorization header fields, +// and sends it to S3 (or the simulator), attaching +// the specified stream if any to the request. Opens a +// connection to the server if necessary, which may +// throw a ConnectionException. Returns the HTTP +// response returned by S3, which may be a 500 error. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, + const std::string& rRequestURI, IOStream* pStreamToSend, + const char* pStreamContentType) +{ + HTTPRequest request(Method, rRequestURI); + request.SetHostName(mHostName); + + std::ostringstream date; + time_t tt = time(NULL); + struct tm *tp = gmtime(&tt); + if (!tp) + { + BOX_ERROR("Failed to get current time"); + THROW_EXCEPTION(HTTPException, Internal); + } + const char *dow[] = {"Sun","Mon","Tue","Wed","Thu","Fri","Sat"}; + date << dow[tp->tm_wday] << ", "; + const char *month[] = {"Jan","Feb","Mar","Apr","May","Jun", + "Jul","Aug","Sep","Oct","Nov","Dec"}; + date << std::internal << std::setfill('0') << + std::setw(2) << tp->tm_mday << " " << + month[tp->tm_mon] << " " << + (tp->tm_year + 1900) << " "; + date << std::setw(2) << tp->tm_hour << ":" << + std::setw(2) << tp->tm_min << ":" << + std::setw(2) << tp->tm_sec << " GMT"; + request.AddHeader("Date", date.str()); + + if (pStreamContentType) + { + request.AddHeader("Content-Type", pStreamContentType); + } + + std::string s3suffix = ".s3.amazonaws.com"; + std::string bucket; + if (mHostName.size() > s3suffix.size()) + { + std::string suffix = mHostName.substr(mHostName.size() - + s3suffix.size(), s3suffix.size()); + if (suffix == s3suffix) + { + bucket = mHostName.substr(0, mHostName.size() - + s3suffix.size()); + } + } + + std::ostringstream data; + data << request.GetVerb() << "\n"; + data << "\n"; /* Content-MD5 */ + data << request.GetContentType() << "\n"; + data << date.str() << "\n"; + + if (! bucket.empty()) + { + data << "/" << bucket; + } + + data << request.GetRequestURI(); + std::string data_string = data.str(); + + unsigned char digest_buffer[EVP_MAX_MD_SIZE]; + unsigned int digest_size = sizeof(digest_buffer); + /* unsigned char* mac = */ HMAC(EVP_sha1(), + mSecretKey.c_str(), mSecretKey.size(), + (const unsigned char*)data_string.c_str(), + data_string.size(), digest_buffer, &digest_size); + std::string digest((const char *)digest_buffer, digest_size); + + base64::encoder encoder; + std::string auth_code = "AWS " + mAccessKey + ":" + + encoder.encode(digest); + + if (auth_code[auth_code.size() - 1] == '\n') + { + auth_code = auth_code.substr(0, auth_code.size() - 1); + } + + request.AddHeader("Authorization", auth_code); + + if (mpSimulator) + { + if (pStreamToSend) + { + pStreamToSend->CopyStreamTo(request); + } + + request.SetForReading(); + CollectInBufferStream response_buffer; + HTTPResponse response(&response_buffer); + + mpSimulator->Handle(request, response); + return response; + } + else + { + try + { + if (!mapClientSocket.get()) + { + mapClientSocket.reset(new SocketStream()); + mapClientSocket->Open(Socket::TypeINET, + mHostName, mPort); + } + return SendRequest(request, pStreamToSend, + pStreamContentType); + } + catch (ConnectionException &ce) + { + if (ce.GetType() == ConnectionException::SocketWriteError) + { + // server may have disconnected us, + // try to reconnect, just once + mapClientSocket->Open(Socket::TypeINET, + mHostName, mPort); + return SendRequest(request, pStreamToSend, + pStreamContentType); + } + else + { + throw; + } + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::SendRequest(HTTPRequest& rRequest, +// IOStream* pStreamToSend, +// const char* pStreamContentType) +// Purpose: Internal method which sends a pre-existing HTTP +// request to S3. Attaches the specified stream if any +// to the request. Opens a connection to the server if +// necessary, which may throw a ConnectionException. +// Returns the HTTP response returned by S3, which may +// be a 500 error. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::SendRequest(HTTPRequest& rRequest, + IOStream* pStreamToSend, const char* pStreamContentType) +{ + HTTPResponse response; + + if (pStreamToSend) + { + rRequest.SendWithStream(*mapClientSocket, + 30000 /* milliseconds */, + pStreamToSend, response); + } + else + { + rRequest.Send(*mapClientSocket, 30000 /* milliseconds */); + response.Receive(*mapClientSocket, 30000 /* milliseconds */); + } + + return response; +} diff --git a/lib/httpserver/S3Client.h b/lib/httpserver/S3Client.h new file mode 100644 index 00000000..3c4126ac --- /dev/null +++ b/lib/httpserver/S3Client.h @@ -0,0 +1,72 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Client.h +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#ifndef S3CLIENT__H +#define S3CLIENT__H + +#include <string> +#include <map> + +#include "HTTPRequest.h" +#include "SocketStream.h" + +class HTTPResponse; +class HTTPServer; +class IOStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: S3Client +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- +class S3Client +{ + public: + S3Client(HTTPServer* pSimulator, const std::string& rHostName, + const std::string& rAccessKey, const std::string& rSecretKey) + : mpSimulator(pSimulator), + mHostName(rHostName), + mAccessKey(rAccessKey), + mSecretKey(rSecretKey) + { } + + S3Client(std::string HostName, int Port, const std::string& rAccessKey, + const std::string& rSecretKey) + : mpSimulator(NULL), + mHostName(HostName), + mPort(Port), + mAccessKey(rAccessKey), + mSecretKey(rSecretKey) + { } + + HTTPResponse GetObject(const std::string& rObjectURI); + HTTPResponse PutObject(const std::string& rObjectURI, + IOStream& rStreamToSend, const char* pContentType = NULL); + + private: + HTTPServer* mpSimulator; + std::string mHostName; + int mPort; + std::auto_ptr<SocketStream> mapClientSocket; + std::string mAccessKey, mSecretKey; + + HTTPResponse FinishAndSendRequest(HTTPRequest::Method Method, + const std::string& rRequestURI, + IOStream* pStreamToSend = NULL, + const char* pStreamContentType = NULL); + HTTPResponse SendRequest(HTTPRequest& rRequest, + IOStream* pStreamToSend = NULL, + const char* pStreamContentType = NULL); +}; + +#endif // S3CLIENT__H + diff --git a/lib/httpserver/cdecode.cpp b/lib/httpserver/cdecode.cpp new file mode 100644 index 00000000..e632f182 --- /dev/null +++ b/lib/httpserver/cdecode.cpp @@ -0,0 +1,92 @@ +/* +cdecoder.c - c source to a base64 decoding algorithm implementation + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +extern "C" +{ + +#include "cdecode.h" + +int base64_decode_value(char value_in) +{ + static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; + static const char decoding_size = sizeof(decoding); + value_in -= 43; + if (value_in < 0 || value_in > decoding_size) return -1; + return decoding[(int)value_in]; +} + +void base64_init_decodestate(base64_decodestate* state_in) +{ + state_in->step = step_a; + state_in->plainchar = 0; +} + +int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in) +{ + const char* codechar = code_in; + char* plainchar = plaintext_out; + char fragment; + + *plainchar = state_in->plainchar; + + switch (state_in->step) + { + while (1) + { + case step_a: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_a; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar = (fragment & 0x03f) << 2; + case step_b: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_b; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x030) >> 4; + *plainchar = (fragment & 0x00f) << 4; + case step_c: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_c; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x03c) >> 2; + *plainchar = (fragment & 0x003) << 6; + case step_d: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_d; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x03f); + } + } + /* control should not reach here */ + return plainchar - plaintext_out; +} + +} diff --git a/lib/httpserver/cdecode.h b/lib/httpserver/cdecode.h new file mode 100644 index 00000000..d0d7f489 --- /dev/null +++ b/lib/httpserver/cdecode.h @@ -0,0 +1,28 @@ +/* +cdecode.h - c header for a base64 decoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_CDECODE_H +#define BASE64_CDECODE_H + +typedef enum +{ + step_a, step_b, step_c, step_d +} base64_decodestep; + +typedef struct +{ + base64_decodestep step; + char plainchar; +} base64_decodestate; + +void base64_init_decodestate(base64_decodestate* state_in); + +int base64_decode_value(char value_in); + +int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in); + +#endif /* BASE64_CDECODE_H */ diff --git a/lib/httpserver/cencode.cpp b/lib/httpserver/cencode.cpp new file mode 100644 index 00000000..b33c0683 --- /dev/null +++ b/lib/httpserver/cencode.cpp @@ -0,0 +1,113 @@ +/* +cencoder.c - c source to a base64 encoding algorithm implementation + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +extern "C" +{ + +#include "cencode.h" + +const int CHARS_PER_LINE = 72; + +void base64_init_encodestate(base64_encodestate* state_in) +{ + state_in->step = step_A; + state_in->result = 0; + state_in->stepcount = 0; +} + +char base64_encode_value(char value_in) +{ + static const char* encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + if (value_in > 63) return '='; + return encoding[(int)value_in]; +} + +int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in) +{ + const char* plainchar = plaintext_in; + const char* const plaintextend = plaintext_in + length_in; + char* codechar = code_out; + char result; + char fragment; + + result = state_in->result; + + switch (state_in->step) + { + while (1) + { + case step_A: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_A; + return codechar - code_out; + } + fragment = *plainchar++; + result = (fragment & 0x0fc) >> 2; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x003) << 4; + case step_B: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_B; + return codechar - code_out; + } + fragment = *plainchar++; + result |= (fragment & 0x0f0) >> 4; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x00f) << 2; + case step_C: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_C; + return codechar - code_out; + } + fragment = *plainchar++; + result |= (fragment & 0x0c0) >> 6; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x03f) >> 0; + *codechar++ = base64_encode_value(result); + + ++(state_in->stepcount); + if (state_in->stepcount == CHARS_PER_LINE/4) + { + *codechar++ = '\n'; + state_in->stepcount = 0; + } + } + } + /* control should not reach here */ + return codechar - code_out; +} + +int base64_encode_blockend(char* code_out, base64_encodestate* state_in) +{ + char* codechar = code_out; + + switch (state_in->step) + { + case step_B: + *codechar++ = base64_encode_value(state_in->result); + *codechar++ = '='; + *codechar++ = '='; + break; + case step_C: + *codechar++ = base64_encode_value(state_in->result); + *codechar++ = '='; + break; + case step_A: + break; + } + *codechar++ = '\n'; + + return codechar - code_out; +} + +} diff --git a/lib/httpserver/cencode.h b/lib/httpserver/cencode.h new file mode 100644 index 00000000..cf321312 --- /dev/null +++ b/lib/httpserver/cencode.h @@ -0,0 +1,32 @@ +/* +cencode.h - c header for a base64 encoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_CENCODE_H +#define BASE64_CENCODE_H + +typedef enum +{ + step_A, step_B, step_C +} base64_encodestep; + +typedef struct +{ + base64_encodestep step; + char result; + int stepcount; +} base64_encodestate; + +void base64_init_encodestate(base64_encodestate* state_in); + +char base64_encode_value(char value_in); + +int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in); + +int base64_encode_blockend(char* code_out, base64_encodestate* state_in); + +#endif /* BASE64_CENCODE_H */ + diff --git a/lib/httpserver/decode.h b/lib/httpserver/decode.h new file mode 100644 index 00000000..fe59ef7a --- /dev/null +++ b/lib/httpserver/decode.h @@ -0,0 +1,77 @@ +// :mode=c++: +/* +decode.h - c++ wrapper for a base64 decoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_DECODE_H +#define BASE64_DECODE_H + +#include <iostream> + +namespace base64 +{ + + extern "C" + { + #include "cdecode.h" + } + + struct decoder + { + base64_decodestate _state; + int _buffersize; + + decoder(int buffersize_in = 4096) + : _buffersize(buffersize_in) + {} + int decode(char value_in) + { + return base64_decode_value(value_in); + } + int decode(const char* code_in, const int length_in, char* plaintext_out) + { + return base64_decode_block(code_in, length_in, plaintext_out, &_state); + } + std::string decode(const std::string& input) + { + base64_init_decodestate(&_state); + char* output = new char[2*input.size()]; + int outlength = decode(input.c_str(), input.size(), + output); + std::string output_string(output, outlength); + base64_init_decodestate(&_state); + delete [] output; + return output_string; + } + void decode(std::istream& istream_in, std::ostream& ostream_in) + { + base64_init_decodestate(&_state); + // + const int N = _buffersize; + char* code = new char[N]; + char* plaintext = new char[N]; + int codelength; + int plainlength; + + do + { + istream_in.read((char*)code, N); + codelength = istream_in.gcount(); + plainlength = decode(code, codelength, plaintext); + ostream_in.write((const char*)plaintext, plainlength); + } + while (istream_in.good() && codelength > 0); + // + base64_init_decodestate(&_state); + + delete [] code; + delete [] plaintext; + } + }; + +} // namespace base64 + +#endif // BASE64_DECODE_H diff --git a/lib/httpserver/encode.h b/lib/httpserver/encode.h new file mode 100644 index 00000000..81957a0f --- /dev/null +++ b/lib/httpserver/encode.h @@ -0,0 +1,87 @@ +// :mode=c++: +/* +encode.h - c++ wrapper for a base64 encoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_ENCODE_H +#define BASE64_ENCODE_H + +#include <iostream> + +namespace base64 +{ + + extern "C" + { + #include "cencode.h" + } + + struct encoder + { + base64_encodestate _state; + int _buffersize; + + encoder(int buffersize_in = 4096) + : _buffersize(buffersize_in) + {} + int encode(char value_in) + { + return base64_encode_value(value_in); + } + int encode(const char* code_in, const int length_in, char* plaintext_out) + { + return base64_encode_block(code_in, length_in, plaintext_out, &_state); + } + int encode_end(char* plaintext_out) + { + return base64_encode_blockend(plaintext_out, &_state); + } + std::string encode(const std::string& input) + { + base64_init_encodestate(&_state); + char* output = new char[2*input.size()]; + int outlength = encode(input.c_str(), input.size(), + output); + outlength += encode_end(output + outlength); + std::string output_string(output, outlength); + base64_init_encodestate(&_state); + delete [] output; + return output_string; + } + void encode(std::istream& istream_in, std::ostream& ostream_in) + { + base64_init_encodestate(&_state); + // + const int N = _buffersize; + char* plaintext = new char[N]; + char* code = new char[2*N]; + int plainlength; + int codelength; + + do + { + istream_in.read(plaintext, N); + plainlength = istream_in.gcount(); + // + codelength = encode(plaintext, plainlength, code); + ostream_in.write(code, codelength); + } + while (istream_in.good() && plainlength > 0); + + codelength = encode_end(code); + ostream_in.write(code, codelength); + // + base64_init_encodestate(&_state); + + delete [] code; + delete [] plaintext; + } + }; + +} // namespace base64 + +#endif // BASE64_ENCODE_H + diff --git a/lib/intercept/intercept.cpp b/lib/intercept/intercept.cpp index 65514ae2..7a33b610 100644 --- a/lib/intercept/intercept.cpp +++ b/lib/intercept/intercept.cpp @@ -22,6 +22,7 @@ #endif #include <errno.h> +#include <stdarg.h> #ifdef HAVE_DLFCN_H #include <dlfcn.h> @@ -84,6 +85,12 @@ static closedir_t* closedir_real = NULL; static lstat_t* lstat_real = NULL; static lstat_t* lstat_hook = NULL; static const char* lstat_file = NULL; +static lstat_t* stat_real = NULL; +static lstat_t* stat_hook = NULL; +static const char* stat_file = NULL; + +static lstat_post_hook_t* lstat_post_hook = NULL; +static lstat_post_hook_t* stat_post_hook = NULL; #define SIZE_ALWAYS_ERROR -773 @@ -97,7 +104,10 @@ void intercept_clear_setup() intercept_filepos = 0; intercept_delay_ms = 0; readdir_hook = NULL; + stat_hook = NULL; lstat_hook = NULL; + stat_post_hook = NULL; + lstat_post_hook = NULL; } bool intercept_triggered() @@ -222,8 +232,16 @@ int intercept_reterr() } \ } +#if defined _FILE_OFFSET_BITS && _FILE_OFFSET_BITS == 64 + #define DEFINE_ONLY_OPEN64 +#endif + extern "C" int -open(const char *path, int flags, mode_t mode) +#ifdef DEFINE_ONLY_OPEN64 + open64(const char *path, int flags, ...) +#else + open(const char *path, int flags, ...) +#endif // DEFINE_ONLY_OPEN64 { if(intercept_count > 0) { @@ -236,6 +254,15 @@ open(const char *path, int flags, mode_t mode) } } + mode_t mode = 0; + if (flags & O_CREAT) + { + va_list ap; + va_start(ap, flags); + mode = va_arg(ap, int); + va_end(ap); + } + #ifdef PLATFORM_NO_SYSCALL int r = TEST_open(path, flags, mode); #else @@ -257,14 +284,27 @@ open(const char *path, int flags, mode_t mode) return r; } +#ifndef DEFINE_ONLY_OPEN64 extern "C" int -open64(const char *path, int flags, mode_t mode) +// open64(const char *path, int flags, mode_t mode) +// open64(const char *path, int flags, ...) +open64 (__const char *path, int flags, ...) { + mode_t mode = 0; + if (flags & O_CREAT) + { + va_list ap; + va_start(ap, flags); + mode = va_arg(ap, int); + va_end(ap); + } + // With _FILE_OFFSET_BITS set to 64 this should really use (flags | // O_LARGEFILE) here, but not actually necessary for the tests and not // worth the trouble finding O_LARGEFILE return open(path, flags, mode); } +#endif // !DEFINE_ONLY_OPEN64 extern "C" int close(int d) @@ -375,12 +415,12 @@ void intercept_setup_readdir_hook(const char *dirname, readdir_t hookfn) if (hookfn != NULL) { - TRACE2("readdir hooked to %p for %s\n", hookfn, dirname); + BOX_TRACE("readdir hooked to " << hookfn << " for " << dirname); } else if (intercept_filename != NULL) { - TRACE2("readdir unhooked from %p for %s\n", readdir_hook, - intercept_filename); + BOX_TRACE("readdir unhooked from " << readdir_hook << + " for " << intercept_filename); } intercept_filename = dirname; @@ -392,11 +432,11 @@ void intercept_setup_lstat_hook(const char *filename, lstat_t hookfn) /* if (hookfn != NULL) { - TRACE2("lstat hooked to %p for %s\n", hookfn, filename); + BOX_TRACE("lstat hooked to " << hookfn << " for " << filename); } else { - TRACE2("lstat unhooked from %p for %s\n", lstat_hook, + BOX_TRACE("lstat unhooked from " << lstat_hook << " for " << lstat_file); } */ @@ -405,6 +445,40 @@ void intercept_setup_lstat_hook(const char *filename, lstat_t hookfn) lstat_hook = hookfn; } +void intercept_setup_lstat_post_hook(lstat_post_hook_t hookfn) +{ + /* + if (hookfn != NULL) + { + BOX_TRACE("lstat hooked to " << hookfn << " for " << filename); + } + else + { + BOX_TRACE("lstat unhooked from " << lstat_hook << " for " << + lstat_file); + } + */ + + lstat_post_hook = hookfn; +} + +void intercept_setup_stat_post_hook(lstat_post_hook_t hookfn) +{ + /* + if (hookfn != NULL) + { + BOX_TRACE("lstat hooked to " << hookfn << " for " << filename); + } + else + { + BOX_TRACE("lstat unhooked from " << lstat_hook << " for " << + lstat_file); + } + */ + + stat_post_hook = hookfn; +} + static void * find_function(const char *pName) { dlerror(); @@ -534,10 +608,15 @@ lstat(const char *file_name, STAT_STRUCT *buf) if (lstat_hook == NULL || strcmp(file_name, lstat_file) != 0) { #ifdef LINUX_WEIRD_LSTAT - return lstat_real(ver, file_name, buf); + int ret = lstat_real(ver, file_name, buf); #else - return lstat_real(file_name, buf); + int ret = lstat_real(file_name, buf); #endif + if (lstat_post_hook != NULL) + { + ret = lstat_post_hook(ret, file_name, buf); + } + return ret; } #ifdef LINUX_WEIRD_LSTAT @@ -547,4 +626,48 @@ lstat(const char *file_name, STAT_STRUCT *buf) #endif } +extern "C" int +#ifdef LINUX_WEIRD_LSTAT +__xstat(int ver, const char *file_name, STAT_STRUCT *buf) +#else +stat(const char *file_name, STAT_STRUCT *buf) +#endif +{ + if (stat_real == NULL) + { + #ifdef LINUX_WEIRD_LSTAT + stat_real = (lstat_t*)find_function("__xstat"); + #else + stat_real = (lstat_t*)find_function("stat"); + #endif + } + + if (stat_real == NULL) + { + perror("cannot find real stat"); + errno = ENOSYS; + return -1; + } + + if (stat_hook == NULL || strcmp(file_name, stat_file) != 0) + { + #ifdef LINUX_WEIRD_LSTAT + int ret = stat_real(ver, file_name, buf); + #else + int ret = stat_real(file_name, buf); + #endif + if (stat_post_hook != NULL) + { + ret = stat_post_hook(ret, file_name, buf); + } + return ret; + } + + #ifdef LINUX_WEIRD_LSTAT + return stat_hook(ver, file_name, buf); + #else + return stat_hook(file_name, buf); + #endif +} + #endif // n PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE diff --git a/lib/intercept/intercept.h b/lib/intercept/intercept.h index bc6557f3..80a17d3f 100644 --- a/lib/intercept/intercept.h +++ b/lib/intercept/intercept.h @@ -23,17 +23,20 @@ extern "C" typedef struct dirent *(readdir_t) (DIR *dir); typedef int (closedir_t)(DIR *dir); #if defined __GNUC__ && __GNUC__ >= 2 -#define LINUX_WEIRD_LSTAT -#define STAT_STRUCT struct stat /* should be stat64 */ - typedef int (lstat_t) (int ver, const char *file_name, - STAT_STRUCT *buf); + #define LINUX_WEIRD_LSTAT + #define STAT_STRUCT struct stat /* should be stat64 */ + typedef int (lstat_t) (int ver, const char *file_name, + STAT_STRUCT *buf); #else -#define STAT_STRUCT struct stat - typedef int (lstat_t) (const char *file_name, - STAT_STRUCT *buf); + #define STAT_STRUCT struct stat + typedef int (lstat_t) (const char *file_name, + STAT_STRUCT *buf); #endif } +typedef int (lstat_post_hook_t) (int old_ret, const char *file_name, + struct stat *buf); + void intercept_setup_error(const char *filename, unsigned int errorafter, int errortoreturn, int syscalltoerror); void intercept_setup_delay(const char *filename, unsigned int delay_after, @@ -42,6 +45,10 @@ bool intercept_triggered(); void intercept_setup_readdir_hook(const char *dirname, readdir_t hookfn); void intercept_setup_lstat_hook (const char *filename, lstat_t hookfn); +void intercept_setup_lstat_post_hook(lstat_post_hook_t hookfn); +void intercept_setup_stat_post_hook (lstat_post_hook_t hookfn); + +void intercept_clear_setup(); #endif // !PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE #endif // !INTERCEPT_H diff --git a/lib/raidfile/RaidFileController.cpp b/lib/raidfile/RaidFileController.cpp index 0cc2ede7..2cc6976b 100644 --- a/lib/raidfile/RaidFileController.cpp +++ b/lib/raidfile/RaidFileController.cpp @@ -70,11 +70,14 @@ void RaidFileController::Initialise(const std::string& rConfigFilename) static const ConfigurationVerifyKey verifykeys[] = { - {"SetNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"BlockSize", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"Dir0", 0, ConfigTest_Exists, 0}, - {"Dir1", 0, ConfigTest_Exists, 0}, - {"Dir2", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("SetNumber", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("BlockSize", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("Dir0", ConfigTest_Exists), + ConfigurationVerifyKey("Dir1", ConfigTest_Exists), + ConfigurationVerifyKey("Dir2", + ConfigTest_Exists | ConfigTest_LastEntry) }; static const ConfigurationVerify subverify = @@ -105,6 +108,10 @@ void RaidFileController::Initialise(const std::string& rConfigFilename) BOX_ERROR("RaidFile configuration file errors: " << err); THROW_EXCEPTION(RaidFileException, BadConfigFile) } + + // Allow reinitializing the controller by remove any existing + // disc sets. Used by Boxi unit tests. + mSetList.clear(); // Use the values int expectedSetNum = 0; diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp index 187270f9..0a79be57 100644 --- a/lib/raidfile/RaidFileRead.cpp +++ b/lib/raidfile/RaidFileRead.cpp @@ -25,10 +25,11 @@ #include <dirent.h> #endif -#include <stdio.h> -#include <string.h> -#include <memory> +#include <cstdio> +#include <cstdlib> +#include <cstring> #include <map> +#include <memory> #include "RaidFileRead.h" #include "RaidFileException.h" @@ -1021,6 +1022,7 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, Filename, &startDisc, &existingFiles, pRevisionID); if(existance == RaidFileUtil::NoFile) { + BOX_ERROR("Expected raidfile " << Filename << " does not exist"); THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist) } else if(existance == RaidFileUtil::NonRaid) diff --git a/lib/raidfile/RaidFileUtil.cpp b/lib/raidfile/RaidFileUtil.cpp index da23cef5..7c6299ec 100644 --- a/lib/raidfile/RaidFileUtil.cpp +++ b/lib/raidfile/RaidFileUtil.cpp @@ -22,17 +22,21 @@ // -------------------------------------------------------------------------- // // Function -// Name: RaidFileUtil::RaidFileExists(RaidFileDiscSet &, const std::string &) -// Purpose: Check to see the state of a RaidFile on disc (doesn't look at contents, -// just at existense of files) +// Name: RaidFileUtil::RaidFileExists(RaidFileDiscSet &, +// const std::string &, int *, int *, int64_t *) +// Purpose: Check to see the state of a RaidFile on disc +// (doesn't look at contents, just at existence of +// files) // Created: 2003/07/11 // // -------------------------------------------------------------------------- -RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pStartDisc, int *pExisitingFiles, int64_t *pRevisionID) +RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, + const std::string &rFilename, int *pStartDisc, int *pExistingFiles, + int64_t *pRevisionID) { - if(pExisitingFiles) + if(pExistingFiles) { - *pExisitingFiles = 0; + *pExistingFiles = 0; } // For stat call, although the results are not examined @@ -53,11 +57,20 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, // Get unique ID if(pRevisionID != 0) { - (*pRevisionID) = FileModificationTime(st); -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC - // On linux, the time resolution is very low for modification times. - // So add the size to it to give a bit more chance of it changing. + #ifdef WIN32 + *pRevisionID = st.st_mtime; + #else + *pRevisionID = FileModificationTime(st); + #endif + +#ifdef BOX_RELEASE_BUILD + // The resolution of timestamps may be very + // low, e.g. 1 second. So add the size to it + // to give a bit more chance of it changing. // TODO: Make this better. + // Disabled in debug mode, to simulate + // filesystem with 1-second timestamp + // resolution, e.g. MacOS X HFS, Linux. (*pRevisionID) += st.st_size; #endif } @@ -71,10 +84,10 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, int64_t revisionID = 0; int setSize = rDiscSet.size(); int rfCount = 0; -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC + // TODO: replace this with better linux revision ID detection int64_t revisionIDplus = 0; -#endif + for(int f = 0; f < setSize; ++f) { std::string componentFile(RaidFileUtil::MakeRaidComponentName(rDiscSet, rFilename, (f + startDisc) % setSize)); @@ -83,25 +96,34 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, // Component file exists, add to count rfCount++; // Set flags for existance? - if(pExisitingFiles) + if(pExistingFiles) { - (*pExisitingFiles) |= (1 << f); + (*pExistingFiles) |= (1 << f); } // Revision ID if(pRevisionID != 0) { - int64_t rid = FileModificationTime(st); + #ifdef WIN32 + int64_t rid = st.st_mtime; + #else + int64_t rid = FileModificationTime(st); + #endif + if(rid > revisionID) revisionID = rid; -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC revisionIDplus += st.st_size; -#endif } } } if(pRevisionID != 0) { (*pRevisionID) = revisionID; -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC +#ifdef BOX_RELEASE_BUILD + // The resolution of timestamps may be very low, e.g. + // 1 second. So add the size to it to give a bit more + // chance of it changing. + // TODO: Make this better. + // Disabled in debug mode, to simulate filesystem with + // 1-second timestamp resolution, e.g. MacOS X HFS, Linux. (*pRevisionID) += revisionIDplus; #endif } diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp index 66ab81c8..efec43a2 100644 --- a/lib/raidfile/RaidFileWrite.cpp +++ b/lib/raidfile/RaidFileWrite.cpp @@ -96,7 +96,8 @@ void RaidFileWrite::Open(bool AllowOverwrite) RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename); if(existance != RaidFileUtil::NoFile) { - TRACE2("Trying to overwrite raidfile %d %s\n", mSetNumber, mFilename.c_str()); + BOX_ERROR("Attempted to overwrite raidfile " << + mSetNumber << " " << mFilename); THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile) } } @@ -178,7 +179,8 @@ void RaidFileWrite::Write(const void *pBuffer, int Length) int written = ::write(mOSFileHandle, pBuffer, Length); if(written != Length) { - TRACE3("RaidFileWrite::Write: Write failure, Length = %d, written = %d, errno = %d\n", Length, written, errno); + BOX_LOG_SYS_ERROR("RaidFileWrite failed, Length = " << + Length << ", written = " << written); THROW_EXCEPTION(RaidFileException, OSError) } } @@ -272,6 +274,7 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) if(::unlink(renameTo.c_str()) != 0 && GetLastError() != ERROR_FILE_NOT_FOUND) { + BOX_LOG_WIN_ERROR("failed to delete file: " << renameTo); THROW_EXCEPTION(RaidFileException, OSError) } #endif @@ -387,7 +390,7 @@ void RaidFileWrite::TransformToRaidStorage() // // DEBUG MODE -- check file system size block size is same as block size for files // // doesn't really apply, as space benefits of using fragment size are worth efficiency, // // and anyway, it'll be buffered eventually so it won't matter. -// #ifndef NDEBUG +// #ifndef BOX_RELEASE_BUILD // { // if(writeFileStat.st_blksize != blockSize) // { @@ -779,7 +782,7 @@ int RaidFileWrite::Read(void *pBuffer, int NBytes, int Timeout) // -------------------------------------------------------------------------- void RaidFileWrite::Close() { - TRACE0("Warning: RaidFileWrite::Close() called, discarding file\n"); + BOX_WARNING("RaidFileWrite::Close() called, discarding file"); if(mOSFileHandle != -1) { Discard(); diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp index bdbbb433..d868774f 100644 --- a/lib/server/Daemon.cpp +++ b/lib/server/Daemon.cpp @@ -47,22 +47,21 @@ Daemon *Daemon::spDaemon = 0; // // -------------------------------------------------------------------------- Daemon::Daemon() - : mpConfiguration(NULL), - mReloadConfigWanted(false), + : mReloadConfigWanted(false), mTerminateWanted(false), + #ifdef WIN32 + mSingleProcess(true), + mRunInForeground(true), + mKeepConsoleOpenAfterFork(true), + #else mSingleProcess(false), mRunInForeground(false), mKeepConsoleOpenAfterFork(false), + #endif mHaveConfigFile(false), mAppName(DaemonName()) { - if(spDaemon != NULL) - { - THROW_EXCEPTION(ServerException, AlreadyDaemonConstructed) - } - spDaemon = this; - - // And in debug builds, we'll switch on assert failure logging to syslog + // In debug builds, switch on assert failure logging to syslog ASSERT_FAILS_TO_SYSLOG_ON // And trace goes to syslog too TRACE_TO_SYSLOG(true) @@ -78,14 +77,6 @@ Daemon::Daemon() // -------------------------------------------------------------------------- Daemon::~Daemon() { - if(mpConfiguration) - { - delete mpConfiguration; - mpConfiguration = 0; - } - - ASSERT(spDaemon == this); - spDaemon = NULL; } // -------------------------------------------------------------------------- @@ -104,9 +95,9 @@ std::string Daemon::GetOptionString() { return "c:" #ifndef WIN32 - "DFk" + "DFK" #endif - "hqvVt:T"; + "hkPqQt:TUvVW:"; } void Daemon::Usage() @@ -123,13 +114,20 @@ void Daemon::Usage() #ifndef WIN32 " -D Debugging mode, do not fork, one process only, one client only\n" " -F Do not fork into background, but fork to serve multiple clients\n" +#endif " -k Keep console open after fork, keep writing log messages to it\n" +#ifndef WIN32 + " -K Stop writing log messages to console while daemon is running\n" + " -P Show process ID (PID) in console output\n" #endif " -q Run more quietly, reduce verbosity level by one, can repeat\n" + " -Q Run at minimum verbosity, log nothing\n" " -v Run more verbosely, increase verbosity level by one, can repeat\n" - " -V Run at maximum verbosity\n" + " -V Run at maximum verbosity, log everything\n" + " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n" " -t <tag> Tag console output with specified marker\n" - " -T Timestamp console output\n"; + " -T Timestamp console output\n" + " -U Timestamp console output with microseconds\n"; } // -------------------------------------------------------------------------- @@ -166,13 +164,19 @@ int Daemon::ProcessOption(signed int option) mRunInForeground = true; } break; +#endif // !WIN32 case 'k': { mKeepConsoleOpenAfterFork = true; } break; -#endif + + case 'K': + { + mKeepConsoleOpenAfterFork = false; + } + break; case 'h': { @@ -181,6 +185,12 @@ int Daemon::ProcessOption(signed int option) } break; + case 'P': + { + Console::SetShowPID(true); + } + break; + case 'q': { if(mLogLevel == Log::NOTHING) @@ -194,6 +204,13 @@ int Daemon::ProcessOption(signed int option) } break; + case 'Q': + { + mLogLevel = Log::NOTHING; + } + break; + + case 'v': { if(mLogLevel == Log::EVERYTHING) @@ -213,9 +230,21 @@ int Daemon::ProcessOption(signed int option) } break; + case 'W': + { + mLogLevel = Logging::GetNamedLevel(optarg); + if (mLogLevel == Log::INVALID) + { + BOX_FATAL("Invalid logging level"); + return 2; + } + } + break; + case 't': { - Console::SetTag(optarg); + Logging::SetProgramName(optarg); + Console::SetShowTag(true); } break; @@ -225,6 +254,13 @@ int Daemon::ProcessOption(signed int option) } break; + case 'U': + { + Console::SetShowTime(true); + Console::SetShowTimeMicros(true); + } + break; + case '?': { BOX_FATAL("Unknown option on command line: " @@ -260,7 +296,7 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) mConfigFileName = DefaultConfigFile; mAppName = argv[0]; - #ifdef NDEBUG + #ifdef BOX_RELEASE_BUILD mLogLevel = Log::NOTICE; // need an int to do math with #else mLogLevel = Log::INFO; // need an int to do math with @@ -277,7 +313,7 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) // reset getopt, just in case anybody used it before. // unfortunately glibc and BSD differ on this point! // http://www.ussg.iu.edu/hypermail/linux/kernel/0305.3/0262.html - #if HAVE_DECL_OPTRESET == 1 + #if HAVE_DECL_OPTRESET == 1 || defined WIN32 optind = 1; optreset = 1; #elif defined __GLIBC__ @@ -314,7 +350,8 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) return 2; } - Logging::SetGlobalLevel((Log::Level)mLogLevel); + Logging::FilterConsole((Log::Level)mLogLevel); + Logging::FilterSyslog ((Log::Level)mLogLevel); return Main(mConfigFileName); } @@ -322,6 +359,98 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) // -------------------------------------------------------------------------- // // Function +// Name: Daemon::Configure(const std::string& rConfigFileName) +// Purpose: Loads daemon configuration. Useful when you have +// a local Daemon object and don't intend to fork() +// or call Main(). +// Created: 2008/04/19 +// +// -------------------------------------------------------------------------- + +bool Daemon::Configure(const std::string& rConfigFileName) +{ + // Load the configuration file. + std::string errors; + std::auto_ptr<Configuration> apConfig; + + try + { + apConfig = Configuration::LoadAndVerify(rConfigFileName, + GetConfigVerify(), errors); + } + catch(BoxException &e) + { + if(e.GetType() == CommonException::ExceptionType && + e.GetSubType() == CommonException::OSFileOpenError) + { + BOX_ERROR("Failed to open configuration file: " << + rConfigFileName); + return false; + } + + throw; + } + + // Got errors? + if(apConfig.get() == 0) + { + BOX_ERROR("Failed to load or verify configuration file"); + return false; + } + + if(!Configure(*apConfig)) + { + BOX_ERROR("Failed to verify configuration file"); + return false; + } + + // Store configuration + mConfigFileName = rConfigFileName; + mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Configure(const Configuration& rConfig) +// Purpose: Loads daemon configuration. Useful when you have +// a local Daemon object and don't intend to fork() +// or call Main(). +// Created: 2008/08/12 +// +// -------------------------------------------------------------------------- + +bool Daemon::Configure(const Configuration& rConfig) +{ + std::string errors; + + // Verify() may modify the configuration, e.g. adding default values + // for required keys, so need to make a copy here + std::auto_ptr<Configuration> apConf(new Configuration(rConfig)); + apConf->Verify(*GetConfigVerify(), errors); + + // Got errors? + if(!errors.empty()) + { + BOX_ERROR("Configuration errors: " << errors); + return false; + } + + // Store configuration + mapConfiguration = apConf; + + // Let the derived class have a go at setting up stuff + // in the initial process + SetupInInitialProcess(); + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function // Name: Daemon::Main(const std::string& rConfigFileName) // Purpose: Starts the daemon off -- equivalent of C main() function // Created: 2003/07/29 @@ -331,69 +460,38 @@ int Daemon::Main(const std::string &rConfigFileName) { // Banner (optional) { - #ifndef NDEBUG - BOX_NOTICE(DaemonBanner()); - #endif + BOX_SYSLOG(Log::NOTICE, DaemonBanner()); } std::string pidFileName; - mConfigFileName = rConfigFileName; - - bool asDaemon = !mSingleProcess && !mRunInForeground; + bool asDaemon = !mSingleProcess && !mRunInForeground; try { - // Load the configuration file. - std::string errors; - std::auto_ptr<Configuration> pconfig; - - try - { - pconfig = Configuration::LoadAndVerify( - mConfigFileName.c_str(), - GetConfigVerify(), errors); - } - catch(BoxException &e) - { - if(e.GetType() == CommonException::ExceptionType && - e.GetSubType() == CommonException::OSFileOpenError) - { - BOX_FATAL("Failed to start: failed to open " - "configuration file: " - << mConfigFileName); - return 1; - } - - throw; - } - - // Got errors? - if(pconfig.get() == 0 || !errors.empty()) + if (!Configure(rConfigFileName)) { - // Tell user about errors - BOX_FATAL("Failed to start: errors in configuration " - "file: " << mConfigFileName << ": " << errors); - // And give up + BOX_FATAL("Failed to start: failed to load " + "configuration file: " << rConfigFileName); return 1; } - // Store configuration - mpConfiguration = pconfig.release(); - mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); - - // Let the derived class have a go at setting up stuff in the initial process - SetupInInitialProcess(); - // Server configuration const Configuration &serverConfig( - mpConfiguration->GetSubConfiguration("Server")); + mapConfiguration->GetSubConfiguration("Server")); + + if(serverConfig.KeyExists("LogFacility")) + { + std::string facility = + serverConfig.GetKeyValue("LogFacility"); + Logging::SetFacility(Syslog::GetNamedFacility(facility)); + } // Open PID file for writing pidFileName = serverConfig.GetKeyValue("PidFile"); FileHandleGuard<(O_WRONLY | O_CREAT | O_TRUNC), (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)> pidFile(pidFileName.c_str()); -#ifndef WIN32 +#ifndef WIN32 // Handle changing to a different user if(serverConfig.KeyExists("User")) { @@ -409,7 +507,7 @@ int Daemon::Main(const std::string &rConfigFileName) // Change the process ID daemonUser.ChangeProcessUser(); } - + if(asDaemon) { // Let's go... Daemonise... @@ -436,8 +534,7 @@ int Daemon::Main(const std::string &rConfigFileName) // Set new session if(::setsid() == -1) { - BOX_ERROR("Failed to setsid(): " << - strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to setsid()"); THROW_EXCEPTION(ServerException, DaemoniseFailed) } @@ -446,6 +543,7 @@ int Daemon::Main(const std::string &rConfigFileName) { case -1: // error + BOX_LOG_SYS_ERROR("Failed to fork() a child"); THROW_EXCEPTION(ServerException, DaemoniseFailed) break; @@ -460,7 +558,17 @@ int Daemon::Main(const std::string &rConfigFileName) break; } } - +#endif // !WIN32 + + // Must set spDaemon before installing signal handler, + // otherwise the handler will crash if invoked too soon. + if(spDaemon != NULL) + { + THROW_EXCEPTION(ServerException, AlreadyDaemonConstructed) + } + spDaemon = this; + +#ifndef WIN32 // Set signal handler // Don't do this in the parent, since it might be anything // (e.g. test/bbackupd) @@ -468,29 +576,24 @@ int Daemon::Main(const std::string &rConfigFileName) struct sigaction sa; sa.sa_handler = SignalHandler; sa.sa_flags = 0; - sigemptyset(&sa.sa_mask); // macro - if(::sigaction(SIGHUP, &sa, NULL) != 0 || ::sigaction(SIGTERM, &sa, NULL) != 0) + sigemptyset(&sa.sa_mask); // macro + if(::sigaction(SIGHUP, &sa, NULL) != 0 || + ::sigaction(SIGTERM, &sa, NULL) != 0) { + BOX_LOG_SYS_ERROR("Failed to set signal handlers"); THROW_EXCEPTION(ServerException, DaemoniseFailed) } #endif // !WIN32 - // Log the start message - BOX_NOTICE("Starting daemon, version " << BOX_VERSION - << ", config: " << mConfigFileName); - // Write PID to file char pid[32]; -#ifdef WIN32 - int pidsize = sprintf(pid, "%d", (int)GetCurrentProcessId()); -#else int pidsize = sprintf(pid, "%d", (int)getpid()); -#endif if(::write(pidFile, pid, pidsize) != pidsize) { - BOX_FATAL("can't write pid file"); + BOX_LOG_SYS_FATAL("Failed to write PID file: " << + pidFileName); THROW_EXCEPTION(ServerException, DaemoniseFailed) } @@ -515,6 +618,7 @@ int Daemon::Main(const std::string &rConfigFileName) int devnull = ::open(PLATFORM_DEV_NULL, O_RDWR, 0); if(devnull == -1) { + BOX_LOG_SYS_ERROR("Failed to open /dev/null"); THROW_EXCEPTION(CommonException, OSFileError); } // Then duplicate them to all three handles @@ -530,9 +634,13 @@ int Daemon::Main(const std::string &rConfigFileName) // And definitely don't try and send anything to those file descriptors // -- this has in the past sent text to something which isn't expecting it. TRACE_TO_STDOUT(false); - Logging::ToConsole(false); #endif // ! WIN32 - } + Logging::ToConsole(false); + } + + // Log the start message + BOX_NOTICE("Starting daemon, version: " << BOX_VERSION); + BOX_NOTICE("Using configuration file: " << mConfigFileName); } catch(BoxException &e) { @@ -581,10 +689,10 @@ int Daemon::Main(const std::string &rConfigFileName) BOX_NOTICE("Reloading configuration file: " << mConfigFileName); std::string errors; - std::auto_ptr<Configuration> pconfig = + std::auto_ptr<Configuration> pconfig( Configuration::LoadAndVerify( mConfigFileName.c_str(), - GetConfigVerify(), errors); + GetConfigVerify(), errors)); // Got errors? if(pconfig.get() == 0 || !errors.empty()) @@ -598,12 +706,8 @@ int Daemon::Main(const std::string &rConfigFileName) break; } - // delete old configuration - delete mpConfiguration; - mpConfiguration = 0; - // Store configuration - mpConfiguration = pconfig.release(); + mapConfiguration = pconfig; mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); @@ -638,8 +742,21 @@ int Daemon::Main(const std::string &rConfigFileName) #ifdef WIN32 WSACleanup(); +#else + // Should clean up here, but it breaks memory leak tests. + /* + if(asDaemon) + { + // we are running in the child by now, and should not return + mapConfiguration.reset(); + exit(0); + } + */ #endif - + + ASSERT(spDaemon == this); + spDaemon = NULL; + return retcode; } @@ -647,7 +764,8 @@ int Daemon::Main(const std::string &rConfigFileName) // // Function // Name: Daemon::EnterChild() -// Purpose: Sets up for a child task of the main server. Call just after fork() +// Purpose: Sets up for a child task of the main server. Call +// just after fork(). // Created: 2003/07/31 // // -------------------------------------------------------------------------- @@ -789,13 +907,13 @@ const ConfigurationVerify *Daemon::GetConfigVerify() const // -------------------------------------------------------------------------- const Configuration &Daemon::GetConfiguration() const { - if(mpConfiguration == 0) + if(mapConfiguration.get() == 0) { // Shouldn't get anywhere near this if a configuration file can't be loaded THROW_EXCEPTION(ServerException, Internal) } - return *mpConfiguration; + return *mapConfiguration; } @@ -803,8 +921,10 @@ const Configuration &Daemon::GetConfiguration() const // // Function // Name: Daemon::SetupInInitialProcess() -// Purpose: A chance for the daemon to do something initial setting up in the process which -// initiates everything, and after the configuration file has been read and verified. +// Purpose: A chance for the daemon to do something initial +// setting up in the process which initiates +// everything, and after the configuration file has +// been read and verified. // Created: 2003/08/20 // // -------------------------------------------------------------------------- @@ -849,14 +969,16 @@ void Daemon::SetProcessTitle(const char *format, ...) box_time_t Daemon::GetConfigFileModifiedTime() const { - struct stat st; + EMU_STRUCT_STAT st; - if(::stat(GetConfigFileName().c_str(), &st) != 0) + if(EMU_STAT(GetConfigFileName().c_str(), &st) != 0) { if (errno == ENOENT) { return 0; } + BOX_LOG_SYS_ERROR("Failed to stat configuration file: " << + GetConfigFileName()); THROW_EXCEPTION(CommonException, OSFileError) } diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h index 482f926e..a3212a00 100644 --- a/lib/server/Daemon.h +++ b/lib/server/Daemon.h @@ -19,8 +19,8 @@ #include <string> #include "BoxTime.h" +#include "Configuration.h" -class Configuration; class ConfigurationVerify; // -------------------------------------------------------------------------- @@ -40,7 +40,8 @@ private: Daemon(const Daemon &rToCopy); public: - int Main(const char *DefaultConfigFile, int argc, const char *argv[]); + virtual int Main(const char *DefaultConfigFile, int argc, + const char *argv[]); /* override this Main() if you want custom option processing: */ virtual int Main(const std::string &rConfigFile); @@ -53,7 +54,10 @@ public: virtual std::string DaemonBanner() const; virtual const ConfigurationVerify *GetConfigVerify() const; virtual void Usage(); - + + virtual bool Configure(const std::string& rConfigFileName); + virtual bool Configure(const Configuration& rConfig); + bool StopRun() {return mReloadConfigWanted | mTerminateWanted;} bool IsReloadConfigWanted() {return mReloadConfigWanted;} bool IsTerminateWanted() {return mTerminateWanted;} @@ -62,12 +66,20 @@ public: void SetReloadConfigWanted() {mReloadConfigWanted = true;} void SetTerminateWanted() {mTerminateWanted = true;} - virtual void SetupInInitialProcess(); virtual void EnterChild(); static void SetProcessTitle(const char *format, ...); + void SetRunInForeground(bool foreground) + { + mRunInForeground = foreground; + } + void SetSingleProcess(bool value) + { + mSingleProcess = value; + } protected: + virtual void SetupInInitialProcess(); box_time_t GetLoadedConfigModifiedTime() const; bool IsSingleProcess() { return mSingleProcess; } virtual std::string GetOptionString(); @@ -78,7 +90,7 @@ private: box_time_t GetConfigFileModifiedTime() const; std::string mConfigFileName; - Configuration *mpConfiguration; + std::auto_ptr<Configuration> mapConfiguration; box_time_t mLoadedConfigModifiedTime; bool mReloadConfigWanted; bool mTerminateWanted; @@ -91,8 +103,10 @@ private: std::string mAppName; }; -#define DAEMON_VERIFY_SERVER_KEYS {"PidFile", 0, ConfigTest_Exists, 0}, \ - {"User", 0, ConfigTest_LastEntry, 0} +#define DAEMON_VERIFY_SERVER_KEYS \ + ConfigurationVerifyKey("PidFile", ConfigTest_Exists), \ + ConfigurationVerifyKey("LogFacility", 0), \ + ConfigurationVerifyKey("User", ConfigTest_LastEntry) #endif // DAEMON__H diff --git a/lib/server/OverlappedIO.h b/lib/server/OverlappedIO.h new file mode 100644 index 00000000..12495053 --- /dev/null +++ b/lib/server/OverlappedIO.h @@ -0,0 +1,42 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: OverlappedIO.h +// Purpose: Windows overlapped IO handle guard +// Created: 2008/09/30 +// +// -------------------------------------------------------------------------- + +#ifndef OVERLAPPEDIO__H +#define OVERLAPPEDIO__H + +class OverlappedIO +{ +public: + OVERLAPPED mOverlapped; + + OverlappedIO() + { + ZeroMemory(&mOverlapped, sizeof(mOverlapped)); + mOverlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, + NULL); + if (mOverlapped.hEvent == INVALID_HANDLE_VALUE) + { + BOX_LOG_WIN_ERROR("Failed to create event for " + "overlapped I/O"); + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + } + + ~OverlappedIO() + { + if (CloseHandle(mOverlapped.hEvent) != TRUE) + { + BOX_LOG_WIN_ERROR("Failed to delete event for " + "overlapped I/O"); + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + } +}; + +#endif // !OVERLAPPEDIO__H diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp index 4398a58f..5dc5d0b1 100644 --- a/lib/server/Protocol.cpp +++ b/lib/server/Protocol.cpp @@ -26,7 +26,7 @@ #include "MemLeakFindOn.h" -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 1024 #else // #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 1024 @@ -670,9 +670,18 @@ std::auto_ptr<IOStream> Protocol::ReceiveStream() InformStreamReceiving(streamSize); // Return a stream object - return std::auto_ptr<IOStream>((streamSize == ProtocolStream_SizeUncertain)? - ((IOStream*)(new ProtocolUncertainStream(mrStream))) - :((IOStream*)(new PartialReadStream(mrStream, streamSize)))); + if(streamSize == ProtocolStream_SizeUncertain) + { + BOX_TRACE("Receiving stream, size uncertain"); + return std::auto_ptr<IOStream>( + new ProtocolUncertainStream(mrStream)); + } + else + { + BOX_TRACE("Receiving stream, size " << streamSize << " bytes"); + return std::auto_ptr<IOStream>( + new PartialReadStream(mrStream, streamSize)); + } } // -------------------------------------------------------------------------- @@ -749,8 +758,10 @@ void Protocol::SendStream(IOStream &rStream) } // Send final byte to finish the stream + BOX_TRACE("Sending end of stream byte"); uint8_t endOfStream = ProtocolStreamHeader_EndOfStream; mrStream.Write(&endOfStream, 1); + BOX_TRACE("Sent end of stream byte"); } catch(...) { @@ -788,6 +799,7 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) // Quick sanity check if(BytesInBlock == 0) { + BOX_TRACE("Zero size block, not sending anything"); return 0; } @@ -813,6 +825,8 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) } } ASSERT(header > 0); + BOX_TRACE("Sending header byte " << (int)header << " plus " << + writeSize << " bytes to stream"); // Store the header Block[-1] = header; @@ -820,6 +834,7 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) // Write everything out mrStream.Write(Block - 1, writeSize + 1); + BOX_TRACE("Sent " << (writeSize+1) << " bytes to stream"); // move the remainer to the beginning of the block for the next time round if(writeSize != BytesInBlock) { diff --git a/lib/server/ProtocolUncertainStream.cpp b/lib/server/ProtocolUncertainStream.cpp index 60c1fa1d..84a213a8 100644 --- a/lib/server/ProtocolUncertainStream.cpp +++ b/lib/server/ProtocolUncertainStream.cpp @@ -41,7 +41,8 @@ ProtocolUncertainStream::~ProtocolUncertainStream() { if(!mFinished) { - TRACE0("ProtocolUncertainStream::~ProtocolUncertainStream() destroyed when stream not complete\n"); + BOX_WARNING("ProtocolUncertainStream destroyed before " + "stream finished"); } } @@ -76,11 +77,15 @@ int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) toRead = mBytesLeftInCurrentBlock; } + BOX_TRACE("Reading " << toRead << " bytes from stream"); + // Read it int r = mrSource.Read(((uint8_t*)pBuffer) + read, toRead, Timeout); // Give up now if it didn't return anything if(r == 0) { + BOX_TRACE("Read " << r << " bytes from " + "stream, returning"); return read; } @@ -91,16 +96,22 @@ int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) // stop now if the stream returned less than we asked for -- avoid blocking if(r != toRead) { + BOX_TRACE("Read " << r << " bytes from " + "stream, returning"); return read; } } else { - // Read the header byte to find out how much there is in the next block + // Read the header byte to find out how much there is + // in the next block uint8_t header; if(mrSource.Read(&header, 1, Timeout) == 0) { // Didn't get the byte, return now + BOX_TRACE("Read 0 bytes of block header, " + "returning with " << read << " bytes " + "read this time"); return read; } @@ -109,6 +120,8 @@ int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) { // All done. mFinished = true; + BOX_TRACE("Stream finished, returning with " << + read << " bytes read this time"); return read; } else if(header <= Protocol::ProtocolStreamHeader_MaxEncodedSizeValue) @@ -126,6 +139,10 @@ int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) // Bad. It used the reserved values. THROW_EXCEPTION(ServerException, ProtocolUncertainStreamBadBlockHeader) } + + BOX_TRACE("Read header byte " << (int)header << ", " + "next block has " << + mBytesLeftInCurrentBlock << " bytes"); } } diff --git a/lib/server/SSLLib.cpp b/lib/server/SSLLib.cpp index e9c990b9..de7a941b 100644 --- a/lib/server/SSLLib.cpp +++ b/lib/server/SSLLib.cpp @@ -14,12 +14,16 @@ #include <openssl/err.h> #include <openssl/rand.h> +#ifdef WIN32 + #include <wincrypt.h> +#endif + #include "SSLLib.h" #include "ServerException.h" #include "MemLeakFindOn.h" -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD bool SSLLib__TraceErrors = false; #endif @@ -35,7 +39,7 @@ void SSLLib::Initialise() { if(!::SSL_library_init()) { - LogError("Initialisation"); + LogError("initialising OpenSSL"); THROW_EXCEPTION(ServerException, SSLLibraryInitialisationError) } @@ -43,7 +47,37 @@ void SSLLib::Initialise() ::SSL_load_error_strings(); // Extra seeding over and above what's already done by the library -#ifdef HAVE_RANDOM_DEVICE +#ifdef WIN32 + HCRYPTPROV provider; + if(!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT)) + { + BOX_LOG_WIN_ERROR("Failed to acquire crypto context"); + BOX_WARNING("No random device -- additional seeding of " + "random number generator not performed."); + } + else + { + // must free provider + BYTE buf[1024]; + + if(!CryptGenRandom(provider, sizeof(buf), buf)) + { + BOX_LOG_WIN_ERROR("Failed to get random data"); + BOX_WARNING("No random device -- additional seeding of " + "random number generator not performed."); + } + else + { + RAND_seed(buf, sizeof(buf)); + } + + if(!CryptReleaseContext(provider, 0)) + { + BOX_LOG_WIN_ERROR("Failed to release crypto context"); + } + } +#elif HAVE_RANDOM_DEVICE if(::RAND_load_file(RANDOM_DEVICE, 1024) != 1024) { THROW_EXCEPTION(ServerException, SSLRandomInitFailed) @@ -63,14 +97,14 @@ void SSLLib::Initialise() // Created: 2003/08/06 // // -------------------------------------------------------------------------- -void SSLLib::LogError(const char *ErrorDuringAction) +void SSLLib::LogError(const std::string& rErrorDuringAction) { unsigned long errcode; char errname[256]; // SSL docs say at least 120 bytes while((errcode = ERR_get_error()) != 0) { ::ERR_error_string_n(errcode, errname, sizeof(errname)); - BOX_ERROR("SSL error during " << ErrorDuringAction << ": " << + BOX_ERROR("SSL error while " << rErrorDuringAction << ": " << errname); } } diff --git a/lib/server/SSLLib.h b/lib/server/SSLLib.h index cdff4f04..ff4aab19 100644 --- a/lib/server/SSLLib.h +++ b/lib/server/SSLLib.h @@ -10,7 +10,7 @@ #ifndef SSLLIB__H #define SSLLIB__H -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD extern bool SSLLib__TraceErrors; #define SET_DEBUG_SSLLIB_TRACE_ERRORS {SSLLib__TraceErrors = true;} #else @@ -29,7 +29,7 @@ namespace SSLLib { void Initialise(); - void LogError(const char *ErrorDuringAction); + void LogError(const std::string& rErrorDuringAction); }; #endif // SSLLIB__H diff --git a/lib/server/ServerControl.cpp b/lib/server/ServerControl.cpp new file mode 100644 index 00000000..c4668b57 --- /dev/null +++ b/lib/server/ServerControl.cpp @@ -0,0 +1,227 @@ +#include "Box.h" + +#include <errno.h> + +#ifdef HAVE_SYS_TYPES_H + #include <sys/types.h> +#endif + +#ifdef HAVE_SYS_WAIT_H + #include <sys/wait.h> +#endif + +#ifdef HAVE_SIGNAL_H + #include <signal.h> +#endif + +#include "ServerControl.h" +#include "Test.h" + +#ifdef WIN32 + +#include "WinNamedPipeStream.h" +#include "IOStreamGetLine.h" +#include "BoxPortsAndFiles.h" + +static std::string sPipeName; + +void SetNamedPipeName(const std::string& rPipeName) +{ + sPipeName = rPipeName; +} + +bool SendCommands(const std::string& rCmd) +{ + WinNamedPipeStream connection; + + try + { + connection.Connect(sPipeName); + } + catch(...) + { + BOX_ERROR("Failed to connect to daemon control socket"); + return false; + } + + // For receiving data + IOStreamGetLine getLine(connection); + + // Wait for the configuration summary + std::string configSummary; + if(!getLine.GetLine(configSummary)) + { + BOX_ERROR("Failed to receive configuration summary from daemon"); + return false; + } + + // Was the connection rejected by the server? + if(getLine.IsEOF()) + { + BOX_ERROR("Server rejected the connection"); + return false; + } + + // Decode it + int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait; + if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", + &autoBackup, &updateStoreInterval, + &minimumFileAge, &maxUploadWait) != 4) + { + BOX_ERROR("Config summary didn't decode"); + return false; + } + + std::string cmds; + bool expectResponse; + + if (rCmd != "") + { + cmds = rCmd; + cmds += "\nquit\n"; + expectResponse = true; + } + else + { + cmds = "quit\n"; + expectResponse = false; + } + + connection.Write(cmds.c_str(), cmds.size()); + + // Read the response + std::string line; + bool statusOk = !expectResponse; + + while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line)) + { + // Is this an OK or error line? + if (line == "ok") + { + statusOk = true; + } + else if (line == "error") + { + BOX_ERROR(rCmd); + break; + } + else + { + BOX_WARNING("Unexpected response to command '" << + rCmd << "': " << line) + } + } + + return statusOk; +} + +bool HUPServer(int pid) +{ + return SendCommands("reload"); +} + +bool KillServerInternal(int pid) +{ + HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, false, pid); + if (hProcess == NULL) + { + BOX_ERROR("Failed to open process " << pid << ": " << + GetErrorMessage(GetLastError())); + return false; + } + + if (!TerminateProcess(hProcess, 1)) + { + BOX_ERROR("Failed to terminate process " << pid << ": " << + GetErrorMessage(GetLastError())); + CloseHandle(hProcess); + return false; + } + + CloseHandle(hProcess); + return true; +} + +#else // !WIN32 + +bool HUPServer(int pid) +{ + if(pid == 0) return false; + return ::kill(pid, SIGHUP) == 0; +} + +bool KillServerInternal(int pid) +{ + if(pid == 0 || pid == -1) return false; + bool killed = (::kill(pid, SIGTERM) == 0); + if (!killed) + { + BOX_LOG_SYS_ERROR("Failed to kill process " << pid); + } + TEST_THAT(killed); + return killed; +} + +#endif // WIN32 + +bool KillServer(int pid, bool WaitForProcess) +{ + if (!KillServerInternal(pid)) + { + return false; + } + + #ifdef HAVE_WAITPID + if (WaitForProcess) + { + int status, result; + + result = waitpid(pid, &status, 0); + if (result != pid) + { + BOX_LOG_SYS_ERROR("waitpid failed"); + } + TEST_THAT(result == pid); + + TEST_THAT(WIFEXITED(status)); + if (WIFEXITED(status)) + { + if (WEXITSTATUS(status) != 0) + { + BOX_WARNING("process exited with code " << + WEXITSTATUS(status)); + } + TEST_THAT(WEXITSTATUS(status) == 0); + } + } + #endif + + for (int i = 0; i < 30; i++) + { + if (i == 0) + { + printf("Waiting for server to die (pid %d): ", pid); + } + + printf("."); + fflush(stdout); + + if (!ServerIsAlive(pid)) break; + ::sleep(1); + if (!ServerIsAlive(pid)) break; + } + + if (!ServerIsAlive(pid)) + { + printf(" done.\n"); + } + else + { + printf(" failed!\n"); + } + + fflush(stdout); + + return !ServerIsAlive(pid); +} + diff --git a/lib/server/ServerControl.h b/lib/server/ServerControl.h index ce5620c2..b2e51864 100644 --- a/lib/server/ServerControl.h +++ b/lib/server/ServerControl.h @@ -3,188 +3,16 @@ #include "Test.h" -#ifdef WIN32 - -#include "WinNamedPipeStream.h" -#include "IOStreamGetLine.h" -#include "BoxPortsAndFiles.h" - -static std::string sPipeName; - -static void SetNamedPipeName(const std::string& rPipeName) -{ - sPipeName = rPipeName; -} - -static bool SendCommands(const std::string& rCmd) -{ - WinNamedPipeStream connection; - - try - { - connection.Connect(sPipeName); - } - catch(...) - { - BOX_ERROR("Failed to connect to daemon control socket"); - return false; - } - - // For receiving data - IOStreamGetLine getLine(connection); - - // Wait for the configuration summary - std::string configSummary; - if(!getLine.GetLine(configSummary)) - { - BOX_ERROR("Failed to receive configuration summary from daemon"); - return false; - } - - // Was the connection rejected by the server? - if(getLine.IsEOF()) - { - BOX_ERROR("Server rejected the connection"); - return false; - } - - // Decode it - int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait; - if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", - &autoBackup, &updateStoreInterval, - &minimumFileAge, &maxUploadWait) != 4) - { - BOX_ERROR("Config summary didn't decode"); - return false; - } - - std::string cmds; - bool expectResponse; - - if (rCmd != "") - { - cmds = rCmd; - cmds += "\nquit\n"; - expectResponse = true; - } - else - { - cmds = "quit\n"; - expectResponse = false; - } - - connection.Write(cmds.c_str(), cmds.size()); - - // Read the response - std::string line; - bool statusOk = !expectResponse; - - while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line)) - { - // Is this an OK or error line? - if (line == "ok") - { - statusOk = true; - } - else if (line == "error") - { - BOX_ERROR(rCmd); - break; - } - else - { - BOX_WARNING("Unexpected response to command '" << - rCmd << "': " << line) - } - } - - return statusOk; -} - -inline bool HUPServer(int pid) -{ - return SendCommands("reload"); -} +bool HUPServer(int pid); +bool KillServer(int pid, bool WaitForProcess = false); -inline bool KillServerInternal(int pid) -{ - HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, false, pid); - if (hProcess == NULL) - { - BOX_ERROR("Failed to open process " << pid << ": " << - GetErrorMessage(GetLastError())); - return false; - } - - if (!TerminateProcess(hProcess, 1)) - { - BOX_ERROR("Failed to terminate process " << pid << ": " << - GetErrorMessage(GetLastError())); - CloseHandle(hProcess); - return false; - } - - CloseHandle(hProcess); - return true; -} - -#else // !WIN32 - -inline bool HUPServer(int pid) -{ - if(pid == 0) return false; - return ::kill(pid, SIGHUP) == 0; -} - -inline bool KillServerInternal(int pid) -{ - if(pid == 0 || pid == -1) return false; - bool killed = (::kill(pid, SIGTERM) == 0); - if (!killed) - { - BOX_ERROR("Failed to kill process " << pid << ": " << - strerror(errno)); - } - TEST_THAT(killed); - return killed; -} +#ifdef WIN32 + #include "WinNamedPipeStream.h" + #include "IOStreamGetLine.h" + #include "BoxPortsAndFiles.h" + void SetNamedPipeName(const std::string& rPipeName); + // bool SendCommands(const std::string& rCmd); #endif // WIN32 -inline bool KillServer(int pid) -{ - if (!KillServerInternal(pid)) - { - return false; - } - - for (int i = 0; i < 30; i++) - { - if (i == 0) - { - printf("Waiting for server to die: "); - } - - printf("."); - fflush(stdout); - - if (!ServerIsAlive(pid)) break; - ::sleep(1); - if (!ServerIsAlive(pid)) break; - } - - if (!ServerIsAlive(pid)) - { - printf(" done.\n"); - } - else - { - printf(" failed!\n"); - } - - fflush(stdout); - - return !ServerIsAlive(pid); -} - #endif // SERVER_CONTROL_H diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h index 32a57bac..34de7def 100644 --- a/lib/server/ServerStream.h +++ b/lib/server/ServerStream.h @@ -108,7 +108,7 @@ public: { // Child task, dump leaks to trace, which we make sure is on #ifdef BOX_MEMORY_LEAK_TESTING - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD TRACE_TO_SYSLOG(true); TRACE_TO_STDOUT(true); #endif @@ -120,13 +120,19 @@ public: } } +protected: + virtual void NotifyListenerIsReady() { } + +public: virtual void Run2(bool &rChildExit) { try { - // Wait object with a timeout of 10 seconds, which is a reasonable time to wait before - // cleaning up finished child processes. - WaitForEvent connectionWait(10000); + // Wait object with a timeout of 1 second, which + // is a reasonable time to wait before cleaning up + // finished child processes, and allows the daemon + // to terminate reasonably quickly on request. + WaitForEvent connectionWait(1000); // BLOCK { @@ -218,6 +224,8 @@ public: } } } + + NotifyListenerIsReady(); while(!StopRun()) { @@ -273,7 +281,7 @@ public: } // Log it - BOX_WARNING("Message from child process " << pid << ": " << logMessage); + BOX_NOTICE("Message from child process " << pid << ": " << logMessage); } else { @@ -365,8 +373,8 @@ private: }; #define SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ - {"ListenAddresses", DEFAULT_ADDRESSES, 0, 0}, \ - DAEMON_VERIFY_SERVER_KEYS + ConfigurationVerifyKey("ListenAddresses", 0, DEFAULT_ADDRESSES), \ + DAEMON_VERIFY_SERVER_KEYS #include "MemLeakFindOff.h" diff --git a/lib/server/ServerTLS.h b/lib/server/ServerTLS.h index 71d35380..a74a671e 100644 --- a/lib/server/ServerTLS.h +++ b/lib/server/ServerTLS.h @@ -55,7 +55,8 @@ public: mContext.Initialise(true /* as server */, certFile.c_str(), keyFile.c_str(), caFile.c_str()); // Then do normal stream server stuff - ServerStream<SocketStreamTLS, Port, ListenBacklog>::Run2(rChildExit); + ServerStream<SocketStreamTLS, Port, ListenBacklog, + ForkToHandleRequests>::Run2(rChildExit); } virtual void HandleConnection(SocketStreamTLS &rStream) @@ -70,11 +71,10 @@ private: }; #define SERVERTLS_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ - {"CertificateFile", 0, ConfigTest_Exists, 0}, \ - {"PrivateKeyFile", 0, ConfigTest_Exists, 0}, \ - {"TrustedCAsFile", 0, ConfigTest_Exists, 0}, \ - SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) - + ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), \ + ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), \ + ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), \ + SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) #endif // SERVERTLS__H diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp index 28dae69f..4a83bdb0 100644 --- a/lib/server/Socket.cpp +++ b/lib/server/Socket.cpp @@ -32,12 +32,15 @@ // -------------------------------------------------------------------------- // // Function -// Name: Socket::NameLookupToSockAddr(SocketAllAddr &, int, char *, int) +// Name: Socket::NameLookupToSockAddr(SocketAllAddr &, int, +// char *, int) // Purpose: Sets up a sockaddr structure given a name and type // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type, const char *Name, int Port, int &rSockAddrLenOut) +void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, + enum Type Type, const std::string& rName, int Port, + int &rSockAddrLenOut) { int sockAddrLen = 0; @@ -47,7 +50,7 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type sockDomain = AF_INET; { // Lookup hostname - struct hostent *phost = ::gethostbyname(Name); + struct hostent *phost = ::gethostbyname(rName.c_str()); if(phost != NULL) { if(phost->h_addr_list[0] != 0) @@ -81,7 +84,7 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type sockDomain = AF_UNIX; { // Check length of name is OK - unsigned int nameLen = ::strlen(Name); + unsigned int nameLen = rName.length(); if(nameLen >= (sizeof(addr.sa_unix.sun_path) - 1)) { THROW_EXCEPTION(ServerException, SocketNameUNIXPathTooLong); @@ -91,7 +94,9 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type addr.sa_unix.sun_len = sockAddrLen; #endif addr.sa_unix.sun_family = PF_UNIX; - ::strcpy(addr.sa_unix.sun_path, Name); + ::strncpy(addr.sa_unix.sun_path, rName.c_str(), + sizeof(addr.sa_unix.sun_path) - 1); + addr.sa_unix.sun_path[sizeof(addr.sa_unix.sun_path)-1] = 0; } break; #endif diff --git a/lib/server/Socket.h b/lib/server/Socket.h index 057e4cad..5034dbd8 100644 --- a/lib/server/Socket.h +++ b/lib/server/Socket.h @@ -39,13 +39,15 @@ typedef union { // -------------------------------------------------------------------------- namespace Socket { - enum + enum Type { TypeINET = 1, TypeUNIX = 2 }; - void NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type, const char *Name, int Port, int &rSockAddrLenOut); + void NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, + enum Type type, const std::string& rName, int Port, + int &rSockAddrLenOut); void LogIncomingConnection(const struct sockaddr *addr, socklen_t addrlen); std::string IncomingConnectionLogMessage(const struct sockaddr *addr, socklen_t addrlen); }; diff --git a/lib/server/SocketListen.h b/lib/server/SocketListen.h index ff08fb8f..586adf22 100644 --- a/lib/server/SocketListen.h +++ b/lib/server/SocketListen.h @@ -108,45 +108,57 @@ public: if(::close(mSocketHandle) == -1) #endif { - THROW_EXCEPTION(ServerException, SocketCloseError) + BOX_LOG_SYS_ERROR("Failed to close network " + "socket"); + THROW_EXCEPTION(ServerException, + SocketCloseError) } } mSocketHandle = -1; } - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ // // Function // Name: SocketListen::Listen(int, char*, int) // Purpose: Initialises, starts the socket listening. // Created: 2003/07/31 // - // -------------------------------------------------------------------------- - void Listen(int Type, const char *Name, int Port = 0) + // ------------------------------------------------------------------ + void Listen(Socket::Type Type, const char *Name, int Port = 0) { - if(mSocketHandle != -1) {THROW_EXCEPTION(ServerException, SocketAlreadyOpen)} + if(mSocketHandle != -1) + { + THROW_EXCEPTION(ServerException, SocketAlreadyOpen); + } // Setup parameters based on type, looking up names if required int sockDomain = 0; SocketAllAddr addr; int addrLen = 0; - Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, Port, addrLen); + Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, + Port, addrLen); // Create the socket - mSocketHandle = ::socket(sockDomain, SOCK_STREAM, 0 /* let OS choose protocol */); + mSocketHandle = ::socket(sockDomain, SOCK_STREAM, + 0 /* let OS choose protocol */); if(mSocketHandle == -1) { + BOX_LOG_SYS_ERROR("Failed to create a network socket"); THROW_EXCEPTION(ServerException, SocketOpenError) } // Set an option to allow reuse (useful for -HUP situations!) #ifdef WIN32 - if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, "", 0) == -1) + if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, "", + 0) == -1) #else int option = true; - if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, &option, sizeof(option)) == -1) + if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, + &option, sizeof(option)) == -1) #endif { + BOX_LOG_SYS_ERROR("Failed to set socket options"); THROW_EXCEPTION(ServerException, SocketOpenError) } @@ -161,19 +173,25 @@ public: } } - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ // // Function // Name: SocketListen::Accept(int) - // Purpose: Accepts a connection, returning a pointer to a class of - // the specified type. May return a null pointer if a signal happens, - // or there's a timeout. Timeout specified in milliseconds, defaults to infinite time. + // Purpose: Accepts a connection, returning a pointer to + // a class of the specified type. May return a + // null pointer if a signal happens, or there's + // a timeout. Timeout specified in + // milliseconds, defaults to infinite time. // Created: 2003/07/31 // - // -------------------------------------------------------------------------- - std::auto_ptr<SocketType> Accept(int Timeout = INFTIM, std::string *pLogMsg = 0) + // ------------------------------------------------------------------ + std::auto_ptr<SocketType> Accept(int Timeout = INFTIM, + std::string *pLogMsg = 0) { - if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)} + if(mSocketHandle == -1) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } // Do the accept, using the supplied locking type int sock; @@ -185,8 +203,10 @@ public: if(!socklock.HaveLock()) { - // Didn't get the lock for some reason. Wait a while, then - // return nothing. + // Didn't get the lock for some reason. + // Wait a while, then return nothing. + BOX_ERROR("Failed to get a lock on incoming " + "connection"); ::sleep(1); return std::auto_ptr<SocketType>(); } @@ -202,12 +222,18 @@ public: // signal? if(errno == EINTR) { + BOX_ERROR("Failed to accept " + "connection: interrupted by " + "signal"); // return nothing return std::auto_ptr<SocketType>(); } else { - THROW_EXCEPTION(ServerException, SocketPollError) + BOX_LOG_SYS_ERROR("Failed to poll " + "connection"); + THROW_EXCEPTION(ServerException, + SocketPollError) } break; case 0: // timed out @@ -220,16 +246,19 @@ public: sock = ::accept(mSocketHandle, &addr, &addrlen); } - // Got socket (or error), unlock (implcit in destruction) + + // Got socket (or error), unlock (implicit in destruction) if(sock == -1) { + BOX_LOG_SYS_ERROR("Failed to accept connection"); THROW_EXCEPTION(ServerException, SocketAcceptError) } // Log it if(pLogMsg) { - *pLogMsg = Socket::IncomingConnectionLogMessage(&addr, addrlen); + *pLogMsg = Socket::IncomingConnectionLogMessage(&addr, + addrlen); } else { @@ -243,27 +272,29 @@ public: // Functions to allow adding to WaitForEvent class, for efficient waiting // on multiple sockets. #ifdef HAVE_KQUEUE - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ // // Function // Name: SocketListen::FillInKEevent // Purpose: Fills in a kevent structure for this socket // Created: 9/3/04 // - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ void FillInKEvent(struct kevent &rEvent, int Flags = 0) const { - EV_SET(&rEvent, mSocketHandle, EVFILT_READ, 0, 0, 0, (void*)this); + EV_SET(&rEvent, mSocketHandle, EVFILT_READ, 0, 0, 0, + (void*)this); } #else - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ // // Function // Name: SocketListen::FillInPoll - // Purpose: Fills in the data necessary for a poll operation + // Purpose: Fills in the data necessary for a poll + // operation // Created: 9/3/04 // - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ void FillInPoll(int &fd, short &events, int Flags = 0) const { fd = mSocketHandle; diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp index 5cb252bd..95b4b4f4 100644 --- a/lib/server/SocketStream.cpp +++ b/lib/server/SocketStream.cpp @@ -18,7 +18,11 @@ #include <string.h> #ifndef WIN32 -#include <poll.h> + #include <poll.h> +#endif + +#ifdef HAVE_UCRED_H + #include <ucred.h> #endif #include "SocketStream.h" @@ -123,20 +127,23 @@ void SocketStream::Attach(int socket) THROW_EXCEPTION(ServerException, SocketAlreadyOpen) } - mSocketHandle = socket; ResetCounters(); + + mSocketHandle = socket; + mReadClosed = false; + mWriteClosed = false; } // -------------------------------------------------------------------------- // // Function -// Name: SocketStream::Open(int, char *, int) +// Name: SocketStream::Open(Socket::Type, char *, int) // Purpose: Opens a connection to a listening socket (INET or UNIX) // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void SocketStream::Open(int Type, const char *Name, int Port) +void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) { if(mSocketHandle != INVALID_SOCKET_VALUE) { @@ -147,12 +154,14 @@ void SocketStream::Open(int Type, const char *Name, int Port) int sockDomain = 0; SocketAllAddr addr; int addrLen = 0; - Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, Port, addrLen); + Socket::NameLookupToSockAddr(addr, sockDomain, Type, rName, Port, addrLen); // Create the socket - mSocketHandle = ::socket(sockDomain, SOCK_STREAM, 0 /* let OS choose protocol */); + mSocketHandle = ::socket(sockDomain, SOCK_STREAM, + 0 /* let OS choose protocol */); if(mSocketHandle == INVALID_SOCKET_VALUE) { + BOX_LOG_SYS_ERROR("Failed to create a network socket"); THROW_EXCEPTION(ServerException, SocketOpenError) } @@ -163,28 +172,24 @@ void SocketStream::Open(int Type, const char *Name, int Port) #ifdef WIN32 DWORD err = WSAGetLastError(); ::closesocket(mSocketHandle); -#else - int err = errno; + BOX_LOG_WIN_ERROR_NUMBER("Failed to connect to socket " + "(type " << Type << ", name " << rName << + ", port " << Port << ")", err); +#else // !WIN32 + BOX_LOG_SYS_ERROR("Failed to connect to socket (type " << + Type << ", name " << rName << ", port " << Port << + ")"); ::close(mSocketHandle); -#endif - -#ifdef WIN32 - BOX_ERROR("Failed to connect to socket (type " << Type << - ", name " << Name << ", port " << Port << "): " << - GetErrorMessage(err) - ); -#else - BOX_ERROR("Failed to connect to socket (type " << Type << - ", name " << Name << ", port " << Port << "): " << - strerror(err) << " (" << err << ")" - ); -#endif +#endif // WIN32 mSocketHandle = INVALID_SOCKET_VALUE; THROW_EXCEPTION(ConnectionException, Conn_SocketConnectError) } ResetCounters(); + + mReadClosed = false; + mWriteClosed = false; } // -------------------------------------------------------------------------- @@ -220,7 +225,9 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) else { // Bad! - THROW_EXCEPTION(ServerException, SocketPollError) + BOX_LOG_SYS_ERROR("Failed to poll socket"); + THROW_EXCEPTION(ServerException, + SocketPollError) } break; @@ -250,9 +257,12 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) else { // Other error - THROW_EXCEPTION(ConnectionException, Conn_SocketReadError) + BOX_LOG_SYS_ERROR("Failed to read from socket"); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError); } } + // Closed for reading? if(r == 0) { @@ -297,7 +307,9 @@ void SocketStream::Write(const void *pBuffer, int NBytes) { // Error. mWriteClosed = true; // assume can't write again - THROW_EXCEPTION(ConnectionException, Conn_SocketWriteError) + BOX_LOG_SYS_ERROR("Failed to write to socket"); + THROW_EXCEPTION(ConnectionException, + Conn_SocketWriteError); } // Knock off bytes sent @@ -310,7 +322,9 @@ void SocketStream::Write(const void *pBuffer, int NBytes) // Need to wait until it can send again? if(bytesLeft > 0) { - TRACE3("Waiting to send data on socket %d, (%d to send of %d)\n", mSocketHandle, bytesLeft, NBytes); + BOX_TRACE("Waiting to send data on socket " << + mSocketHandle << " (" << bytesLeft << + " of " << NBytes << " bytes left)"); // Wait for data to send. struct pollfd p; @@ -323,7 +337,10 @@ void SocketStream::Write(const void *pBuffer, int NBytes) // Don't exception if it's just a signal if(errno != EINTR) { - THROW_EXCEPTION(ServerException, SocketPollError) + BOX_LOG_SYS_ERROR("Failed to poll " + "socket"); + THROW_EXCEPTION(ServerException, + SocketPollError) } } } @@ -350,7 +367,9 @@ void SocketStream::Close() if(::close(mSocketHandle) == -1) #endif { - THROW_EXCEPTION(ServerException, SocketCloseError) + BOX_LOG_SYS_ERROR("Failed to close socket"); + // don't throw an exception here, assume that the socket was + // already closed or closing. } mSocketHandle = INVALID_SOCKET_VALUE; } @@ -380,6 +399,7 @@ void SocketStream::Shutdown(bool Read, bool Write) // Shut it down! if(::shutdown(mSocketHandle, how) == -1) { + BOX_LOG_SYS_ERROR("Failed to shutdown socket"); THROW_EXCEPTION(ConnectionException, Conn_SocketShutdownError) } } @@ -458,18 +478,37 @@ bool SocketStream::GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut) struct ucred cred; socklen_t credLen = sizeof(cred); - if(::getsockopt(mSocketHandle, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) + if(::getsockopt(mSocketHandle, SOL_SOCKET, SO_PEERCRED, &cred, + &credLen) == 0) { rUidOut = cred.uid; rGidOut = cred.gid; return true; } + + BOX_LOG_SYS_ERROR("Failed to get peer credentials on socket"); +#endif + +#if defined HAVE_UCRED_H && HAVE_GETPEERUCRED + ucred_t *pucred = NULL; + if(::getpeerucred(mSocketHandle, &pucred) == 0) + { + rUidOut = ucred_geteuid(pucred); + rGidOut = ucred_getegid(pucred); + ucred_free(pucred); + if (rUidOut == -1 || rGidOut == -1) + { + BOX_ERROR("Failed to get peer credentials on " + "socket: insufficient information"); + return false; + } + return true; + } + + BOX_LOG_SYS_ERROR("Failed to get peer credentials on socket"); #endif // Not available return false; } - - - diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h index 51f2e306..2b582f21 100644 --- a/lib/server/SocketStream.h +++ b/lib/server/SocketStream.h @@ -11,6 +11,7 @@ #define SOCKETSTREAM__H #include "IOStream.h" +#include "Socket.h" #ifdef WIN32 typedef SOCKET tOSSocketHandle; @@ -36,7 +37,7 @@ public: SocketStream(const SocketStream &rToCopy); ~SocketStream(); - void Open(int Type, const char *Name, int Port = 0); + void Open(Socket::Type Type, const std::string& rName, int Port = 0); void Attach(int socket); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); diff --git a/lib/server/SocketStreamTLS.cpp b/lib/server/SocketStreamTLS.cpp index 58dc5754..19fdadd4 100644 --- a/lib/server/SocketStreamTLS.cpp +++ b/lib/server/SocketStreamTLS.cpp @@ -99,9 +99,10 @@ SocketStreamTLS::~SocketStreamTLS() // Created: 2003/08/06 // // -------------------------------------------------------------------------- -void SocketStreamTLS::Open(const TLSContext &rContext, int Type, const char *Name, int Port) +void SocketStreamTLS::Open(const TLSContext &rContext, Socket::Type Type, + const std::string& rName, int Port) { - SocketStream::Open(Type, Name, Port); + SocketStream::Open(Type, rName, Port); Handshake(rContext); ResetCounters(); } @@ -123,7 +124,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) mpBIO = ::BIO_new(::BIO_s_socket()); if(mpBIO == 0) { - SSLLib::LogError("Create socket bio"); + SSLLib::LogError("creating socket bio"); THROW_EXCEPTION(ServerException, TLSAllocationFailed) } @@ -134,7 +135,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) mpSSL = ::SSL_new(rContext.GetRawContext()); if(mpSSL == 0) { - SSLLib::LogError("Create ssl"); + SSLLib::LogError("creating SSL object"); THROW_EXCEPTION(ServerException, TLSAllocationFailed) } @@ -202,12 +203,12 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) // Error occured if(IsServer) { - SSLLib::LogError("Accept"); + SSLLib::LogError("accepting connection"); THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) } else { - SSLLib::LogError("Connect"); + SSLLib::LogError("connecting"); THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) } } @@ -334,7 +335,7 @@ int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) break; default: - SSLLib::LogError("Read"); + SSLLib::LogError("reading"); THROW_EXCEPTION(ConnectionException, Conn_TLSReadFailed) break; } @@ -390,7 +391,7 @@ void SocketStreamTLS::Write(const void *pBuffer, int NBytes) case SSL_ERROR_WANT_WRITE: // wait for the requried data { - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD bool conditionmet = #endif WaitWhenRetryRequired(se, IOStream::TimeOutInfinite); @@ -399,7 +400,7 @@ void SocketStreamTLS::Write(const void *pBuffer, int NBytes) break; default: - SSLLib::LogError("Write"); + SSLLib::LogError("writing"); THROW_EXCEPTION(ConnectionException, Conn_TLSWriteFailed) break; } @@ -441,7 +442,7 @@ void SocketStreamTLS::Shutdown(bool Read, bool Write) if(::SSL_shutdown(mpSSL) < 0) { - SSLLib::LogError("Shutdown"); + SSLLib::LogError("shutting down"); THROW_EXCEPTION(ConnectionException, Conn_TLSShutdownFailed) } diff --git a/lib/server/SocketStreamTLS.h b/lib/server/SocketStreamTLS.h index 64e52833..bb40ed10 100644 --- a/lib/server/SocketStreamTLS.h +++ b/lib/server/SocketStreamTLS.h @@ -38,7 +38,8 @@ private: SocketStreamTLS(const SocketStreamTLS &rToCopy); public: - void Open(const TLSContext &rContext, int Type, const char *Name, int Port = 0); + void Open(const TLSContext &rContext, Socket::Type Type, + const std::string& rName, int Port = 0); void Handshake(const TLSContext &rContext, bool IsServer = false); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp index 49143801..ebc7384a 100644 --- a/lib/server/TLSContext.cpp +++ b/lib/server/TLSContext.cpp @@ -75,19 +75,25 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c // Setup our identity if(::SSL_CTX_use_certificate_chain_file(mpContext, CertificatesFile) != 1) { - SSLLib::LogError("Load certificates"); + std::string msg = "loading certificates from "; + msg += CertificatesFile; + SSLLib::LogError(msg); THROW_EXCEPTION(ServerException, TLSLoadCertificatesFailed) } if(::SSL_CTX_use_PrivateKey_file(mpContext, PrivateKeyFile, SSL_FILETYPE_PEM) != 1) { - SSLLib::LogError("Load private key"); + std::string msg = "loading private key from "; + msg += PrivateKeyFile; + SSLLib::LogError(msg); THROW_EXCEPTION(ServerException, TLSLoadPrivateKeyFailed) } // Setup the identify of CAs we trust if(::SSL_CTX_load_verify_locations(mpContext, TrustedCAsFile, NULL) != 1) { - SSLLib::LogError("Load CA cert"); + std::string msg = "loading CA cert from "; + msg += TrustedCAsFile; + SSLLib::LogError(msg); THROW_EXCEPTION(ServerException, TLSLoadTrustedCAsFailed) } @@ -99,7 +105,7 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c // Setup allowed ciphers if(::SSL_CTX_set_cipher_list(mpContext, CIPHER_LIST) != 1) { - SSLLib::LogError("Set cipher list"); + SSLLib::LogError("setting cipher list to " CIPHER_LIST); THROW_EXCEPTION(ServerException, TLSSetCiphersFailed) } } diff --git a/lib/server/WinNamedPipeListener.h b/lib/server/WinNamedPipeListener.h new file mode 100644 index 00000000..26e76e3d --- /dev/null +++ b/lib/server/WinNamedPipeListener.h @@ -0,0 +1,232 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: WinNamedPipeListener.h +// Purpose: Windows named pipe socket connection listener +// for server +// Created: 2008/09/30 +// +// -------------------------------------------------------------------------- + +#ifndef WINNAMEDPIPELISTENER__H +#define WINNAMEDPIPELISTENER__H + +#include <OverlappedIO.h> +#include <WinNamedPipeStream.h> + +#include "ServerException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: WinNamedPipeListener +// Purpose: +// Created: 2008/09/30 +// +// -------------------------------------------------------------------------- +template<int ListenBacklog = 128> +class WinNamedPipeListener +{ +private: + std::auto_ptr<std::string> mapPipeName; + std::auto_ptr<OverlappedIO> mapOverlapConnect; + HANDLE mPipeHandle; + +public: + // Initialise + WinNamedPipeListener() + : mPipeHandle(INVALID_HANDLE_VALUE) + { } + +private: + WinNamedPipeListener(const WinNamedPipeListener &rToCopy) + { /* forbidden */ } + + HANDLE CreatePipeHandle(const std::string& rName) + { + std::string socket = WinNamedPipeStream::sPipeNamePrefix + + rName; + + HANDLE handle = CreateNamedPipeA( + socket.c_str(), // pipe name + PIPE_ACCESS_DUPLEX | // read/write access + FILE_FLAG_OVERLAPPED, // enabled overlapped I/O + PIPE_TYPE_BYTE | // message type pipe + PIPE_READMODE_BYTE | // message-read mode + PIPE_WAIT, // blocking mode + ListenBacklog + 1, // max. instances + 4096, // output buffer size + 4096, // input buffer size + NMPWAIT_USE_DEFAULT_WAIT, // client time-out + NULL); // default security attribute + + if (handle == INVALID_HANDLE_VALUE) + { + BOX_LOG_WIN_ERROR("Failed to create named pipe " << + socket); + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + return handle; + } + +public: + ~WinNamedPipeListener() + { + Close(); + } + + void Close() + { + if (mPipeHandle != INVALID_HANDLE_VALUE) + { + if (mapOverlapConnect.get()) + { + // outstanding connect in progress + if (CancelIo(mPipeHandle) != TRUE) + { + BOX_LOG_WIN_ERROR("Failed to cancel " + "outstanding connect request " + "on named pipe"); + } + + mapOverlapConnect.reset(); + } + + if (CloseHandle(mPipeHandle) != TRUE) + { + BOX_LOG_WIN_ERROR("Failed to close named pipe " + "handle"); + } + + mPipeHandle = INVALID_HANDLE_VALUE; + } + } + + // ------------------------------------------------------------------ + // + // Function + // Name: WinNamedPipeListener::Listen(std::string name) + // Purpose: Initialises socket name + // Created: 2003/07/31 + // + // ------------------------------------------------------------------ + void Listen(const std::string& rName) + { + Close(); + mapPipeName.reset(new std::string(rName)); + mPipeHandle = CreatePipeHandle(rName); + } + + // ------------------------------------------------------------------ + // + // Function + // Name: WinNamedPipeListener::Accept(int) + // Purpose: Accepts a connection, returning a pointer to + // a class of the specified type. May return a + // null pointer if a signal happens, or there's + // a timeout. Timeout specified in + // milliseconds, defaults to infinite time. + // Created: 2003/07/31 + // + // ------------------------------------------------------------------ + std::auto_ptr<WinNamedPipeStream> Accept(int Timeout = INFTIM, + const char* pLogMsgOut = NULL) + { + if(!mapPipeName.get()) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + + BOOL connected = FALSE; + std::auto_ptr<WinNamedPipeStream> mapStream; + + if (!mapOverlapConnect.get()) + { + // start a new connect operation + mapOverlapConnect.reset(new OverlappedIO()); + connected = ConnectNamedPipe(mPipeHandle, + &mapOverlapConnect->mOverlapped); + + if (connected == FALSE) + { + if (GetLastError() == ERROR_PIPE_CONNECTED) + { + connected = TRUE; + } + else if (GetLastError() != ERROR_IO_PENDING) + { + BOX_LOG_WIN_ERROR("Failed to connect " + "named pipe"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + } + } + + if (connected == FALSE) + { + // wait for connection + DWORD result = WaitForSingleObject( + mapOverlapConnect->mOverlapped.hEvent, + (Timeout == INFTIM) ? INFINITE : Timeout); + + if (result == WAIT_OBJECT_0) + { + DWORD dummy; + + if (!GetOverlappedResult(mPipeHandle, + &mapOverlapConnect->mOverlapped, + &dummy, TRUE)) + { + BOX_LOG_WIN_ERROR("Failed to get " + "overlapped connect result"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + + connected = TRUE; + } + else if (result == WAIT_TIMEOUT) + { + return mapStream; // contains NULL + } + else if (result == WAIT_ABANDONED) + { + BOX_ERROR("Wait for named pipe connection " + "was abandoned by the system"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + else if (result == WAIT_FAILED) + { + BOX_LOG_WIN_ERROR("Failed to wait for named " + "pipe connection"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + else + { + BOX_ERROR("Failed to wait for named pipe " + "connection: unknown return code " << + result); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + } + + ASSERT(connected == TRUE); + + mapStream.reset(new WinNamedPipeStream(mPipeHandle)); + mPipeHandle = CreatePipeHandle(*mapPipeName); + mapOverlapConnect.reset(); + + return mapStream; + } +}; + +#include "MemLeakFindOff.h" + +#endif // WINNAMEDPIPELISTENER__H diff --git a/lib/server/WinNamedPipeStream.cpp b/lib/server/WinNamedPipeStream.cpp index fedb57d8..1179516e 100644 --- a/lib/server/WinNamedPipeStream.cpp +++ b/lib/server/WinNamedPipeStream.cpp @@ -44,7 +44,55 @@ WinNamedPipeStream::WinNamedPipeStream() mWriteClosed(false), mIsServer(false), mIsConnected(false) -{ +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::WinNamedPipeStream(HANDLE) +// Purpose: Constructor (with already-connected pipe handle) +// Created: 2008/10/01 +// +// -------------------------------------------------------------------------- +WinNamedPipeStream::WinNamedPipeStream(HANDLE hNamedPipe) + : mSocketHandle(hNamedPipe), + mReadableEvent(INVALID_HANDLE_VALUE), + mBytesInBuffer(0), + mReadClosed(false), + mWriteClosed(false), + mIsServer(true), + mIsConnected(true) +{ + // create the Readable event + mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + + if (mReadableEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to create the Readable event: " << + GetErrorMessage(GetLastError())); + Close(); + THROW_EXCEPTION(CommonException, Internal) + } + + // initialise the OVERLAPPED structure + memset(&mReadOverlap, 0, sizeof(mReadOverlap)); + mReadOverlap.hEvent = mReadableEvent; + + // start the first overlapped read + if (!ReadFile(mSocketHandle, mReadBuffer, sizeof(mReadBuffer), + NULL, &mReadOverlap)) + { + DWORD err = GetLastError(); + + if (err != ERROR_IO_PENDING) + { + BOX_ERROR("Failed to start overlapped read: " << + GetErrorMessage(err)); + Close(); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } } // -------------------------------------------------------------------------- @@ -80,33 +128,17 @@ WinNamedPipeStream::~WinNamedPipeStream() // Created: 2005/12/07 // // -------------------------------------------------------------------------- -void WinNamedPipeStream::Accept(const std::string& rName) +/* +void WinNamedPipeStream::Accept() { - if (mSocketHandle != INVALID_HANDLE_VALUE || mIsConnected) + if (mSocketHandle == INVALID_HANDLE_VALUE) { - THROW_EXCEPTION(ServerException, SocketAlreadyOpen) + THROW_EXCEPTION(ServerException, BadSocketHandle); } - std::string socket = sPipeNamePrefix + rName; - - mSocketHandle = CreateNamedPipeA( - socket.c_str(), // pipe name - PIPE_ACCESS_DUPLEX | // read/write access - FILE_FLAG_OVERLAPPED, // enabled overlapped I/O - PIPE_TYPE_BYTE | // message type pipe - PIPE_READMODE_BYTE | // message-read mode - PIPE_WAIT, // blocking mode - 1, // max. instances - 4096, // output buffer size - 4096, // input buffer size - NMPWAIT_USE_DEFAULT_WAIT, // client time-out - NULL); // default security attribute - - if (mSocketHandle == INVALID_HANDLE_VALUE) + if (mIsConnected) { - BOX_ERROR("Failed to CreateNamedPipeA(" << socket << "): " << - GetErrorMessage(GetLastError())); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_EXCEPTION(ServerException, SocketAlreadyOpen); } bool connected = ConnectNamedPipe(mSocketHandle, (LPOVERLAPPED) NULL); @@ -156,6 +188,7 @@ void WinNamedPipeStream::Accept(const std::string& rName) } } } +*/ // -------------------------------------------------------------------------- // @@ -217,7 +250,7 @@ void WinNamedPipeStream::Connect(const std::string& rName) int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) { // TODO no support for timeouts yet - if (Timeout != IOStream::TimeOutInfinite) + if (!mIsServer && Timeout != IOStream::TimeOutInfinite) { THROW_EXCEPTION(CommonException, AssertFailed) } @@ -249,8 +282,29 @@ int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) { // overlapped I/O completed successfully? // (wait if needed) + DWORD waitResult = WaitForSingleObject( + mReadOverlap.hEvent, Timeout); - if (GetOverlappedResult(mSocketHandle, + if (waitResult == WAIT_ABANDONED) + { + BOX_ERROR("Wait for command socket read " + "abandoned by system"); + THROW_EXCEPTION(ServerException, + BadSocketHandle); + } + else if (waitResult == WAIT_TIMEOUT) + { + // wait timed out, nothing to read + NumBytesRead = 0; + } + else if (waitResult != WAIT_OBJECT_0) + { + BOX_ERROR("Failed to wait for command " + "socket read: unknown result " << + waitResult); + } + // object is ready to read from + else if (GetOverlappedResult(mSocketHandle, &mReadOverlap, &NumBytesRead, TRUE)) { needAnotherRead = true; @@ -267,7 +321,7 @@ int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) { if (err == ERROR_BROKEN_PIPE) { - BOX_ERROR("Control client " + BOX_NOTICE("Control client " "disconnected"); } else @@ -342,29 +396,6 @@ int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) Conn_SocketReadError) } } - - // If the read succeeded immediately, leave the event - // signaled, so that we will be called again to process - // the newly read data and start another overlapped read. - if (needAnotherRead && !mReadClosed) - { - // leave signalled - } - else if (!needAnotherRead && mBytesInBuffer > 0) - { - // leave signalled - } - else - { - // nothing left to read, reset the event - ResetEvent(mReadableEvent); - // FIXME: a pending read could have signalled - // the event (again) while we were busy reading. - // that signal would be lost, and the reading - // thread would block. Should be pretty obvious - // if this happens in practice: control client - // hangs. - } } else { @@ -440,23 +471,21 @@ void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) if (!Success) { - DWORD err = GetLastError(); - BOX_ERROR("Failed to write to control socket: " << - GetErrorMessage(err)); - Close(); - // ERROR_NO_DATA is a strange name for - // "The pipe is being closed". No exception wanted. + // "The pipe is being closed". - if (err == ERROR_NO_DATA) - { - return; - } - else + DWORD err = GetLastError(); + + if (err != ERROR_NO_DATA) { - THROW_EXCEPTION(ConnectionException, - Conn_SocketWriteError) + BOX_ERROR("Failed to write to control " + "socket: " << GetErrorMessage(err)); } + + Close(); + + THROW_EXCEPTION(ConnectionException, + Conn_SocketWriteError) } NumBytesWrittenTotal += NumBytesWrittenThisTime; diff --git a/lib/server/WinNamedPipeStream.h b/lib/server/WinNamedPipeStream.h index 6acd48f6..386ff7e3 100644 --- a/lib/server/WinNamedPipeStream.h +++ b/lib/server/WinNamedPipeStream.h @@ -24,10 +24,11 @@ class WinNamedPipeStream : public IOStream { public: WinNamedPipeStream(); + WinNamedPipeStream(HANDLE hNamedPipe); ~WinNamedPipeStream(); // server side - create the named pipe and listen for connections - void Accept(const std::string& rName); + // use WinNamedPipeListener to do this instead. // client side - connect to a waiting server void Connect(const std::string& rName); @@ -40,9 +41,6 @@ public: virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); - bool IsConnected() { return mIsConnected; } - HANDLE GetSocketHandle() { return mSocketHandle; } - HANDLE GetReadableEvent() { return mReadableEvent; } protected: void MarkAsReadClosed() {mReadClosed = true;} @@ -62,6 +60,7 @@ private: bool mIsServer; bool mIsConnected; +public: static std::string sPipeNamePrefix; }; diff --git a/lib/server/makeprotocol.pl.in b/lib/server/makeprotocol.pl.in index 269ff3f5..91ba55b0 100755 --- a/lib/server/makeprotocol.pl.in +++ b/lib/server/makeprotocol.pl.in @@ -178,6 +178,9 @@ print CPP <<__E; // Auto-generated file -- do not edit #include "Box.h" + +#include <sstream> + #include "$h_filename" #include "IOStream.h" @@ -273,7 +276,7 @@ __E if($derive_objects_from ne 'ProtocolObject') { - # output a definition for the protocol object derviced class + # output a definition for the protocol object derived class print H <<__E; class ${protocol_name}ProtocolServer; @@ -338,6 +341,7 @@ __E if(obj_is_type($cmd,'IsError')) { print H "\tbool IsError(int &rTypeOut, int &rSubTypeOut) const;\n"; + print H "\tstd::string GetMessage() const;\n"; } if($type eq 'Server' && obj_is_type($cmd, 'Command')) { @@ -498,6 +502,27 @@ bool ${class}IsError(int &rTypeOut, int &rSubTypeOut) const rSubTypeOut = m$mem_subtype; return true; } +std::string ${class}GetMessage() const +{ + switch(m$mem_subtype) + { +__E + foreach my $const (@{$cmd_constants{$cmd}}) + { + next unless $const =~ /^Err_(.*)/; + my $shortname = $1; + $const =~ s/ = .*//; + print CPP <<__E; + case $const: return "$shortname"; +__E + } + print CPP <<__E; + default: + std::ostringstream out; + out << "Unknown subtype " << m$mem_subtype; + return out.str(); + } +} __E } @@ -513,11 +538,13 @@ __E } if($implement_filelog) { - my ($format,$args) = make_log_strings($cmd); + my ($log) = make_log_strings_framework($cmd); print CPP <<__E; void ${class}LogFile(const char *Action, FILE *File) const { - ::fprintf(File,"%s $format\\n",Action$args); + std::ostringstream oss; + oss << $log; + ::fprintf(File, "%s\\n", oss.str().c_str()); ::fflush(File); } __E @@ -620,16 +647,16 @@ protected: __E -my $construtor_extra = ''; -$construtor_extra .= ', mLogToSysLog(false)' if $implement_syslog; -$construtor_extra .= ', mLogToFile(0)' if $implement_filelog; +my $constructor_extra = ''; +$constructor_extra .= ', mLogToSysLog(false)' if $implement_syslog; +$constructor_extra .= ', mLogToFile(0)' if $implement_filelog; my $destructor_extra = ($type eq 'Server')?"\n\tDeleteStreamsToSend();":''; my $prefix = $classname_base.'::'; print CPP <<__E; $prefix$classname_base(IOStream &rStream) - : Protocol(rStream)$construtor_extra + : Protocol(rStream)$constructor_extra { } $prefix~$classname_base() @@ -661,7 +688,7 @@ print CPP <<__E; } } __E -# write receieve and send functions +# write receive and send functions print CPP <<__E; std::auto_ptr<$derive_objects_from> ${prefix}Receive() { @@ -734,51 +761,9 @@ void ${prefix}DoServer($context_class &rContext) // Get an object from the conversation std::auto_ptr<${derive_objects_from}> pobj(Receive()); -__E - if($implement_syslog) - { - print CPP <<__E; - if(mLogToSysLog) - { - pobj->LogSysLog("Receive"); - } -__E - } - if($implement_filelog) - { - print CPP <<__E; - if(mLogToFile != 0) - { - pobj->LogFile("Receive", mLogToFile); - } -__E - } - print CPP <<__E; - // Run the command std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(pobj->DoCommand(*this, rContext).release())); -__E - if($implement_syslog) - { - print CPP <<__E; - if(mLogToSysLog) - { - preply->LogSysLog("Send"); - } -__E - } - if($implement_filelog) - { - print CPP <<__E; - if(mLogToFile != 0) - { - preply->LogFile("Send", mLogToFile); - } -__E - } - print CPP <<__E; - // Send the reply Send(*(preply.get())); @@ -824,13 +809,57 @@ if($implement_filelog || $implement_syslog) if($implement_syslog) { - $fR .= qq~\tif(mLogToSysLog) { ::syslog(LOG_INFO, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain":"Receiving stream, size %d", Size); }\n~; - $fS .= qq~\tif(mLogToSysLog) { ::syslog(LOG_INFO, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain":"Sending stream, size %d", Size); }\n~; + $fR .= <<__E; + if(mLogToSysLog) + { + if(Size==Protocol::ProtocolStream_SizeUncertain) + { + BOX_TRACE("Receiving stream, size uncertain"); + } + else + { + BOX_TRACE("Receiving stream, size " << Size); + } } +__E + + $fS .= <<__E; + if(mLogToSysLog) + { + if(Size==Protocol::ProtocolStream_SizeUncertain) + { + BOX_TRACE("Sending stream, size uncertain"); + } + else + { + BOX_TRACE("Sending stream, size " << Size); + } + } +__E + } + if($implement_filelog) { - $fR .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain\\n":"Receiving stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; - $fS .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain\\n":"Sending stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; + $fR .= <<__E; + if(mLogToFile) + { + ::fprintf(mLogToFile, + (Size==Protocol::ProtocolStream_SizeUncertain) + ?"Receiving stream, size uncertain\\n" + :"Receiving stream, size %d\\n", Size); + ::fflush(mLogToFile); + } +__E + $fS .= <<__E; + if(mLogToFile) + { + ::fprintf(mLogToFile, + (Size==Protocol::ProtocolStream_SizeUncertain) + ?"Sending stream, size uncertain\\n" + :"Sending stream, size %d\\n", Size); + ::fflush(mLogToFile); + } +__E } print CPP <<__E; @@ -888,11 +917,15 @@ std::auto_ptr<$classname_base$reply> ${classname_base}::Query(const $classname_b if(preply->IsError(type, subType)) { SetError(type, subType); - TRACE2("Protocol: Error received %d/%d\\n", type, subType); + BOX_WARNING("$cmd command failed: received error " << + ((${classname_base}Error&)*preply).GetMessage()); } else { SetError(Protocol::UnknownError, Protocol::UnknownError); + BOX_WARNING("$cmd command failed: received " + "unexpected response type " << + preply->GetType()); } // Throw an exception @@ -1020,10 +1053,22 @@ sub make_log_strings_framework my ($format,$arg) = @{$log_display_types{$ty}}; $arg =~ s/VAR/m$nm/g; - if ($format =~ m'x$') + if ($format eq '\\"%s\\"') + { + $arg = "\"\\\"\" << $arg << \"\\\"\""; + } + elsif ($format =~ m'x$') { - $arg = "std::hex << std::showbase " . - "<< $arg << std::dec"; + # my $width = 0; + # $ty =~ /^int(\d+)$/ and $width = $1 / 4; + $arg = "($arg == 0 ? \"0x\" : \"\") " . + "<< std::hex " . + "<< std::showbase " . + # "<< std::setw($width) " . + # "<< std::setfill('0') " . + # "<< std::internal " . + "<< $arg " . + "<< std::dec"; } push @args, $arg; diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index 071dc788..ad8c3041 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -21,173 +21,36 @@ #include <sstream> // message resource definitions for syslog() - #include "messages.h" -// our implementation for a timer, based on a -// simple thread which sleeps for a period of time - -static bool gTimerInitialised = false; -static bool gFinishTimer; -static CRITICAL_SECTION gLock; - -typedef struct -{ - int countDown; - int interval; -} -Timer_t; - -std::list<Timer_t> gTimerList; -static void (__cdecl *gTimerFunc) (int) = NULL; - -int setitimer(int type, struct itimerval *timeout, void *arg) -{ - assert(gTimerInitialised); - - if (ITIMER_REAL != type) - { - errno = ENOSYS; - return -1; - } - - EnterCriticalSection(&gLock); - - // we only need seconds for the mo! - if (timeout->it_value.tv_sec == 0 && - timeout->it_value.tv_usec == 0) - { - gTimerList.clear(); - } - else - { - Timer_t ourTimer; - ourTimer.countDown = timeout->it_value.tv_sec; - ourTimer.interval = timeout->it_interval.tv_sec; - gTimerList.push_back(ourTimer); - } - - LeaveCriticalSection(&gLock); - - // indicate success - return 0; -} - -static unsigned int WINAPI RunTimer(LPVOID lpParameter) -{ - gFinishTimer = false; - - while (!gFinishTimer) - { - std::list<Timer_t>::iterator it; - EnterCriticalSection(&gLock); - - for (it = gTimerList.begin(); it != gTimerList.end(); it++) - { - Timer_t& rTimer(*it); - - rTimer.countDown --; - if (rTimer.countDown == 0) - { - if (gTimerFunc != NULL) - { - gTimerFunc(0); - } - if (rTimer.interval) - { - rTimer.countDown = rTimer.interval; - } - else - { - // mark for deletion - rTimer.countDown = -1; - } - } - } - - for (it = gTimerList.begin(); it != gTimerList.end(); it++) - { - Timer_t& rTimer(*it); - - if (rTimer.countDown == -1) - { - gTimerList.erase(it); - - // the iterator is now invalid, so restart search - it = gTimerList.begin(); - - // if the list is now empty, don't try to increment - // the iterator again - if (it == gTimerList.end()) break; - } - } - - LeaveCriticalSection(&gLock); - // we only need to have a 1 second resolution - Sleep(1000); - } - - return 0; -} - -int SetTimerHandler(void (__cdecl *func ) (int)) -{ - gTimerFunc = func; - return 0; -} - -void InitTimer(void) -{ - assert(!gTimerInitialised); - - InitializeCriticalSection(&gLock); - - // create our thread - HANDLE ourThread = (HANDLE)_beginthreadex(NULL, 0, RunTimer, 0, - CREATE_SUSPENDED, NULL); - SetThreadPriority(ourThread, THREAD_PRIORITY_LOWEST); - ResumeThread(ourThread); - - gTimerInitialised = true; -} - -void FiniTimer(void) -{ - assert(gTimerInitialised); - gFinishTimer = true; - EnterCriticalSection(&gLock); - DeleteCriticalSection(&gLock); - gTimerInitialised = false; -} - -//Our constants we need to keep track of -//globals +DWORD winerrno; struct passwd gTempPasswd; -bool EnableBackupRights( void ) +bool EnableBackupRights() { HANDLE hToken; TOKEN_PRIVILEGES token_priv; //open current process to adjust privileges - if( !OpenProcessToken( GetCurrentProcess( ), - TOKEN_ADJUST_PRIVILEGES, - &hToken )) + if(!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, + &hToken)) { - printf( "Cannot open process token: error %d\n", - (int)GetLastError() ); + ::syslog(LOG_ERR, "Failed to open process token: %s", + GetErrorMessage(GetLastError()).c_str()); return false; } //let's build the token privilege struct - //first, look up the LUID for the backup privilege - if( !LookupPrivilegeValue( NULL, //this system + if (!LookupPrivilegeValue( + NULL, //this system SE_BACKUP_NAME, //the name of the privilege - &( token_priv.Privileges[0].Luid )) ) //result + &( token_priv.Privileges[0].Luid ))) //result { - printf( "Cannot lookup backup privilege: error %d\n", - (int)GetLastError( ) ); + ::syslog(LOG_ERR, "Failed to lookup backup privilege: %s", + GetErrorMessage(GetLastError()).c_str()); + CloseHandle(hToken); return false; } @@ -198,24 +61,25 @@ bool EnableBackupRights( void ) // because we're going exit right after dumping the streams, there isn't // any need to save current state - if( !AdjustTokenPrivileges( hToken, //our process token + if (!AdjustTokenPrivileges( + hToken, //our process token false, //we're not disabling everything &token_priv, //address of structure - sizeof( token_priv ), //size of structure - NULL, NULL )) //don't save current state + sizeof(token_priv), //size of structure + NULL, NULL)) //don't save current state { //this function is a little tricky - if we were adjusting //more than one privilege, it could return success but not //adjust them all - in the general case, you need to trap this - printf( "Could not enable backup privileges: error %d\n", - (int)GetLastError( ) ); + ::syslog(LOG_ERR, "Failed to enable backup privilege: %s", + GetErrorMessage(GetLastError()).c_str()); + CloseHandle(hToken); return false; } - else - { - return true; - } + + CloseHandle(hToken); + return true; } // forward declaration @@ -267,7 +131,8 @@ std::string GetDefaultConfigFilePath(const std::string& rName) // Created: 4th February 2006 // // -------------------------------------------------------------------------- -WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) +WCHAR* ConvertToWideString(const char* pString, unsigned int codepage, + bool logErrors) { int len = MultiByteToWideChar ( @@ -282,9 +147,13 @@ WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) if (len == 0) { - ::syslog(LOG_WARNING, - "Failed to convert string to wide string: " - "error %d", GetLastError()); + winerrno = GetLastError(); + if (logErrors) + { + ::syslog(LOG_WARNING, + "Failed to convert string to wide string: " + "%s", GetErrorMessage(winerrno).c_str()); + } errno = EINVAL; return NULL; } @@ -293,9 +162,13 @@ WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) if (buffer == NULL) { - ::syslog(LOG_WARNING, - "Failed to convert string to wide string: " - "out of memory"); + if (logErrors) + { + ::syslog(LOG_WARNING, + "Failed to convert string to wide string: " + "out of memory"); + } + winerrno = ERROR_OUTOFMEMORY; errno = ENOMEM; return NULL; } @@ -312,9 +185,13 @@ WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) if (len == 0) { - ::syslog(LOG_WARNING, - "Failed to convert string to wide string: " - "error %i", GetLastError()); + winerrno = GetLastError(); + if (logErrors) + { + ::syslog(LOG_WARNING, + "Failed to convert string to wide string: " + "%s", GetErrorMessage(winerrno).c_str()); + } errno = EACCES; delete [] buffer; return NULL; @@ -336,7 +213,7 @@ WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) // -------------------------------------------------------------------------- WCHAR* ConvertUtf8ToWideString(const char* pString) { - return ConvertToWideString(pString, CP_UTF8); + return ConvertToWideString(pString, CP_UTF8, true); } // -------------------------------------------------------------------------- @@ -425,7 +302,8 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) bool ConvertEncoding(const std::string& rSource, int sourceCodePage, std::string& rDest, int destCodePage) { - WCHAR* pWide = ConvertToWideString(rSource.c_str(), sourceCodePage); + WCHAR* pWide = ConvertToWideString(rSource.c_str(), sourceCodePage, + true); if (pWide == NULL) { ::syslog(LOG_ERR, "Failed to convert string '%s' from " @@ -450,24 +328,26 @@ bool ConvertEncoding(const std::string& rSource, int sourceCodePage, return true; } -bool ConvertToUtf8(const char* pString, std::string& rDest, int sourceCodePage) +bool ConvertToUtf8(const std::string& rSource, std::string& rDest, + int sourceCodePage) { - return ConvertEncoding(pString, sourceCodePage, rDest, CP_UTF8); + return ConvertEncoding(rSource, sourceCodePage, rDest, CP_UTF8); } -bool ConvertFromUtf8(const char* pString, std::string& rDest, int destCodePage) +bool ConvertFromUtf8(const std::string& rSource, std::string& rDest, + int destCodePage) { - return ConvertEncoding(pString, CP_UTF8, rDest, destCodePage); + return ConvertEncoding(rSource, CP_UTF8, rDest, destCodePage); } -bool ConvertConsoleToUtf8(const char* pString, std::string& rDest) +bool ConvertConsoleToUtf8(const std::string& rSource, std::string& rDest) { - return ConvertEncoding(pString, GetConsoleCP(), rDest, CP_UTF8); + return ConvertToUtf8(rSource, rDest, GetConsoleCP()); } -bool ConvertUtf8ToConsole(const char* pString, std::string& rDest) +bool ConvertUtf8ToConsole(const std::string& rSource, std::string& rDest) { - return ConvertEncoding(pString, CP_UTF8, rDest, GetConsoleOutputCP()); + return ConvertFromUtf8(rSource, rDest, GetConsoleOutputCP()); } // -------------------------------------------------------------------------- @@ -506,6 +386,7 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName) "Failed to open '%s': path too long", pFileName); errno = ENAMETOOLONG; + winerrno = ERROR_INVALID_NAME; tmpStr = ""; return tmpStr; } @@ -581,6 +462,8 @@ std::string GetErrorMessage(DWORD errorCode) // -------------------------------------------------------------------------- HANDLE openfile(const char *pFileName, int flags, int mode) { + winerrno = ERROR_INVALID_FUNCTION; + std::string AbsPathWithUnicode = ConvertPathToAbsoluteUnicode(pFileName); @@ -654,12 +537,37 @@ HANDLE openfile(const char *pFileName, int flags, int mode) if (hdir == INVALID_HANDLE_VALUE) { + winerrno = GetLastError(); + switch(winerrno) + { + case ERROR_SHARING_VIOLATION: + errno = EBUSY; + break; + + default: + errno = EINVAL; + } + ::syslog(LOG_WARNING, "Failed to open file '%s': " "%s", pFileName, GetErrorMessage(GetLastError()).c_str()); + return INVALID_HANDLE_VALUE; } + if (flags & O_APPEND) + { + if (SetFilePointer(hdir, 0, NULL, FILE_END) == + INVALID_SET_FILE_POINTER) + { + winerrno = GetLastError(); + errno = EINVAL; + CloseHandle(hdir); + return INVALID_HANDLE_VALUE; + } + } + + winerrno = NO_ERROR; return hdir; } @@ -667,11 +575,13 @@ HANDLE openfile(const char *pFileName, int flags, int mode) // // Function // Name: emu_fstat -// Purpose: replacement for fstat supply a windows handle +// Purpose: replacement for fstat. Supply a windows handle. +// Returns a struct emu_stat to have room for 64-bit +// file identifier in st_ino (mingw allows only 16!) // Created: 25th October 2004 // // -------------------------------------------------------------------------- -int emu_fstat(HANDLE hdir, struct stat * st) +int emu_fstat(HANDLE hdir, struct emu_stat * st) { if (hdir == INVALID_HANDLE_VALUE) { @@ -702,8 +612,8 @@ int emu_fstat(HANDLE hdir, struct stat * st) // This is how we get our INODE (equivalent) information ULARGE_INTEGER conv; conv.HighPart = fi.nFileIndexHigh; - conv.LowPart = fi.nFileIndexLow; - st->st_ino = (_ino_t)conv.QuadPart; + conv.LowPart = fi.nFileIndexLow; + st->st_ino = conv.QuadPart; // get the time information st->st_ctime = ConvertFileTimeToTime_t(&fi.ftCreationTime); @@ -716,20 +626,8 @@ int emu_fstat(HANDLE hdir, struct stat * st) } else { - // size of the file - LARGE_INTEGER st_size; - memset(&st_size, 0, sizeof(st_size)); - - if (!GetFileSizeEx(hdir, &st_size)) - { - ::syslog(LOG_WARNING, "Failed to get file size: " - "%s", GetErrorMessage(GetLastError()).c_str()); - errno = EACCES; - return -1; - } - - conv.HighPart = st_size.HighPart; - conv.LowPart = st_size.LowPart; + conv.HighPart = fi.nFileSizeHigh; + conv.LowPart = fi.nFileSizeLow; st->st_size = (_off_t)conv.QuadPart; } @@ -862,12 +760,14 @@ HANDLE OpenFileByNameUtf8(const char* pFileName, DWORD flags) // // Function // Name: emu_stat -// Purpose: replacement for the lstat and stat functions, -// works with unicode filenames supplied in utf8 format +// Purpose: replacement for the lstat and stat functions. +// Works with unicode filenames supplied in utf8. +// Returns a struct emu_stat to have room for 64-bit +// file identifier in st_ino (mingw allows only 16!) // Created: 25th October 2004 // // -------------------------------------------------------------------------- -int emu_stat(const char * pName, struct stat * st) +int emu_stat(const char * pName, struct emu_stat * st) { HANDLE handle = OpenFileByNameUtf8(pName, FILE_READ_ATTRIBUTES | FILE_READ_EA); @@ -1078,7 +978,7 @@ DIR *opendir(const char *name) } pDir->name = ConvertUtf8ToWideString(dirName.c_str()); - // We are responsible for freeing dir->name + // We are responsible for freeing dir->name with delete[] if (pDir->name == NULL) { @@ -1396,16 +1296,20 @@ static bool sHaveWarnedEventLogFull = false; void openlog(const char * daemonName, int, int) { + std::string nameStr = "Box Backup ("; + nameStr += daemonName; + nameStr += ")"; + // register a default event source, so that we can // log errors with the process of adding or registering our own. gSyslogH = RegisterEventSource( NULL, // uses local computer - daemonName); // source name + nameStr.c_str()); // source name if (gSyslogH == NULL) { } - char* name = strdup(daemonName); + char* name = strdup(nameStr.c_str()); BOOL success = AddEventSource(name, 0); free(name); @@ -1415,7 +1319,7 @@ void openlog(const char * daemonName, int, int) return; } - HANDLE newSyslogH = RegisterEventSource(NULL, daemonName); + HANDLE newSyslogH = RegisterEventSource(NULL, nameStr.c_str()); if (newSyslogH == NULL) { ::syslog(LOG_ERR, "Failed to register our own event source: " @@ -1484,8 +1388,6 @@ void syslog(int loglevel, const char *frmt, ...) va_end(args); - LPCSTR strings[] = { buffer, NULL }; - if (gSyslogH == 0) { printf("%s\r\n", buffer); @@ -1493,17 +1395,45 @@ void syslog(int loglevel, const char *frmt, ...) return; } - if (!ReportEvent(gSyslogH, // event log handle - errinfo, // event type - 0, // category zero - MSG_ERR, // event identifier - - // we will call them all the same - NULL, // no user security identifier - 1, // one substitution string - 0, // no data - strings, // pointer to string array - NULL)) // pointer to data + WCHAR* pWide = ConvertToWideString(buffer, CP_UTF8, false); + // must delete[] pWide + DWORD result; + + if (pWide == NULL) + { + std::string buffer2 = buffer; + buffer2 += " (failed to convert string encoding)"; + LPCSTR strings[] = { buffer2.c_str(), NULL }; + + result = ReportEventA(gSyslogH, // event log handle + errinfo, // event type + 0, // category zero + MSG_ERR, // event identifier - + // we will call them all the same + NULL, // no user security identifier + 1, // one substitution string + 0, // no data + strings, // pointer to string array + NULL); // pointer to data + } + else + { + LPCWSTR strings[] = { pWide, NULL }; + result = ReportEventW(gSyslogH, // event log handle + errinfo, // event type + 0, // category zero + MSG_ERR, // event identifier - + // we will call them all the same + NULL, // no user security identifier + 1, // one substitution string + 0, // no data + strings, // pointer to string array + NULL); // pointer to data + delete [] pWide; + } + + if (result == 0) { DWORD err = GetLastError(); if (err == ERROR_LOG_FILE_FULL) @@ -1527,9 +1457,6 @@ void syslog(int loglevel, const char *frmt, ...) { sHaveWarnedEventLogFull = false; } - - // printf("%s\r\n", buffer); - // fflush(stdout); } int emu_chdir(const char* pDirName) diff --git a/lib/win32/emu.h b/lib/win32/emu.h index 8ab74130..811e6495 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -1,5 +1,17 @@ // emulates unix syscalls to win32 functions +#ifdef WIN32 + #define EMU_STRUCT_STAT struct emu_stat + #define EMU_STAT emu_stat + #define EMU_FSTAT emu_fstat + #define EMU_LSTAT emu_stat +#else + #define EMU_STRUCT_STAT struct stat + #define EMU_STAT ::stat + #define EMU_FSTAT ::fstat + #define EMU_LSTAT ::lstat +#endif + #if ! defined EMU_INCLUDE && defined WIN32 #define EMU_INCLUDE @@ -28,11 +40,6 @@ #else typedef unsigned int mode_t; typedef unsigned int pid_t; - - // must define _INO_T_DEFINED before including <sys/types.h> - // to replace it with our own. - typedef u_int64_t _ino_t; - #define _INO_T_DEFINED #endif // set up to include the necessary parts of Windows headers @@ -78,11 +85,6 @@ #define fileno(struct_file) _fileno(struct_file) #endif -int SetTimerHandler(void (__cdecl *func ) (int)); -int setitimer(int type, struct itimerval *timeout, void *arg); -void InitTimer(void); -void FiniTimer(void); - struct passwd { char *pw_name; char *pw_passwd; @@ -184,13 +186,6 @@ inline int geteuid(void) #define timespec timeval -//not available in win32 -struct itimerval -{ - timeval it_interval; - timeval it_value; -}; - //win32 deals in usec not nsec - so need to ensure this follows through #define tv_nsec tv_usec @@ -205,6 +200,7 @@ struct itimerval //again need to verify these #define S_IFLNK 1 +#define S_IFSOCK 0 #define S_ISLNK(x) ( false ) @@ -244,6 +240,7 @@ int closedir(DIR *dp); // local constant to open file exclusively without shared access #define O_LOCK 0x10000 +extern DWORD winerrno; /* used to report errors from openfile() */ HANDLE openfile(const char *filename, int flags, int mode); #define LOG_DEBUG LOG_INFO @@ -260,6 +257,15 @@ void openlog (const char * daemonName, int, int); void closelog(void); void syslog (int loglevel, const char *fmt, ...); +#define LOG_LOCAL0 0 +#define LOG_LOCAL1 0 +#define LOG_LOCAL2 0 +#define LOG_LOCAL3 0 +#define LOG_LOCAL4 0 +#define LOG_LOCAL5 0 +#define LOG_LOCAL6 0 +#define LOG_DAEMON 0 + #ifndef __MINGW32__ #define strtoll _strtoi64 #endif @@ -292,6 +298,11 @@ inline int ioctl(SOCKET sock, int flag, int * something) return 0; } +extern "C" inline int getpid() +{ + return (int)GetCurrentProcessId(); +} + inline int waitpid(pid_t pid, int *status, int) { return 0; @@ -303,27 +314,19 @@ struct statfs TCHAR f_mntonname[MAX_PATH]; }; -#if 0 -// I think this should get us going -// Although there is a warning about -// mount points in win32 can now exists - which means inode number can be -// duplicated, so potential of a problem - perhaps this needs to be -// implemented with a little more thought... TODO - -struct stat { - //_dev_t st_dev; - u_int64_t st_ino; +struct emu_stat { + int st_dev; + uint64_t st_ino; DWORD st_mode; short st_nlink; short st_uid; short st_gid; //_dev_t st_rdev; - u_int64_t st_size; + uint64_t st_size; time_t st_atime; time_t st_mtime; time_t st_ctime; }; -#endif // 0 // need this for conversions time_t ConvertFileTimeToTime_t(FILETIME *fileTime); @@ -332,8 +335,8 @@ bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo); int emu_chdir (const char* pDirName); int emu_mkdir (const char* pPathName); int emu_unlink (const char* pFileName); -int emu_fstat (HANDLE file, struct stat* st); -int emu_stat (const char* pName, struct stat* st); +int emu_fstat (HANDLE file, struct emu_stat* st); +int emu_stat (const char* pName, struct emu_stat* st); int emu_utimes (const char* pName, const struct timeval[]); int emu_chmod (const char* pName, mode_t mode); char* emu_getcwd (char* pBuffer, int BufSize); @@ -342,14 +345,22 @@ int emu_rename (const char* pOldName, const char* pNewName); #define chdir(directory) emu_chdir (directory) #define mkdir(path, mode) emu_mkdir (path) #define unlink(file) emu_unlink (file) -#define stat(filename, struct) emu_stat (filename, struct) -#define lstat(filename, struct) emu_stat (filename, struct) -#define fstat(handle, struct) emu_fstat (handle, struct) #define utimes(buffer, times) emu_utimes (buffer, times) #define chmod(file, mode) emu_chmod (file, mode) #define getcwd(buffer, size) emu_getcwd (buffer, size) #define rename(oldname, newname) emu_rename (oldname, newname) +// Not safe to replace stat/fstat/lstat on mingw at least, as struct stat +// has a 16-bit st_ino and we need a 64-bit one. +// +// #define stat(filename, struct) emu_stat (filename, struct) +// #define lstat(filename, struct) emu_stat (filename, struct) +// #define fstat(handle, struct) emu_fstat (handle, struct) +// +// But lstat doesn't exist on Windows, so we have to provide something: + +#define lstat(filename, struct) stat(filename, struct) + int statfs(const char * name, struct statfs * s); int poll(struct pollfd *ufds, unsigned long nfds, int timeout); @@ -374,8 +385,8 @@ bool ConvertToUtf8 (const std::string& rSource, std::string& rDest, int sourceCodePage); bool ConvertFromUtf8 (const std::string& rSource, std::string& rDest, int destCodePage); -bool ConvertUtf8ToConsole(const char* pString, std::string& rDest); -bool ConvertConsoleToUtf8(const char* pString, std::string& rDest); +bool ConvertUtf8ToConsole(const std::string& rSource, std::string& rDest); +bool ConvertConsoleToUtf8(const std::string& rSource, std::string& rDest); // Utility function which returns a default config file name, // based on the path of the current executable. diff --git a/lib/win32/getopt_long.cxx b/lib/win32/getopt_long.cpp index a24930aa..5d910e1b 100755 --- a/lib/win32/getopt_long.cxx +++ b/lib/win32/getopt_long.cpp @@ -69,7 +69,6 @@ #if defined _MSC_VER || defined __MINGW32__
#define REPLACE_GETOPT /* use this getopt as the system getopt(3) */
-#endif
#ifdef REPLACE_GETOPT
int opterr = 1; /* if error message should be printed */
@@ -548,3 +547,4 @@ getopt_long_only(int nargc, char * const *nargv, const char *options, FLAG_PERMUTE|FLAG_LONGONLY));
}
+#endif // defined _MSC_VER || defined __MINGW32__
|