From a84d45498bd861c9225080232948a99c2e317bb8 Mon Sep 17 00:00:00 2001 From: Reinhard Tartler Date: Thu, 2 Apr 2009 13:58:11 +0200 Subject: Import upstream version 0.11~rc3~r2491 --- .hgignore | 37 + .svnrevision | 2 +- LICENSE.txt | 4 +- bin/bbackupctl/bbackupctl.cpp | 19 +- bin/bbackupd/BackupClientContext.cpp | 27 +- bin/bbackupd/BackupClientContext.h | 19 +- bin/bbackupd/BackupClientDeleteList.cpp | 80 +- bin/bbackupd/BackupClientDeleteList.h | 34 +- bin/bbackupd/BackupClientDirectoryRecord.cpp | 576 ++-- bin/bbackupd/BackupClientDirectoryRecord.h | 159 +- bin/bbackupd/BackupClientInodeToIDMap.cpp | 11 +- bin/bbackupd/BackupDaemon.cpp | 1699 +++++----- bin/bbackupd/BackupDaemon.h | 148 +- bin/bbackupd/BackupDaemonInterface.h | 164 + bin/bbackupd/Win32ServiceFunctions.cpp | 8 +- bin/bbackupd/bbackupd-config.in | 6 +- bin/bbackupd/bbackupd.cpp | 2 +- bin/bbackupd/win32/NotifySysAdmin.vbs | 62 +- bin/bbackupd/win32/bbackupd.conf | 2 +- bin/bbackupquery/BackupQueries.cpp | 1240 +++---- bin/bbackupquery/BackupQueries.h | 280 +- bin/bbackupquery/BoxBackupCompareParams.h | 107 + bin/bbackupquery/bbackupquery.cpp | 79 +- bin/bbackupquery/documentation.txt | 22 +- bin/bbstoreaccounts/bbstoreaccounts.cpp | 112 +- bin/bbstored/BBStoreDHousekeeping.cpp | 16 +- bin/bbstored/BackupCommands.cpp | 165 +- bin/bbstored/BackupConstants.h | 2 - bin/bbstored/BackupContext.cpp | 1661 ---------- bin/bbstored/BackupContext.h | 149 - bin/bbstored/BackupStoreContext.cpp | 1752 ++++++++++ bin/bbstored/BackupStoreContext.h | 183 ++ bin/bbstored/BackupStoreDaemon.cpp | 34 +- bin/bbstored/BackupStoreDaemon.h | 15 +- bin/bbstored/HousekeepStoreAccount.cpp | 274 +- bin/bbstored/HousekeepStoreAccount.h | 3 + bin/bbstored/backupprotocol.txt | 8 +- bin/bbstored/bbstored-config.in | 4 +- bin/bbstored/bbstored.cpp | 2 +- cleanupforcvs.pl | 4 +- configure.ac | 105 +- contrib/bbadmin/accounts.cgi | 580 ++++ contrib/bbadmin/apache.conf | 14 + contrib/bbadmin/bb.css | 70 + contrib/bbreporter/LICENSE | 674 ++++ contrib/bbreporter/bbreporter.py | 538 ++++ contrib/debian/bbackupd.in | 63 +- contrib/debian/bbstored.in | 57 +- contrib/mac_osx/org.boxbackup.bbackupd.plist.in | 20 + contrib/mac_osx/org.boxbackup.bbstored.plist.in | 21 + contrib/redhat/bbackupd.in | 8 +- contrib/redhat/bbstored.in | 8 +- contrib/rpm/boxbackup.spec | 20 +- contrib/solaris/bbackupd-smf-method.in | 2 +- contrib/solaris/bbstored-smf-method.in | 2 +- contrib/suse/bbackupd.in | 4 +- contrib/suse/bbstored.in | 4 +- contrib/windows/installer/boxbackup.mpi.in | 3392 ++++++++++++++++++++ contrib/windows/installer/tools/InstallService.bat | 3 + .../windows/installer/tools/KillBackupProcess.bat | 3 + contrib/windows/installer/tools/QueryOutputAll.bat | 5 + .../windows/installer/tools/QueryOutputCurrent.bat | 5 + contrib/windows/installer/tools/ReloadConfig.bat | 3 + contrib/windows/installer/tools/RemoteControl.exe | Bin 0 -> 156536 bytes contrib/windows/installer/tools/RemoveService.bat | 3 + contrib/windows/installer/tools/RestartService.bat | 5 + contrib/windows/installer/tools/ShowUsage.bat | 3 + contrib/windows/installer/tools/StartService.bat | 3 + contrib/windows/installer/tools/StopService.bat | 3 + contrib/windows/installer/tools/Sync.bat | 3 + distribution/COMMON-MANIFEST.txt | 17 +- distribution/boxbackup/CONTACT.txt | 6 +- distribution/boxbackup/DISTRIBUTION-MANIFEST.txt | 42 +- distribution/boxbackup/DOCUMENTATION.txt | 2 +- distribution/boxbackup/VERSION.txt | 2 +- docs/Makefile | 129 + docs/api-notes/INDEX.txt | 61 + docs/api-notes/Win32_Clients.txt | 13 + docs/api-notes/backup_encryption.txt | 109 + docs/api-notes/bin_bbackupd.txt | 88 + docs/api-notes/bin_bbstored.txt | 54 + docs/api-notes/common/lib_common.txt | 52 + docs/api-notes/common/lib_common/BoxTime.txt | 7 + .../common/lib_common/CollectInBufferStream.txt | 26 + docs/api-notes/common/lib_common/Configuration.txt | 102 + docs/api-notes/common/lib_common/Conversion.txt | 14 + docs/api-notes/common/lib_common/ExcludeList.txt | 21 + docs/api-notes/common/lib_common/FdGetLine.txt | 11 + docs/api-notes/common/lib_common/Guards.txt | 5 + docs/api-notes/common/lib_common/IOStream.txt | 89 + .../common/lib_common/IOStreamGetLine.txt | 29 + docs/api-notes/common/lib_common/MainHelper.txt | 4 + docs/api-notes/common/lib_common/WaitForEvent.txt | 16 + docs/api-notes/common/lib_common/xStream.txt | 40 + docs/api-notes/common/lib_compress.txt | 8 + .../common/lib_compress/CompressStream.txt | 27 + docs/api-notes/common/lib_crypto.txt | 28 + docs/api-notes/common/lib_crypto/CipherContext.txt | 28 + .../common/lib_crypto/RollingChecksum.txt | 36 + docs/api-notes/common/lib_server.txt | 9 + docs/api-notes/common/lib_server/Daemon.txt | 96 + docs/api-notes/common/lib_server/Protocol.txt | 120 + docs/api-notes/common/lib_server/ServerStream.txt | 29 + docs/api-notes/common/lib_server/ServerTLS.txt | 6 + docs/api-notes/common/lib_server/SocketStream.txt | 8 + .../common/lib_server/SocketStreamTLS.txt | 11 + docs/api-notes/common/lib_server/TLSContext.txt | 16 + docs/api-notes/common/memory_leaks.txt | 44 + docs/api-notes/encrypt_rsync.txt | 66 + docs/api-notes/lib_backupclient.txt | 46 + docs/api-notes/lib_backupstore.txt | 30 + docs/api-notes/raidfile/RaidFileRead.txt | 14 + docs/api-notes/raidfile/RaidFileWrite.txt | 36 + docs/api-notes/raidfile/lib_raidfile.txt | 73 + .../win32_build_on_cygwin_using_mingw.txt | 107 + .../api-notes/win32_build_on_linux_using_mingw.txt | 108 + docs/api-notes/windows_porting.txt | 100 + docs/backup/INDEX.txt | 61 - docs/backup/Win32_Clients.txt | 13 - docs/backup/backup_encryption.txt | 109 - docs/backup/bin_bbackupd.txt | 88 - docs/backup/bin_bbstored.txt | 54 - docs/backup/encrypt_rsync.txt | 66 - docs/backup/lib_backupclient.txt | 46 - docs/backup/lib_backupstore.txt | 30 - docs/backup/win32_build_on_cygwin_using_mingw.txt | 53 - docs/backup/win32_build_on_linux_using_mingw.txt | 64 - docs/backup/windows_porting.txt | 100 - docs/common/lib_common.txt | 52 - docs/common/lib_common/BoxTime.txt | 7 - docs/common/lib_common/CollectInBufferStream.txt | 26 - docs/common/lib_common/Configuration.txt | 102 - docs/common/lib_common/Conversion.txt | 14 - docs/common/lib_common/ExcludeList.txt | 21 - docs/common/lib_common/FdGetLine.txt | 11 - docs/common/lib_common/Guards.txt | 5 - docs/common/lib_common/IOStream.txt | 89 - docs/common/lib_common/IOStreamGetLine.txt | 29 - docs/common/lib_common/MainHelper.txt | 4 - docs/common/lib_common/WaitForEvent.txt | 16 - docs/common/lib_common/xStream.txt | 40 - docs/common/lib_compress.txt | 8 - docs/common/lib_compress/CompressStream.txt | 27 - docs/common/lib_crypto.txt | 28 - docs/common/lib_crypto/CipherContext.txt | 28 - docs/common/lib_crypto/RollingChecksum.txt | 36 - docs/common/lib_server.txt | 9 - docs/common/lib_server/Daemon.txt | 96 - docs/common/lib_server/Protocol.txt | 120 - docs/common/lib_server/ServerStream.txt | 29 - docs/common/lib_server/ServerTLS.txt | 6 - docs/common/lib_server/SocketStream.txt | 8 - docs/common/lib_server/SocketStreamTLS.txt | 11 - docs/common/lib_server/TLSContext.txt | 16 - docs/common/memory_leaks.txt | 44 - docs/docbook/adminguide.xml | 1981 ++++++++++++ docs/docbook/bb-book.xsl | 17 + docs/docbook/bb-man.xsl.tmpl | 9 + docs/docbook/bb-nochunk-book.xsl | 17 + docs/docbook/bbackupctl.xml | 205 ++ docs/docbook/bbackupd-config.xml | 153 + docs/docbook/bbackupd.conf.xml | 479 +++ docs/docbook/bbackupd.xml | 209 ++ docs/docbook/bbackupquery.xml | 506 +++ docs/docbook/bbstoreaccounts.xml | 386 +++ docs/docbook/bbstored-certs.xml | 180 ++ docs/docbook/bbstored-config.xml | 148 + docs/docbook/bbstored.conf.xml | 211 ++ docs/docbook/bbstored.xml | 90 + docs/docbook/html/bbdoc-man.css | 104 + docs/docbook/html/bbdoc.css | 112 + docs/docbook/html/favicon.ico | Bin 0 -> 67646 bytes docs/docbook/html/images/arrow.png | Bin 0 -> 197 bytes docs/docbook/html/images/bblogo.png | Bin 0 -> 5882 bytes docs/docbook/html/images/stepahead.png | Bin 0 -> 298 bytes docs/docbook/instguide.xml | 766 +++++ docs/docbook/raidfile-config.xml | 198 ++ docs/docbook/raidfile.conf.xml | 143 + docs/images/bblogo-alpha.xcf | Bin 0 -> 93980 bytes docs/raidfile/lib_raidfile.txt | 73 - docs/raidfile/lib_raidfile/RaidFileRead.txt | 14 - docs/raidfile/lib_raidfile/RaidFileWrite.txt | 36 - docs/tools/generate_except_xml.pl | 74 + documentation/Makefile | 66 - documentation/adminguide.xml | 1981 ------------ documentation/bb-book.xsl | 17 - documentation/bb-man.xsl | 9 - documentation/bb-nochunk-book.xsl | 17 - documentation/bbackupctl.xml | 147 - documentation/bbackupquery.xml | 380 --- documentation/bbstoreaccounts.xml | 290 -- documentation/bbstored-certs.xml | 125 - documentation/bbstored-config.xml | 140 - documentation/generate_except_xml.pl | 74 - documentation/html/bbdoc-man.css | 104 - documentation/html/bbdoc.css | 112 - documentation/html/images/arrow.png | Bin 197 -> 0 bytes documentation/html/images/bblogo.png | Bin 5882 -> 0 bytes documentation/html/images/stepahead.png | Bin 298 -> 0 bytes documentation/instguide.xml | 766 ----- documentation/raidfile-config.xml | 143 - infrastructure/BoxPlatform.pm.in | 50 +- infrastructure/buildenv-testmain-template.cpp | 103 +- infrastructure/m4/ax_bswap64.m4 | 12 +- infrastructure/m4/ax_check_define_pragma.m4 | 6 +- infrastructure/m4/ax_check_dirent_d_type.m4 | 12 +- infrastructure/m4/ax_check_malloc_workaround.m4 | 12 +- infrastructure/m4/ax_check_nonaligned_access.m4 | 18 +- infrastructure/m4/ax_check_syscall_lseek.m4 | 12 +- infrastructure/m4/ax_func_syscall.m4 | 23 +- infrastructure/makebuildenv.pl.in | 125 +- infrastructure/makedistribution.pl.in | 67 +- infrastructure/makeparcels.pl.in | 185 +- infrastructure/msvc/2003/bbackupctl.vcproj | 4 +- infrastructure/msvc/2003/bbackupd.vcproj | 4 +- infrastructure/msvc/2003/boxquery.vcproj | 4 +- infrastructure/msvc/2003/common.vcproj | 4 +- infrastructure/msvc/2003/win32test.vcproj | 2 +- infrastructure/msvc/2005/bbackupctl.vcproj | 4 +- infrastructure/msvc/2005/bbackupd.vcproj | 4 +- infrastructure/msvc/2005/boxquery.vcproj | 6 +- infrastructure/msvc/2005/common.vcproj | 4 +- infrastructure/msvc/2005/win32test.vcproj | 4 +- infrastructure/parcelpath.pl | 17 + infrastructure/printversion.pl | 12 + lib/backupclient/BackupClientCryptoKeys.cpp | 52 +- lib/backupclient/BackupClientCryptoKeys.h | 2 +- lib/backupclient/BackupClientFileAttributes.cpp | 64 +- lib/backupclient/BackupClientFileAttributes.h | 8 +- lib/backupclient/BackupClientRestore.cpp | 289 +- lib/backupclient/BackupClientRestore.h | 19 +- lib/backupclient/BackupDaemonConfigVerify.cpp | 97 +- lib/backupclient/BackupStoreFile.cpp | 27 +- lib/backupclient/BackupStoreFile.h | 17 +- lib/backupclient/BackupStoreFileDiff.cpp | 69 +- lib/backupclient/BackupStoreFileEncodeStream.cpp | 69 +- lib/backupclient/BackupStoreFileEncodeStream.h | 10 +- lib/backupclient/BackupStoreFilename.cpp | 32 +- lib/backupclient/BackupStoreFilename.h | 24 +- lib/backupclient/BackupStoreFilenameClear.cpp | 23 +- lib/backupclient/BackupStoreObjectDump.cpp | 21 +- lib/backupclient/RunStatusProvider.h | 29 + lib/backupstore/BackupStoreAccountDatabase.cpp | 4 +- lib/backupstore/BackupStoreCheck.cpp | 3 +- lib/backupstore/BackupStoreCheck.h | 4 +- lib/backupstore/BackupStoreCheck2.cpp | 271 +- lib/backupstore/BackupStoreCheckData.cpp | 15 +- lib/backupstore/BackupStoreConfigVerify.cpp | 17 +- lib/backupstore/BackupStoreInfo.cpp | 4 +- lib/backupstore/StoreStructure.h | 2 +- lib/common/Box.h | 59 +- lib/common/BoxException.h | 1 + lib/common/BoxPlatform.h | 37 +- lib/common/BoxTime.cpp | 46 +- lib/common/BoxTime.h | 3 + lib/common/Configuration.cpp | 335 +- lib/common/Configuration.h | 76 +- lib/common/DebugAssertFailed.cpp | 4 +- lib/common/DebugMemLeakFinder.cpp | 58 +- lib/common/DebugPrintf.cpp | 4 +- lib/common/EventWatchFilesystemObject.cpp | 43 +- lib/common/FdGetLine.h | 2 +- lib/common/FileModificationTime.h | 23 +- lib/common/FileStream.cpp | 110 +- lib/common/FileStream.h | 21 +- lib/common/Guards.h | 4 +- lib/common/IOStream.cpp | 36 +- lib/common/IOStream.h | 10 +- lib/common/IOStreamGetLine.h | 2 +- lib/common/Logging.cpp | 233 +- lib/common/Logging.h | 138 +- lib/common/NamedLock.cpp | 8 +- lib/common/PartialReadStream.cpp | 3 +- lib/common/ReadLoggingStream.cpp | 28 +- lib/common/ReadLoggingStream.h | 19 +- lib/common/SelfFlushingStream.h | 71 + lib/common/Test.cpp | 418 +++ lib/common/Test.h | 430 +-- lib/common/Timer.cpp | 308 +- lib/common/Timer.h | 36 +- lib/common/Utils.cpp | 179 +- lib/common/Utils.h | 5 + lib/common/WaitForEvent.h | 2 + lib/common/makeexception.pl.in | 12 +- lib/compress/Compress.h | 8 +- lib/compress/CompressStream.cpp | 4 +- lib/crypto/CipherContext.cpp | 15 +- lib/httpserver/HTTPException.txt | 16 + lib/httpserver/HTTPQueryDecoder.cpp | 159 + lib/httpserver/HTTPQueryDecoder.h | 47 + lib/httpserver/HTTPRequest.cpp | 783 +++++ lib/httpserver/HTTPRequest.h | 174 + lib/httpserver/HTTPResponse.cpp | 648 ++++ lib/httpserver/HTTPResponse.h | 175 + lib/httpserver/HTTPServer.cpp | 249 ++ lib/httpserver/HTTPServer.h | 79 + lib/httpserver/Makefile.extra | 7 + lib/httpserver/S3Client.cpp | 243 ++ lib/httpserver/S3Client.h | 72 + lib/httpserver/cdecode.cpp | 92 + lib/httpserver/cdecode.h | 28 + lib/httpserver/cencode.cpp | 113 + lib/httpserver/cencode.h | 32 + lib/httpserver/decode.h | 77 + lib/httpserver/encode.h | 87 + lib/intercept/intercept.cpp | 141 +- lib/intercept/intercept.h | 21 +- lib/raidfile/RaidFileController.cpp | 17 +- lib/raidfile/RaidFileRead.cpp | 8 +- lib/raidfile/RaidFileUtil.cpp | 58 +- lib/raidfile/RaidFileWrite.cpp | 11 +- lib/server/Daemon.cpp | 328 +- lib/server/Daemon.h | 28 +- lib/server/OverlappedIO.h | 42 + lib/server/Protocol.cpp | 23 +- lib/server/ProtocolUncertainStream.cpp | 21 +- lib/server/SSLLib.cpp | 44 +- lib/server/SSLLib.h | 4 +- lib/server/ServerControl.cpp | 227 ++ lib/server/ServerControl.h | 188 +- lib/server/ServerStream.h | 22 +- lib/server/ServerTLS.h | 12 +- lib/server/Socket.cpp | 15 +- lib/server/Socket.h | 6 +- lib/server/SocketListen.h | 85 +- lib/server/SocketStream.cpp | 101 +- lib/server/SocketStream.h | 3 +- lib/server/SocketStreamTLS.cpp | 21 +- lib/server/SocketStreamTLS.h | 3 +- lib/server/TLSContext.cpp | 14 +- lib/server/WinNamedPipeListener.h | 232 ++ lib/server/WinNamedPipeStream.cpp | 153 +- lib/server/WinNamedPipeStream.h | 7 +- lib/server/makeprotocol.pl.in | 161 +- lib/win32/emu.cpp | 363 +-- lib/win32/emu.h | 83 +- lib/win32/getopt_long.cpp | 550 ++++ lib/win32/getopt_long.cxx | 550 ---- modules.txt | 4 + parcels.txt | 25 +- runtest.pl.in | 21 + test/backupdiff/testbackupdiff.cpp | 83 +- test/backupstore/testbackupstore.cpp | 52 +- test/backupstorefix/testbackupstorefix.cpp | 9 +- .../testfiles/testbackupstorefix.pl.in | 16 +- test/basicserver/testbasicserver.cpp | 4 +- test/bbackupd/Makefile.extra | 15 +- test/bbackupd/testbbackupd.cpp | 1751 +++++++--- test/bbackupd/testfiles/bbackupd-exclude.conf.in | 47 + test/bbackupd/testfiles/bbackupd-snapshot.conf.in | 56 + test/bbackupd/testfiles/bbackupd-symlink.conf.in | 55 + test/bbackupd/testfiles/bbackupd-temploc.conf | 3 +- test/bbackupd/testfiles/bbackupd.conf.in | 1 + test/bbackupd/testfiles/bbstored.conf | 2 +- test/bbackupd/testfiles/extcheck1.pl.in | 5 +- test/bbackupd/testfiles/extcheck2.pl.in | 5 +- test/bbackupd/testfiles/notifyscript.pl.in | 11 +- test/common/testcommon.cpp | 57 +- test/compress/testcompress.cpp | 3 +- test/httpserver/testfiles/httpserver.conf | 8 + test/httpserver/testfiles/photos/puppy.jpg | 1 + test/httpserver/testfiles/testrequests.pl | 143 + test/httpserver/testhttpserver.cpp | 661 ++++ test/raidfile/testraidfile.cpp | 3 +- 364 files changed, 29736 insertions(+), 13410 deletions(-) create mode 100644 .hgignore create mode 100644 bin/bbackupd/BackupDaemonInterface.h create mode 100644 bin/bbackupquery/BoxBackupCompareParams.h delete mode 100644 bin/bbstored/BackupContext.cpp delete mode 100644 bin/bbstored/BackupContext.h create mode 100644 bin/bbstored/BackupStoreContext.cpp create mode 100644 bin/bbstored/BackupStoreContext.h create mode 100755 contrib/bbadmin/accounts.cgi create mode 100644 contrib/bbadmin/apache.conf create mode 100644 contrib/bbadmin/bb.css create mode 100644 contrib/bbreporter/LICENSE create mode 100755 contrib/bbreporter/bbreporter.py create mode 100644 contrib/mac_osx/org.boxbackup.bbackupd.plist.in create mode 100644 contrib/mac_osx/org.boxbackup.bbstored.plist.in create mode 100755 contrib/windows/installer/boxbackup.mpi.in create mode 100755 contrib/windows/installer/tools/InstallService.bat create mode 100755 contrib/windows/installer/tools/KillBackupProcess.bat create mode 100755 contrib/windows/installer/tools/QueryOutputAll.bat create mode 100755 contrib/windows/installer/tools/QueryOutputCurrent.bat create mode 100755 contrib/windows/installer/tools/ReloadConfig.bat create mode 100755 contrib/windows/installer/tools/RemoteControl.exe create mode 100755 contrib/windows/installer/tools/RemoveService.bat create mode 100755 contrib/windows/installer/tools/RestartService.bat create mode 100755 contrib/windows/installer/tools/ShowUsage.bat create mode 100755 contrib/windows/installer/tools/StartService.bat create mode 100755 contrib/windows/installer/tools/StopService.bat create mode 100755 contrib/windows/installer/tools/Sync.bat create mode 100644 docs/Makefile create mode 100644 docs/api-notes/INDEX.txt create mode 100644 docs/api-notes/Win32_Clients.txt create mode 100644 docs/api-notes/backup_encryption.txt create mode 100644 docs/api-notes/bin_bbackupd.txt create mode 100644 docs/api-notes/bin_bbstored.txt create mode 100644 docs/api-notes/common/lib_common.txt create mode 100644 docs/api-notes/common/lib_common/BoxTime.txt create mode 100644 docs/api-notes/common/lib_common/CollectInBufferStream.txt create mode 100644 docs/api-notes/common/lib_common/Configuration.txt create mode 100644 docs/api-notes/common/lib_common/Conversion.txt create mode 100644 docs/api-notes/common/lib_common/ExcludeList.txt create mode 100644 docs/api-notes/common/lib_common/FdGetLine.txt create mode 100644 docs/api-notes/common/lib_common/Guards.txt create mode 100644 docs/api-notes/common/lib_common/IOStream.txt create mode 100644 docs/api-notes/common/lib_common/IOStreamGetLine.txt create mode 100644 docs/api-notes/common/lib_common/MainHelper.txt create mode 100644 docs/api-notes/common/lib_common/WaitForEvent.txt create mode 100644 docs/api-notes/common/lib_common/xStream.txt create mode 100644 docs/api-notes/common/lib_compress.txt create mode 100644 docs/api-notes/common/lib_compress/CompressStream.txt create mode 100644 docs/api-notes/common/lib_crypto.txt create mode 100644 docs/api-notes/common/lib_crypto/CipherContext.txt create mode 100644 docs/api-notes/common/lib_crypto/RollingChecksum.txt create mode 100644 docs/api-notes/common/lib_server.txt create mode 100644 docs/api-notes/common/lib_server/Daemon.txt create mode 100644 docs/api-notes/common/lib_server/Protocol.txt create mode 100644 docs/api-notes/common/lib_server/ServerStream.txt create mode 100644 docs/api-notes/common/lib_server/ServerTLS.txt create mode 100644 docs/api-notes/common/lib_server/SocketStream.txt create mode 100644 docs/api-notes/common/lib_server/SocketStreamTLS.txt create mode 100644 docs/api-notes/common/lib_server/TLSContext.txt create mode 100644 docs/api-notes/common/memory_leaks.txt create mode 100644 docs/api-notes/encrypt_rsync.txt create mode 100644 docs/api-notes/lib_backupclient.txt create mode 100644 docs/api-notes/lib_backupstore.txt create mode 100644 docs/api-notes/raidfile/RaidFileRead.txt create mode 100644 docs/api-notes/raidfile/RaidFileWrite.txt create mode 100644 docs/api-notes/raidfile/lib_raidfile.txt create mode 100644 docs/api-notes/win32_build_on_cygwin_using_mingw.txt create mode 100644 docs/api-notes/win32_build_on_linux_using_mingw.txt create mode 100644 docs/api-notes/windows_porting.txt delete mode 100644 docs/backup/INDEX.txt delete mode 100644 docs/backup/Win32_Clients.txt delete mode 100644 docs/backup/backup_encryption.txt delete mode 100644 docs/backup/bin_bbackupd.txt delete mode 100644 docs/backup/bin_bbstored.txt delete mode 100644 docs/backup/encrypt_rsync.txt delete mode 100644 docs/backup/lib_backupclient.txt delete mode 100644 docs/backup/lib_backupstore.txt delete mode 100644 docs/backup/win32_build_on_cygwin_using_mingw.txt delete mode 100644 docs/backup/win32_build_on_linux_using_mingw.txt delete mode 100644 docs/backup/windows_porting.txt delete mode 100644 docs/common/lib_common.txt delete mode 100644 docs/common/lib_common/BoxTime.txt delete mode 100644 docs/common/lib_common/CollectInBufferStream.txt delete mode 100644 docs/common/lib_common/Configuration.txt delete mode 100644 docs/common/lib_common/Conversion.txt delete mode 100644 docs/common/lib_common/ExcludeList.txt delete mode 100644 docs/common/lib_common/FdGetLine.txt delete mode 100644 docs/common/lib_common/Guards.txt delete mode 100644 docs/common/lib_common/IOStream.txt delete mode 100644 docs/common/lib_common/IOStreamGetLine.txt delete mode 100644 docs/common/lib_common/MainHelper.txt delete mode 100644 docs/common/lib_common/WaitForEvent.txt delete mode 100644 docs/common/lib_common/xStream.txt delete mode 100644 docs/common/lib_compress.txt delete mode 100644 docs/common/lib_compress/CompressStream.txt delete mode 100644 docs/common/lib_crypto.txt delete mode 100644 docs/common/lib_crypto/CipherContext.txt delete mode 100644 docs/common/lib_crypto/RollingChecksum.txt delete mode 100644 docs/common/lib_server.txt delete mode 100644 docs/common/lib_server/Daemon.txt delete mode 100644 docs/common/lib_server/Protocol.txt delete mode 100644 docs/common/lib_server/ServerStream.txt delete mode 100644 docs/common/lib_server/ServerTLS.txt delete mode 100644 docs/common/lib_server/SocketStream.txt delete mode 100644 docs/common/lib_server/SocketStreamTLS.txt delete mode 100644 docs/common/lib_server/TLSContext.txt delete mode 100644 docs/common/memory_leaks.txt create mode 100644 docs/docbook/adminguide.xml create mode 100644 docs/docbook/bb-book.xsl create mode 100644 docs/docbook/bb-man.xsl.tmpl create mode 100644 docs/docbook/bb-nochunk-book.xsl create mode 100644 docs/docbook/bbackupctl.xml create mode 100644 docs/docbook/bbackupd-config.xml create mode 100644 docs/docbook/bbackupd.conf.xml create mode 100644 docs/docbook/bbackupd.xml create mode 100644 docs/docbook/bbackupquery.xml create mode 100644 docs/docbook/bbstoreaccounts.xml create mode 100644 docs/docbook/bbstored-certs.xml create mode 100644 docs/docbook/bbstored-config.xml create mode 100644 docs/docbook/bbstored.conf.xml create mode 100644 docs/docbook/bbstored.xml create mode 100644 docs/docbook/html/bbdoc-man.css create mode 100644 docs/docbook/html/bbdoc.css create mode 100644 docs/docbook/html/favicon.ico create mode 100644 docs/docbook/html/images/arrow.png create mode 100644 docs/docbook/html/images/bblogo.png create mode 100644 docs/docbook/html/images/stepahead.png create mode 100644 docs/docbook/instguide.xml create mode 100644 docs/docbook/raidfile-config.xml create mode 100644 docs/docbook/raidfile.conf.xml create mode 100644 docs/images/bblogo-alpha.xcf delete mode 100644 docs/raidfile/lib_raidfile.txt delete mode 100644 docs/raidfile/lib_raidfile/RaidFileRead.txt delete mode 100644 docs/raidfile/lib_raidfile/RaidFileWrite.txt create mode 100644 docs/tools/generate_except_xml.pl delete mode 100644 documentation/Makefile delete mode 100644 documentation/adminguide.xml delete mode 100644 documentation/bb-book.xsl delete mode 100644 documentation/bb-man.xsl delete mode 100644 documentation/bb-nochunk-book.xsl delete mode 100644 documentation/bbackupctl.xml delete mode 100644 documentation/bbackupquery.xml delete mode 100644 documentation/bbstoreaccounts.xml delete mode 100644 documentation/bbstored-certs.xml delete mode 100644 documentation/bbstored-config.xml delete mode 100644 documentation/generate_except_xml.pl delete mode 100644 documentation/html/bbdoc-man.css delete mode 100644 documentation/html/bbdoc.css delete mode 100644 documentation/html/images/arrow.png delete mode 100644 documentation/html/images/bblogo.png delete mode 100644 documentation/html/images/stepahead.png delete mode 100644 documentation/instguide.xml delete mode 100644 documentation/raidfile-config.xml create mode 100644 infrastructure/parcelpath.pl create mode 100644 infrastructure/printversion.pl create mode 100644 lib/backupclient/RunStatusProvider.h create mode 100644 lib/common/SelfFlushingStream.h create mode 100644 lib/common/Test.cpp create mode 100644 lib/httpserver/HTTPException.txt create mode 100644 lib/httpserver/HTTPQueryDecoder.cpp create mode 100644 lib/httpserver/HTTPQueryDecoder.h create mode 100644 lib/httpserver/HTTPRequest.cpp create mode 100644 lib/httpserver/HTTPRequest.h create mode 100644 lib/httpserver/HTTPResponse.cpp create mode 100644 lib/httpserver/HTTPResponse.h create mode 100644 lib/httpserver/HTTPServer.cpp create mode 100644 lib/httpserver/HTTPServer.h create mode 100644 lib/httpserver/Makefile.extra create mode 100644 lib/httpserver/S3Client.cpp create mode 100644 lib/httpserver/S3Client.h create mode 100644 lib/httpserver/cdecode.cpp create mode 100644 lib/httpserver/cdecode.h create mode 100644 lib/httpserver/cencode.cpp create mode 100644 lib/httpserver/cencode.h create mode 100644 lib/httpserver/decode.h create mode 100644 lib/httpserver/encode.h create mode 100644 lib/server/OverlappedIO.h create mode 100644 lib/server/ServerControl.cpp create mode 100644 lib/server/WinNamedPipeListener.h create mode 100755 lib/win32/getopt_long.cpp delete mode 100755 lib/win32/getopt_long.cxx create mode 100644 test/bbackupd/testfiles/bbackupd-exclude.conf.in create mode 100644 test/bbackupd/testfiles/bbackupd-snapshot.conf.in create mode 100644 test/bbackupd/testfiles/bbackupd-symlink.conf.in create mode 100644 test/httpserver/testfiles/httpserver.conf create mode 100644 test/httpserver/testfiles/photos/puppy.jpg create mode 100755 test/httpserver/testfiles/testrequests.pl create mode 100644 test/httpserver/testhttpserver.cpp diff --git a/.hgignore b/.hgignore new file mode 100644 index 00000000..c21cefc6 --- /dev/null +++ b/.hgignore @@ -0,0 +1,37 @@ +.svn +BoxConfig.h +BoxConfig.h.in +BoxPlatform.pm +BoxPortsAndFiles.h +ExceptionCodes.txt +Makefile +_main.cpp +_t +_t-gdb +aclocal.m4 +autogen_* +autom4te.cache +bbackupd-config +bbackupd.conf +bbstored-certs +bbstored-config +config.log +config.status +configure +debug +extcheck1.pl +extcheck2.pl +local +makebuildenv.pl +makedistribution.pl +makedocumentation.pl +makeexception.pl +makeparcels.pl +makeprotocol.pl +notifyscript.pl +parcels +raidfile-config +release +runtest.pl +syncallowscript.pl +testbackupstorefix.pl diff --git a/.svnrevision b/.svnrevision index 4adb23b1..951ba797 100644 --- a/.svnrevision +++ b/.svnrevision @@ -1 +1 @@ -2072 +2491 diff --git a/LICENSE.txt b/LICENSE.txt index d08aa509..d53bfd71 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ -Box Backup, http://www.fluffy.co.uk/boxbackup +Box Backup, http://www.boxbackup.org/ -Copyright (c) 2003-2007 Ben Summers and contributors. All rights reserved. +Copyright (c) 2003-2008 Ben Summers and contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions diff --git a/bin/bbackupctl/bbackupctl.cpp b/bin/bbackupctl/bbackupctl.cpp index 3795cbed..2c41f614 100644 --- a/bin/bbackupctl/bbackupctl.cpp +++ b/bin/bbackupctl/bbackupctl.cpp @@ -15,6 +15,8 @@ #include #endif +#include + #include "MainHelper.h" #include "BoxPortsAndFiles.h" #include "BackupDaemonConfigVerify.h" @@ -61,9 +63,7 @@ int main(int argc, const char *argv[]) MAINHELPER_START -#if defined WIN32 && ! defined NDEBUG - ::openlog("Box Backup (bbackupctl)", 0, 0); -#endif + Logging::SetProgramName("bbackupctl"); // Filename for configuration file? std::string configFilename; @@ -256,7 +256,8 @@ int main(int argc, const char *argv[]) case SyncAndWaitForEnd: { // send a sync command - std::string cmd("force-sync\n"); + commandName = "force-sync"; + std::string cmd = commandName + "\n"; connection.Write(cmd.c_str(), cmd.size()); connection.WriteAllBuffered(); @@ -336,13 +337,17 @@ int main(int argc, const char *argv[]) { if(!quiet) { - BOX_INFO("Succeeded."); + BOX_INFO("Control command " + "sent: " << + commandName); } finished = true; } else if(line == "error") { - BOX_ERROR("Check command spelling"); + BOX_ERROR("Control command failed: " << + commandName << ". Check " + "command spelling."); returnCode = 1; finished = true; } @@ -352,7 +357,7 @@ int main(int argc, const char *argv[]) MAINHELPER_END -#if defined WIN32 && ! defined NDEBUG +#if defined WIN32 && ! defined BOX_RELEASE_BUILD closelog(); #endif diff --git a/bin/bbackupd/BackupClientContext.cpp b/bin/bbackupd/BackupClientContext.cpp index 4b4efd90..b978f54c 100644 --- a/bin/bbackupd/BackupClientContext.cpp +++ b/bin/bbackupd/BackupClientContext.cpp @@ -41,17 +41,20 @@ // -------------------------------------------------------------------------- BackupClientContext::BackupClientContext ( - BackupDaemon &rDaemon, + LocationResolver &rResolver, TLSContext &rTLSContext, const std::string &rHostname, + int Port, int32_t AccountNumber, bool ExtendedLogging, bool ExtendedLogToFile, - std::string ExtendedLogFile + std::string ExtendedLogFile, + ProgressNotifier& rProgressNotifier ) - : mrDaemon(rDaemon), + : mrResolver(rResolver), mrTLSContext(rTLSContext), mHostname(rHostname), + mPort(Port), mAccountNumber(AccountNumber), mpSocket(0), mpConnection(0), @@ -66,8 +69,9 @@ BackupClientContext::BackupClientContext mStorageLimitExceeded(false), mpExcludeFiles(0), mpExcludeDirs(0), - mKeepAliveTimer(0), - mbIsManaged(false) + mKeepAliveTimer(0, "KeepAliveTime"), + mbIsManaged(false), + mrProgressNotifier(rProgressNotifier) { } @@ -129,7 +133,8 @@ BackupProtocolClient &BackupClientContext::GetConnection() mHostname << "'..."); // Connect! - mpSocket->Open(mrTLSContext, Socket::TypeINET, mHostname.c_str(), BOX_PORT_BBSTORED); + mpSocket->Open(mrTLSContext, Socket::TypeINET, + mHostname.c_str(), mPort); // And create a procotol object mpConnection = new BackupProtocolClient(*mpSocket); @@ -146,8 +151,8 @@ BackupProtocolClient &BackupClientContext::GetConnection() if (!mpExtendedLogFileHandle) { - BOX_ERROR("Failed to open extended log " - "file: " << strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to open extended " + "log file: " << mExtendedLogFile); } else { @@ -465,7 +470,7 @@ bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirec { // Location name -- look up in daemon's records std::string locPath; - if(!mrDaemon.FindLocationPathName(elementName.GetClearFilename(), locPath)) + if(!mrResolver.FindLocationPathName(elementName.GetClearFilename(), locPath)) { // Didn't find the location... so can't give the local filename return false; @@ -504,7 +509,7 @@ void BackupClientContext::SetKeepAliveTime(int iSeconds) { mKeepAliveTime = iSeconds < 0 ? 0 : iSeconds; BOX_TRACE("Set keep-alive time to " << mKeepAliveTime << " seconds"); - mKeepAliveTimer = Timer(mKeepAliveTime); + mKeepAliveTimer = Timer(mKeepAliveTime, "KeepAliveTime"); } // -------------------------------------------------------------------------- @@ -564,7 +569,7 @@ void BackupClientContext::DoKeepAlive() BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); mpConnection->QueryGetIsAlive(); - mKeepAliveTimer = Timer(mKeepAliveTime); + mKeepAliveTimer = Timer(mKeepAliveTime, "KeepAliveTime"); } int BackupClientContext::GetMaximumDiffingTime() diff --git a/bin/bbackupd/BackupClientContext.h b/bin/bbackupd/BackupClientContext.h index 152d8556..4665df2b 100644 --- a/bin/bbackupd/BackupClientContext.h +++ b/bin/bbackupd/BackupClientContext.h @@ -12,6 +12,8 @@ #include "BoxTime.h" #include "BackupClientDeleteList.h" +#include "BackupClientDirectoryRecord.h" +#include "BackupDaemonInterface.h" #include "BackupStoreFile.h" #include "ExcludeList.h" #include "Timer.h" @@ -25,6 +27,7 @@ class BackupStoreFilenameClear; #include + // -------------------------------------------------------------------------- // // Class @@ -38,13 +41,15 @@ class BackupClientContext : public DiffTimer public: BackupClientContext ( - BackupDaemon &rDaemon, + LocationResolver &rResolver, TLSContext &rTLSContext, const std::string &rHostname, + int32_t Port, int32_t AccountNumber, bool ExtendedLogging, bool ExtendedLogToFile, - std::string ExtendedLogFile + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier ); virtual ~BackupClientContext(); private: @@ -69,6 +74,7 @@ public: int64_t GetClientStoreMarker() const {return mClientStoreMarker;} bool StorageLimitExceeded() {return mStorageLimitExceeded;} + void SetStorageLimitExceeded() {mStorageLimitExceeded = true;} // -------------------------------------------------------------------------- // @@ -197,10 +203,16 @@ public: virtual int GetMaximumDiffingTime(); virtual bool IsManaged() { return mbIsManaged; } + ProgressNotifier& GetProgressNotifier() const + { + return mrProgressNotifier; + } + private: - BackupDaemon &mrDaemon; + LocationResolver &mrResolver; TLSContext &mrTLSContext; std::string mHostname; + int mPort; int32_t mAccountNumber; SocketStreamTLS *mpSocket; BackupProtocolClient *mpConnection; @@ -219,6 +231,7 @@ private: bool mbIsManaged; int mKeepAliveTime; int mMaximumDiffingTime; + ProgressNotifier &mrProgressNotifier; }; #endif // BACKUPCLIENTCONTEXT__H diff --git a/bin/bbackupd/BackupClientDeleteList.cpp b/bin/bbackupd/BackupClientDeleteList.cpp index f6d8e0dc..b9b5b53e 100644 --- a/bin/bbackupd/BackupClientDeleteList.cpp +++ b/bin/bbackupd/BackupClientDeleteList.cpp @@ -42,21 +42,38 @@ BackupClientDeleteList::~BackupClientDeleteList() { } +BackupClientDeleteList::FileToDelete::FileToDelete(int64_t DirectoryID, + const BackupStoreFilename& rFilename, + const std::string& rLocalPath) +: mDirectoryID(DirectoryID), + mFilename(rFilename), + mLocalPath(rLocalPath) +{ } + +BackupClientDeleteList::DirToDelete::DirToDelete(int64_t ObjectID, + const std::string& rLocalPath) +: mObjectID(ObjectID), + mLocalPath(rLocalPath) +{ } + // -------------------------------------------------------------------------- // // Function -// Name: BackupClientDeleteList::AddDirectoryDelete(int64_t) +// Name: BackupClientDeleteList::AddDirectoryDelete(int64_t, +// const BackupStoreFilename&) // Purpose: Add a directory to the list of directories to be deleted. // Created: 10/11/03 // // -------------------------------------------------------------------------- -void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID) +void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID, + const std::string& rLocalPath) { // Only add the delete to the list if it's not in the "no delete" set - if(mDirectoryNoDeleteList.find(ObjectID) == mDirectoryNoDeleteList.end()) + if(mDirectoryNoDeleteList.find(ObjectID) == + mDirectoryNoDeleteList.end()) { // Not in the list, so should delete it - mDirectoryList.push_back(ObjectID); + mDirectoryList.push_back(DirToDelete(ObjectID, rLocalPath)); } } @@ -64,18 +81,22 @@ void BackupClientDeleteList::AddDirectoryDelete(int64_t ObjectID) // -------------------------------------------------------------------------- // // Function -// Name: BackupClientDeleteList::AddFileDelete(int64_t, BackupStoreFilenameClear &) +// Name: BackupClientDeleteList::AddFileDelete(int64_t, +// const BackupStoreFilename &) // Purpose: // Created: 10/11/03 // // -------------------------------------------------------------------------- -void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, const BackupStoreFilename &rFilename) +void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, + const BackupStoreFilename &rFilename, const std::string& rLocalPath) { // Try to find it in the no delete list - std::vector >::iterator delEntry(mFileNoDeleteList.begin()); + std::vector >::iterator + delEntry(mFileNoDeleteList.begin()); while(delEntry != mFileNoDeleteList.end()) { - if((delEntry)->first == DirectoryID && (delEntry)->second == rFilename) + if((delEntry)->first == DirectoryID + && (delEntry)->second == rFilename) { // Found! break; @@ -86,7 +107,8 @@ void BackupClientDeleteList::AddFileDelete(int64_t DirectoryID, const BackupStor // Only add it to the delete list if it wasn't in the no delete list if(delEntry == mFileNoDeleteList.end()) { - mFileList.push_back(std::pair(DirectoryID, rFilename)); + mFileList.push_back(FileToDelete(DirectoryID, rFilename, + rLocalPath)); } } @@ -113,18 +135,24 @@ void BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) BackupProtocolClient &connection(rContext.GetConnection()); // Do the deletes - for(std::vector::iterator i(mDirectoryList.begin()); i != mDirectoryList.end(); ++i) + for(std::vector::iterator i(mDirectoryList.begin()); + i != mDirectoryList.end(); ++i) { - connection.QueryDeleteDirectory(*i); + connection.QueryDeleteDirectory(i->mObjectID); + rContext.GetProgressNotifier().NotifyDirectoryDeleted( + i->mObjectID, i->mLocalPath); } // Clear the directory list mDirectoryList.clear(); // Delete the files - for(std::vector >::iterator i(mFileList.begin()); i != mFileList.end(); ++i) + for(std::vector::iterator i(mFileList.begin()); + i != mFileList.end(); ++i) { - connection.QueryDeleteFile(i->first, i->second); + connection.QueryDeleteFile(i->mDirectoryID, i->mFilename); + rContext.GetProgressNotifier().NotifyFileDeleted( + i->mDirectoryID, i->mLocalPath); } } @@ -140,7 +168,15 @@ void BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID) { // First of all, is it in the delete vector? - std::vector::iterator delEntry(std::find(mDirectoryList.begin(), mDirectoryList.end(), ObjectID)); + std::vector::iterator delEntry(mDirectoryList.begin()); + for(; delEntry != mDirectoryList.end(); delEntry++) + { + if(delEntry->mObjectID == ObjectID) + { + // Found! + break; + } + } if(delEntry != mDirectoryList.end()) { // erase this entry @@ -148,7 +184,8 @@ void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID) } else { - // Haven't been asked to delete it yet, put it in the no delete list + // Haven't been asked to delete it yet, put it in the + // no delete list mDirectoryNoDeleteList.insert(ObjectID); } } @@ -162,13 +199,15 @@ void BackupClientDeleteList::StopDirectoryDeletion(int64_t ObjectID) // Created: 19/11/03 // // -------------------------------------------------------------------------- -void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, const BackupStoreFilename &rFilename) +void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, + const BackupStoreFilename &rFilename) { // Find this in the delete list - std::vector >::iterator delEntry(mFileList.begin()); + std::vector::iterator delEntry(mFileList.begin()); while(delEntry != mFileList.end()) { - if((delEntry)->first == DirectoryID && (delEntry)->second == rFilename) + if(delEntry->mDirectoryID == DirectoryID + && delEntry->mFilename == rFilename) { // Found! break; @@ -186,10 +225,5 @@ void BackupClientDeleteList::StopFileDeletion(int64_t DirectoryID, const BackupS // Haven't been asked to delete it yet, put it in the no delete list mFileNoDeleteList.push_back(std::pair(DirectoryID, rFilename)); } - } - - - - diff --git a/bin/bbackupd/BackupClientDeleteList.h b/bin/bbackupd/BackupClientDeleteList.h index 5940cf50..b0fbf51a 100644 --- a/bin/bbackupd/BackupClientDeleteList.h +++ b/bin/bbackupd/BackupClientDeleteList.h @@ -28,22 +28,46 @@ class BackupClientContext; // -------------------------------------------------------------------------- class BackupClientDeleteList { +private: + class FileToDelete + { + public: + int64_t mDirectoryID; + BackupStoreFilename mFilename; + std::string mLocalPath; + FileToDelete(int64_t DirectoryID, + const BackupStoreFilename& rFilename, + const std::string& rLocalPath); + }; + + class DirToDelete + { + public: + int64_t mObjectID; + std::string mLocalPath; + DirToDelete(int64_t ObjectID, const std::string& rLocalPath); + }; + public: BackupClientDeleteList(); ~BackupClientDeleteList(); - void AddDirectoryDelete(int64_t ObjectID); - void AddFileDelete(int64_t DirectoryID, const BackupStoreFilename &rFilename); + void AddDirectoryDelete(int64_t ObjectID, + const std::string& rLocalPath); + void AddFileDelete(int64_t DirectoryID, + const BackupStoreFilename &rFilename, + const std::string& rLocalPath); void StopDirectoryDeletion(int64_t ObjectID); - void StopFileDeletion(int64_t DirectoryID, const BackupStoreFilename &rFilename); + void StopFileDeletion(int64_t DirectoryID, + const BackupStoreFilename &rFilename); void PerformDeletions(BackupClientContext &rContext); private: - std::vector mDirectoryList; + std::vector mDirectoryList; std::set mDirectoryNoDeleteList; // note: things only get in this list if they're not present in mDirectoryList when they are 'added' - std::vector > mFileList; + std::vector mFileList; std::vector > mFileNoDeleteList; }; diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/bin/bbackupd/BackupClientDirectoryRecord.cpp index 0a0703c2..b8d42d47 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/bin/bbackupd/BackupClientDirectoryRecord.cpp @@ -2,7 +2,8 @@ // // File // Name: BackupClientDirectoryRecord.cpp -// Purpose: Implementation of record about directory for backup client +// Purpose: Implementation of record about directory for +// backup client // Created: 2003/10/08 // // -------------------------------------------------------------------------- @@ -100,16 +101,27 @@ void BackupClientDirectoryRecord::DeleteSubDirectories() // -------------------------------------------------------------------------- // // Function -// Name: BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &, int64_t, const std::string &, bool) -// Purpose: Syncronise, recusively, a local directory with the server. +// Name: BackupClientDirectoryRecord::SyncDirectory(i +// BackupClientDirectoryRecord::SyncParams &, +// int64_t, const std::string &, +// const std::string &, bool) +// Purpose: Recursively synchronise a local directory +// with the server. // Created: 2003/10/08 // // -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::SyncParams &rParams, int64_t ContainingDirectoryID, - const std::string &rLocalPath, bool ThisDirHasJustBeenCreated) +void BackupClientDirectoryRecord::SyncDirectory( + BackupClientDirectoryRecord::SyncParams &rParams, + int64_t ContainingDirectoryID, + const std::string &rLocalPath, + const std::string &rRemotePath, + bool ThisDirHasJustBeenCreated) { + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + // Signal received by daemon? - if(rParams.mrDaemon.StopRun()) + if(rParams.mrRunStatusProvider.StopRun()) { // Yes. Stop now. THROW_EXCEPTION(BackupStoreException, SignalReceived) @@ -118,49 +130,66 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Start by making some flag changes, marking this sync as not done, // and on the immediate sub directories. mSyncDone = false; - for(std::map::iterator i = mSubDirectories.begin(); + for(std::map::iterator + i = mSubDirectories.begin(); i != mSubDirectories.end(); ++i) { i->second->mSyncDone = false; } - // Work out the time in the future after which the file should be uploaded regardless. - // This is a simple way to avoid having too many problems with file servers when they have - // clients with badly out of sync clocks. - rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + rParams.mMaxFileTimeInFuture; + // Work out the time in the future after which the file should + // be uploaded regardless. This is a simple way to avoid having + // too many problems with file servers when they have clients + // with badly out of sync clocks. + rParams.mUploadAfterThisTimeInTheFuture = GetCurrentBoxTime() + + rParams.mMaxFileTimeInFuture; - // Build the current state checksum to compare against while getting info from dirs - // Note checksum is used locally only, so byte order isn't considered. + // Build the current state checksum to compare against while + // getting info from dirs. Note checksum is used locally only, + // so byte order isn't considered. MD5Digest currentStateChecksum; + EMU_STRUCT_STAT dest_st; // Stat the directory, to get attribute info + // If it's a symbolic link, we want the link target here + // (as we're about to back up the contents of the directory) { - struct stat st; - if(::stat(rLocalPath.c_str(), &st) != 0) + if(EMU_STAT(rLocalPath.c_str(), &dest_st) != 0) { - // The directory has probably been deleted, so just ignore this error. - // In a future scan, this deletion will be noticed, deleted from server, and this object deleted. - rParams.GetProgressNotifier().NotifyDirStatFailed( - this, rLocalPath, strerror(errno)); + // The directory has probably been deleted, so + // just ignore this error. In a future scan, this + // deletion will be noticed, deleted from server, + // and this object deleted. + rNotifier.NotifyDirStatFailed(this, rLocalPath, + strerror(errno)); return; } - // Store inode number in map so directories are tracked in case they're renamed + // Store inode number in map so directories are tracked + // in case they're renamed { - BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap()); - idMap.AddToMap(st.st_ino, mObjectID, ContainingDirectoryID); + BackupClientInodeToIDMap &idMap( + rParams.mrContext.GetNewIDMap()); + idMap.AddToMap(dest_st.st_ino, mObjectID, + ContainingDirectoryID); } // Add attributes to checksum - currentStateChecksum.Add(&st.st_mode, sizeof(st.st_mode)); - currentStateChecksum.Add(&st.st_uid, sizeof(st.st_uid)); - currentStateChecksum.Add(&st.st_gid, sizeof(st.st_gid)); + currentStateChecksum.Add(&dest_st.st_mode, + sizeof(dest_st.st_mode)); + currentStateChecksum.Add(&dest_st.st_uid, + sizeof(dest_st.st_uid)); + currentStateChecksum.Add(&dest_st.st_gid, + sizeof(dest_st.st_gid)); // Inode to be paranoid about things moving around - currentStateChecksum.Add(&st.st_ino, sizeof(st.st_ino)); + currentStateChecksum.Add(&dest_st.st_ino, + sizeof(dest_st.st_ino)); #ifdef HAVE_STRUCT_STAT_ST_FLAGS - currentStateChecksum.Add(&st.st_flags, sizeof(st.st_flags)); + currentStateChecksum.Add(&dest_st.st_flags, + sizeof(dest_st.st_flags)); #endif StreamableMemBlock xattr; - BackupClientFileAttributes::FillExtendedAttr(xattr, rLocalPath.c_str()); + BackupClientFileAttributes::FillExtendedAttr(xattr, + rLocalPath.c_str()); currentStateChecksum.Add(xattr.GetBuffer(), xattr.GetSize()); } @@ -170,13 +199,13 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn std::vector files; bool downloadDirectoryRecordBecauseOfFutureFiles = false; - struct stat dir_st; - if(::lstat(rLocalPath.c_str(), &dir_st) != 0) + EMU_STRUCT_STAT link_st; + if(EMU_LSTAT(rLocalPath.c_str(), &link_st) != 0) { // Report the error (logs and // eventual email to administrator) - rParams.GetProgressNotifier().NotifyFileStatFailed(this, - rLocalPath, strerror(errno)); + rNotifier.NotifyFileStatFailed(this, rLocalPath, + strerror(errno)); // FIXME move to NotifyFileStatFailed() SetErrorWhenReadingFilesystemObject(rParams, @@ -192,8 +221,7 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn DIR *dirHandle = 0; try { - rParams.GetProgressNotifier().NotifyScanDirectory( - this, rLocalPath); + rNotifier.NotifyScanDirectory(this, rLocalPath); dirHandle = ::opendir(rLocalPath.c_str()); if(dirHandle == 0) @@ -202,17 +230,19 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // eventual email to administrator) if (errno == EACCES) { - rParams.GetProgressNotifier().NotifyDirListFailed( - this, rLocalPath, "Access denied"); + rNotifier.NotifyDirListFailed(this, + rLocalPath, "Access denied"); } else { - rParams.GetProgressNotifier().NotifyDirListFailed(this, + rNotifier.NotifyDirListFailed(this, rLocalPath, strerror(errno)); } - // Report the error (logs and eventual email to administrator) - SetErrorWhenReadingFilesystemObject(rParams, rLocalPath.c_str()); + // Report the error (logs and eventual email + // to administrator) + SetErrorWhenReadingFilesystemObject(rParams, + rLocalPath.c_str()); // Ignore this directory for now. return; } @@ -228,14 +258,17 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn ::memset(&checksum_info, 0, sizeof(checksum_info)); struct dirent *en = 0; - struct stat st; + EMU_STRUCT_STAT file_st; std::string filename; while((en = ::readdir(dirHandle)) != 0) { rParams.mrContext.DoKeepAlive(); - // Don't need to use LinuxWorkaround_FinishDirentStruct(en, rLocalPath.c_str()); - // on Linux, as a stat is performed to get all this info + // Don't need to use + // LinuxWorkaround_FinishDirentStruct(en, + // rLocalPath.c_str()); + // on Linux, as a stat is performed to + // get all this info if(en->d_name[0] == '.' && (en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0'))) @@ -259,11 +292,11 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // prefer S_IFREG, S_IFDIR... int type = en->d_type; #else - if(::lstat(filename.c_str(), &st) != 0) + if(EMU_LSTAT(filename.c_str(), &file_st) != 0) { // Report the error (logs and // eventual email to administrator) - rParams.GetProgressNotifier().NotifyFileStatFailed(this, + rNotifier.NotifyFileStatFailed(this, filename, strerror(errno)); // FIXME move to NotifyFileStatFailed() @@ -274,19 +307,18 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn continue; } - if(st.st_dev != dir_st.st_dev) + if(file_st.st_dev != dest_st.st_dev) { if(!(rParams.mrContext.ExcludeDir( filename))) { - rParams.GetProgressNotifier() - .NotifyMountPointSkipped( - this, filename); + rNotifier.NotifyMountPointSkipped( + this, filename); } continue; } - int type = st.st_mode & S_IFMT; + int type = file_st.st_mode & S_IFMT; #endif if(type == S_IFREG || type == S_IFLNK) @@ -296,8 +328,7 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Exclude it? if(rParams.mrContext.ExcludeFile(filename)) { - rParams.GetProgressNotifier() - .NotifyFileExcluded( + rNotifier.NotifyFileExcluded( this, filename); @@ -315,8 +346,7 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Exclude it? if(rParams.mrContext.ExcludeDir(filename)) { - rParams.GetProgressNotifier() - .NotifyDirExcluded( + rNotifier.NotifyDirExcluded( this, filename); @@ -327,19 +357,22 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Store on list dirs.push_back(std::string(en->d_name)); } + else if (type == S_IFSOCK || type == S_IFIFO) + { + // removed notification for these types + // see Debian bug 479145, no objections + } else { if(rParams.mrContext.ExcludeFile(filename)) { - rParams.GetProgressNotifier() - .NotifyFileExcluded( + rNotifier.NotifyFileExcluded( this, filename); } else { - rParams.GetProgressNotifier() - .NotifyUnsupportedFileType( + rNotifier.NotifyUnsupportedFileType( this, filename); SetErrorWhenReadingFilesystemObject( rParams, filename.c_str()); @@ -354,10 +387,9 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn #ifdef WIN32 // We didn't stat the file before, // but now we need the information. - if(::lstat(filename.c_str(), &st) != 0) + if(emu_stat(filename.c_str(), &file_st) != 0) { - rParams.GetProgressNotifier() - .NotifyFileStatFailed(this, + rNotifier.NotifyFileStatFailed(this, filename, strerror(errno)); @@ -370,18 +402,17 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn continue; } - if(st.st_dev != dir_st.st_dev) + if(file_st.st_dev != link_st.st_dev) { - rParams.GetProgressNotifier() - .NotifyMountPointSkipped(this, + rNotifier.NotifyMountPointSkipped(this, filename); continue; } #endif - checksum_info.mModificationTime = FileModificationTime(st); - checksum_info.mAttributeModificationTime = FileAttrModificationTime(st); - checksum_info.mSize = st.st_size; + checksum_info.mModificationTime = FileModificationTime(file_st); + checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st); + checksum_info.mSize = file_st.st_size; currentStateChecksum.Add(&checksum_info, sizeof(checksum_info)); currentStateChecksum.Add(en->d_name, strlen(en->d_name)); @@ -394,7 +425,7 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn // Log that this has happened if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) { - rParams.GetProgressNotifier().NotifyFileModifiedInFuture( + rNotifier.NotifyFileModifiedInFuture( this, filename); rParams.mHaveLoggedWarningAboutFutureFileTimes = true; } @@ -468,7 +499,8 @@ void BackupClientDirectoryRecord::SyncDirectory(BackupClientDirectoryRecord::Syn } // Do the directory reading - bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, pdirOnStore, entriesLeftOver, files, dirs); + bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, + rRemotePath, pdirOnStore, entriesLeftOver, files, dirs); // LAST THING! (think exception safety) // Store the new checksum -- don't fetch things unnecessarily in the future @@ -604,11 +636,18 @@ void BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord:: // Created: 2003/10/09 // // -------------------------------------------------------------------------- -bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncParams &rParams, - const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore, +bool BackupClientDirectoryRecord::UpdateItems( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string &rLocalPath, + const std::string &rRemotePath, + BackupStoreDirectory *pDirOnStore, std::vector &rEntriesLeftOver, - std::vector &rFiles, const std::vector &rDirs) + std::vector &rFiles, + const std::vector &rDirs) { + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + bool allUpdatedSuccessfully = true; // Decrypt all the directory entries. @@ -634,7 +673,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP f != rFiles.end(); ++f) { // Send keep-alive message if needed - rParams.mrContext.DoKeepAlive(); + rContext.DoKeepAlive(); // Filename of this file std::string filename(MakeFullPath(rLocalPath, *f)); @@ -648,10 +687,10 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // BLOCK { // Stat the file - struct stat st; - if(::lstat(filename.c_str(), &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(filename.c_str(), &st) != 0) { - rParams.GetProgressNotifier().NotifyFileStatFailed(this, + rNotifier.NotifyFileStatFailed(this, filename, strerror(errno)); // Report the error (logs and @@ -689,7 +728,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == 0)) { // Directory exists in the place of this file -- sort it out - RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, en->GetObjectID(), *f); + RemoveDirectoryInPlaceOfFile(rParams, pDirOnStore, + en, *f); en = 0; } @@ -701,7 +741,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // 2) It's not in the store // Do we know about the inode number? - const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); + const BackupClientInodeToIDMap &idMap(rContext.GetCurrentIDMap()); int64_t renameObjectID = 0, renameInDirectory = 0; if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) { @@ -711,24 +751,24 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP bool isCurrentVersion = false; box_time_t srvModTime = 0, srvAttributesHash = 0; BackupStoreFilenameClear oldLeafname; - if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname)) + if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname)) { // Only interested if it's a file and the latest version if(!isDir && isCurrentVersion) { // Check that the object we found in the ID map doesn't exist on disc - struct stat st; - if(::stat(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) + EMU_STRUCT_STAT st; + if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) { // Doesn't exist locally, but does exist on the server. // Therefore we can safely rename it to this new file. // Get the connection to the server - BackupProtocolClient &connection(rParams.mrContext.GetConnection()); + BackupProtocolClient &connection(rContext.GetConnection()); // Only do this step if there is room on the server. // This step will be repeated later when there is space available - if(!rParams.mrContext.StorageLimitExceeded()) + if(!rContext.StorageLimitExceeded()) { // Rename the existing files (ie include old versions) on the server connection.QueryMoveObject(renameObjectID, renameInDirectory, mObjectID /* move to this directory */, @@ -736,7 +776,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP storeFilename); // Stop the attempt to delete the file in the original location - BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList()); + BackupClientDeleteList &rdelList(rContext.GetDeleteList()); rdelList.StopFileDeletion(renameInDirectory, oldLeafname); // Create new entry in the directory for it @@ -799,13 +839,15 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP if (pDirOnStore != 0 && en == 0) { doUpload = true; - BOX_TRACE(filename << ": will upload " + BOX_TRACE("Upload decision: " << + filename << ": will upload " "(not on server)"); } else if (modTime >= rParams.mSyncPeriodStart) { doUpload = true; - BOX_TRACE(filename << ": will upload " + BOX_TRACE("Upload decision: " << + filename << ": will upload " "(modified since last sync)"); } } @@ -823,7 +865,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP > rParams.mMaxUploadWait) { doUpload = true; - BOX_TRACE(filename << ": will upload " + BOX_TRACE("Upload decision: " << + filename << ": will upload " "(continually modified)"); } @@ -840,7 +883,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP en->GetModificationTime() != modTime) { doUpload = true; - BOX_TRACE(filename << ": will upload " + BOX_TRACE("Upload decision: " << + filename << ": will upload " "(mod time changed)"); } @@ -852,64 +896,121 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP rParams.mUploadAfterThisTimeInTheFuture) { doUpload = true; - BOX_TRACE(filename << ": will upload " + BOX_TRACE("Upload decision: " << + filename << ": will upload " "(mod time in the future)"); } } - - if (!doUpload) + + if (en != 0 && en->GetModificationTime() == modTime) { - BOX_TRACE(filename << ": will not upload " - "(no reason to upload, mod time is " - << modTime << " versus sync period " - << rParams.mSyncPeriodStart << " to " - << rParams.mSyncPeriodEnd << ")"); + BOX_TRACE("Upload decision: " << + filename << ": will not upload " + "(not modified since last upload)"); + } + else if (!doUpload) + { + if (modTime > rParams.mSyncPeriodEnd) + { + box_time_t now = GetCurrentBoxTime(); + int age = BoxTimeToSeconds(now - + modTime); + BOX_TRACE("Upload decision: " << + filename << ": will not upload " + "(modified too recently: " + "only " << age << "seconds ago)"); + } + else + { + BOX_TRACE("Upload decision: " << + filename << ": will not upload " + "(mod time is " << modTime << + " which is outside sync window, " + << rParams.mSyncPeriodStart << " to " + << rParams.mSyncPeriodEnd << ")"); + } } + bool fileSynced = true; + if (doUpload) { + // Upload needed, don't mark sync success until + // we've actually done it + fileSynced = false; + // Make sure we're connected -- must connect here so we know whether // the storage limit has been exceeded, and hence whether or not // to actually upload the file. - rParams.mrContext.GetConnection(); + rContext.GetConnection(); // Only do this step if there is room on the server. // This step will be repeated later when there is space available - if(!rParams.mrContext.StorageLimitExceeded()) + if(!rContext.StorageLimitExceeded()) { - // Upload the file to the server, recording the object ID it returns - bool noPreviousVersionOnServer = ((pDirOnStore != 0) && (en == 0)); + // Upload the file to the server, recording the + // object ID it returns + bool noPreviousVersionOnServer = + ((pDirOnStore != 0) && (en == 0)); - // Surround this in a try/catch block, to catch errrors, but still continue + // Surround this in a try/catch block, to + // catch errors, but still continue bool uploadSuccess = false; try { - latestObjectID = UploadFile(rParams, filename, storeFilename, fileSize, modTime, attributesHash, noPreviousVersionOnServer); - uploadSuccess = true; + latestObjectID = UploadFile(rParams, + filename, storeFilename, + fileSize, modTime, + attributesHash, + noPreviousVersionOnServer); + + if (latestObjectID == 0) + { + // storage limit exceeded + rParams.mrContext.SetStorageLimitExceeded(); + uploadSuccess = false; + allUpdatedSuccessfully = false; + } + else + { + uploadSuccess = true; + } } catch(ConnectionException &e) { - // Connection errors should just be passed on to the main handler, retries - // would probably just cause more problems. - rParams.GetProgressNotifier() - .NotifyFileUploadException( - this, filename, e); + // Connection errors should just be + // passed on to the main handler, + // retries would probably just cause + // more problems. + rNotifier.NotifyFileUploadException( + this, filename, e); throw; } catch(BoxException &e) { - // an error occured -- make return code false, to show error in directory + if (e.GetType() == BackupStoreException::ExceptionType && + e.GetSubType() == BackupStoreException::SignalReceived) + { + // abort requested, pass the + // exception on up. + throw; + } + + // an error occured -- make return + // code false, to show error in directory allUpdatedSuccessfully = false; // Log it. SetErrorWhenReadingFilesystemObject(rParams, filename.c_str()); - rParams.GetProgressNotifier() - .NotifyFileUploadException( - this, filename, e); + rNotifier.NotifyFileUploadException( + this, filename, e); } - // Update structures if the file was uploaded successfully. + // Update structures if the file was uploaded + // successfully. if(uploadSuccess) { + fileSynced = true; + // delete from pending entries if(pendingFirstSeenTime != 0 && mpPendingEntries != 0) { @@ -919,28 +1020,41 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP } else { - rParams.GetProgressNotifier().NotifyFileSkippedServerFull(this, + rNotifier.NotifyFileSkippedServerFull(this, filename); } } else if(en != 0 && en->GetAttributesHash() != attributesHash) { // Attributes have probably changed, upload them again. - // If the attributes have changed enough, the directory hash will have changed too, - // and so the dir will have been downloaded, and the entry will be available. + // If the attributes have changed enough, the directory + // hash will have changed too, and so the dir will have + // been downloaded, and the entry will be available. // Get connection - BackupProtocolClient &connection(rParams.mrContext.GetConnection()); + BackupProtocolClient &connection(rContext.GetConnection()); // Only do this step if there is room on the server. - // This step will be repeated later when there is space available - if(!rParams.mrContext.StorageLimitExceeded()) + // This step will be repeated later when there is + // space available + if(!rContext.StorageLimitExceeded()) { - // Update store - BackupClientFileAttributes attr; - attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */); - MemBlockStream attrStream(attr); - connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); + try + { + // Update store + BackupClientFileAttributes attr; + attr.ReadAttributes(filename.c_str(), false /* put mod times in the attributes, please */); + MemBlockStream attrStream(attr); + connection.QuerySetReplacementFileAttributes(mObjectID, attributesHash, storeFilename, attrStream); + fileSynced = true; + } + catch (BoxException &e) + { + BOX_ERROR("Failed to read or store " + "file attributes for '" << + filename << "', will try " + "again later"); + } } } @@ -976,36 +1090,56 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP if(fileSize >= rParams.mFileTrackingSizeThreshold) { // Get the map - BackupClientInodeToIDMap &idMap(rParams.mrContext.GetNewIDMap()); + BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap()); // Need to get an ID from somewhere... if(latestObjectID != 0) { // Use this one + BOX_TRACE("Storing uploaded file ID " << + inodeNum << " (" << filename << ") " + "in ID map as object " << + latestObjectID << " with parent " << + mObjectID); idMap.AddToMap(inodeNum, latestObjectID, mObjectID /* containing directory */); } else { // Don't know it -- haven't sent anything to the store, and didn't get a listing. // Look it up in the current map, and if it's there, use that. - const BackupClientInodeToIDMap ¤tIDMap(rParams.mrContext.GetCurrentIDMap()); + const BackupClientInodeToIDMap ¤tIDMap(rContext.GetCurrentIDMap()); int64_t objid = 0, dirid = 0; if(currentIDMap.Lookup(inodeNum, objid, dirid)) { // Found + if (dirid != mObjectID) + { + BOX_WARNING("Found conflicting parent ID for file ID " << inodeNum << " (" << filename << "): expected " << mObjectID << " but found " << dirid << " (same directory used in two different locations?)"); + } + ASSERT(dirid == mObjectID); + // NOTE: If the above assert fails, an inode number has been reused by the OS, // or there is a problem somewhere. If this happened on a short test run, look // into it. However, in a long running process this may happen occasionally and - // not indiciate anything wrong. + // not indicate anything wrong. // Run the release version for real life use, where this check is not made. - idMap.AddToMap(inodeNum, objid, mObjectID /* containing directory */); + BOX_TRACE("Storing found file ID " << + inodeNum << " (" << filename << + ") in ID map as object " << + objid << " with parent " << + mObjectID); + idMap.AddToMap(inodeNum, objid, + mObjectID /* containing directory */); } } } - rParams.GetProgressNotifier().NotifyFileSynchronised(this, - filename, fileSize); + if (fileSynced) + { + rNotifier.NotifyFileSynchronised(this, filename, + fileSize); + } } // Erase contents of files to save space when recursing @@ -1014,7 +1148,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // Delete the pending entries, if the map is entry if(mpPendingEntries != 0 && mpPendingEntries->size() == 0) { - TRACE1("Deleting mpPendingEntries from dir ID %lld\n", mObjectID); + BOX_TRACE("Deleting mpPendingEntries from dir ID " << + BOX_FORMAT_OBJECTID(mObjectID)); delete mpPendingEntries; mpPendingEntries = 0; } @@ -1024,7 +1159,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP d != rDirs.end(); ++d) { // Send keep-alive message if needed - rParams.mrContext.DoKeepAlive(); + rContext.DoKeepAlive(); // Get the local filename std::string dirname(MakeFullPath(rLocalPath, *d)); @@ -1044,21 +1179,27 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // Check that the entry which might have been found is in fact a directory if((en != 0) && ((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == 0)) { - // Entry exists, but is not a directory. Bad. Get rid of it. - BackupProtocolClient &connection(rParams.mrContext.GetConnection()); + // Entry exists, but is not a directory. Bad. + // Get rid of it. + BackupProtocolClient &connection(rContext.GetConnection()); connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); + rNotifier.NotifyFileDeleted(en->GetObjectID(), + storeFilename.GetClearFilename()); // Nothing found en = 0; } - // Flag for having created directory, so can optimise the recusive call not to - // read it again, because we know it's empty. + // Flag for having created directory, so can optimise the + // recursive call not to read it again, because we know + // it's empty. bool haveJustCreatedDirOnServer = false; // Next, see if it's in the list of sub directories BackupClientDirectoryRecord *psubDirRecord = 0; - std::map::iterator e(mSubDirectories.find(*d)); + std::map::iterator + e(mSubDirectories.find(*d)); + if(e != mSubDirectories.end()) { // In the list, just use this pointer @@ -1080,7 +1221,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // No. Exists on the server, and we know about it from the listing. subDirObjectID = en->GetObjectID(); } - else if(rParams.mrContext.StorageLimitExceeded()) + else if(rContext.StorageLimitExceeded()) // know we've got a connection if we get this far, // as dir will have been modified. { @@ -1098,29 +1239,47 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP box_time_t attrModTime = 0; InodeRefType inodeNum = 0; BackupClientFileAttributes attr; - attr.ReadAttributes(dirname.c_str(), true /* directories have zero mod times */, - 0 /* not interested in mod time */, &attrModTime, 0 /* not file size */, - &inodeNum); + bool failedToReadAttributes = false; + + try + { + attr.ReadAttributes(dirname.c_str(), + true /* directories have zero mod times */, + 0 /* not interested in mod time */, + &attrModTime, 0 /* not file size */, + &inodeNum); + } + catch (BoxException &e) + { + BOX_WARNING("Failed to read attributes " + "of directory, cannot check " + "for rename, assuming new: '" + << dirname << "'"); + failedToReadAttributes = true; + } // Check to see if the directory been renamed // First, do we have a record in the ID map? int64_t renameObjectID = 0, renameInDirectory = 0; bool renameDir = false; - const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); - if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) + const BackupClientInodeToIDMap &idMap( + rContext.GetCurrentIDMap()); + + if(!failedToReadAttributes && idMap.Lookup(inodeNum, + renameObjectID, renameInDirectory)) { // Look up on the server to get the name, to build the local filename std::string localPotentialOldName; bool isDir = false; bool isCurrentVersion = false; - if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion)) + if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion)) { // Only interested if it's a directory if(isDir && isCurrentVersion) { // Check that the object doesn't exist already - struct stat st; - if(::stat(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) + EMU_STRUCT_STAT st; + if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) { // Doesn't exist locally, but does exist on the server. // Therefore we can safely rename it. @@ -1131,7 +1290,7 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP } // Get connection - BackupProtocolClient &connection(rParams.mrContext.GetConnection()); + BackupProtocolClient &connection(rContext.GetConnection()); // Don't do a check for storage limit exceeded here, because if we get to this // stage, a connection will have been opened, and the status known, so the check @@ -1151,7 +1310,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream); // Stop it being deleted later - BackupClientDeleteList &rdelList(rParams.mrContext.GetDeleteList()); + BackupClientDeleteList &rdelList( + rContext.GetDeleteList()); rdelList.StopDirectoryDeletion(renameObjectID); // This is the ID for the renamed directory @@ -1188,12 +1348,14 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP } } - ASSERT(psubDirRecord != 0 || rParams.mrContext.StorageLimitExceeded()); + ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded()); if(psubDirRecord) { // Sync this sub directory too - psubDirRecord->SyncDirectory(rParams, mObjectID, dirname, haveJustCreatedDirOnServer); + psubDirRecord->SyncDirectory(rParams, mObjectID, + dirname, rRemotePath + "/" + *d, + haveJustCreatedDirOnServer); } // Zero pointer in rEntriesLeftOver, if we have a pointer to zero @@ -1222,20 +1384,27 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // to a list, which is actually deleted at the very end of the session. // If there's an error during the process, it doesn't matter if things // aren't actually deleted, as the whole state will be reset anyway. - BackupClientDeleteList &rdel(rParams.mrContext.GetDeleteList()); + BackupClientDeleteList &rdel(rContext.GetDeleteList()); + + BackupStoreFilenameClear clear(en->GetName()); + std::string localName = MakeFullPath(rLocalPath, + clear.GetClearFilename()); // Delete this entry -- file or directory? if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) { // Set a pending deletion for the file - rdel.AddFileDelete(mObjectID, en->GetName()); + rdel.AddFileDelete(mObjectID, en->GetName(), + localName); } else if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) { // Set as a pending deletion for the directory - rdel.AddDirectoryDelete(en->GetObjectID()); + rdel.AddDirectoryDelete(en->GetObjectID(), + localName); - // If there's a directory record for it in the sub directory map, delete it now + // If there's a directory record for it in + // the sub directory map, delete it now BackupStoreFilenameClear dirname(en->GetName()); std::map::iterator e(mSubDirectories.find(dirname.GetClearFilename())); if(e != mSubDirectories.end()) @@ -1249,8 +1418,8 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP rLocalPath, dirname.GetClearFilename()); - TRACE1("Deleted directory record for " - "%s\n", name.c_str()); + BOX_TRACE("Deleted directory record " + "for " << name); } } } @@ -1270,14 +1439,24 @@ bool BackupClientDirectoryRecord::UpdateItems(BackupClientDirectoryRecord::SyncP // Created: 9/7/04 // // -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename) +void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( + SyncParams &rParams, + BackupStoreDirectory* pDirOnStore, + BackupStoreDirectory::Entry* pEntry, + const std::string &rFilename) { // First, delete the directory BackupProtocolClient &connection(rParams.mrContext.GetConnection()); - connection.QueryDeleteDirectory(ObjectID); + connection.QueryDeleteDirectory(pEntry->GetObjectID()); + + BackupStoreFilenameClear clear(pEntry->GetName()); + rParams.mrContext.GetProgressNotifier().NotifyDirectoryDeleted( + pEntry->GetObjectID(), clear.GetClearFilename()); // Then, delete any directory record - std::map::iterator e(mSubDirectories.find(rFilename)); + std::map::iterator + e(mSubDirectories.find(rFilename)); + if(e != mSubDirectories.end()) { // A record exists for this, remove it @@ -1294,16 +1473,30 @@ void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile(SyncParams &rPara // -------------------------------------------------------------------------- // // Function -// Name: BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &, const std::string &, const BackupStoreFilename &, int64_t, box_time_t, box_time_t, bool) -// Purpose: Private. Upload a file to the server -- may send a patch instead of the whole thing +// Name: BackupClientDirectoryRecord::UploadFile( +// BackupClientDirectoryRecord::SyncParams &, +// const std::string &, +// const BackupStoreFilename &, +// int64_t, box_time_t, box_time_t, bool) +// Purpose: Private. Upload a file to the server. May send +// a patch instead of the whole thing // Created: 20/1/04 // // -------------------------------------------------------------------------- -int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename, - int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer) +int64_t BackupClientDirectoryRecord::UploadFile( + BackupClientDirectoryRecord::SyncParams &rParams, + const std::string &rFilename, + const BackupStoreFilename &rStoreFilename, + int64_t FileSize, + box_time_t ModificationTime, + box_time_t AttributesHash, + bool NoPreviousVersionOnServer) { + BackupClientContext& rContext(rParams.mrContext); + ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); + // Get the connection - BackupProtocolClient &connection(rParams.mrContext.GetConnection()); + BackupProtocolClient &connection(rContext.GetConnection()); // Info int64_t objID = 0; @@ -1312,8 +1505,10 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn // Use a try block to catch store full errors try { - // Might an old version be on the server, and is the file size over the diffing threshold? - if(!NoPreviousVersionOnServer && FileSize >= rParams.mDiffingUploadSizeThreshold) + // Might an old version be on the server, and is the file + // size over the diffing threshold? + if(!NoPreviousVersionOnServer && + FileSize >= rParams.mDiffingUploadSizeThreshold) { // YES -- try to do diff, if possible // First, query the server to see if there's an old version available @@ -1323,7 +1518,7 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn if(diffFromID != 0) { // Found an old version - rParams.GetProgressNotifier().NotifyFileUploadingPatch(this, + rNotifier.NotifyFileUploadingPatch(this, rFilename); // Get the index @@ -1333,7 +1528,7 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn // Diff the file // - rParams.mrContext.ManageDiffProcess(); + rContext.ManageDiffProcess(); bool isCompletelyDifferent = false; std::auto_ptr patchStream( @@ -1342,11 +1537,11 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn mObjectID, /* containing directory */ rStoreFilename, diffFromID, *blockIndexStream, connection.GetTimeout(), - &rParams.mrContext, // DiffTimer implementation + &rContext, // DiffTimer implementation 0 /* not interested in the modification time */, &isCompletelyDifferent)); - rParams.mrContext.UnManageDiffProcess(); + rContext.UnManageDiffProcess(); // // Upload the patch to the store @@ -1354,6 +1549,9 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn std::auto_ptr stored(connection.QueryStoreFile(mObjectID, ModificationTime, AttributesHash, isCompletelyDifferent?(0):(diffFromID), rStoreFilename, *patchStream)); + // Get object ID from the result + objID = stored->GetObjectID(); + // Don't attempt to upload it again! doNormalUpload = false; } @@ -1362,13 +1560,14 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn if(doNormalUpload) { // below threshold or nothing to diff from, so upload whole - rParams.GetProgressNotifier().NotifyFileUploading(this, - rFilename); + rNotifier.NotifyFileUploading(this, rFilename); // Prepare to upload, getting a stream which will encode the file as we go along std::auto_ptr upload( BackupStoreFile::EncodeFile(rFilename.c_str(), - mObjectID, rStoreFilename)); + mObjectID, rStoreFilename, NULL, + &rParams, + &(rParams.mrRunStatusProvider))); // Send to store std::auto_ptr stored( @@ -1384,9 +1583,10 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn } catch(BoxException &e) { - rParams.mrContext.UnManageDiffProcess(); + rContext.UnManageDiffProcess(); - if(e.GetType() == ConnectionException::ExceptionType && e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) + if(e.GetType() == ConnectionException::ExceptionType && + e.GetSubType() == ConnectionException::Protocol_UnexpectedReply) { // Check and see what error the protocol has, // this is more useful to users than the exception. @@ -1397,11 +1597,15 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn && subtype == BackupProtocolClientError::Err_StorageLimitExceeded) { // The hard limit was exceeded on the server, notify! - rParams.mrDaemon.NotifySysadmin(BackupDaemon::NotifyEvent_StoreFull); + rParams.mrSysadminNotifier.NotifySysadmin( + SysadminNotifier::StoreFull); + // return an error code instead of + // throwing an exception that we + // can't debug. + return 0; } - rParams.GetProgressNotifier() - .NotifyFileUploadServerError( - this, rFilename, type, subtype); + rNotifier.NotifyFileUploadServerError(this, + rFilename, type, subtype); } } @@ -1409,7 +1613,7 @@ int64_t BackupClientDirectoryRecord::UploadFile(BackupClientDirectoryRecord::Syn throw; } - rParams.GetProgressNotifier().NotifyFileUploaded(this, rFilename, FileSize); + rNotifier.NotifyFileUploaded(this, rFilename, FileSize); // Return the new object ID of this file return objID; @@ -1450,16 +1654,20 @@ void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(BackupClie // Created: 8/3/04 // // -------------------------------------------------------------------------- -BackupClientDirectoryRecord::SyncParams::SyncParams(BackupDaemon &rDaemon, - ProgressNotifier &rProgressNotifier, BackupClientContext &rContext) - : mrProgressNotifier(rProgressNotifier), - mSyncPeriodStart(0), +BackupClientDirectoryRecord::SyncParams::SyncParams( + RunStatusProvider &rRunStatusProvider, + SysadminNotifier &rSysadminNotifier, + ProgressNotifier &rProgressNotifier, + BackupClientContext &rContext) + : mSyncPeriodStart(0), mSyncPeriodEnd(0), mMaxUploadWait(0), mMaxFileTimeInFuture(99999999999999999LL), mFileTrackingSizeThreshold(16*1024), mDiffingUploadSizeThreshold(16*1024), - mrDaemon(rDaemon), + mrRunStatusProvider(rRunStatusProvider), + mrSysadminNotifier(rSysadminNotifier), + mrProgressNotifier(rProgressNotifier), mrContext(rContext), mReadErrorsOnFilesystemObjects(false), mUploadAfterThisTimeInTheFuture(99999999999999999LL), diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/bin/bbackupd/BackupClientDirectoryRecord.h index 9e4dda7a..fce3fc04 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.h +++ b/bin/bbackupd/BackupClientDirectoryRecord.h @@ -13,91 +13,18 @@ #include #include -#include "BoxTime.h" #include "BackupClientFileAttributes.h" +#include "BackupDaemonInterface.h" #include "BackupStoreDirectory.h" +#include "BoxTime.h" #include "MD5Digest.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" class Archive; class BackupClientContext; class BackupDaemon; -// -------------------------------------------------------------------------- -// -// Class -// Name: ProgressNotifier -// Purpose: Provides methods for the backup library to inform the user -// interface about its progress with the backup -// Created: 2005/11/20 -// -// -------------------------------------------------------------------------- -class BackupClientDirectoryRecord; - -class ProgressNotifier -{ - public: - virtual ~ProgressNotifier() { } - virtual void NotifyScanDirectory( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyDirStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyFileStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyDirListFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyMountPointSkipped( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyDirExcluded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyUnsupportedFileType( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileReadFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) = 0; - virtual void NotifyFileModifiedInFuture( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileSkippedServerFull( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileUploadException( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const BoxException& rException) = 0; - virtual void NotifyFileUploadServerError( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int type, int subtype) = 0; - virtual void NotifyFileUploading( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileUploadingPatch( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; - virtual void NotifyFileUploaded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize) = 0; - virtual void NotifyFileSynchronised( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize) = 0; -}; - // -------------------------------------------------------------------------- // // Class @@ -132,11 +59,12 @@ public: // Created: 8/3/04 // // -------------------------------------------------------------------------- - class SyncParams + class SyncParams : public ReadLoggingStream::Logger { public: SyncParams( - BackupDaemon &rDaemon, + RunStatusProvider &rRunStatusProvider, + SysadminNotifier &rSysadminNotifier, ProgressNotifier &rProgressNotifier, BackupClientContext &rContext); ~SyncParams(); @@ -144,7 +72,6 @@ public: // No copying SyncParams(const SyncParams&); SyncParams &operator=(const SyncParams&); - ProgressNotifier &mrProgressNotifier; public: // Data members are public, as accessors are not justified here @@ -154,7 +81,9 @@ public: box_time_t mMaxFileTimeInFuture; int32_t mFileTrackingSizeThreshold; int32_t mDiffingUploadSizeThreshold; - BackupDaemon &mrDaemon; + RunStatusProvider &mrRunStatusProvider; + SysadminNotifier &mrSysadminNotifier; + ProgressNotifier &mrProgressNotifier; BackupClientContext &mrContext; bool mReadErrorsOnFilesystemObjects; @@ -162,41 +91,79 @@ public: box_time_t mUploadAfterThisTimeInTheFuture; bool mHaveLoggedWarningAboutFutureFileTimes; + bool StopRun() { return mrRunStatusProvider.StopRun(); } + void NotifySysadmin(SysadminNotifier::EventCode Event) + { + mrSysadminNotifier.NotifySysadmin(Event); + } ProgressNotifier& GetProgressNotifier() const { return mrProgressNotifier; } + + /* ReadLoggingStream::Logger implementation */ + virtual void Log(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset, + length, elapsed, finish); + } + virtual void Log(int64_t readSize, int64_t offset, + int64_t length) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset, + length); + } + virtual void Log(int64_t readSize, int64_t offset) + { + mrProgressNotifier.NotifyReadProgress(readSize, offset); + } }; - void SyncDirectory(SyncParams &rParams, int64_t ContainingDirectoryID, const std::string &rLocalPath, + void SyncDirectory(SyncParams &rParams, + int64_t ContainingDirectoryID, + const std::string &rLocalPath, + const std::string &rRemotePath, bool ThisDirHasJustBeenCreated = false); private: void DeleteSubDirectories(); BackupStoreDirectory *FetchDirectoryListing(SyncParams &rParams); - void UpdateAttributes(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, const std::string &rLocalPath); - bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath, BackupStoreDirectory *pDirOnStore, + void UpdateAttributes(SyncParams &rParams, + BackupStoreDirectory *pDirOnStore, + const std::string &rLocalPath); + bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath, + const std::string &rRemotePath, + BackupStoreDirectory *pDirOnStore, std::vector &rEntriesLeftOver, - std::vector &rFiles, const std::vector &rDirs); - int64_t UploadFile(SyncParams &rParams, const std::string &rFilename, const BackupStoreFilename &rStoreFilename, - int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer); - void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, const char *Filename); - void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, int64_t ObjectID, const std::string &rFilename); + std::vector &rFiles, + const std::vector &rDirs); + int64_t UploadFile(SyncParams &rParams, + const std::string &rFilename, + const BackupStoreFilename &rStoreFilename, + int64_t FileSize, box_time_t ModificationTime, + box_time_t AttributesHash, bool NoPreviousVersionOnServer); + void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, + const char *Filename); + void RemoveDirectoryInPlaceOfFile(SyncParams &rParams, + BackupStoreDirectory* pDirOnStore, + BackupStoreDirectory::Entry* pEntry, + const std::string &rFilename); private: - int64_t mObjectID; + int64_t mObjectID; std::string mSubDirName; - bool mInitialSyncDone; - bool mSyncDone; + bool mInitialSyncDone; + bool mSyncDone; // Checksum of directory contents and attributes, used to detect changes uint8_t mStateChecksum[MD5Digest::DigestLength]; - std::map *mpPendingEntries; - std::map mSubDirectories; + std::map *mpPendingEntries; + std::map mSubDirectories; // mpPendingEntries is a pointer rather than simple a member - // variables, because most of the time it'll be empty. This would waste a lot - // of memory because of STL allocation policies. + // variable, because most of the time it'll be empty. This would + // waste a lot of memory because of STL allocation policies. }; #endif // BACKUPCLIENTDIRECTORYRECORD__H diff --git a/bin/bbackupd/BackupClientInodeToIDMap.cpp b/bin/bbackupd/BackupClientInodeToIDMap.cpp index 0d4fd507..b9f56c5a 100644 --- a/bin/bbackupd/BackupClientInodeToIDMap.cpp +++ b/bin/bbackupd/BackupClientInodeToIDMap.cpp @@ -217,13 +217,16 @@ void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, // -------------------------------------------------------------------------- // // Function -// Name: BackupClientInodeToIDMap::Lookup(InodeRefType, int64_t &, int64_t &) const -// Purpose: Looks up an inode in the map, returning true if it exists, and the object -// ids of it and the directory it's in the reference arguments. +// Name: BackupClientInodeToIDMap::Lookup(InodeRefType, +// int64_t &, int64_t &) const +// Purpose: Looks up an inode in the map, returning true if it +// exists, and the object ids of it and the directory +// it's in the reference arguments. // Created: 11/11/03 // // -------------------------------------------------------------------------- -bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const +bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, + int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const { #ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION std::map >::const_iterator i(mMap.find(InodeRef)); diff --git a/bin/bbackupd/BackupDaemon.cpp b/bin/bbackupd/BackupDaemon.cpp index e762bbdc..3615b848 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/bin/bbackupd/BackupDaemon.cpp @@ -10,6 +10,7 @@ #include "Box.h" #include +#include #include #ifdef HAVE_UNISTD_H @@ -47,36 +48,34 @@ #include "BoxPortsAndFiles.h" #include "SSLLib.h" -#include "TLSContext.h" -#include "BackupDaemon.h" -#include "BackupDaemonConfigVerify.h" +#include "autogen_BackupProtocolClient.h" +#include "autogen_ClientException.h" +#include "autogen_ConversionException.h" +#include "Archive.h" #include "BackupClientContext.h" +#include "BackupClientCryptoKeys.h" #include "BackupClientDirectoryRecord.h" -#include "BackupStoreDirectory.h" #include "BackupClientFileAttributes.h" -#include "BackupStoreFilenameClear.h" #include "BackupClientInodeToIDMap.h" -#include "autogen_BackupProtocolClient.h" -#include "autogen_ConversionException.h" -#include "BackupClientCryptoKeys.h" -#include "BannerText.h" +#include "BackupClientMakeExcludeList.h" +#include "BackupDaemon.h" +#include "BackupDaemonConfigVerify.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "Random.h" +#include "BackupStoreFilenameClear.h" +#include "BannerText.h" +#include "Conversion.h" #include "ExcludeList.h" -#include "BackupClientMakeExcludeList.h" -#include "IOStreamGetLine.h" -#include "Utils.h" #include "FileStream.h" -#include "BackupStoreException.h" -#include "BackupStoreConstants.h" -#include "LocalProcessStream.h" #include "IOStreamGetLine.h" -#include "Conversion.h" -#include "Archive.h" -#include "Timer.h" +#include "LocalProcessStream.h" #include "Logging.h" -#include "autogen_ClientException.h" +#include "Random.h" +#include "Timer.h" +#include "Utils.h" #ifdef WIN32 #include "Win32ServiceFunctions.h" @@ -93,25 +92,6 @@ static const time_t MAX_SLEEP_TIME = 1024; // This prevents repetative cycles of load on the server #define SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY 6 -#ifdef WIN32 -// -------------------------------------------------------------------------- -// -// Function -// Name: HelperThread() -// Purpose: Background thread function, called by Windows, -// calls the BackupDaemon's RunHelperThread method -// to listen for and act on control communications -// Created: 18/2/04 -// -// -------------------------------------------------------------------------- -unsigned int WINAPI HelperThread(LPVOID lpParam) -{ - ((BackupDaemon *)lpParam)->RunHelperThread(); - - return 0; -} -#endif - // -------------------------------------------------------------------------- // // Function @@ -122,9 +102,23 @@ unsigned int WINAPI HelperThread(LPVOID lpParam) // -------------------------------------------------------------------------- BackupDaemon::BackupDaemon() : mState(BackupDaemon::State_Initialising), - mpCommandSocketInfo(0), + mDeleteRedundantLocationsAfter(0), + mLastNotifiedEvent(SysadminNotifier::MAX), mDeleteUnusedRootDirEntriesAfter(0), - mLogAllFileAccess(false) + mClientStoreMarker(BackupClientContext::ClientStoreMarker_NotKnown), + mStorageLimitExceeded(false), + mReadErrorsOnFilesystemObjects(false), + mLastSyncTime(0), + mNextSyncTime(0), + mCurrentSyncStartTime(0), + mUpdateStoreInterval(0), + mDeleteStoreObjectInfoFile(false), + mDoSyncForcedByPreviousSyncError(false), + mLogAllFileAccess(false), + mpProgressNotifier(this), + mpLocationResolver(this), + mpRunStatusProvider(this), + mpSysadminNotifier(this) #ifdef WIN32 , mInstallService(false), mRemoveService(false), @@ -134,40 +128,6 @@ BackupDaemon::BackupDaemon() { // Only ever one instance of a daemon SSLLib::Initialise(); - - // Initialise notification sent status - for(int l = 0; l < NotifyEvent__MAX; ++l) - { - mNotificationsSent[l] = false; - } - - #ifdef WIN32 - // Create the event object to signal from main thread to - // worker when new messages are queued to be sent to the - // command socket. - - mhMessageToSendEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - if(mhMessageToSendEvent == INVALID_HANDLE_VALUE) - { - BOX_ERROR("Failed to create event object: error " << - GetLastError()); - exit(1); - } - - // Create the event object to signal from worker to main thread - // when a command has been received on the command socket. - - mhCommandReceivedEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - if(mhCommandReceivedEvent == INVALID_HANDLE_VALUE) - { - BOX_ERROR("Failed to create event object: error " << - GetLastError()); - exit(1); - } - - // Create the critical section to protect the message queue - InitializeCriticalSection(&mMessageQueueLock); - #endif } // -------------------------------------------------------------------------- @@ -182,12 +142,6 @@ BackupDaemon::~BackupDaemon() { DeleteAllLocations(); DeleteAllIDMaps(); - - if(mpCommandSocketInfo != 0) - { - delete mpCommandSocketInfo; - mpCommandSocketInfo = 0; - } } // -------------------------------------------------------------------------- @@ -262,12 +216,12 @@ void BackupDaemon::SetupInInitialProcess() if(GetConfiguration().KeyExists("CommandSocket")) { BOX_WARNING( - "==============================================================================\n" - "SECURITY WARNING: This platform cannot check the credentials of connections to\n" - "the command socket. This is a potential DoS security problem.\n" - "Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n" - "is not used.\n" - "==============================================================================\n" + "==============================================================================\n" + "SECURITY WARNING: This platform cannot check the credentials of connections to\n" + "the command socket. This is a potential DoS security problem.\n" + "Remove the CommandSocket directive from the bbackupd.conf file if bbackupctl\n" + "is not used.\n" + "==============================================================================\n" ); } } @@ -294,7 +248,7 @@ void BackupDaemon::DeleteAllLocations() // Clear the contents of the map, so it is empty mLocations.clear(); - // And delete everything from the assoicated mount vector + // And delete everything from the associated mount vector mIDMapMounts.clear(); } @@ -322,6 +276,7 @@ int BackupDaemon::ProcessOption(signed int option) case 'S': { mServiceName = optarg; + Logging::SetProgramName(mServiceName); return 0; } @@ -356,8 +311,6 @@ int BackupDaemon::Main(const std::string &rConfigFileName) return RemoveService(mServiceName); } - Logging::SetProgramName("Box Backup (" + mServiceName + ")"); - int returnCode; if (mRunAsService) @@ -377,220 +330,6 @@ int BackupDaemon::Main(const std::string &rConfigFileName) return returnCode; } - -void BackupDaemon::RunHelperThread(void) -{ - const Configuration &conf(GetConfiguration()); - mpCommandSocketInfo = new CommandSocketInfo; - WinNamedPipeStream& rSocket(mpCommandSocketInfo->mListeningSocket); - - // loop until the parent process exits, or we decide - // to kill the thread ourselves - while (!IsTerminateWanted()) - { - try - { - std::string socket = conf.GetKeyValue("CommandSocket"); - rSocket.Accept(socket); - } - catch (BoxException &e) - { - BOX_ERROR("Failed to open command socket: " << - e.what()); - SetTerminateWanted(); - break; // this is fatal to listening thread - } - catch(std::exception &e) - { - BOX_ERROR("Failed to open command socket: " << - e.what()); - SetTerminateWanted(); - break; // this is fatal to listening thread - } - catch(...) - { - BOX_ERROR("Failed to open command socket: " - "unknown error"); - SetTerminateWanted(); - break; // this is fatal to listening thread - } - - try - { - // Errors here do not kill the thread, - // only the current connection. - - // This next section comes from Ben's original function - // Log - BOX_INFO("Connection from command socket"); - - // Send a header line summarising the configuration - // and current state - char summary[256]; - size_t summarySize = sprintf(summary, - "bbackupd: %d %d %d %d\nstate %d\n", - conf.GetKeyValueBool("AutomaticBackup"), - conf.GetKeyValueInt("UpdateStoreInterval"), - conf.GetKeyValueInt("MinimumFileAge"), - conf.GetKeyValueInt("MaxUploadWait"), - mState); - - rSocket.Write(summary, summarySize); - rSocket.Write("ping\n", 5); - - // old queued messages are not useful - EnterCriticalSection(&mMessageQueueLock); - mMessageList.clear(); - ResetEvent(mhMessageToSendEvent); - LeaveCriticalSection(&mMessageQueueLock); - - IOStreamGetLine readLine(rSocket); - std::string command; - - while (rSocket.IsConnected() && !IsTerminateWanted()) - { - HANDLE handles[2]; - handles[0] = mhMessageToSendEvent; - handles[1] = rSocket.GetReadableEvent(); - - BOX_TRACE("Received command '" << command - << "' over command socket"); - - DWORD result = WaitForMultipleObjects( - sizeof(handles)/sizeof(*handles), - handles, FALSE, 1000); - - if(result == 0) - { - ResetEvent(mhMessageToSendEvent); - - EnterCriticalSection(&mMessageQueueLock); - try - { - while (mMessageList.size() > 0) - { - std::string message = *(mMessageList.begin()); - mMessageList.erase(mMessageList.begin()); - printf("Sending '%s' to waiting client... ", message.c_str()); - message += "\n"; - rSocket.Write(message.c_str(), - message.length()); - - printf("done.\n"); - } - } - catch (...) - { - LeaveCriticalSection(&mMessageQueueLock); - throw; - } - LeaveCriticalSection(&mMessageQueueLock); - continue; - } - else if(result == WAIT_TIMEOUT) - { - continue; - } - else if(result != 1) - { - BOX_ERROR("WaitForMultipleObjects returned invalid result " << result); - continue; - } - - if(!readLine.GetLine(command)) - { - BOX_ERROR("Failed to read line"); - continue; - } - - BOX_INFO("Received command " << command << - " from client"); - - bool sendOK = false; - bool sendResponse = true; - bool disconnect = false; - - // Command to process! - if(command == "quit" || command == "") - { - // Close the socket. - disconnect = true; - sendResponse = false; - } - else if(command == "sync") - { - // Sync now! - this->mDoSyncFlagOut = true; - this->mSyncIsForcedOut = false; - sendOK = true; - SetEvent(mhCommandReceivedEvent); - } - else if(command == "force-sync") - { - // Sync now (forced -- overrides any SyncAllowScript) - this->mDoSyncFlagOut = true; - this->mSyncIsForcedOut = true; - sendOK = true; - SetEvent(mhCommandReceivedEvent); - } - else if(command == "reload") - { - // Reload the configuration - SetReloadConfigWanted(); - sendOK = true; - SetEvent(mhCommandReceivedEvent); - } - else if(command == "terminate") - { - // Terminate the daemon cleanly - SetTerminateWanted(); - sendOK = true; - SetEvent(mhCommandReceivedEvent); - } - else - { - BOX_ERROR("Received unknown command " - "'" << command << "' " - "from client"); - sendResponse = true; - sendOK = false; - } - - // Send a response back? - if(sendResponse) - { - const char* response = sendOK ? "ok\n" : "error\n"; - rSocket.Write( - response, strlen(response)); - } - - if(disconnect) - { - break; - } - } - - rSocket.Close(); - } - catch(BoxException &e) - { - BOX_ERROR("Communication error with " - "control client: " << e.what()); - } - catch(std::exception &e) - { - BOX_ERROR("Internal error in command socket " - "thread: " << e.what()); - } - catch(...) - { - BOX_ERROR("Communication error with control client"); - } - } - - CloseHandle(mhCommandReceivedEvent); - CloseHandle(mhMessageToSendEvent); -} #endif // -------------------------------------------------------------------------- @@ -606,36 +345,29 @@ void BackupDaemon::Run() // initialise global timer mechanism Timers::Init(); - #ifdef WIN32 - // Create a thread to handle the named pipe - HANDLE hThread; - unsigned int dwThreadId; - - hThread = (HANDLE) _beginthreadex( - NULL, // default security attributes - 0, // use default stack size - HelperThread, // thread function - this, // argument to thread function - 0, // use default creation flags - &dwThreadId); // returns the thread identifier - #else + #ifndef WIN32 // Ignore SIGPIPE so that if a command connection is broken, // the daemon doesn't terminate. ::signal(SIGPIPE, SIG_IGN); + #endif - // Create a command socket? - const Configuration &conf(GetConfiguration()); - if(conf.KeyExists("CommandSocket")) - { - // Yes, create a local UNIX socket - mpCommandSocketInfo = new CommandSocketInfo; - const char *socketName = - conf.GetKeyValue("CommandSocket").c_str(); + // Create a command socket? + const Configuration &conf(GetConfiguration()); + if(conf.KeyExists("CommandSocket")) + { + // Yes, create a local UNIX socket + mapCommandSocketInfo.reset(new CommandSocketInfo); + const char *socketName = + conf.GetKeyValue("CommandSocket").c_str(); + #ifdef WIN32 + mapCommandSocketInfo->mListeningSocket.Listen( + socketName); + #else ::unlink(socketName); - mpCommandSocketInfo->mListeningSocket.Listen( + mapCommandSocketInfo->mListeningSocket.Listen( Socket::TypeUNIX, socketName); - } - #endif // !WIN32 + #endif + } // Handle things nicely on exceptions try @@ -644,16 +376,11 @@ void BackupDaemon::Run() } catch(...) { - #ifdef WIN32 - // Don't delete the socket, as the helper thread - // is probably still using it. Let Windows clean - // up after us. - #else - if(mpCommandSocketInfo != 0) + if(mapCommandSocketInfo.get()) { try { - delete mpCommandSocketInfo; + mapCommandSocketInfo.reset(); } catch(std::exception &e) { @@ -666,91 +393,63 @@ void BackupDaemon::Run() BOX_WARNING("Error closing command socket " "after exception, ignored."); } - mpCommandSocketInfo = 0; } - #endif // WIN32 Timers::Cleanup(); throw; } - #ifndef WIN32 - // Clean up - if(mpCommandSocketInfo != 0) - { - delete mpCommandSocketInfo; - mpCommandSocketInfo = 0; - } - #endif - + // Clean up + mapCommandSocketInfo.reset(); Timers::Cleanup(); } -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupDaemon::Run2() -// Purpose: Run function for daemon (second stage) -// Created: 2003/10/08 -// -// -------------------------------------------------------------------------- -void BackupDaemon::Run2() +void BackupDaemon::InitCrypto() { // Read in the certificates creating a TLS context - TLSContext tlsContext; const Configuration &conf(GetConfiguration()); std::string certFile(conf.GetKeyValue("CertificateFile")); std::string keyFile(conf.GetKeyValue("PrivateKeyFile")); std::string caFile(conf.GetKeyValue("TrustedCAsFile")); - tlsContext.Initialise(false /* as client */, certFile.c_str(), keyFile.c_str(), caFile.c_str()); + mTlsContext.Initialise(false /* as client */, certFile.c_str(), + keyFile.c_str(), caFile.c_str()); // Set up the keys for various things BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str()); +} - // Setup various timings - int maximumDiffingTime = 600; - int keepAliveTime = 60; +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupDaemon::Run2() +// Purpose: Run function for daemon (second stage) +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +void BackupDaemon::Run2() +{ + InitCrypto(); - // max diffing time, keep-alive time - if(conf.KeyExists("MaximumDiffingTime")) - { - maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); - } - if(conf.KeyExists("KeepAliveTime")) - { - keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); - } + const Configuration &conf(GetConfiguration()); // How often to connect to the store (approximate) - box_time_t updateStoreInterval = SecondsToBoxTime(conf.GetKeyValueInt("UpdateStoreInterval")); + mUpdateStoreInterval = SecondsToBoxTime( + conf.GetKeyValueInt("UpdateStoreInterval")); // But are we connecting automatically? bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup"); - // The minimum age a file needs to be before it will be considered for uploading - box_time_t minimumFileAge = SecondsToBoxTime(conf.GetKeyValueInt("MinimumFileAge")); - - // The maximum time we'll wait to upload a file, regardless of how often it's modified - box_time_t maxUploadWait = SecondsToBoxTime(conf.GetKeyValueInt("MaxUploadWait")); - // Adjust by subtracting the minimum file age, so is relative to sync period end in comparisons - maxUploadWait = (maxUploadWait > minimumFileAge)?(maxUploadWait - minimumFileAge):(0); - // When the next sync should take place -- which is ASAP - box_time_t nextSyncTime = 0; + mNextSyncTime = 0; // When the last sync started (only updated if the store was not full when the sync ended) - box_time_t lastSyncTime = 0; + mLastSyncTime = 0; // -------------------------------------------------------------------------------------------- - // And what's the current client store marker? - int64_t clientStoreMarker = - BackupClientContext::ClientStoreMarker_NotKnown; - // haven't contacted the store yet - - bool deleteStoreObjectInfoFile = DeserializeStoreObjectInfo( - clientStoreMarker, lastSyncTime, nextSyncTime); + mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo( + mLastSyncTime, mNextSyncTime); // -------------------------------------------------------------------------------------------- @@ -758,91 +457,98 @@ void BackupDaemon::Run2() // Set state SetState(State_Idle); + mDoSyncForcedByPreviousSyncError = false; + // Loop around doing backups do { // Flags used below bool storageLimitExceeded = false; bool doSync = false; - bool doSyncForcedByCommand = false; + bool mDoSyncForcedByCommand = false; // Is a delay necessary? + box_time_t currentTime; + + do { - box_time_t currentTime; - do + // Check whether we should be stopping, + // and don't run a sync if so. + if(StopRun()) break; + + currentTime = GetCurrentBoxTime(); + + // Pause a while, but no more than + // MAX_SLEEP_TIME seconds (use the conditional + // because times are unsigned) + box_time_t requiredDelay = + (mNextSyncTime < currentTime) + ? (0) + : (mNextSyncTime - currentTime); + + // If there isn't automatic backup happening, + // set a long delay. And limit delays at the + // same time. + if(!automaticBackup && !mDoSyncForcedByPreviousSyncError) { - // Check whether we should be stopping, - // and don't run a sync if so. - if(StopRun()) break; - - currentTime = GetCurrentBoxTime(); - - // Pause a while, but no more than - // MAX_SLEEP_TIME seconds (use the conditional - // because times are unsigned) - box_time_t requiredDelay = - (nextSyncTime < currentTime) - ? (0) - : (nextSyncTime - currentTime); - - // If there isn't automatic backup happening, - // set a long delay. And limit delays at the - // same time. - if(!automaticBackup || requiredDelay > - SecondsToBoxTime(MAX_SLEEP_TIME)) + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + else if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) + { + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } + + // Only delay if necessary + if(requiredDelay > 0) + { + // Sleep somehow. There are choices + // on how this should be done, + // depending on the state of the + // control connection + if(mapCommandSocketInfo.get() != 0) { - requiredDelay = SecondsToBoxTime( - MAX_SLEEP_TIME); + // A command socket exists, + // so sleep by waiting on it + WaitOnCommandSocket(requiredDelay, + doSync, mDoSyncForcedByCommand); } - - // Only delay if necessary - if(requiredDelay > 0) + else { - // Sleep somehow. There are choices - // on how this should be done, - // depending on the state of the - // control connection - if(mpCommandSocketInfo != 0) - { - // A command socket exists, - // so sleep by waiting on it - WaitOnCommandSocket( - requiredDelay, doSync, - doSyncForcedByCommand); - } - else - { - // No command socket or - // connection, just do a - // normal sleep - time_t sleepSeconds = - BoxTimeToSeconds( - requiredDelay); - ::sleep((sleepSeconds <= 0) - ? 1 - : sleepSeconds); - } + // No command socket or + // connection, just do a + // normal sleep + time_t sleepSeconds = + BoxTimeToSeconds(requiredDelay); + ::sleep((sleepSeconds <= 0) + ? 1 : sleepSeconds); } - - } while((!automaticBackup || (currentTime < nextSyncTime)) && !doSync && !StopRun()); + } + + if ((automaticBackup || mDoSyncForcedByPreviousSyncError) + && currentTime >= mNextSyncTime) + { + doSync = true; + } } + while(!doSync && !StopRun()); // Time of sync start, and if it's time for another sync // (and we're doing automatic syncs), set the flag - box_time_t currentSyncStartTime = GetCurrentBoxTime(); - if(automaticBackup && currentSyncStartTime >= nextSyncTime) + mCurrentSyncStartTime = GetCurrentBoxTime(); + if((automaticBackup || mDoSyncForcedByPreviousSyncError) && + mCurrentSyncStartTime >= mNextSyncTime) { doSync = true; } // Use a script to see if sync is allowed now? - if(!doSyncForcedByCommand && doSync && !StopRun()) + if(!mDoSyncForcedByCommand && doSync && !StopRun()) { int d = UseScriptToSeeIfSyncAllowed(); if(d > 0) { // Script has asked for a delay - nextSyncTime = GetCurrentBoxTime() + + mNextSyncTime = GetCurrentBoxTime() + SecondsToBoxTime(d); doSync = false; } @@ -852,383 +558,448 @@ void BackupDaemon::Run2() // to be stopping) if(doSync && !StopRun()) { - // Touch a file to record times in filesystem - TouchFileInWorkingDir("last_sync_start"); + RunSyncNowWithExceptionHandling(); + } - // Tell anything connected to the command socket - SendSyncStartOrFinish(true /* start */); - - // Reset statistics on uploads - BackupStoreFile::ResetStats(); - - // Calculate the sync period of files to examine - box_time_t syncPeriodStart = lastSyncTime; - box_time_t syncPeriodEnd = currentSyncStartTime - - minimumFileAge; + // Set state + SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); - if(syncPeriodStart >= syncPeriodEnd && - syncPeriodStart - syncPeriodEnd < minimumFileAge) - { - // This can happen if we receive a force-sync - // command less than minimumFileAge after - // the last sync. Deal with it by moving back - // syncPeriodStart, which should not do any - // damage. - syncPeriodStart = syncPeriodEnd - - SecondsToBoxTime(1); - } + } while(!StopRun()); + + // Make sure we have a clean start next time round (if restart) + DeleteAllLocations(); + DeleteAllIDMaps(); +} - if(syncPeriodStart >= syncPeriodEnd) - { - BOX_ERROR("Invalid (negative) sync period: " - "perhaps your clock is going " - "backwards (" << syncPeriodStart << - " to " << syncPeriodEnd << ")"); - THROW_EXCEPTION(ClientException, - ClockWentBackwards); - } +void BackupDaemon::RunSyncNowWithExceptionHandling() +{ + OnBackupStart(); - // Check logic - ASSERT(syncPeriodEnd > syncPeriodStart); - // Paranoid check on sync times - if(syncPeriodStart >= syncPeriodEnd) continue; - - // Adjust syncPeriodEnd to emulate snapshot - // behaviour properly - box_time_t syncPeriodEndExtended = syncPeriodEnd; - // Using zero min file age? - if(minimumFileAge == 0) - { - // Add a year on to the end of the end time, - // to make sure we sync files which are - // modified after the scan run started. - // Of course, they may be eligible to be - // synced again the next time round, - // but this should be OK, because the changes - // only upload should upload no data. - syncPeriodEndExtended += SecondsToBoxTime( - (time_t)(356*24*3600)); - } + // Do sync + bool errorOccurred = false; + int errorCode = 0, errorSubCode = 0; + const char* errorString = "unknown"; - // Delete the serialised store object file, - // so that we don't try to reload it after a - // partially completed backup - if(deleteStoreObjectInfoFile && - !DeleteStoreObjectInfo()) - { - BOX_ERROR("Failed to delete the " - "StoreObjectInfoFile, backup cannot " - "continue safely."); - THROW_EXCEPTION(ClientException, - FailedToDeleteStoreObjectInfoFile); - } + try + { + RunSyncNow(); + } + catch(BoxException &e) + { + errorOccurred = true; + errorString = e.what(); + errorCode = e.GetType(); + errorSubCode = e.GetSubType(); + } + catch(std::exception &e) + { + BOX_ERROR("Internal error during backup run: " << e.what()); + errorOccurred = true; + errorString = e.what(); + } + catch(...) + { + // TODO: better handling of exceptions here... + // need to be very careful + errorOccurred = true; + } - // In case the backup throws an exception, - // we should not try to delete the store info - // object file again. - deleteStoreObjectInfoFile = false; - - // Do sync - bool errorOccurred = false; - int errorCode = 0, errorSubCode = 0; - const char* errorString = "unknown"; + // do not retry immediately without a good reason + mDoSyncForcedByPreviousSyncError = false; + + if(errorOccurred) + { + // Is it a berkely db failure? + bool isBerkelyDbFailure = false; - try - { - // Set state and log start - SetState(State_Connected); - BOX_NOTICE("Beginning scan of local files"); + if (errorCode == BackupStoreException::ExceptionType + && errorSubCode == BackupStoreException::BerkelyDBFailure) + { + isBerkelyDbFailure = true; + } - std::string extendedLogFile; - if (conf.KeyExists("ExtendedLogFile")) - { - extendedLogFile = conf.GetKeyValue( - "ExtendedLogFile"); - } - - if (conf.KeyExists("LogAllFileAccess")) - { - mLogAllFileAccess = - conf.GetKeyValueBool( - "LogAllFileAccess"); - } - - // Then create a client context object (don't - // just connect, as this may be unnecessary) - BackupClientContext clientContext - ( - *this, - tlsContext, - conf.GetKeyValue("StoreHostname"), - conf.GetKeyValueInt("AccountNumber"), - conf.GetKeyValueBool("ExtendedLogging"), - conf.KeyExists("ExtendedLogFile"), - extendedLogFile - ); - - // Set up the sync parameters - BackupClientDirectoryRecord::SyncParams params( - *this, *this, clientContext); - params.mSyncPeriodStart = syncPeriodStart; - params.mSyncPeriodEnd = syncPeriodEndExtended; - // use potentially extended end time - params.mMaxUploadWait = maxUploadWait; - params.mFileTrackingSizeThreshold = - conf.GetKeyValueInt( - "FileTrackingSizeThreshold"); - params.mDiffingUploadSizeThreshold = - conf.GetKeyValueInt( - "DiffingUploadSizeThreshold"); - params.mMaxFileTimeInFuture = - SecondsToBoxTime( - conf.GetKeyValueInt( - "MaxFileTimeInFuture")); - mDeleteRedundantLocationsAfter = - conf.GetKeyValueInt( - "DeleteRedundantLocationsAfter"); - - clientContext.SetMaximumDiffingTime(maximumDiffingTime); - clientContext.SetKeepAliveTime(keepAliveTime); - - // Set store marker - clientContext.SetClientStoreMarker(clientStoreMarker); - - // Set up the locations, if necessary -- - // need to do it here so we have a - // (potential) connection to use - if(mLocations.empty()) - { - const Configuration &locations( - conf.GetSubConfiguration( - "BackupLocations")); - - // Make sure all the directory records - // are set up - SetupLocations(clientContext, locations); - } - - // Get some ID maps going - SetupIDMapsForSync(); - - // Delete any unused directories? - DeleteUnusedRootDirEntries(clientContext); - - // Notify administrator - NotifySysadmin(NotifyEvent_BackupStart); - - // Go through the records, syncing them - for(std::vector::const_iterator - i(mLocations.begin()); - i != mLocations.end(); ++i) - { - // Set current and new ID map pointers - // in the context - clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], mNewIDMaps[(*i)->mIDMapIndex]); - - // Set exclude lists (context doesn't - // take ownership) - clientContext.SetExcludeLists( - (*i)->mpExcludeFiles, - (*i)->mpExcludeDirs); - - // Sync the directory - (*i)->mpDirectoryRecord->SyncDirectory( - params, - BackupProtocolClientListDirectory::RootDirectory, - (*i)->mPath); + if(isBerkelyDbFailure) + { + // Delete corrupt files + DeleteCorruptBerkelyDbFiles(); + } - // Unset exclude lists (just in case) - clientContext.SetExcludeLists(0, 0); - } - - // Errors reading any files? - if(params.mReadErrorsOnFilesystemObjects) - { - // Notify administrator - NotifySysadmin(NotifyEvent_ReadError); - } - else - { - // Unset the read error flag, so the // error is reported again if it - // happens again - mNotificationsSent[NotifyEvent_ReadError] = false; - } - - // Perform any deletions required -- these are - // delayed until the end to allow renaming to - // happen neatly. - clientContext.PerformDeletions(); + // Clear state data + // Go back to beginning of time + mLastSyncTime = 0; + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything + DeleteAllLocations(); + DeleteAllIDMaps(); - // Close any open connection - clientContext.CloseAnyOpenConnection(); - - // Get the new store marker - clientStoreMarker = clientContext.GetClientStoreMarker(); - - // Check the storage limit - if(clientContext.StorageLimitExceeded()) - { - // Tell the sysadmin about this - NotifySysadmin(NotifyEvent_StoreFull); - } - else - { - // The start time of the next run is - // the end time of this run. - // This is only done if the storage - // limit wasn't exceeded (as things - // won't have been done properly if - // it was) - lastSyncTime = syncPeriodEnd; - - // unflag the storage full notify flag - // so that next time the store is full, - // an alert will be sent - mNotificationsSent[NotifyEvent_StoreFull] = false; - } - - // Calculate when the next sync run should be - nextSyncTime = currentSyncStartTime + - updateStoreInterval + - Random::RandomInt(updateStoreInterval >> + // Handle restart? + if(StopRun()) + { + BOX_NOTICE("Exception (" << errorCode + << "/" << errorSubCode + << ") due to signal"); + OnBackupFinish(); + return; + } + + NotifySysadmin(SysadminNotifier::BackupError); + + // If the Berkely db files get corrupted, + // delete them and try again immediately. + if(isBerkelyDbFailure) + { + BOX_ERROR("Berkely db inode map files corrupted, " + "deleting and restarting scan. Renamed files " + "and directories will not be tracked until " + "after this scan."); + ::sleep(1); + } + else + { + // Not restart/terminate, pause and retry + // Notify administrator + SetState(State_Error); + BOX_ERROR("Exception caught (" << errorString << + " " << errorCode << "/" << errorSubCode << + "), reset state and waiting to retry..."); + ::sleep(10); + mNextSyncTime = mCurrentSyncStartTime + + SecondsToBoxTime(100) + + Random::RandomInt(mUpdateStoreInterval >> SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); - - // Commit the ID Maps - CommitIDMapsAfterSync(); + } + } + // Notify system administrator about the final state of the backup + else if(mReadErrorsOnFilesystemObjects) + { + NotifySysadmin(SysadminNotifier::ReadError); + } + else if(mStorageLimitExceeded) + { + NotifySysadmin(SysadminNotifier::StoreFull); + } + else + { + NotifySysadmin(SysadminNotifier::BackupOK); + } + + // If we were retrying after an error, and this backup succeeded, + // then now would be a good time to stop :-) + mDoSyncForcedByPreviousSyncError = errorOccurred; - // Log - BOX_NOTICE("Finished scan of local files"); + OnBackupFinish(); +} - // Notify administrator - NotifySysadmin(NotifyEvent_BackupFinish); +void BackupDaemon::RunSyncNow() +{ + // Delete the serialised store object file, + // so that we don't try to reload it after a + // partially completed backup + if(mDeleteStoreObjectInfoFile && + !DeleteStoreObjectInfo()) + { + BOX_ERROR("Failed to delete the StoreObjectInfoFile, " + "backup cannot continue safely."); + THROW_EXCEPTION(ClientException, + FailedToDeleteStoreObjectInfoFile); + } - // -------------------------------------------------------------------------------------------- + // In case the backup throws an exception, + // we should not try to delete the store info + // object file again. + mDeleteStoreObjectInfoFile = false; - // We had a successful backup, save the store - // info. If we save successfully, we must - // delete the file next time we start a backup + const Configuration &conf(GetConfiguration()); - deleteStoreObjectInfoFile = - SerializeStoreObjectInfo( - clientStoreMarker, - lastSyncTime, nextSyncTime); + std::auto_ptr fileLogger; - // -------------------------------------------------------------------------------------------- - } - catch(BoxException &e) - { - errorOccurred = true; - errorString = e.what(); - errorCode = e.GetType(); - errorSubCode = e.GetSubType(); - } - catch(std::exception &e) - { - BOX_ERROR("Internal error during " - "backup run: " << e.what()); - errorOccurred = true; - errorString = e.what(); - } - catch(...) - { - // TODO: better handling of exceptions here... - // need to be very careful - errorOccurred = true; - } - - if(errorOccurred) - { - // Is it a berkely db failure? - bool isBerkelyDbFailure = false; + if (conf.KeyExists("LogFile")) + { + Log::Level level = Log::INFO; + if (conf.KeyExists("LogFileLevel")) + { + level = Logging::GetNamedLevel( + conf.GetKeyValue("LogFileLevel")); + } + fileLogger.reset(new FileLogger(conf.GetKeyValue("LogFile"), + level)); + } - if (errorCode == BackupStoreException::ExceptionType - && errorSubCode == BackupStoreException::BerkelyDBFailure) - { - isBerkelyDbFailure = true; - } + std::string extendedLogFile; + if (conf.KeyExists("ExtendedLogFile")) + { + extendedLogFile = conf.GetKeyValue("ExtendedLogFile"); + } + + if (conf.KeyExists("LogAllFileAccess")) + { + mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess"); + } + + // Then create a client context object (don't + // just connect, as this may be unnecessary) + BackupClientContext clientContext + ( + *mpLocationResolver, + mTlsContext, + conf.GetKeyValue("StoreHostname"), + conf.GetKeyValueInt("StorePort"), + conf.GetKeyValueInt("AccountNumber"), + conf.GetKeyValueBool("ExtendedLogging"), + conf.KeyExists("ExtendedLogFile"), + extendedLogFile, *mpProgressNotifier + ); + + // The minimum age a file needs to be before it will be + // considered for uploading + box_time_t minimumFileAge = SecondsToBoxTime( + conf.GetKeyValueInt("MinimumFileAge")); - if(isBerkelyDbFailure) - { - // Delete corrupt files - DeleteCorruptBerkelyDbFiles(); - } + // The maximum time we'll wait to upload a file, regardless + // of how often it's modified + box_time_t maxUploadWait = SecondsToBoxTime( + conf.GetKeyValueInt("MaxUploadWait")); + // Adjust by subtracting the minimum file age, so is relative + // to sync period end in comparisons + if (maxUploadWait > minimumFileAge) + { + maxUploadWait -= minimumFileAge; + } + else + { + maxUploadWait = 0; + } - // Clear state data - syncPeriodStart = 0; - // go back to beginning of time - clientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; // no store marker, so download everything - DeleteAllLocations(); - DeleteAllIDMaps(); + // Calculate the sync period of files to examine + box_time_t syncPeriodStart = mLastSyncTime; + box_time_t syncPeriodEnd = GetCurrentBoxTime() - minimumFileAge; - // Handle restart? - if(StopRun()) - { - BOX_NOTICE("Exception (" << errorCode - << "/" << errorSubCode - << ") due to signal"); - return; - } + if(syncPeriodStart >= syncPeriodEnd && + syncPeriodStart - syncPeriodEnd < minimumFileAge) + { + // This can happen if we receive a force-sync command less + // than minimumFileAge after the last sync. Deal with it by + // moving back syncPeriodStart, which should not do any + // damage. + syncPeriodStart = syncPeriodEnd - + SecondsToBoxTime(1); + } - // If the Berkely db files get corrupted, delete them and try again immediately - if(isBerkelyDbFailure) - { - BOX_ERROR("Berkely db inode map files corrupted, deleting and restarting scan. Renamed files and directories will not be tracked until after this scan."); - ::sleep(1); - } - else - { - // Not restart/terminate, pause and retry - // Notify administrator - NotifySysadmin(NotifyEvent_BackupError); - SetState(State_Error); - BOX_ERROR("Exception caught (" - << errorString - << " " << errorCode - << "/" << errorSubCode - << "), reset state and " - "waiting to retry..."); - ::sleep(10); - nextSyncTime = currentSyncStartTime + - SecondsToBoxTime(90) + - Random::RandomInt( - updateStoreInterval >> - SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); - } - } + if(syncPeriodStart >= syncPeriodEnd) + { + BOX_ERROR("Invalid (negative) sync period: " + "perhaps your clock is going " + "backwards (" << syncPeriodStart << + " to " << syncPeriodEnd << ")"); + THROW_EXCEPTION(ClientException, + ClockWentBackwards); + } - // Log the stats - BOX_NOTICE("File statistics: total file size uploaded " - << BackupStoreFile::msStats.mBytesInEncodedFiles - << ", bytes already on server " - << BackupStoreFile::msStats.mBytesAlreadyOnServer - << ", encoded size " - << BackupStoreFile::msStats.mTotalFileStreamSize); - BackupStoreFile::ResetStats(); + // Check logic + ASSERT(syncPeriodEnd > syncPeriodStart); + // Paranoid check on sync times + if(syncPeriodStart >= syncPeriodEnd) return; + + // Adjust syncPeriodEnd to emulate snapshot + // behaviour properly + box_time_t syncPeriodEndExtended = syncPeriodEnd; + + // Using zero min file age? + if(minimumFileAge == 0) + { + // Add a year on to the end of the end time, + // to make sure we sync files which are + // modified after the scan run started. + // Of course, they may be eligible to be + // synced again the next time round, + // but this should be OK, because the changes + // only upload should upload no data. + syncPeriodEndExtended += SecondsToBoxTime( + (time_t)(356*24*3600)); + } + + // Set up the sync parameters + BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider, + *mpSysadminNotifier, *mpProgressNotifier, clientContext); + params.mSyncPeriodStart = syncPeriodStart; + params.mSyncPeriodEnd = syncPeriodEndExtended; + // use potentially extended end time + params.mMaxUploadWait = maxUploadWait; + params.mFileTrackingSizeThreshold = + conf.GetKeyValueInt("FileTrackingSizeThreshold"); + params.mDiffingUploadSizeThreshold = + conf.GetKeyValueInt("DiffingUploadSizeThreshold"); + params.mMaxFileTimeInFuture = + SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); + mDeleteRedundantLocationsAfter = + conf.GetKeyValueInt("DeleteRedundantLocationsAfter"); + mStorageLimitExceeded = false; + mReadErrorsOnFilesystemObjects = false; - // Tell anything connected to the command socket - SendSyncStartOrFinish(false /* finish */); + // Setup various timings + int maximumDiffingTime = 600; + int keepAliveTime = 60; - // Touch a file to record times in filesystem - TouchFileInWorkingDir("last_sync_finish"); - } + // max diffing time, keep-alive time + if(conf.KeyExists("MaximumDiffingTime")) + { + maximumDiffingTime = conf.GetKeyValueInt("MaximumDiffingTime"); + } + if(conf.KeyExists("KeepAliveTime")) + { + keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); + } + + clientContext.SetMaximumDiffingTime(maximumDiffingTime); + clientContext.SetKeepAliveTime(keepAliveTime); + + // Set store marker + clientContext.SetClientStoreMarker(mClientStoreMarker); + + // Set up the locations, if necessary -- + // need to do it here so we have a + // (potential) connection to use + if(mLocations.empty()) + { + const Configuration &locations( + conf.GetSubConfiguration( + "BackupLocations")); - // Set state - SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); + // Make sure all the directory records + // are set up + SetupLocations(clientContext, locations); + } + + mpProgressNotifier->NotifyIDMapsSetup(clientContext); + + // Get some ID maps going + SetupIDMapsForSync(); - } while(!StopRun()); + // Delete any unused directories? + DeleteUnusedRootDirEntries(clientContext); + + // Go through the records, syncing them + for(std::vector::const_iterator + i(mLocations.begin()); + i != mLocations.end(); ++i) + { + // Set current and new ID map pointers + // in the context + clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], + mNewIDMaps[(*i)->mIDMapIndex]); - // Make sure we have a clean start next time round (if restart) - DeleteAllLocations(); - DeleteAllIDMaps(); + // Set exclude lists (context doesn't + // take ownership) + clientContext.SetExcludeLists( + (*i)->mpExcludeFiles, + (*i)->mpExcludeDirs); + + // Sync the directory + (*i)->mpDirectoryRecord->SyncDirectory( + params, + BackupProtocolClientListDirectory::RootDirectory, + (*i)->mPath, std::string("/") + (*i)->mName); + + // Unset exclude lists (just in case) + clientContext.SetExcludeLists(0, 0); + } + + // Perform any deletions required -- these are + // delayed until the end to allow renaming to + // happen neatly. + clientContext.PerformDeletions(); + + // Close any open connection + clientContext.CloseAnyOpenConnection(); + + // Get the new store marker + mClientStoreMarker = clientContext.GetClientStoreMarker(); + mStorageLimitExceeded = clientContext.StorageLimitExceeded(); + mReadErrorsOnFilesystemObjects = + params.mReadErrorsOnFilesystemObjects; + + if(!mStorageLimitExceeded) + { + // The start time of the next run is the end time of this + // run. This is only done if the storage limit wasn't + // exceeded (as things won't have been done properly if + // it was) + mLastSyncTime = syncPeriodEnd; + } + + // Commit the ID Maps + CommitIDMapsAfterSync(); + + // Calculate when the next sync run should be + mNextSyncTime = mCurrentSyncStartTime + + mUpdateStoreInterval + + Random::RandomInt(mUpdateStoreInterval >> + SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); + + // -------------------------------------------------------------------------------------------- + + // We had a successful backup, save the store + // info. If we save successfully, we must + // delete the file next time we start a backup + + mDeleteStoreObjectInfoFile = + SerializeStoreObjectInfo(mLastSyncTime, + mNextSyncTime); + + // -------------------------------------------------------------------------------------------- } +void BackupDaemon::OnBackupStart() +{ + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_start"); + + // Reset statistics on uploads + BackupStoreFile::ResetStats(); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(true /* start */); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupStart); + + // Set state and log start + SetState(State_Connected); + BOX_NOTICE("Beginning scan of local files"); +} + +void BackupDaemon::OnBackupFinish() +{ + // Log + BOX_NOTICE("Finished scan of local files"); + + // Log the stats + BOX_NOTICE("File statistics: total file size uploaded " + << BackupStoreFile::msStats.mBytesInEncodedFiles + << ", bytes already on server " + << BackupStoreFile::msStats.mBytesAlreadyOnServer + << ", encoded size " + << BackupStoreFile::msStats.mTotalFileStreamSize); + + // Reset statistics again + BackupStoreFile::ResetStats(); + + // Notify administrator + NotifySysadmin(SysadminNotifier::BackupFinish); + + // Tell anything connected to the command socket + SendSyncStartOrFinish(false /* finish */); + + // Touch a file to record times in filesystem + TouchFileInWorkingDir("last_sync_finish"); +} // -------------------------------------------------------------------------- // // Function // Name: BackupDaemon::UseScriptToSeeIfSyncAllowed() -// Purpose: Private. Use a script to see if the sync should be allowed (if configured) -// Returns -1 if it's allowed, time in seconds to wait otherwise. +// Purpose: Private. Use a script to see if the sync should be +// allowed now (if configured). Returns -1 if it's +// allowed, time in seconds to wait otherwise. // Created: 21/6/04 // // -------------------------------------------------------------------------- @@ -1250,7 +1021,8 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() pid_t pid = 0; try { - std::auto_ptr pscript(LocalProcessStream(conf.GetKeyValue("SyncAllowScript").c_str(), pid)); + std::auto_ptr pscript(LocalProcessStream( + conf.GetKeyValue("SyncAllowScript").c_str(), pid)); // Read in the result IOStreamGetLine getLine(*pscript); @@ -1323,33 +1095,13 @@ int BackupDaemon::UseScriptToSeeIfSyncAllowed() // -------------------------------------------------------------------------- void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) { -#ifdef WIN32 - DWORD requiredDelayMs = BoxTimeToMilliSeconds(RequiredDelay); - - DWORD result = WaitForSingleObject(mhCommandReceivedEvent, - (DWORD)requiredDelayMs); - - if(result == WAIT_OBJECT_0) - { - DoSyncFlagOut = this->mDoSyncFlagOut; - SyncIsForcedOut = this->mSyncIsForcedOut; - ResetEvent(mhCommandReceivedEvent); - } - else if(result == WAIT_TIMEOUT) + ASSERT(mapCommandSocketInfo.get()); + if(!mapCommandSocketInfo.get()) { - DoSyncFlagOut = false; - SyncIsForcedOut = false; - } - else - { - BOX_ERROR("Unexpected result from WaitForSingleObject: " - "error " << GetLastError()); + // failure case isn't too bad + ::sleep(1); + return; } - - return; -#else // ! WIN32 - ASSERT(mpCommandSocketInfo != 0); - if(mpCommandSocketInfo == 0) {::sleep(1); return;} // failure case isn't too bad BOX_TRACE("Wait on command socket, delay = " << RequiredDelay); @@ -1362,12 +1114,12 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla if(timeout == INFTIM) timeout = 100000; // Wait for socket connection, or handle a command? - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { // No connection, listen for a new one - mpCommandSocketInfo->mpConnectedSocket.reset(mpCommandSocketInfo->mListeningSocket.Accept(timeout).release()); + mapCommandSocketInfo->mpConnectedSocket.reset(mapCommandSocketInfo->mListeningSocket.Accept(timeout).release()); - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { // If a connection didn't arrive, there was a timeout, which means we've // waited long enough and it's time to go. @@ -1386,7 +1138,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla { uid_t remoteEUID = 0xffff; gid_t remoteEGID = 0xffff; - if(mpCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID)) + if(mapCommandSocketInfo->mpConnectedSocket->GetPeerCredentials(remoteEUID, remoteEGID)) { // Credentials are available -- check UID if(remoteEUID == ::getuid()) @@ -1403,7 +1155,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla { // Dump the connection BOX_ERROR("Incoming command connection from peer had different user ID than this process, or security check could not be completed."); - mpCommandSocketInfo->mpConnectedSocket.reset(); + mapCommandSocketInfo->mpConnectedSocket.reset(); return; } else @@ -1420,7 +1172,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla conf.GetKeyValueInt("MinimumFileAge"), conf.GetKeyValueInt("MaxUploadWait"), mState); - mpCommandSocketInfo->mpConnectedSocket->Write(summary, summarySize); + mapCommandSocketInfo->mpConnectedSocket->Write(summary, summarySize); // Set the timeout to something very small, so we don't wait too long on waiting // for any incoming data @@ -1430,22 +1182,22 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla } // So there must be a connection now. - ASSERT(mpCommandSocketInfo->mpConnectedSocket.get() != 0); + ASSERT(mapCommandSocketInfo->mpConnectedSocket.get() != 0); // Is there a getline object ready? - if(mpCommandSocketInfo->mpGetLine == 0) + if(mapCommandSocketInfo->mpGetLine == 0) { // Create a new one - mpCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mpCommandSocketInfo->mpConnectedSocket.get())); + mapCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mapCommandSocketInfo->mpConnectedSocket.get())); } // Ping the remote side, to provide errors which will mean the socket gets closed - mpCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5); + mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5); // Wait for a command or something on the socket std::string command; - while(mpCommandSocketInfo->mpGetLine != 0 && !mpCommandSocketInfo->mpGetLine->IsEOF() - && mpCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) + while(mapCommandSocketInfo->mpGetLine != 0 && !mapCommandSocketInfo->mpGetLine->IsEOF() + && mapCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) { BOX_TRACE("Receiving command '" << command << "' over command socket"); @@ -1490,7 +1242,7 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla // Send a response back? if(sendResponse) { - mpCommandSocketInfo->mpConnectedSocket->Write(sendOK?"ok\n":"error\n", sendOK?3:6); + mapCommandSocketInfo->mpConnectedSocket->Write(sendOK?"ok\n":"error\n", sendOK?3:6); } // Set timeout to something very small, so this just checks for data which is waiting @@ -1498,18 +1250,39 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla } // Close on EOF? - if(mpCommandSocketInfo->mpGetLine != 0 && mpCommandSocketInfo->mpGetLine->IsEOF()) + if(mapCommandSocketInfo->mpGetLine != 0 && mapCommandSocketInfo->mpGetLine->IsEOF()) { CloseCommandConnection(); } } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write to command socket: " << ce.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) + { + throw; // thread will die + } + else + { + // Close socket and ignore error + CloseCommandConnection(); + } + } catch(std::exception &e) { - BOX_ERROR("Internal error in command socket thread: " - << e.what()); - // If an error occurs, and there is a connection active, just close that - // connection and continue. Otherwise, let the error propagate. - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + BOX_ERROR("Failed to write to command socket: " << + e.what()); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { throw; // thread will die } @@ -1521,9 +1294,13 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla } catch(...) { - // If an error occurs, and there is a connection active, just close that - // connection and continue. Otherwise, let the error propagate. - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + BOX_ERROR("Failed to write to command socket: unknown error"); + + // If an error occurs, and there is a connection active, + // just close that connection and continue. Otherwise, + // let the error propagate. + + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { throw; // thread will die } @@ -1533,7 +1310,6 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla CloseCommandConnection(); } } -#endif // WIN32 } @@ -1547,17 +1323,16 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla // -------------------------------------------------------------------------- void BackupDaemon::CloseCommandConnection() { -#ifndef WIN32 try { BOX_TRACE("Closing command connection"); - if(mpCommandSocketInfo->mpGetLine) + if(mapCommandSocketInfo->mpGetLine) { - delete mpCommandSocketInfo->mpGetLine; - mpCommandSocketInfo->mpGetLine = 0; + delete mapCommandSocketInfo->mpGetLine; + mapCommandSocketInfo->mpGetLine = 0; } - mpCommandSocketInfo->mpConnectedSocket.reset(); + mapCommandSocketInfo->mpConnectedSocket.reset(); } catch(std::exception &e) { @@ -1568,7 +1343,6 @@ void BackupDaemon::CloseCommandConnection() { // Ignore any errors } -#endif } @@ -1586,27 +1360,15 @@ void BackupDaemon::SendSyncStartOrFinish(bool SendStart) // The bbackupctl program can't rely on a state change, because it // may never change if the server doesn't need to be contacted. - if(mpCommandSocketInfo != NULL && -#ifdef WIN32 - mpCommandSocketInfo->mListeningSocket.IsConnected() -#else - mpCommandSocketInfo->mpConnectedSocket.get() != 0 -#endif - ) + if(mapCommandSocketInfo.get() && + mapCommandSocketInfo->mpConnectedSocket.get() != 0) { std::string message = SendStart ? "start-sync" : "finish-sync"; try { -#ifdef WIN32 - EnterCriticalSection(&mMessageQueueLock); - mMessageList.push_back(message); - SetEvent(mhMessageToSendEvent); - LeaveCriticalSection(&mMessageQueueLock); -#else message += "\n"; - mpCommandSocketInfo->mpConnectedSocket->Write( + mapCommandSocketInfo->mpConnectedSocket->Write( message.c_str(), message.size()); -#endif } catch(std::exception &e) { @@ -1668,14 +1430,20 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // Just a check to make sure it's right. DeleteAllLocations(); - // Going to need a copy of the root directory. Get a connection, and fetch it. + // Going to need a copy of the root directory. Get a connection, + // and fetch it. BackupProtocolClient &connection(rClientContext.GetConnection()); - // Ask server for a list of everything in the root directory, which is a directory itself - std::auto_ptr dirreply(connection.QueryListDirectory( + // Ask server for a list of everything in the root directory, + // which is a directory itself + std::auto_ptr dirreply( + connection.QueryListDirectory( BackupProtocolClientListDirectory::RootDirectory, - BackupProtocolClientListDirectory::Flags_Dir, // only directories - BackupProtocolClientListDirectory::Flags_Deleted | BackupProtocolClientListDirectory::Flags_OldVersion, // exclude old/deleted stuff + // only directories + BackupProtocolClientListDirectory::Flags_Dir, + // exclude old/deleted stuff + BackupProtocolClientListDirectory::Flags_Deleted | + BackupProtocolClientListDirectory::Flags_OldVersion, false /* no attributes */)); // Retrieve the directory from the stream following @@ -1755,33 +1523,41 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con #endif // HAVE_STRUCT_MNTENT_MNT_DIR // Check sorting and that things are as we expect ASSERT(mountPoints.size() > 0); -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD { std::set::reverse_iterator i(mountPoints.rbegin()); ASSERT(*i == "/"); } -#endif // n NDEBUG +#endif // n BOX_RELEASE_BUILD #endif // n HAVE_STRUCT_STATFS_F_MNTONNAME || n HAVE_STRUCT_STATVFS_F_MNTONNAME #endif // HAVE_MOUNTS // Then... go through each of the entries in the configuration, // making sure there's a directory created for it. - for(std::list >::const_iterator i = rLocationsConf.mSubConfigurations.begin(); - i != rLocationsConf.mSubConfigurations.end(); ++i) - { - BOX_TRACE("new location: " << i->first); + std::vector locNames = + rLocationsConf.GetSubConfigurationNames(); + + for(std::vector::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) + { + const Configuration& rConfig( + rLocationsConf.GetSubConfiguration(*pLocName)); + BOX_TRACE("new location: " << *pLocName); + // Create a record for it std::auto_ptr apLoc(new Location); try { // Setup names in the location record - apLoc->mName = i->first; - apLoc->mPath = i->second.GetKeyValue("Path"); + apLoc->mName = *pLocName; + apLoc->mPath = rConfig.GetKeyValue("Path"); // Read the exclude lists from the Configuration - apLoc->mpExcludeFiles = BackupClientMakeExcludeList_Files(i->second); - apLoc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(i->second); + apLoc->mpExcludeFiles = BackupClientMakeExcludeList_Files(rConfig); + apLoc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(rConfig); // Does this exist on the server? // Remove from dir object early, so that if we fail @@ -1814,10 +1590,9 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con if(::statfs(apLoc->mPath.c_str(), &s) != 0) #endif // HAVE_STRUCT_STATVFS_F_MNTONNAME { - BOX_WARNING("Failed to stat location " + BOX_LOG_SYS_WARNING("Failed to stat location " "path '" << apLoc->mPath << - "' (" << strerror(errno) << - "), skipping location '" << + "', skipping location '" << apLoc->mName << "'"); continue; } @@ -1929,7 +1704,8 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // Create and store the directory object for the root of this location ASSERT(oid != 0); - BackupClientDirectoryRecord *precord = new BackupClientDirectoryRecord(oid, i->first); + BackupClientDirectoryRecord *precord = + new BackupClientDirectoryRecord(oid, *pLocName); apLoc->mpDirectoryRecord.reset(precord); // Push it back on the vector of locations @@ -2007,8 +1783,8 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // -------------------------------------------------------------------------- void BackupDaemon::SetupIDMapsForSync() { - // Need to do different things depending on whether it's an in memory implementation, - // or whether it's all stored on disc. + // Need to do different things depending on whether it's an + // in memory implementation, or whether it's all stored on disc. #ifdef BACKIPCLIENTINODETOIDMAP_IN_MEMORY_IMPLEMENTATION @@ -2016,8 +1792,8 @@ void BackupDaemon::SetupIDMapsForSync() DeleteIDMapVector(mNewIDMaps); FillIDMapVector(mNewIDMaps, true /* new maps */); - // Then make sure that the current maps have objects, even if they are empty - // (for the very first run) + // Then make sure that the current maps have objects, + // even if they are empty (for the very first run) if(mCurrentIDMaps.empty()) { FillIDMapVector(mCurrentIDMaps, false /* current maps */); @@ -2191,9 +1967,8 @@ void BackupDaemon::CommitIDMapsAfterSync() #endif if(::rename(newmap.c_str(), target.c_str()) != 0) { - BOX_ERROR("failed to rename ID map: " << newmap - << " to " << target << ": " - << strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to rename ID map: " << + newmap << " to " << target); THROW_EXCEPTION(CommonException, OSFileError) } } @@ -2281,20 +2056,14 @@ void BackupDaemon::SetState(int State) sprintf(newState, "state %d", State); std::string message = newState; -#ifdef WIN32 - EnterCriticalSection(&mMessageQueueLock); - mMessageList.push_back(newState); - SetEvent(mhMessageToSendEvent); - LeaveCriticalSection(&mMessageQueueLock); -#else message += "\n"; - if(mpCommandSocketInfo == 0) + if(!mapCommandSocketInfo.get()) { return; } - if(mpCommandSocketInfo->mpConnectedSocket.get() == 0) + if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { return; } @@ -2302,22 +2071,27 @@ void BackupDaemon::SetState(int State) // Something connected to the command socket, tell it about the new state try { - mpCommandSocketInfo->mpConnectedSocket->Write(message.c_str(), + mapCommandSocketInfo->mpConnectedSocket->Write(message.c_str(), message.length()); } + catch(ConnectionException &ce) + { + BOX_NOTICE("Failed to write state to command socket: " << + ce.what()); + CloseCommandConnection(); + } catch(std::exception &e) { - BOX_ERROR("Internal error while writing state " - "to command socket: " << e.what()); + BOX_ERROR("Failed to write state to command socket: " << + e.what()); CloseCommandConnection(); } catch(...) { - BOX_ERROR("Internal error while writing state " - "to command socket: unknown error"); + BOX_ERROR("Failed to write state to command socket: " + "unknown error"); CloseCommandConnection(); } -#endif } @@ -2351,7 +2125,7 @@ void BackupDaemon::TouchFileInWorkingDir(const char *Filename) // Created: 25/2/04 // // -------------------------------------------------------------------------- -void BackupDaemon::NotifySysadmin(int Event) +void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event) { static const char *sEventNames[] = { @@ -2360,31 +2134,47 @@ void BackupDaemon::NotifySysadmin(int Event) "backup-error", "backup-start", "backup-finish", + "backup-ok", 0 }; - BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); - BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); - BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); - ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == NotifyEvent__MAX + 1); + // BOX_TRACE("sizeof(sEventNames) == " << sizeof(sEventNames)); + // BOX_TRACE("sizeof(*sEventNames) == " << sizeof(*sEventNames)); + // BOX_TRACE("NotifyEvent__MAX == " << NotifyEvent__MAX); + ASSERT((sizeof(sEventNames)/sizeof(*sEventNames)) == SysadminNotifier::MAX + 1); - BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << - sEventNames[Event]); - - if(Event < 0 || Event >= NotifyEvent__MAX) + if(Event < 0 || Event >= SysadminNotifier::MAX) { + BOX_ERROR("BackupDaemon::NotifySysadmin() called for " + "invalid event code " << Event); THROW_EXCEPTION(BackupStoreException, BadNotifySysadminEventCode); } - // Don't send lots of repeated messages - if(mNotificationsSent[Event] && - Event != NotifyEvent_BackupStart && - Event != NotifyEvent_BackupFinish) + BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << + sEventNames[Event]); + + if(!GetConfiguration().KeyExists("NotifyAlways") || + !GetConfiguration().GetKeyValueBool("NotifyAlways")) { - BOX_WARNING("Suppressing duplicate notification about " << - sEventNames[Event]); - return; + // Don't send lots of repeated messages + // Note: backup-start and backup-finish will always be + // logged, because mLastNotifiedEvent is never set to + // these values and therefore they are never "duplicates". + if(mLastNotifiedEvent == Event) + { + if(Event == SysadminNotifier::BackupOK) + { + BOX_INFO("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + else + { + BOX_WARNING("Suppressing duplicate notification " + "about " << sEventNames[Event]); + } + return; + } } // Is there a notification script? @@ -2392,10 +2182,10 @@ void BackupDaemon::NotifySysadmin(int Event) if(!conf.KeyExists("NotifyScript")) { // Log, and then return - if(Event != NotifyEvent_BackupStart && - Event != NotifyEvent_BackupFinish) + if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) { - BOX_ERROR("Not notifying administrator about event " + BOX_INFO("Not notifying administrator about event " << sEventNames[Event] << " -- set NotifyScript " "to do this in future"); } @@ -2407,20 +2197,22 @@ void BackupDaemon::NotifySysadmin(int Event) sEventNames[Event]); // Log what we're about to do - BOX_NOTICE("About to notify administrator about event " + BOX_INFO("About to notify administrator about event " << sEventNames[Event] << ", running script '" << script << "'"); // Then do it - if(::system(script.c_str()) != 0) + int returnCode = ::system(script.c_str()); + if(returnCode != 0) { - BOX_ERROR("Notify script returned an error code. ('" - << script << "')"); + BOX_WARNING("Notify script returned error code: " << + returnCode << " ('" << script << "')"); + } + else if(Event != SysadminNotifier::BackupStart && + Event != SysadminNotifier::BackupFinish) + { + mLastNotifiedEvent = Event; } - - // Flag that this is done so the administrator isn't constantly - // bombarded with lots of errors - mNotificationsSent[Event] = true; } @@ -2461,13 +2253,13 @@ void BackupDaemon::DeleteUnusedRootDirEntries(BackupClientContext &rContext) // Entries to delete, and it's the right time to do so... BOX_NOTICE("Deleting unused locations from store root..."); BackupProtocolClient &connection(rContext.GetConnection()); - for(std::vector >::iterator i(mUnusedRootDirEntries.begin()); i != mUnusedRootDirEntries.end(); ++i) + for(std::vector >::iterator + i(mUnusedRootDirEntries.begin()); + i != mUnusedRootDirEntries.end(); ++i) { connection.QueryDeleteDirectory(i->first); - - // Log this - BOX_NOTICE("Deleted " << i->second << " (ID " << i->first - << ") from store root"); + rContext.GetProgressNotifier().NotifyFileDeleted( + i->first, i->second); } // Reset state @@ -2738,9 +2530,12 @@ BackupDaemon::CommandSocketInfo::~CommandSocketInfo() // -------------------------------------------------------------------------- // // Function -// Name: BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) -// Purpose: Serializes remote directory and file information into a stream of bytes, using an Archive abstraction. -// +// Name: BackupDaemon::SerializeStoreObjectInfo( +// box_time_t theLastSyncTime, +// box_time_t theNextSyncTime) +// Purpose: Serializes remote directory and file information +// into a stream of bytes, using an Archive +// abstraction. // Created: 2005/04/11 // // -------------------------------------------------------------------------- @@ -2749,7 +2544,8 @@ static const int STOREOBJECTINFO_MAGIC_ID_VALUE = 0x7777525F; static const std::string STOREOBJECTINFO_MAGIC_ID_STRING = "BBACKUPD-STATE"; static const int STOREOBJECTINFO_VERSION = 2; -bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time_t theLastSyncTime, box_time_t theNextSyncTime) const +bool BackupDaemon::SerializeStoreObjectInfo(box_time_t theLastSyncTime, + box_time_t theNextSyncTime) const { if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) { @@ -2778,7 +2574,7 @@ bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time anArchive.Write(STOREOBJECTINFO_MAGIC_ID_STRING); anArchive.Write(STOREOBJECTINFO_VERSION); anArchive.Write(GetLoadedConfigModifiedTime()); - anArchive.Write(aClientStoreMarker); + anArchive.Write(mClientStoreMarker); anArchive.Write(theLastSyncTime); anArchive.Write(theNextSyncTime); @@ -2830,15 +2626,13 @@ bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time } catch(std::exception &e) { - BOX_ERROR("Internal error writing store object " - "info file (" << StoreObjectInfoFile << "): " - << e.what()); + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": " << e.what()); } catch(...) { - BOX_ERROR("Internal error writing store object " - "info file (" << StoreObjectInfoFile << "): " - "unknown error"); + BOX_ERROR("Failed to write StoreObjectInfoFile: " << + StoreObjectInfoFile << ": unknown error"); } return created; @@ -2847,13 +2641,17 @@ bool BackupDaemon::SerializeStoreObjectInfo(int64_t aClientStoreMarker, box_time // -------------------------------------------------------------------------- // // Function -// Name: BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime) -// Purpose: Deserializes remote directory and file information from a stream of bytes, using an Archive abstraction. -// +// Name: BackupDaemon::DeserializeStoreObjectInfo( +// box_time_t & theLastSyncTime, +// box_time_t & theNextSyncTime) +// Purpose: Deserializes remote directory and file information +// from a stream of bytes, using an Archive +// abstraction. // Created: 2005/04/11 // // -------------------------------------------------------------------------- -bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_time_t & theLastSyncTime, box_time_t & theNextSyncTime) +bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, + box_time_t & theNextSyncTime) { // // @@ -2945,7 +2743,7 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ // // this is it, go at it // - anArchive.Read(aClientStoreMarker); + anArchive.Read(mClientStoreMarker); anArchive.Read(theLastSyncTime); anArchive.Read(theNextSyncTime); @@ -3023,7 +2821,7 @@ bool BackupDaemon::DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, box_ DeleteAllLocations(); - aClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; + mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; theLastSyncTime = 0; theNextSyncTime = 0; @@ -3057,9 +2855,10 @@ bool BackupDaemon::DeleteStoreObjectInfo() const // Check to see if the file exists if(!FileExists(storeObjectInfoFile.c_str())) { - // File doesn't exist -- so can't be deleted. But something isn't quite right, so log a message - BOX_WARNING("Store object info file did not exist when it " - "was supposed to. (" << storeObjectInfoFile << ")"); + // File doesn't exist -- so can't be deleted. But something + // isn't quite right, so log a message + BOX_WARNING("StoreObjectInfoFile did not exist when it " + "was supposed to: " << storeObjectInfoFile); // Return true to stop things going around in a loop return true; @@ -3068,8 +2867,8 @@ bool BackupDaemon::DeleteStoreObjectInfo() const // Actually delete it if(::unlink(storeObjectInfoFile.c_str()) != 0) { - BOX_ERROR("Failed to delete the old store object info file: " - << storeObjectInfoFile << ": "<< strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to delete the old " + "StoreObjectInfoFile: " << storeObjectInfoFile); return false; } diff --git a/bin/bbackupd/BackupDaemon.h b/bin/bbackupd/BackupDaemon.h index 62f9c393..0c864abd 100644 --- a/bin/bbackupd/BackupDaemon.h +++ b/bin/bbackupd/BackupDaemon.h @@ -14,16 +14,20 @@ #include #include +#include "BackupClientContext.h" +#include "BackupClientDirectoryRecord.h" #include "BoxTime.h" #include "Daemon.h" -#include "BackupClientDirectoryRecord.h" +#include "Logging.h" #include "Socket.h" #include "SocketListen.h" #include "SocketStream.h" -#include "Logging.h" +#include "TLSContext.h" + #include "autogen_BackupProtocolClient.h" #ifdef WIN32 + #include "WinNamedPipeListener.h" #include "WinNamedPipeStream.h" #endif @@ -43,7 +47,8 @@ class Archive; // Created: 2003/10/08 // // -------------------------------------------------------------------------- -class BackupDaemon : public Daemon, ProgressNotifier +class BackupDaemon : public Daemon, ProgressNotifier, LocationResolver, +RunStatusProvider, SysadminNotifier { public: BackupDaemon(); @@ -52,10 +57,10 @@ public: private: // methods below do partial (specialized) serialization of // client state only - bool SerializeStoreObjectInfo(int64_t aClientStoreMarker, - box_time_t theLastSyncTime, box_time_t theNextSyncTime) const; - bool DeserializeStoreObjectInfo(int64_t & aClientStoreMarker, - box_time_t & theLastSyncTime, box_time_t & theNextSyncTime); + bool SerializeStoreObjectInfo(box_time_t theLastSyncTime, + box_time_t theNextSyncTime) const; + bool DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, + box_time_t & theNextSyncTime); bool DeleteStoreObjectInfo() const; BackupDaemon(const BackupDaemon &); @@ -65,6 +70,14 @@ public: std::string GetOptionString(); int ProcessOption(signed int option); int Main(const std::string &rConfigFileName); + + // This shouldn't be here, but apparently gcc on + // Windows has no idea about inherited methods... + virtual int Main(const char *DefaultConfigFile, int argc, + const char *argv[]) + { + return Daemon::Main(DefaultConfigFile, argc, argv); + } #endif void Run(); @@ -88,21 +101,22 @@ public: int GetState() {return mState;} // Allow other classes to call this too - enum - { - NotifyEvent_StoreFull = 0, - NotifyEvent_ReadError, - NotifyEvent_BackupError, - NotifyEvent_BackupStart, - NotifyEvent_BackupFinish, - NotifyEvent__MAX - // When adding notifications, remember to add strings to NotifySysadmin() - }; - void NotifySysadmin(int Event); + void NotifySysadmin(SysadminNotifier::EventCode Event); private: void Run2(); +public: + void InitCrypto(); + void RunSyncNowWithExceptionHandling(); + void RunSyncNow(); + void OnBackupStart(); + void OnBackupFinish(); + // TouchFileInWorkingDir is only here for use by Boxi. + // This does NOT constitute an API! + void TouchFileInWorkingDir(const char *Filename); + +private: void DeleteAllLocations(); void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf); @@ -126,8 +140,6 @@ private: void CloseCommandConnection(); void SendSyncStartOrFinish(bool SendStart); - void TouchFileInWorkingDir(const char *Filename); - void DeleteUnusedRootDirEntries(BackupClientContext &rContext); #ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET @@ -137,7 +149,7 @@ private: int UseScriptToSeeIfSyncAllowed(); -private: +public: class Location { public: @@ -157,7 +169,11 @@ private: ExcludeList *mpExcludeFiles; ExcludeList *mpExcludeDirs; }; - + + typedef const std::vector Locations; + Locations GetLocations() { return mLocations; } + +private: int mState; // what the daemon is currently doing std::vector mLocations; @@ -179,7 +195,8 @@ private: CommandSocketInfo &operator=(const CommandSocketInfo &); public: #ifdef WIN32 - WinNamedPipeStream mListeningSocket; + WinNamedPipeListener<1 /* listen backlog */> mListeningSocket; + std::auto_ptr mpConnectedSocket; #else SocketListen mListeningSocket; std::auto_ptr mpConnectedSocket; @@ -188,23 +205,51 @@ private: }; // Using a socket? - CommandSocketInfo *mpCommandSocketInfo; + std::auto_ptr mapCommandSocketInfo; // Stop notifications being repeated. - bool mNotificationsSent[NotifyEvent__MAX]; + SysadminNotifier::EventCode mLastNotifiedEvent; // Unused entries in the root directory wait a while before being deleted box_time_t mDeleteUnusedRootDirEntriesAfter; // time to delete them std::vector > mUnusedRootDirEntries; + int64_t mClientStoreMarker; + bool mStorageLimitExceeded; + bool mReadErrorsOnFilesystemObjects; + box_time_t mLastSyncTime, mNextSyncTime; + box_time_t mCurrentSyncStartTime, mUpdateStoreInterval; + TLSContext mTlsContext; + bool mDeleteStoreObjectInfoFile; + bool mDoSyncForcedByPreviousSyncError; + public: bool StopRun() { return this->Daemon::StopRun(); } + bool StorageLimitExceeded() { return mStorageLimitExceeded; } private: bool mLogAllFileAccess; +public: + ProgressNotifier* GetProgressNotifier() { return mpProgressNotifier; } + LocationResolver* GetLocationResolver() { return mpLocationResolver; } + RunStatusProvider* GetRunStatusProvider() { return mpRunStatusProvider; } + SysadminNotifier* GetSysadminNotifier() { return mpSysadminNotifier; } + void SetProgressNotifier (ProgressNotifier* p) { mpProgressNotifier = p; } + void SetLocationResolver (LocationResolver* p) { mpLocationResolver = p; } + void SetRunStatusProvider(RunStatusProvider* p) { mpRunStatusProvider = p; } + void SetSysadminNotifier (SysadminNotifier* p) { mpSysadminNotifier = p; } + +private: + ProgressNotifier* mpProgressNotifier; + LocationResolver* mpLocationResolver; + RunStatusProvider* mpRunStatusProvider; + SysadminNotifier* mpSysadminNotifier; + /* ProgressNotifier implementation */ public: + virtual void NotifyIDMapsSetup(BackupClientContext& rContext) { } + virtual void NotifyScanDirectory( const BackupClientDirectoryRecord* pDirRecord, const std::string& rLocalPath) @@ -387,7 +432,7 @@ public: { if (mLogAllFileAccess) { - BOX_INFO("Uploading complete file: " << rLocalPath); + BOX_NOTICE("Uploading complete file: " << rLocalPath); } } virtual void NotifyFileUploadingPatch( @@ -396,7 +441,7 @@ public: { if (mLogAllFileAccess) { - BOX_INFO("Uploading patch to file: " << rLocalPath); + BOX_NOTICE("Uploading patch to file: " << rLocalPath); } } virtual void NotifyFileUploaded( @@ -406,7 +451,7 @@ public: { if (mLogAllFileAccess) { - BOX_INFO("Uploaded file: " << rLocalPath); + BOX_NOTICE("Uploaded file: " << rLocalPath); } } virtual void NotifyFileSynchronised( @@ -419,18 +464,51 @@ public: BOX_INFO("Synchronised file: " << rLocalPath); } } + virtual void NotifyDirectoryDeleted( + int64_t ObjectID, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Deleted directory: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + } + virtual void NotifyFileDeleted( + int64_t ObjectID, + const std::string& rRemotePath) + { + if (mLogAllFileAccess) + { + BOX_NOTICE("Deleted file: " << rRemotePath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + ")"); + } + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", " << (length - offset) << " remain, eta " << + BoxTimeToSeconds(finish - elapsed) << "s"); + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", " << (length - offset) << " remain"); + } + virtual void NotifyReadProgress(int64_t readSize, int64_t offset) + { + BOX_TRACE("Read " << readSize << " bytes at " << offset << + ", unknown bytes remaining"); + } #ifdef WIN32 - public: - void RunHelperThread(void); - private: - bool mDoSyncFlagOut, mSyncIsForcedOut; bool mInstallService, mRemoveService, mRunAsService; std::string mServiceName; - HANDLE mhMessageToSendEvent, mhCommandReceivedEvent; - CRITICAL_SECTION mMessageQueueLock; - std::vector mMessageList; #endif }; diff --git a/bin/bbackupd/BackupDaemonInterface.h b/bin/bbackupd/BackupDaemonInterface.h new file mode 100644 index 00000000..5bbdd427 --- /dev/null +++ b/bin/bbackupd/BackupDaemonInterface.h @@ -0,0 +1,164 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupDaemonInterface.h +// Purpose: Interfaces for managing a BackupDaemon +// Created: 2008/12/30 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPDAEMONINTERFACE__H +#define BACKUPDAEMONINTERFACE__H + +#include +// #include + +// #include "BackupClientFileAttributes.h" +// #include "BackupStoreDirectory.h" +#include "BoxTime.h" +// #include "MD5Digest.h" +// #include "ReadLoggingStream.h" +// #include "RunStatusProvider.h" + +class Archive; +class BackupClientContext; +class BackupDaemon; + +// -------------------------------------------------------------------------- +// +// Class +// Name: SysadminNotifier +// Purpose: Provides a NotifySysadmin() method to send mail to the sysadmin +// Created: 2005/11/15 +// +// -------------------------------------------------------------------------- +class SysadminNotifier +{ + public: + virtual ~SysadminNotifier() { } + + typedef enum + { + StoreFull = 0, + ReadError, + BackupError, + BackupStart, + BackupFinish, + BackupOK, + MAX + // When adding notifications, remember to add + // strings to NotifySysadmin() + } + EventCode; + + virtual void NotifySysadmin(EventCode Event) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: ProgressNotifier +// Purpose: Provides methods for the backup library to inform the user +// interface about its progress with the backup +// Created: 2005/11/20 +// +// -------------------------------------------------------------------------- + +class BackupClientContext; +class BackupClientDirectoryRecord; + +class ProgressNotifier +{ + public: + virtual ~ProgressNotifier() { } + virtual void NotifyIDMapsSetup(BackupClientContext& rContext) = 0; + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyMountPointSkipped( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyDirExcluded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyUnsupportedFileType( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) = 0; + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) = 0; + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) = 0; + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) = 0; + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) = 0; + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) = 0; + virtual void NotifyDirectoryDeleted( + int64_t ObjectID, + const std::string& rRemotePath) = 0; + virtual void NotifyFileDeleted( + int64_t ObjectID, + const std::string& rRemotePath) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, box_time_t finish) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset, + int64_t length) = 0; + virtual void NotifyReadProgress(int64_t readSize, int64_t offset) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: LocationResolver +// Purpose: Interface for classes that can resolve locations to paths, +// like BackupDaemon +// Created: 2003/10/08 +// +// -------------------------------------------------------------------------- +class LocationResolver +{ +public: + virtual ~LocationResolver() { } + virtual bool FindLocationPathName(const std::string &rLocationName, + std::string &rPathOut) const = 0; +}; + +#endif // BACKUPDAEMONINTERFACE__H diff --git a/bin/bbackupd/Win32ServiceFunctions.cpp b/bin/bbackupd/Win32ServiceFunctions.cpp index a7bf6bd9..2df914a7 100644 --- a/bin/bbackupd/Win32ServiceFunctions.cpp +++ b/bin/bbackupd/Win32ServiceFunctions.cpp @@ -203,12 +203,12 @@ int InstallService(const char* pConfigFileName, const std::string& rServiceName) { if (pConfigFileName != NULL) { - struct stat st; + EMU_STRUCT_STAT st; if (emu_stat(pConfigFileName, &st) != 0) { - BOX_ERROR("Failed to open configuration file '" << - pConfigFileName << "': " << strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to open configuration file " + "'" << pConfigFileName << "'"); return 1; } @@ -221,7 +221,7 @@ int InstallService(const char* pConfigFileName, const std::string& rServiceName) } } - SC_HANDLE scm = OpenSCManager(0,0,SC_MANAGER_CREATE_SERVICE); + SC_HANDLE scm = OpenSCManager(0, 0, SC_MANAGER_CREATE_SERVICE); if (!scm) { diff --git a/bin/bbackupd/bbackupd-config.in b/bin/bbackupd/bbackupd-config.in index 16ddb75c..925dcc3e 100755 --- a/bin/bbackupd/bbackupd-config.in +++ b/bin/bbackupd/bbackupd-config.in @@ -26,7 +26,7 @@ Parameters: explicitly, using bbackupctl sync account-num (hexdecimal) and server-hostname are supplied by the server administrator - working-dir is usually @localstatedir_expanded@ + working-dir is usually @localstatedir_expanded@/bbackupd backup directories is list of directories to back up __E @@ -227,7 +227,7 @@ SUBJECT="BACKUP PROBLEM on host $hostname" SENDTO="$current_username" if [ "\$1" = "" ]; then - echo "Usage: $0 " >&2 + echo "Usage: \$0 " >&2 exit 2 elif [ "\$1" = store-full ]; then $sendmail \$SENDTO < #endif -#include #include #include #include @@ -25,29 +24,33 @@ #include #endif -#include +#include #include +#include +#include +#include +#include "BackupClientFileAttributes.h" +#include "BackupClientMakeExcludeList.h" +#include "BackupClientRestore.h" #include "BackupQueries.h" -#include "Utils.h" -#include "Configuration.h" -#include "autogen_BackupProtocolClient.h" -#include "BackupStoreFilenameClear.h" #include "BackupStoreDirectory.h" -#include "IOStream.h" -#include "BoxTimeToText.h" -#include "FileStream.h" +#include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "TemporaryDirectory.h" -#include "FileModificationTime.h" -#include "BackupClientFileAttributes.h" +#include "BackupStoreFilenameClear.h" +#include "BoxTimeToText.h" #include "CommonException.h" -#include "BackupClientRestore.h" -#include "BackupStoreException.h" +#include "Configuration.h" #include "ExcludeList.h" -#include "BackupClientMakeExcludeList.h" -#include "PathUtils.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "IOStream.h" #include "Logging.h" +#include "PathUtils.h" +#include "SelfFlushingStream.h" +#include "TemporaryDirectory.h" +#include "Utils.h" +#include "autogen_BackupProtocolClient.h" #include "MemLeakFindOn.h" @@ -68,8 +71,10 @@ // Created: 2003/10/10 // // -------------------------------------------------------------------------- -BackupQueries::BackupQueries(BackupProtocolClient &rConnection, const Configuration &rConfiguration) - : mrConnection(rConnection), +BackupQueries::BackupQueries(BackupProtocolClient &rConnection, + const Configuration &rConfiguration, bool readWrite) + : mReadWrite(readWrite), + mrConnection(rConnection), mrConfiguration(rConfiguration), mQuitNow(false), mRunningAsRoot(false), @@ -115,7 +120,13 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) if(Command[0] == 's' && Command[1] == 'h' && Command[2] == ' ' && Command[3] != '\0') { // Yes, run shell command - ::system(Command + 3); + int result = ::system(Command + 3); + if(result != 0) + { + BOX_WARNING("System command returned error code " << + result); + SetReturnCode(ReturnCode::Command_Error); + } return; } @@ -209,28 +220,36 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) { "getobject", "" }, { "get", "i" }, { "compare", "alcqAEQ" }, - { "restore", "dri" }, + { "restore", "drif" }, { "help", "" }, - { "usage", "" }, + { "usage", "m" }, { "undelete", "" }, + { "delete", "" }, { NULL, NULL } }; - #define COMMAND_Quit 0 - #define COMMAND_Exit 1 - #define COMMAND_List 2 - #define COMMAND_pwd 3 - #define COMMAND_cd 4 - #define COMMAND_lcd 5 - #define COMMAND_sh 6 - #define COMMAND_GetObject 7 - #define COMMAND_Get 8 - #define COMMAND_Compare 9 - #define COMMAND_Restore 10 - #define COMMAND_Help 11 - #define COMMAND_Usage 12 - #define COMMAND_Undelete 13 - static const char *alias[] = {"ls", 0}; - static const int aliasIs[] = {COMMAND_List, 0}; + + typedef enum + { + Command_Quit = 0, + Command_Exit, + Command_List, + Command_pwd, + Command_cd, + Command_lcd, + Command_sh, + Command_GetObject, + Command_Get, + Command_Compare, + Command_Restore, + Command_Help, + Command_Usage, + Command_Undelete, + Command_Delete, + } + CommandType; + + static const char *alias[] = {"ls", 0}; + static const int aliasIs[] = {Command_List, 0}; // Work out which command it is... int cmd = 0; @@ -284,25 +303,25 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) } } - if(cmd != COMMAND_Quit && cmd != COMMAND_Exit) + if(cmd != Command_Quit && cmd != Command_Exit) { // If not a quit command, set the return code to zero - SetReturnCode(0); + SetReturnCode(ReturnCode::Command_OK); } // Handle command switch(cmd) { - case COMMAND_Quit: - case COMMAND_Exit: + case Command_Quit: + case Command_Exit: mQuitNow = true; break; - case COMMAND_List: + case Command_List: CommandList(args, opts); break; - case COMMAND_pwd: + case Command_pwd: { // Simple implementation, so do it here BOX_INFO(GetCurrentDirectoryName() << " (" << @@ -310,47 +329,52 @@ void BackupQueries::DoCommand(const char *Command, bool isFromCommandLine) } break; - case COMMAND_cd: + case Command_cd: CommandChangeDir(args, opts); break; - case COMMAND_lcd: + case Command_lcd: CommandChangeLocalDir(args); break; - case COMMAND_sh: + case Command_sh: BOX_ERROR("The command to run must be specified as an argument."); break; - case COMMAND_GetObject: + case Command_GetObject: CommandGetObject(args, opts); break; - case COMMAND_Get: + case Command_Get: CommandGet(args, opts); break; - case COMMAND_Compare: + case Command_Compare: CommandCompare(args, opts); break; - case COMMAND_Restore: + case Command_Restore: CommandRestore(args, opts); break; - case COMMAND_Usage: - CommandUsage(); + case Command_Usage: + CommandUsage(opts); break; - case COMMAND_Help: + case Command_Help: CommandHelp(args); break; - case COMMAND_Undelete: + case Command_Undelete: CommandUndelete(args, opts); break; + case Command_Delete: + CommandDelete(args, opts); + break; + default: + BOX_ERROR("Unknown command: " << Command); break; } } @@ -402,6 +426,7 @@ void BackupQueries::CommandList(const std::vector &args, const bool { BOX_ERROR("Directory '" << args[0] << "' not found " "on store."); + SetReturnCode(ReturnCode::Command_Error); return; } } @@ -427,11 +452,28 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolClientListDirectory::Flags_Deleted; // Do communication - mrConnection.QueryListDirectory( - DirID, - BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, // both files and directories - excludeFlags, - true /* want attributes */); + try + { + mrConnection.QueryListDirectory( + DirID, + BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, + // both files and directories + excludeFlags, + true /* want attributes */); + } + catch (std::exception &e) + { + BOX_ERROR("Failed to list directory: " << e.what()); + SetReturnCode(ReturnCode::Command_Error); + return; + } + catch (...) + { + BOX_ERROR("Failed to list directory: unknown error"); + SetReturnCode(ReturnCode::Command_Error); + return; + } + // Retrieve the directory from the stream following BackupStoreDirectory dir; @@ -574,15 +616,18 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool // -------------------------------------------------------------------------- // // Function -// Name: BackupQueries::FindDirectoryObjectID(const std::string &) -// Purpose: Find the object ID of a directory on the store, or return 0 for not found. -// If pStack != 0, the object is set to the stack of directories. -// Will start from the current directory stack. +// Name: BackupQueries::FindDirectoryObjectID(const +// std::string &) +// Purpose: Find the object ID of a directory on the store, +// or return 0 for not found. If pStack != 0, the +// object is set to the stack of directories. +// Will start from the current directory stack. // Created: 2003/10/10 // // -------------------------------------------------------------------------- -int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, bool AllowOldVersion, - bool AllowDeletedDirs, std::vector > *pStack) +int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, + bool AllowOldVersion, bool AllowDeletedDirs, + std::vector > *pStack) { // Split up string into elements std::vector dirElements; @@ -747,6 +792,7 @@ void BackupQueries::CommandChangeDir(const std::vector &args, const if(args.size() != 1 || args[0].size() == 0) { BOX_ERROR("Incorrect usage. cd [-o] [-d] "); + SetReturnCode(ReturnCode::Command_Error); return; } @@ -764,6 +810,7 @@ void BackupQueries::CommandChangeDir(const std::vector &args, const if(id == 0) { BOX_ERROR("Directory '" << args[0] << "' not found."); + SetReturnCode(ReturnCode::Command_Error); return; } @@ -785,7 +832,7 @@ void BackupQueries::CommandChangeLocalDir(const std::vector &args) if(args.size() != 1 || args[0].size() == 0) { BOX_ERROR("Incorrect usage. lcd "); - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); return; } @@ -795,7 +842,7 @@ void BackupQueries::CommandChangeLocalDir(const std::vector &args) if(!ConvertConsoleToUtf8(args[0].c_str(), dirName)) { BOX_ERROR("Failed to convert path from console encoding."); - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); return; } int result = ::chdir(dirName.c_str()); @@ -810,11 +857,11 @@ void BackupQueries::CommandChangeLocalDir(const std::vector &args) } else { - BOX_ERROR("Error changing to directory '" << - args[0] << ": " << strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to change to directory " + "'" << args[0] << "'"); } - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); return; } @@ -822,9 +869,8 @@ void BackupQueries::CommandChangeLocalDir(const std::vector &args) char wd[PATH_MAX]; if(::getcwd(wd, PATH_MAX) == 0) { - BOX_ERROR("Error getting current directory: " << - strerror(errno)); - SetReturnCode(COMMAND_RETURN_ERROR); + BOX_LOG_SYS_ERROR("Error getting current directory"); + SetReturnCode(ReturnCode::Command_Error); return; } @@ -832,7 +878,7 @@ void BackupQueries::CommandChangeLocalDir(const std::vector &args) if(!ConvertUtf8ToConsole(wd, dirName)) { BOX_ERROR("Failed to convert new path from console encoding."); - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); return; } BOX_INFO("Local current directory is now '" << dirName << "'."); @@ -868,8 +914,8 @@ void BackupQueries::CommandGetObject(const std::vector &args, const } // Does file exist? - struct stat st; - if(::stat(args[1].c_str(), &st) == 0 || errno != ENOENT) + EMU_STRUCT_STAT st; + if(EMU_STAT(args[1].c_str(), &st) == 0 || errno != ENOENT) { BOX_ERROR("The local file '" << args[1] << " already exists."); return; @@ -907,6 +953,117 @@ void BackupQueries::CommandGetObject(const std::vector &args, const } +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::FindFileID(const std::string& +// rNameOrIdString, const bool *options, +// int64_t *pDirIdOut, std::string* pFileNameOut) +// Purpose: Locate a file on the store (either by name or by +// object ID, depending on opts['i'], where name can +// include a path) and return the file ID, placing the +// directory ID in *pDirIdOut and the filename part +// of the path (if not looking up by ID and not NULL) +// in *pFileNameOut. +// Created: 2008-09-12 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::FindFileID(const std::string& rNameOrIdString, + const bool *opts, int64_t *pDirIdOut, std::string* pFileNameOut, + int16_t flagsInclude, int16_t flagsExclude, int16_t* pFlagsOut) +{ + // Find object ID somehow + int64_t fileId; + int64_t dirId = GetCurrentDirectoryID(); + std::string fileName = rNameOrIdString; + + if(!opts['i']) + { + // does this remote filename include a path? + std::string::size_type index = fileName.rfind('/'); + if(index != std::string::npos) + { + std::string dirName(fileName.substr(0, index)); + fileName = fileName.substr(index + 1); + + dirId = FindDirectoryObjectID(dirName); + if(dirId == 0) + { + BOX_ERROR("Directory '" << dirName << + "' not found."); + return 0; + } + } + + if(pFileNameOut) + { + *pFileNameOut = fileName; + } + } + + BackupStoreFilenameClear fn(fileName); + + // Need to look it up in the current directory + mrConnection.QueryListDirectory( + dirId, flagsInclude, flagsExclude, + true /* do want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + BackupStoreDirectory::Entry *en; + + if(opts['i']) + { + // Specified as ID. + fileId = ::strtoll(rNameOrIdString.c_str(), 0, 16); + if(fileId == std::numeric_limits::min() || + fileId == std::numeric_limits::max() || + fileId == 0) + { + BOX_ERROR("Not a valid object ID (specified in hex)."); + return 0; + } + + // Check that the item is actually in the directory + en = dir.FindEntryByID(fileId); + if(en == 0) + { + BOX_ERROR("File ID " << + BOX_FORMAT_OBJECTID(fileId) << + " not found in current directory on store.\n" + "(You can only access files by ID from the " + "current directory.)"); + return 0; + } + } + else + { + // Specified by name, find the object in the directory to get the ID + BackupStoreDirectory::Iterator i(dir); + en = i.FindMatchingClearName(fn); + if(en == 0) + { + BOX_ERROR("Filename '" << rNameOrIdString << "' " + "not found in current directory on store.\n" + "(Subdirectories in path not searched.)"); + return 0; + } + + fileId = en->GetObjectID(); + } + + *pDirIdOut = dirId; + + if(pFlagsOut) + { + *pFlagsOut = en->GetFlags(); + } + + return fileId; +} + // -------------------------------------------------------------------------- // @@ -929,120 +1086,72 @@ void BackupQueries::CommandGet(std::vector args, const bool *opts) } // Find object ID somehow - int64_t fileId; - int64_t dirId = GetCurrentDirectoryID(); + int64_t fileId, dirId; std::string localName; - // BLOCK - { #ifdef WIN32 - for (std::vector::iterator - i = args.begin(); i != args.end(); i++) + for (std::vector::iterator + i = args.begin(); i != args.end(); i++) + { + std::string out; + if(!ConvertConsoleToUtf8(i->c_str(), out)) { - std::string out; - if(!ConvertConsoleToUtf8(i->c_str(), out)) - { - BOX_ERROR("Failed to convert encoding."); - return; - } - *i = out; + BOX_ERROR("Failed to convert encoding."); + return; } + *i = out; + } #endif - std::string fileName(args[0]); + int16_t flagsExclude; - if(!opts['i']) - { - // does this remote filename include a path? - std::string::size_type index = fileName.rfind('/'); - if(index != std::string::npos) - { - std::string dirName(fileName.substr(0, index)); - fileName = fileName.substr(index + 1); - - dirId = FindDirectoryObjectID(dirName); - if(dirId == 0) - { - BOX_ERROR("Directory '" << dirName << - "' not found."); - return; - } - } - } + if(opts['i']) + { + // can retrieve anything by ID + flagsExclude = BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING; + } + else + { + // only current versions by name + flagsExclude = + BackupProtocolClientListDirectory::Flags_OldVersion | + BackupProtocolClientListDirectory::Flags_Deleted; + } - BackupStoreFilenameClear fn(fileName); - // Need to look it up in the current directory - mrConnection.QueryListDirectory( - dirId, - BackupProtocolClientListDirectory::Flags_File, // just files - (opts['i'])?(BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING):(BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted), // only current versions - false /* don't want attributes */); + fileId = FindFileID(args[0], opts, &dirId, &localName, + BackupProtocolClientListDirectory::Flags_File, // just files + flagsExclude, NULL /* don't care about flags found */); - // Retrieve the directory from the stream following - BackupStoreDirectory dir; - std::auto_ptr dirstream(mrConnection.ReceiveStream()); - dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + if (fileId == 0) + { + // error already reported + return; + } - if(opts['i']) + if(opts['i']) + { + // Specified as ID. Must have a local name in the arguments + // (check at beginning of function ensures this) + localName = args[1]; + } + else + { + // Specified by name. Local name already set by FindFileID, + // but may be overridden by user supplying a second argument. + if(args.size() == 2) { - // Specified as ID. - fileId = ::strtoll(args[0].c_str(), 0, 16); - if(fileId == std::numeric_limits::min() || - fileId == std::numeric_limits::max() || - fileId == 0) - { - BOX_ERROR("Not a valid object ID (specified in hex)."); - return; - } - - // Check that the item is actually in the directory - if(dir.FindEntryByID(fileId) == 0) - { - BOX_ERROR("File ID " << - BOX_FORMAT_OBJECTID(fileId) << - " not found in current " - "directory on store.\n" - "(You can only download files by ID " - "from the current directory.)"); - return; - } - - // Must have a local name in the arguments (check at beginning of function ensures this) localName = args[1]; } - else - { - // Specified by name, find the object in the directory to get the ID - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = i.FindMatchingClearName(fn); - - if(en == 0) - { - BOX_ERROR("Filename '" << args[0] << "' " - "not found in current " - "directory on store.\n" - "(Subdirectories in path not " - "searched.)"); - return; - } - - fileId = en->GetObjectID(); - - // Local name is the last argument, which is either - // the looked up filename, or a filename specified - // by the user. - localName = args[args.size() - 1]; - } } // Does local file already exist? (don't want to overwrite) - struct stat st; - if(::stat(localName.c_str(), &st) == 0 || errno != ENOENT) + EMU_STRUCT_STAT st; + if(EMU_STAT(localName.c_str(), &st) == 0 || errno != ENOENT) { BOX_ERROR("The local file " << localName << " already exists, " "will not overwrite it."); - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); return; } @@ -1081,7 +1190,6 @@ void BackupQueries::CommandGet(std::vector args, const bool *opts) } } - // -------------------------------------------------------------------------- // // Function @@ -1090,58 +1198,17 @@ void BackupQueries::CommandGet(std::vector args, const bool *opts) // Created: 29/1/04 // // -------------------------------------------------------------------------- -BackupQueries::CompareParams::CompareParams() - : mQuickCompare(false), - mIgnoreExcludes(false), - mIgnoreAttributes(false), - mDifferences(0), - mDifferencesExplainedByModTime(0), - mUncheckedFiles(0), - mExcludedDirs(0), - mExcludedFiles(0), - mpExcludeFiles(0), - mpExcludeDirs(0), - mLatestFileUploadTime(0) -{ -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CompareParams::~CompareParams() -// Purpose: Destructor -// Created: 29/1/04 -// -// -------------------------------------------------------------------------- -BackupQueries::CompareParams::~CompareParams() -{ - DeleteExcludeLists(); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupQueries::CompareParams::DeleteExcludeLists() -// Purpose: Delete the include lists contained -// Created: 29/1/04 -// -// -------------------------------------------------------------------------- -void BackupQueries::CompareParams::DeleteExcludeLists() -{ - if(mpExcludeFiles != 0) - { - delete mpExcludeFiles; - mpExcludeFiles = 0; - } - if(mpExcludeDirs != 0) - { - delete mpExcludeDirs; - mpExcludeDirs = 0; - } -} - +BackupQueries::CompareParams::CompareParams(bool QuickCompare, + bool IgnoreExcludes, bool IgnoreAttributes, + box_time_t LatestFileUploadTime) +: BoxBackupCompareParams(QuickCompare, IgnoreExcludes, IgnoreAttributes, + LatestFileUploadTime), + mDifferences(0), + mDifferencesExplainedByModTime(0), + mUncheckedFiles(0), + mExcludedDirs(0), + mExcludedFiles(0) +{ } // -------------------------------------------------------------------------- // @@ -1153,24 +1220,19 @@ void BackupQueries::CompareParams::DeleteExcludeLists() // -------------------------------------------------------------------------- void BackupQueries::CommandCompare(const std::vector &args, const bool *opts) { - // Parameters, including count of differences - BackupQueries::CompareParams params; - params.mQuickCompare = opts['q']; - params.mQuietCompare = opts['Q']; - params.mIgnoreExcludes = opts['E']; - params.mIgnoreAttributes = opts['A']; + box_time_t LatestFileUploadTime = GetCurrentBoxTime(); // Try and work out the time before which all files should be on the server { std::string syncTimeFilename(mrConfiguration.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); syncTimeFilename += "last_sync_start"; // Stat it to get file time - struct stat st; - if(::stat(syncTimeFilename.c_str(), &st) == 0) + EMU_STRUCT_STAT st; + if(EMU_STAT(syncTimeFilename.c_str(), &st) == 0) { // Files modified after this time shouldn't be on the server, so report errors slightly differently - params.mLatestFileUploadTime = FileModificationTime(st) - - SecondsToBoxTime(mrConfiguration.GetKeyValueInt("MinimumFileAge")); + LatestFileUploadTime = FileModificationTime(st) - + SecondsToBoxTime(mrConfiguration.GetKeyValueInt("MinimumFileAge")); } else { @@ -1179,8 +1241,16 @@ void BackupQueries::CommandCompare(const std::vector &args, const b } } + // Parameters, including count of differences + BackupQueries::CompareParams params(opts['q'], // quick compare? + opts['E'], // ignore excludes + opts['A'], // ignore attributes + LatestFileUploadTime); + + params.mQuietCompare = opts['Q']; + // Quick compare? - if(params.mQuickCompare) + if(params.QuickCompare()) { BOX_WARNING("Quick compare used -- file attributes are not " "checked."); @@ -1189,11 +1259,16 @@ void BackupQueries::CommandCompare(const std::vector &args, const b if(!opts['l'] && opts['a'] && args.size() == 0) { // Compare all locations - const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations")); - for(std::list >::const_iterator i = locations.mSubConfigurations.begin(); - i != locations.mSubConfigurations.end(); ++i) + const Configuration &rLocations( + mrConfiguration.GetSubConfiguration("BackupLocations")); + std::vector locNames = + rLocations.GetSubConfigurationNames(); + for(std::vector::iterator + pLocName = locNames.begin(); + pLocName != locNames.end(); + pLocName++) { - CompareLocation(i->first, params); + CompareLocation(*pLocName, params); } } else if(opts['l'] && !opts['a'] && args.size() == 1) @@ -1206,7 +1281,7 @@ void BackupQueries::CommandCompare(const std::vector &args, const b // Compare directory to directory // Can't be bothered to do all the hard work to work out which location it's on, and hence which exclude list - if(!params.mIgnoreExcludes) + if(!params.IgnoreExcludes()) { BOX_ERROR("Cannot use excludes on directory to directory comparison -- use -E flag to specify ignored excludes."); return; @@ -1241,15 +1316,15 @@ void BackupQueries::CommandCompare(const std::vector &args, const b { if (params.mUncheckedFiles != 0) { - SetReturnCode(COMPARE_RETURN_ERROR); + SetReturnCode(ReturnCode::Compare_Error); } else if (params.mDifferences != 0) { - SetReturnCode(COMPARE_RETURN_DIFFERENT); + SetReturnCode(ReturnCode::Compare_Different); } else { - SetReturnCode(COMPARE_RETURN_SAME); + SetReturnCode(ReturnCode::Compare_Same); } } } @@ -1263,7 +1338,8 @@ void BackupQueries::CommandCompare(const std::vector &args, const b // Created: 2003/10/13 // // -------------------------------------------------------------------------- -void BackupQueries::CompareLocation(const std::string &rLocation, BackupQueries::CompareParams &rParams) +void BackupQueries::CompareLocation(const std::string &rLocation, + BoxBackupCompareParams &rParams) { // Find the location's sub configuration const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations")); @@ -1287,45 +1363,36 @@ void BackupQueries::CompareLocation(const std::string &rLocation, BackupQueries: } #endif - try - { - // Generate the exclude lists - if(!rParams.mIgnoreExcludes) - { - rParams.mpExcludeFiles = BackupClientMakeExcludeList_Files(loc); - rParams.mpExcludeDirs = BackupClientMakeExcludeList_Dirs(loc); - } - - // Then get it compared - Compare(std::string("/") + rLocation, - loc.GetKeyValue("Path"), rParams); - } - catch(...) + // Generate the exclude lists + if(!rParams.IgnoreExcludes()) { - // Clean up - rParams.DeleteExcludeLists(); - throw; + rParams.LoadExcludeLists(loc); } - - // Delete exclude lists - rParams.DeleteExcludeLists(); + + // Then get it compared + Compare(std::string("/") + rLocation, loc.GetKeyValue("Path"), rParams); } // -------------------------------------------------------------------------- // // Function -// Name: BackupQueries::Compare(const std::string &, const std::string &, BackupQueries::CompareParams &) +// Name: BackupQueries::Compare(const std::string &, +// const std::string &, BackupQueries::CompareParams &) // Purpose: Compare a store directory against a local directory // Created: 2003/10/13 // // -------------------------------------------------------------------------- -void BackupQueries::Compare(const std::string &rStoreDir, const std::string &rLocalDir, BackupQueries::CompareParams &rParams) +void BackupQueries::Compare(const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams) { #ifdef WIN32 + std::string localDirEncoded; std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(rLocalDir.c_str(), localDirEncoded)) return; if(!ConvertConsoleToUtf8(rStoreDir.c_str(), storeDirEncoded)) return; #else + const std::string& localDirEncoded(rLocalDir); const std::string& storeDirEncoded(rStoreDir); #endif @@ -1335,19 +1402,22 @@ void BackupQueries::Compare(const std::string &rStoreDir, const std::string &rLo // Found? if(dirID == 0) { - BOX_WARNING("Local directory '" << rLocalDir << "' exists, " - "but server directory '" << rStoreDir << "' does not " - "exist."); - rParams.mDifferences ++; + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalDir.c_str(), &st) == 0) + { + if(FileAttrModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyRemoteFileMissing(localDirEncoded, + storeDirEncoded, modifiedAfterLastSync); return; } - -#ifdef WIN32 - std::string localDirEncoded; - if(!ConvertConsoleToUtf8(rLocalDir.c_str(), localDirEncoded)) return; -#else - std::string localDirEncoded(rLocalDir); -#endif // Go! Compare(dirID, storeDirEncoded, localDirEncoded, rParams); @@ -1363,56 +1433,36 @@ void BackupQueries::Compare(const std::string &rStoreDir, const std::string &rLo // Created: 2003/10/13 // // -------------------------------------------------------------------------- -void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const std::string &rLocalDir, BackupQueries::CompareParams &rParams) +void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams) { -#ifdef WIN32 - // By this point, rStoreDir and rLocalDir should be in UTF-8 encoding - - std::string localDirDisplay; - std::string storeDirDisplay; - - if(!ConvertUtf8ToConsole(rLocalDir.c_str(), localDirDisplay)) return; - if(!ConvertUtf8ToConsole(rStoreDir.c_str(), storeDirDisplay)) return; -#else - const std::string& localDirDisplay(rLocalDir); - const std::string& storeDirDisplay(rStoreDir); -#endif + rParams.NotifyDirComparing(rLocalDir, rStoreDir); // Get info on the local directory - struct stat st; - if(::lstat(rLocalDir.c_str(), &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(rLocalDir.c_str(), &st) != 0) { // What kind of error? - if(errno == ENOTDIR) - { - BOX_WARNING("Local object '" << localDirDisplay << "' " - "is a file, server object '" << - storeDirDisplay << "' is a directory."); - rParams.mDifferences ++; - } - else if(errno == ENOENT) + if(errno == ENOTDIR || errno == ENOENT) { - BOX_WARNING("Local directory '" << localDirDisplay << - "' does not exist (compared to server " - "directory '" << storeDirDisplay << "')."); - rParams.mDifferences ++; + rParams.NotifyLocalDirMissing(rLocalDir, rStoreDir); } else { - BOX_WARNING("Failed to access local directory '" << - localDirDisplay << ": " << strerror(errno) << - "'."); - rParams.mUncheckedFiles ++; + rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); } return; } // Get the directory listing from the store mrConnection.QueryListDirectory( - DirID, - BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, // get everything - BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted, // except for old versions and deleted files - true /* want attributes */); + DirID, + BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, + // get everything + BackupProtocolClientListDirectory::Flags_OldVersion | + BackupProtocolClientListDirectory::Flags_Deleted, + // except for old versions and deleted files + true /* want attributes */); // Retrieve the directory from the stream following BackupStoreDirectory dir; @@ -1422,8 +1472,7 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s // Test out the attributes if(!dir.HasAttributes()) { - BOX_WARNING("Store directory '" << storeDirDisplay << "' " - "doesn't have attributes."); + rParams.NotifyStoreDirMissingAttributes(rLocalDir, rStoreDir); } else { @@ -1436,12 +1485,27 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s localAttr.ReadAttributes(rLocalDir.c_str(), true /* directories have zero mod times */); - if(!(attr.Compare(localAttr, true, true /* ignore modification times */))) + if(attr.Compare(localAttr, true, true /* ignore modification times */)) { - BOX_WARNING("Local directory '" << localDirDisplay << - "' has different attributes to store " - "directory '" << storeDirDisplay << "'."); - rParams.mDifferences ++; + rParams.NotifyDirCompared(rLocalDir, rStoreDir, + false, false /* actually we didn't check :) */); + } + else + { + bool modifiedAfterLastSync = false; + + EMU_STRUCT_STAT st; + if(EMU_STAT(rLocalDir.c_str(), &st) == 0) + { + if(FileAttrModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + } + + rParams.NotifyDirCompared(rLocalDir, rStoreDir, + true, modifiedAfterLastSync); } } @@ -1449,11 +1513,10 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s DIR *dirhandle = ::opendir(rLocalDir.c_str()); if(dirhandle == 0) { - BOX_WARNING("Failed to open local directory '" << - localDirDisplay << "': " << strerror(errno)); - rParams.mUncheckedFiles ++; + rParams.NotifyLocalDirAccessFailed(rLocalDir, rStoreDir); return; } + try { // Read the files and directories into sets @@ -1484,8 +1547,8 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s #ifndef HAVE_VALID_DIRENT_D_TYPE std::string fn(MakeFullPath (rLocalDir, localDirEn->d_name)); - struct stat st; - if(::lstat(fn.c_str(), &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(fn.c_str(), &st) != 0) { THROW_EXCEPTION(CommonException, OSFileError) } @@ -1518,8 +1581,8 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s // Close directory if(::closedir(dirhandle) != 0) { - BOX_ERROR("Failed to close local directory '" << - localDirDisplay << "': " << strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to close local directory " + "'" << rLocalDir << "'"); } dirhandle = 0; @@ -1557,36 +1620,30 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s for(std::set >::const_iterator i = storeFiles.begin(); i != storeFiles.end(); ++i) { const std::string& fileName(i->first); -#ifdef WIN32 - // File name is also in UTF-8 encoding, - // need to convert to console - std::string fileNameDisplay; - if(!ConvertUtf8ToConsole(i->first.c_str(), - fileNameDisplay)) return; -#else - const std::string& fileNameDisplay(i->first); -#endif - std::string localPath(MakeFullPath - (rLocalDir, fileName)); - std::string localPathDisplay(MakeFullPath - (localDirDisplay, fileNameDisplay)); - std::string storePathDisplay - (storeDirDisplay + "/" + fileNameDisplay); + std::string localPath(MakeFullPath(rLocalDir, fileName)); + std::string storePath(rStoreDir + "/" + fileName); + rParams.NotifyFileComparing(localPath, storePath); + // Does the file exist locally? string_set_iter_t local(localFiles.find(fileName)); if(local == localFiles.end()) { // Not found -- report - BOX_WARNING("Local file '" << - localPathDisplay << "' does not exist, " - "but store file '" << - storePathDisplay << "' does."); - rParams.mDifferences ++; + rParams.NotifyLocalFileMissing(localPath, + storePath); } else - { + { + int64_t fileSize = 0; + + EMU_STRUCT_STAT st; + if(EMU_STAT(localPath.c_str(), &st) == 0) + { + fileSize = st.st_size; + } + try { // Files the same flag? @@ -1594,8 +1651,10 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s // File modified after last sync flag bool modifiedAfterLastSync = false; + + bool hasDifferentAttribs = false; - if(rParams.mQuickCompare) + if(rParams.QuickCompare()) { // Compare file -- fetch it mrConnection.QueryGetBlockIndexByID(i->second->GetObjectID()); @@ -1640,7 +1699,7 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s BackupClientFileAttributes localAttr; box_time_t fileModTime = 0; localAttr.ReadAttributes(localPath.c_str(), false /* don't zero mod times */, &fileModTime); - modifiedAfterLastSync = (fileModTime > rParams.mLatestFileUploadTime); + modifiedAfterLastSync = (fileModTime > rParams.LatestFileUploadTime()); bool ignoreAttrModTime = true; #ifdef WIN32 @@ -1649,7 +1708,7 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s ignoreAttrModTime = false; #endif - if(!rParams.mIgnoreAttributes && + if(!rParams.IgnoreAttributes() && #ifdef PLATFORM_DISABLE_SYMLINK_ATTRIB_COMPARE !fileOnServerStream->IsSymLink() && #endif @@ -1657,128 +1716,43 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s ignoreAttrModTime, fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) { - BOX_WARNING("Local file '" << - localPathDisplay << - "' has different attributes " - "to store file '" << - storePathDisplay << - "'."); - rParams.mDifferences ++; - if(modifiedAfterLastSync) - { - rParams.mDifferencesExplainedByModTime ++; - BOX_INFO("(the file above was modified after the last sync time -- might be reason for difference)"); - } - else if(i->second->HasAttributes()) - { - BOX_INFO("(the file above has had new attributes applied)\n"); - } + hasDifferentAttribs = true; } // Compare contents, if it's a regular file not a link // Remember, we MUST read the entire stream from the server. + SelfFlushingStream flushObject(*objectStream); + if(!fileOnServerStream->IsSymLink()) { + SelfFlushingStream flushFile(*fileOnServerStream); // Open the local file FileStream l(localPath.c_str()); - - // Size - IOStream::pos_type fileSizeLocal = l.BytesLeftToRead(); - IOStream::pos_type fileSizeServer = 0; - - // Test the contents - char buf1[2048]; - char buf2[2048]; - while(fileOnServerStream->StreamDataLeft() && l.StreamDataLeft()) - { - int size = fileOnServerStream->Read(buf1, sizeof(buf1), mrConnection.GetTimeout()); - fileSizeServer += size; - - if(l.Read(buf2, size) != size - || ::memcmp(buf1, buf2, size) != 0) - { - equal = false; - break; - } - } - - // Check read all the data from the server and file -- can't be equal if local and remote aren't the same length - // Can't use StreamDataLeft() test on file, because if it's the same size, it won't know - // it's EOF yet. - if(fileOnServerStream->StreamDataLeft() || fileSizeServer != fileSizeLocal) - { - equal = false; - } - - // Must always read the entire decoded stream, if it's not a symlink - if(fileOnServerStream->StreamDataLeft()) - { - // Absorb all the data remaining - char buffer[2048]; - while(fileOnServerStream->StreamDataLeft()) - { - fileOnServerStream->Read(buffer, sizeof(buffer), mrConnection.GetTimeout()); - } - } - - // Must always read the entire encoded stream - if(objectStream->StreamDataLeft()) - { - // Absorb all the data remaining - char buffer[2048]; - while(objectStream->StreamDataLeft()) - { - objectStream->Read(buffer, sizeof(buffer), mrConnection.GetTimeout()); - } - } + equal = l.CompareWith(*fileOnServerStream, + mrConnection.GetTimeout()); } } - // Report if not equal. - if(!equal) - { - BOX_WARNING("Local file '" << - localPathDisplay << "' " - "has different contents " - "to store file '" << - storePathDisplay << - "'."); - rParams.mDifferences ++; - if(modifiedAfterLastSync) - { - rParams.mDifferencesExplainedByModTime ++; - BOX_INFO("(the file above was modified after the last sync time -- might be reason for difference)"); - } - else if(i->second->HasAttributes()) - { - BOX_INFO("(the file above has had new attributes applied)\n"); - } - } + rParams.NotifyFileCompared(localPath, + storePath, fileSize, + hasDifferentAttribs, !equal, + modifiedAfterLastSync, + i->second->HasAttributes()); } catch(BoxException &e) { - BOX_ERROR("Failed to fetch and compare " - "'" << - storePathDisplay.c_str() << - "': error " << e.what() << - " (" << e.GetType() << - "/" << e.GetSubType() << ")"); - rParams.mUncheckedFiles ++; + rParams.NotifyDownloadFailed(localPath, + storePath, fileSize, e); } catch(std::exception &e) { - BOX_ERROR("Failed to fetch and compare " - "'" << - storePathDisplay.c_str() << - "': " << e.what()); + rParams.NotifyDownloadFailed(localPath, + storePath, fileSize, e); } catch(...) { - BOX_ERROR("Failed to fetch and compare " - "'" << - storePathDisplay.c_str() << - "': unknown error"); - rParams.mUncheckedFiles ++; + rParams.NotifyDownloadFailed(localPath, + storePath, fileSize); } // Remove from set so that we know it's been compared @@ -1786,53 +1760,34 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s } } - // Report any files which exist on the locally, but not on the store + // Report any files which exist locally, but not on the store for(string_set_iter_t i = localFiles.begin(); i != localFiles.end(); ++i) { -#ifdef WIN32 - // File name is also in UTF-8 encoding, - // need to convert to console - std::string fileNameDisplay; - if(!ConvertUtf8ToConsole(i->c_str(), fileNameDisplay)) - return; -#else - const std::string& fileNameDisplay(*i); -#endif - - std::string localPath(MakeFullPath - (rLocalDir, *i)); - std::string localPathDisplay(MakeFullPath - (localDirDisplay, fileNameDisplay)); - std::string storePathDisplay - (storeDirDisplay + "/" + fileNameDisplay); + std::string localPath(MakeFullPath(rLocalDir, *i)); + std::string storePath(rStoreDir + "/" + *i); // Should this be ignored (ie is excluded)? - if(rParams.mpExcludeFiles == 0 || - !(rParams.mpExcludeFiles->IsExcluded(localPath))) + if(!rParams.IsExcludedFile(localPath)) { - BOX_WARNING("Local file '" << - localPathDisplay << - "' exists, but store file '" << - storePathDisplay << - "' does not."); - rParams.mDifferences ++; + bool modifiedAfterLastSync = false; - // Check the file modification time + EMU_STRUCT_STAT st; + if(EMU_STAT(localPath.c_str(), &st) == 0) { - struct stat st; - if(::stat(localPath.c_str(), &st) == 0) + if(FileModificationTime(st) > + rParams.LatestFileUploadTime()) { - if(FileModificationTime(st) > rParams.mLatestFileUploadTime) - { - rParams.mDifferencesExplainedByModTime ++; - BOX_INFO("(the file above was modified after the last sync time -- might be reason for difference)"); - } + modifiedAfterLastSync = true; } } + + rParams.NotifyRemoteFileMissing(localPath, + storePath, modifiedAfterLastSync); } else { - rParams.mExcludedFiles ++; + rParams.NotifyExcludedFile(localPath, + storePath); } } @@ -1843,99 +1798,69 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s // Now do the directories, recursively to check subdirectories for(std::set >::const_iterator i = storeDirs.begin(); i != storeDirs.end(); ++i) { -#ifdef WIN32 - // Directory name is also in UTF-8 encoding, - // need to convert to console - std::string subdirNameDisplay; - if(!ConvertUtf8ToConsole(i->first.c_str(), - subdirNameDisplay)) - return; -#else - const std::string& subdirNameDisplay(i->first); -#endif - - std::string localPath(MakeFullPath - (rLocalDir, i->first)); - std::string localPathDisplay(MakeFullPath - (localDirDisplay, subdirNameDisplay)); - std::string storePathDisplay - (storeDirDisplay + "/" + subdirNameDisplay); + std::string localPath(MakeFullPath(rLocalDir, i->first)); + std::string storePath(rLocalDir + "/" + i->first); // Does the directory exist locally? string_set_iter_t local(localDirs.find(i->first)); if(local == localDirs.end() && - rParams.mpExcludeDirs != NULL && - rParams.mpExcludeDirs->IsExcluded(localPath)) + rParams.IsExcludedDir(localPath)) { - // Not found -- report - BOX_WARNING("Local directory '" << - localPathDisplay << "' is excluded, " - "but store directory '" << - storePathDisplay << "' still exists."); - rParams.mDifferences ++; + rParams.NotifyExcludedFileNotDeleted(localPath, + storePath); } else if(local == localDirs.end()) { // Not found -- report - BOX_WARNING("Local directory '" << - localPathDisplay << "' does not exist, " - "but store directory '" << - storePathDisplay << "' does."); - rParams.mDifferences ++; + rParams.NotifyRemoteFileMissing(localPath, + storePath, false); } - else if(rParams.mpExcludeDirs != NULL && - rParams.mpExcludeDirs->IsExcluded(localPath)) + else if(rParams.IsExcludedDir(localPath)) { // don't recurse into excluded directories } else { // Compare directory - Compare(i->second->GetObjectID(), rStoreDir + "/" + i->first, localPath, rParams); + Compare(i->second->GetObjectID(), + rStoreDir + "/" + i->first, + localPath, rParams); // Remove from set so that we know it's been compared localDirs.erase(local); } } - // Report any files which exist on the locally, but not on the store - for(std::set::const_iterator i = localDirs.begin(); i != localDirs.end(); ++i) + // Report any directories which exist locally, but not on the store + for(std::set::const_iterator + i = localDirs.begin(); + i != localDirs.end(); ++i) { -#ifdef WIN32 - // File name is also in UTF-8 encoding, - // need to convert to console - std::string fileNameDisplay; - if(!ConvertUtf8ToConsole(i->c_str(), fileNameDisplay)) - return; -#else - const std::string& fileNameDisplay(*i); -#endif - - std::string localPath(MakeFullPath - (rLocalDir, *i)); - std::string localPathDisplay(MakeFullPath - (localDirDisplay, fileNameDisplay)); - - std::string storePath - (rStoreDir + "/" + *i); - std::string storePathDisplay - (storeDirDisplay + "/" + fileNameDisplay); + std::string localPath(MakeFullPath(rLocalDir, *i)); + std::string storePath(rStoreDir + "/" + *i); // Should this be ignored (ie is excluded)? - if(rParams.mpExcludeDirs == 0 || !(rParams.mpExcludeDirs->IsExcluded(localPath))) + if(!rParams.IsExcludedDir(localPath)) { - BOX_WARNING("Local directory '" << - localPathDisplay << "' exists, but " - "store directory '" << - storePathDisplay << "' does not."); - rParams.mDifferences ++; + bool modifiedAfterLastSync = false; + + // Check the dir modification time + EMU_STRUCT_STAT st; + if(EMU_STAT(localPath.c_str(), &st) == 0 && + FileModificationTime(st) > + rParams.LatestFileUploadTime()) + { + modifiedAfterLastSync = true; + } + + rParams.NotifyRemoteFileMissing(localPath, + storePath, modifiedAfterLastSync); } else { - rParams.mExcludedDirs ++; + rParams.NotifyExcludedDir(localPath, storePath); } } - } catch(...) { @@ -1943,6 +1868,7 @@ void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const s { ::closedir(dirhandle); } + throw; } } @@ -1960,7 +1886,7 @@ void BackupQueries::CommandRestore(const std::vector &args, const b // Check arguments if(args.size() != 2) { - BOX_ERROR("Incorrect usage. restore [-d] [-r] [-i] "); + BOX_ERROR("Incorrect usage. restore [-drif] "); return; } @@ -2023,18 +1949,19 @@ void BackupQueries::CommandRestore(const std::vector &args, const b localName.c_str(), true /* print progress dots */, restoreDeleted, false /* don't undelete after restore! */, - opts['r'] /* resume? */); + opts['r'] /* resume? */, + opts['f'] /* force continue after errors */); } catch(std::exception &e) { BOX_ERROR("Failed to restore: " << e.what()); - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); return; } catch(...) { BOX_ERROR("Failed to restore: unknown exception"); - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); return; } @@ -2044,33 +1971,38 @@ void BackupQueries::CommandRestore(const std::vector &args, const b BOX_INFO("Restore complete."); break; + case Restore_CompleteWithErrors: + BOX_WARNING("Restore complete, but some files could not be " + "restored."); + break; + case Restore_ResumePossible: - BOX_ERROR("Resume possible -- repeat command with -r flag to resume"); - SetReturnCode(COMMAND_RETURN_ERROR); + BOX_ERROR("Resume possible -- repeat command with -r flag " + "to resume."); + SetReturnCode(ReturnCode::Command_Error); break; case Restore_TargetExists: - BOX_ERROR("The target directory exists. You cannot restore over an existing directory."); - SetReturnCode(COMMAND_RETURN_ERROR); + BOX_ERROR("The target directory exists. You cannot restore " + "over an existing directory."); + SetReturnCode(ReturnCode::Command_Error); break; - #ifdef WIN32 case Restore_TargetPathNotFound: BOX_ERROR("The target directory path does not exist.\n" "To restore to a directory whose parent " "does not exist, create the parent first."); - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); break; - #endif case Restore_UnknownError: BOX_ERROR("Unknown error during restore."); - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); break; default: BOX_ERROR("Unknown restore result " << result << "."); - SetReturnCode(COMMAND_RETURN_ERROR); + SetReturnCode(ReturnCode::Command_Error); break; } } @@ -2131,49 +2063,46 @@ void BackupQueries::CommandHelp(const std::vector &args) // Created: 19/4/04 // // -------------------------------------------------------------------------- -void BackupQueries::CommandUsage() +void BackupQueries::CommandUsage(const bool *opts) { + bool MachineReadable = opts['m']; + // Request full details from the server std::auto_ptr usage(mrConnection.QueryGetAccountUsage()); // Display each entry in turn int64_t hardLimit = usage->GetBlocksHardLimit(); int32_t blockSize = usage->GetBlockSize(); - CommandUsageDisplayEntry("Used", usage->GetBlocksUsed(), hardLimit, blockSize); - CommandUsageDisplayEntry("Old files", usage->GetBlocksInOldFiles(), hardLimit, blockSize); - CommandUsageDisplayEntry("Deleted files", usage->GetBlocksInDeletedFiles(), hardLimit, blockSize); - CommandUsageDisplayEntry("Directories", usage->GetBlocksInDirectories(), hardLimit, blockSize); - CommandUsageDisplayEntry("Soft limit", usage->GetBlocksSoftLimit(), hardLimit, blockSize); - CommandUsageDisplayEntry("Hard limit", hardLimit, hardLimit, blockSize); + CommandUsageDisplayEntry("Used", usage->GetBlocksUsed(), hardLimit, + blockSize, MachineReadable); + CommandUsageDisplayEntry("Old files", usage->GetBlocksInOldFiles(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Deleted files", usage->GetBlocksInDeletedFiles(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Directories", usage->GetBlocksInDirectories(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Soft limit", usage->GetBlocksSoftLimit(), + hardLimit, blockSize, MachineReadable); + CommandUsageDisplayEntry("Hard limit", hardLimit, hardLimit, blockSize, + MachineReadable); } // -------------------------------------------------------------------------- // // Function -// Name: BackupQueries::CommandUsageDisplayEntry(const char *, int64_t, int64_t, int32_t) +// Name: BackupQueries::CommandUsageDisplayEntry(const char *, +// int64_t, int64_t, int32_t, bool) // Purpose: Display an entry in the usage table // Created: 19/4/04 // // -------------------------------------------------------------------------- -void BackupQueries::CommandUsageDisplayEntry(const char *Name, int64_t Size, int64_t HardLimit, int32_t BlockSize) +void BackupQueries::CommandUsageDisplayEntry(const char *Name, int64_t Size, +int64_t HardLimit, int32_t BlockSize, bool MachineReadable) { - // Calculate size in Mb - double mb = (((double)Size) * ((double)BlockSize)) / ((double)(1024*1024)); - int64_t percent = (Size * 100) / HardLimit; - - // Bar graph - char bar[41]; - unsigned int b = (int)((Size * (sizeof(bar)-1)) / HardLimit); - if(b > sizeof(bar)-1) {b = sizeof(bar)-1;} - for(unsigned int l = 0; l < b; l++) - { - bar[l] = '*'; - } - bar[b] = '\0'; - - // Print the entryj - ::printf("%14s %10.1fMb %3d%% %s\n", Name, mb, (int32_t)percent, bar); + std::cout << FormatUsageLineStart(Name, MachineReadable) << + FormatUsageBar(Size, Size * BlockSize, HardLimit * BlockSize, + MachineReadable) << std::endl; } @@ -2187,10 +2116,17 @@ void BackupQueries::CommandUsageDisplayEntry(const char *Name, int64_t Size, int // -------------------------------------------------------------------------- void BackupQueries::CommandUndelete(const std::vector &args, const bool *opts) { + if (!mReadWrite) + { + BOX_ERROR("This command requires a read-write connection. " + "Please reconnect with the -w option."); + return; + } + // Check arguments if(args.size() != 1) { - BOX_ERROR("Incorrect usage. undelete "); + BOX_ERROR("Incorrect usage. undelete or undelete -i "); return; } @@ -2200,23 +2136,133 @@ void BackupQueries::CommandUndelete(const std::vector &args, const #else const std::string& storeDirEncoded(args[0]); #endif - - // Get directory ID - int64_t dirID = FindDirectoryObjectID(storeDirEncoded, - false /* no old versions */, true /* find deleted dirs */); - - // Allowable? - if(dirID == 0) + + // Find object ID somehow + int64_t fileId, parentId; + std::string fileName; + int16_t flagsOut; + + fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, + /* include files and directories */ + BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, + /* include old and deleted files */ + BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, + &flagsOut); + + if (fileId == 0) { - BOX_ERROR("Directory '" << args[0] << "' not found on server."); + // error already reported return; } - if(dirID == BackupProtocolClientListDirectory::RootDirectory) + + // Undelete it on the store + try + { + // Undelete object + if(flagsOut & BackupProtocolClientListDirectory::Flags_File) + { + mrConnection.QueryUndeleteFile(parentId, fileId); + } + else + { + mrConnection.QueryUndeleteDirectory(fileId); + } + } + catch (BoxException &e) + { + BOX_ERROR("Failed to undelete object: " << + e.what()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to undelete object: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Failed to undelete object: unknown error"); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandDelete(const +// std::vector &, const bool *) +// Purpose: Deletes a file +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandDelete(const std::vector &args, + const bool *opts) +{ + if (!mReadWrite) + { + BOX_ERROR("This command requires a read-write connection. " + "Please reconnect with the -w option."); + return; + } + + // Check arguments + if(args.size() != 1) { - BOX_ERROR("Cannot undelete the root directory."); + BOX_ERROR("Incorrect usage. delete "); return; } - // Undelete - mrConnection.QueryUndeleteDirectory(dirID); +#ifdef WIN32 + std::string storeDirEncoded; + if(!ConvertConsoleToUtf8(args[0].c_str(), storeDirEncoded)) return; +#else + const std::string& storeDirEncoded(args[0]); +#endif + + // Find object ID somehow + int64_t fileId, parentId; + std::string fileName; + int16_t flagsOut; + + fileId = FindFileID(storeDirEncoded, opts, &parentId, &fileName, + /* include files and directories */ + BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING, + /* exclude old and deleted files */ + BackupProtocolClientListDirectory::Flags_OldVersion | + BackupProtocolClientListDirectory::Flags_Deleted, + &flagsOut); + + if (fileId == 0) + { + // error already reported + return; + } + + BackupStoreFilenameClear fn(fileName); + + // Delete it on the store + try + { + // Delete object + if(flagsOut & BackupProtocolClientListDirectory::Flags_File) + { + mrConnection.QueryDeleteFile(parentId, fn); + } + else + { + mrConnection.QueryDeleteDirectory(fileId); + } + } + catch (BoxException &e) + { + BOX_ERROR("Failed to delete object: " << + e.what()); + } + catch(std::exception &e) + { + BOX_ERROR("Failed to delete object: " << + e.what()); + } + catch(...) + { + BOX_ERROR("Failed to delete object: unknown error"); + } } diff --git a/bin/bbackupquery/BackupQueries.h b/bin/bbackupquery/BackupQueries.h index b2ef8cc2..392aa428 100644 --- a/bin/bbackupquery/BackupQueries.h +++ b/bin/bbackupquery/BackupQueries.h @@ -14,6 +14,7 @@ #include #include "BoxTime.h" +#include "BoxBackupCompareParams.h" class BackupProtocolClient; class Configuration; @@ -30,7 +31,9 @@ class ExcludeList; class BackupQueries { public: - BackupQueries(BackupProtocolClient &rConnection, const Configuration &rConfiguration); + BackupQueries(BackupProtocolClient &rConnection, + const Configuration &rConfiguration, + bool readWrite); ~BackupQueries(); private: BackupQueries(const BackupQueries &); @@ -54,43 +57,282 @@ private: void CommandCompare(const std::vector &args, const bool *opts); void CommandRestore(const std::vector &args, const bool *opts); void CommandUndelete(const std::vector &args, const bool *opts); - void CommandUsage(); - void CommandUsageDisplayEntry(const char *Name, int64_t Size, int64_t HardLimit, int32_t BlockSize); + void CommandDelete(const std::vector &args, + const bool *opts); + void CommandUsage(const bool *opts); + void CommandUsageDisplayEntry(const char *Name, int64_t Size, + int64_t HardLimit, int32_t BlockSize, bool MachineReadable); void CommandHelp(const std::vector &args); // Implementations - void List(int64_t DirID, const std::string &rListRoot, const bool *opts, bool FirstLevel); - class CompareParams + void List(int64_t DirID, const std::string &rListRoot, const bool *opts, + bool FirstLevel); + +public: + class CompareParams : public BoxBackupCompareParams { public: - CompareParams(); - ~CompareParams(); - void DeleteExcludeLists(); - bool mQuickCompare; + CompareParams(bool QuickCompare, bool IgnoreExcludes, + bool IgnoreAttributes, box_time_t LatestFileUploadTime); + bool mQuietCompare; - bool mIgnoreExcludes; - bool mIgnoreAttributes; int mDifferences; int mDifferencesExplainedByModTime; int mUncheckedFiles; int mExcludedDirs; int mExcludedFiles; - const ExcludeList *mpExcludeFiles; - const ExcludeList *mpExcludeDirs; - box_time_t mLatestFileUploadTime; + + std::string ConvertForConsole(const std::string& rUtf8String) + { + #ifdef WIN32 + std::string output; + + if(!ConvertUtf8ToConsole(rUtf8String.c_str(), output)) + { + BOX_WARNING("Character set conversion failed " + "on string: " << rUtf8String); + return rUtf8String; + } + + return output; + #else + return rUtf8String; + #endif + } + + virtual void NotifyLocalDirMissing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Local directory '" << + ConvertForConsole(rLocalPath) << "' " + "does not exist, but remote directory does."); + mDifferences ++; + } + + virtual void NotifyLocalDirAccessFailed( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_LOG_SYS_WARNING("Failed to access local directory " + "'" << ConvertForConsole(rLocalPath) << "'"); + mUncheckedFiles ++; + } + + virtual void NotifyStoreDirMissingAttributes( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Store directory '" << + ConvertForConsole(rRemotePath) << "' " + "doesn't have attributes."); + } + + virtual void NotifyRemoteFileMissing( + const std::string& rLocalPath, + const std::string& rRemotePath, + bool modifiedAfterLastSync) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "exists, but remote file '" << + ConvertForConsole(rRemotePath) << "' " + "does not."); + mDifferences ++; + + if(modifiedAfterLastSync) + { + mDifferencesExplainedByModTime ++; + BOX_INFO("(the file above was modified after " + "the last sync time -- might be " + "reason for difference)"); + } + } + + virtual void NotifyLocalFileMissing( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Remote file '" << + ConvertForConsole(rRemotePath) << "' " + "exists, but local file '" << + ConvertForConsole(rLocalPath) << "' does not."); + mDifferences ++; + } + + virtual void NotifyExcludedFileNotDeleted( + const std::string& rLocalPath, + const std::string& rRemotePath) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "is excluded, but remote file '" << + ConvertForConsole(rRemotePath) << "' " + "still exists."); + mDifferences ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + BoxException& rException) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath) << "': " << + rException.what() << " (" << + rException.GetType() << "/" << + rException.GetSubType() << ")"); + mUncheckedFiles ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath) << "': " << + rException.what()); + mUncheckedFiles ++; + } + + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) + { + BOX_ERROR("Failed to download remote file '" << + ConvertForConsole(rRemotePath)); + mUncheckedFiles ++; + } + + virtual void NotifyExcludedFile(const std::string& rLocalPath, + const std::string& rRemotePath) + { + mExcludedFiles ++; + } + + virtual void NotifyExcludedDir(const std::string& rLocalPath, + const std::string& rRemotePath) + { + mExcludedDirs ++; + } + + virtual void NotifyDirComparing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + } + + virtual void NotifyDirCompared( + const std::string& rLocalPath, + const std::string& rRemotePath, + bool HasDifferentAttributes, + bool modifiedAfterLastSync) + { + if(HasDifferentAttributes) + { + BOX_WARNING("Local directory '" << + ConvertForConsole(rLocalPath) << "' " + "has different attributes to " + "store directory '" << + ConvertForConsole(rRemotePath) << "'."); + mDifferences ++; + + if(modifiedAfterLastSync) + { + mDifferencesExplainedByModTime ++; + BOX_INFO("(the directory above was " + "modified after the last sync " + "time -- might be reason for " + "difference)"); + } + } + } + + virtual void NotifyFileComparing(const std::string& rLocalPath, + const std::string& rRemotePath) + { + } + + virtual void NotifyFileCompared(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + bool HasDifferentAttributes, bool HasDifferentContents, + bool ModifiedAfterLastSync, bool NewAttributesApplied) + { + int NewDifferences = 0; + + if(HasDifferentAttributes) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "has different attributes to " + "store file '" << + ConvertForConsole(rRemotePath) << "'."); + NewDifferences ++; + } + + if(HasDifferentContents) + { + BOX_WARNING("Local file '" << + ConvertForConsole(rLocalPath) << "' " + "has different contents to " + "store file '" << + ConvertForConsole(rRemotePath) << "'."); + NewDifferences ++; + } + + if(HasDifferentAttributes || HasDifferentContents) + { + if(ModifiedAfterLastSync) + { + mDifferencesExplainedByModTime += + NewDifferences; + BOX_INFO("(the file above was modified " + "after the last sync time -- " + "might be reason for difference)"); + } + else if(NewAttributesApplied) + { + BOX_INFO("(the file above has had new " + "attributes applied)\n"); + } + } + + mDifferences += NewDifferences; + } }; - void CompareLocation(const std::string &rLocation, CompareParams &rParams); - void Compare(const std::string &rStoreDir, const std::string &rLocalDir, CompareParams &rParams); - void Compare(int64_t DirID, const std::string &rStoreDir, const std::string &rLocalDir, CompareParams &rParams); + void CompareLocation(const std::string &rLocation, + BoxBackupCompareParams &rParams); + void Compare(const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams); + void Compare(int64_t DirID, const std::string &rStoreDir, + const std::string &rLocalDir, BoxBackupCompareParams &rParams); + +public: + + class ReturnCode + { + public: + enum { + Command_OK = 0, + Compare_Same = 1, + Compare_Different, + Compare_Error, + Command_Error, + } Type; + }; + +private: // Utility functions - int64_t FindDirectoryObjectID(const std::string &rDirName, bool AllowOldVersion = false, - bool AllowDeletedDirs = false, std::vector > *pStack = 0); + int64_t FindDirectoryObjectID(const std::string &rDirName, + bool AllowOldVersion = false, bool AllowDeletedDirs = false, + std::vector > *pStack = 0); + int64_t FindFileID(const std::string& rNameOrIdString, + const bool *opts, int64_t *pDirIdOut, + std::string* pFileNameOut, int16_t flagsInclude, + int16_t flagsExclude, int16_t* pFlagsOut); int64_t GetCurrentDirectoryID(); std::string GetCurrentDirectoryName(); void SetReturnCode(int code) {mReturnCode = code;} private: + bool mReadWrite; BackupProtocolClient &mrConnection; const Configuration &mrConfiguration; bool mQuitNow; diff --git a/bin/bbackupquery/BoxBackupCompareParams.h b/bin/bbackupquery/BoxBackupCompareParams.h new file mode 100644 index 00000000..c58759a2 --- /dev/null +++ b/bin/bbackupquery/BoxBackupCompareParams.h @@ -0,0 +1,107 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BoxBackupCompareParams.h +// Purpose: Parameters and notifiers for a compare operation +// Created: 2008/12/30 +// +// -------------------------------------------------------------------------- + +#ifndef BOXBACKUPCOMPAREPARAMS__H +#define BOXBACKUPCOMPAREPARAMS__H + +#include +#include + +#include "BoxTime.h" +#include "ExcludeList.h" +#include "BackupClientMakeExcludeList.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: BoxBackupCompareParams +// Purpose: Parameters and notifiers for a compare operation +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +class BoxBackupCompareParams +{ +private: + std::auto_ptr mapExcludeFiles, mapExcludeDirs; + bool mQuickCompare; + bool mIgnoreExcludes; + bool mIgnoreAttributes; + box_time_t mLatestFileUploadTime; + +public: + BoxBackupCompareParams(bool QuickCompare, bool IgnoreExcludes, + bool IgnoreAttributes, box_time_t LatestFileUploadTime) + : mQuickCompare(QuickCompare), + mIgnoreExcludes(IgnoreExcludes), + mIgnoreAttributes(IgnoreAttributes), + mLatestFileUploadTime(LatestFileUploadTime) + { } + + virtual ~BoxBackupCompareParams() { } + + bool QuickCompare() { return mQuickCompare; } + bool IgnoreExcludes() { return mIgnoreExcludes; } + bool IgnoreAttributes() { return mIgnoreAttributes; } + box_time_t LatestFileUploadTime() { return mLatestFileUploadTime; } + + void LoadExcludeLists(const Configuration& rLoc) + { + mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rLoc)); + mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rLoc)); + } + bool IsExcludedFile(const std::string& rLocalPath) + { + if (!mapExcludeFiles.get()) return false; + return mapExcludeFiles->IsExcluded(rLocalPath); + } + bool IsExcludedDir(const std::string& rLocalPath) + { + if (!mapExcludeDirs.get()) return false; + return mapExcludeDirs->IsExcluded(rLocalPath); + } + + virtual void NotifyLocalDirMissing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyLocalDirAccessFailed(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyStoreDirMissingAttributes(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyRemoteFileMissing(const std::string& rLocalPath, + const std::string& rRemotePath, + bool modifiedAfterLastSync) = 0; + virtual void NotifyLocalFileMissing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyExcludedFileNotDeleted(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + BoxException& rException) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + std::exception& rException) = 0; + virtual void NotifyDownloadFailed(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes) = 0; + virtual void NotifyExcludedFile(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyExcludedDir(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirComparing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyDirCompared(const std::string& rLocalPath, + const std::string& rRemotePath, bool HasDifferentAttributes, + bool modifiedAfterLastSync) = 0; + virtual void NotifyFileComparing(const std::string& rLocalPath, + const std::string& rRemotePath) = 0; + virtual void NotifyFileCompared(const std::string& rLocalPath, + const std::string& rRemotePath, int64_t NumBytes, + bool HasDifferentAttributes, bool HasDifferentContents, + bool modifiedAfterLastSync, bool newAttributesApplied) = 0; +}; + +#endif // BOXBACKUPCOMPAREPARAMS__H diff --git a/bin/bbackupquery/bbackupquery.cpp b/bin/bbackupquery/bbackupquery.cpp index c9d6b715..33860dcf 100644 --- a/bin/bbackupquery/bbackupquery.cpp +++ b/bin/bbackupquery/bbackupquery.cpp @@ -37,6 +37,8 @@ #endif #endif +#include + #include "MainHelper.h" #include "BoxPortsAndFiles.h" #include "BackupDaemonConfigVerify.h" @@ -61,10 +63,13 @@ void PrintUsageAndExit() #ifdef WIN32 "[-u] " #endif - "\n\t[-c config_file] [-l log_file] [commands]\n" + "\n" + "\t[-c config_file] [-o log_file] [-O log_file_level]\n" + "\t[-l protocol_log_file] [commands]\n" + "\n" "As many commands as you require.\n" "If commands are multiple words, remember to enclose the command in quotes.\n" - "Remember to use quit command if you don't want to drop into interactive mode.\n"); + "Remember to use the quit command unless you want to end up in interactive mode.\n"); exit(1); } @@ -90,7 +95,7 @@ int main(int argc, const char *argv[]) #endif // Really don't want trace statements happening, even in debug mode - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD BoxDebugTraceOn = false; #endif @@ -106,24 +111,26 @@ int main(int argc, const char *argv[]) #endif // Flags - bool quiet = false; bool readWrite = false; - Logging::SetProgramName("Box Backup (bbackupquery)"); + Logging::SetProgramName("bbackupquery"); - #ifdef NDEBUG + #ifdef BOX_RELEASE_BUILD int masterLevel = Log::NOTICE; // need an int to do math with #else int masterLevel = Log::INFO; // need an int to do math with #endif #ifdef WIN32 - const char* validOpts = "qvwuc:l:"; + const char* validOpts = "qvwuc:l:o:O:W:"; bool unicodeConsole = false; #else - const char* validOpts = "qvwc:l:"; + const char* validOpts = "qvwc:l:o:O:W:"; #endif + std::string fileLogFile; + Log::Level fileLogLevel = Log::INVALID; + // See if there's another entry on the command line int c; while((c = getopt(argc, (char * const *)argv, validOpts)) != -1) @@ -132,9 +139,6 @@ int main(int argc, const char *argv[]) { case 'q': { - // Quiet mode - quiet = true; - if(masterLevel == Log::NOTHING) { BOX_FATAL("Too many '-q': " @@ -159,6 +163,17 @@ int main(int argc, const char *argv[]) } break; + case 'W': + { + masterLevel = Logging::GetNamedLevel(optarg); + if (masterLevel == Log::INVALID) + { + BOX_FATAL("Invalid logging level"); + return 2; + } + } + break; + case 'w': // Read/write mode readWrite = true; @@ -174,8 +189,24 @@ int main(int argc, const char *argv[]) logFile = ::fopen(optarg, "w"); if(logFile == 0) { - BOX_ERROR("Failed to open log file '" << - optarg << "': " << strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to open log file " + "'" << optarg << "'"); + } + break; + + case 'o': + fileLogFile = optarg; + fileLogLevel = Log::EVERYTHING; + break; + + case 'O': + { + fileLogLevel = Logging::GetNamedLevel(optarg); + if (fileLogLevel == Log::INVALID) + { + BOX_FATAL("Invalid logging level"); + return 2; + } } break; @@ -196,6 +227,19 @@ int main(int argc, const char *argv[]) Logging::SetGlobalLevel((Log::Level)masterLevel); + std::auto_ptr fileLogger; + if (fileLogLevel != Log::INVALID) + { + fileLogger.reset(new FileLogger(fileLogFile, fileLogLevel)); + } + + bool quiet = false; + if (masterLevel < Log::NOTICE) + { + // Quiet mode + quiet = true; + } + // Print banner? if(!quiet) { @@ -260,7 +304,9 @@ int main(int argc, const char *argv[]) // 2. Connect to server if(!quiet) BOX_INFO("Connecting to store..."); SocketStreamTLS socket; - socket.Open(tlsContext, Socket::TypeINET, conf.GetKeyValue("StoreHostname").c_str(), BOX_PORT_BBSTORED); + socket.Open(tlsContext, Socket::TypeINET, + conf.GetKeyValue("StoreHostname").c_str(), + conf.GetKeyValueInt("StorePort")); // 3. Make a protocol, and handshake if(!quiet) BOX_INFO("Handshake with store..."); @@ -291,7 +337,7 @@ int main(int argc, const char *argv[]) if(!quiet) printf("Login complete.\n\nType \"help\" for a list of commands.\n\n"); // Set up a context for our work - BackupQueries context(connection, conf); + BackupQueries context(connection, conf, readWrite); // Start running commands... first from the command line { @@ -377,7 +423,8 @@ int main(int argc, const char *argv[]) #ifdef WIN32 // Clean up our sockets - WSACleanup(); + // FIXME we should do this, but I get an abort() when I try + // WSACleanup(); #endif MAINHELPER_END diff --git a/bin/bbackupquery/documentation.txt b/bin/bbackupquery/documentation.txt index 42217edc..d32bf200 100644 --- a/bin/bbackupquery/documentation.txt +++ b/bin/bbackupquery/documentation.txt @@ -116,7 +116,7 @@ compare This can be used for automated tests. < -> restore [-d] [-r] [-i] +> restore [-drif] Restores a directory to the local disc. The local directory specified must not exist (unless a previous restore is being restarted). @@ -126,6 +126,7 @@ compare -d -- restore a deleted directory or deleted files inside -r -- resume an interrupted restoration -i -- directory name is actually an ID + -f -- force restore to continue if errors are encountered If a restore operation is interrupted for any reason, it can be restarted using the -r switch. Restore progress information is saved in a file at @@ -141,10 +142,12 @@ compare stored format, which is encrypted and compressed. < -> usage +> usage [-m] Show space used on the server for this account. + -m -- display the output in machine-readable form + Used: Total amount of space used on the server. Old files: Space used by old files Deleted files: Space used by deleted files @@ -158,6 +161,21 @@ compare files is near zero. < +> undelete +undelete -i + + Removes the deleted flag from the specified directory name (in the + current directory) or hex object ID. Be careful not to use this + command where a directory already exists with the same name which is + not marked as deleted. +< + +> delete + + Sets the deleted flag on the specified file name (in the current + directory, or with a relative path). +< + > quit End session and exit. diff --git a/bin/bbstoreaccounts/bbstoreaccounts.cpp b/bin/bbstoreaccounts/bbstoreaccounts.cpp index 6f079d30..71114ef4 100644 --- a/bin/bbstoreaccounts/bbstoreaccounts.cpp +++ b/bin/bbstoreaccounts/bbstoreaccounts.cpp @@ -9,12 +9,17 @@ #include "Box.h" -#include +#include #include +#include + #include -#include -#include + #include +#include +#include +#include +#include #include "BoxPortsAndFiles.h" #include "BackupStoreConfigVerify.h" @@ -27,12 +32,15 @@ #include "NamedLock.h" #include "UnixUser.h" #include "BackupStoreCheck.h" +#include "Utils.h" #include "MemLeakFindOn.h" // max size of soft limit as percent of hard limit #define MAX_SOFT_LIMIT_SIZE 97 +bool sMachineReadableOutput = false; + void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit) { if(SoftLimit >= HardLimit) @@ -62,22 +70,11 @@ int BlockSizeOfDiscSet(int DiscSet) return controller.GetDiscSet(DiscSet).GetBlockSize(); } -const char *BlockSizeToString(int64_t Blocks, int DiscSet) +std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int DiscSet) { - // Not reentrant, nor can be used in the same function call twice, etc. - static char string[256]; - - // Work out size in Mb. - double mb = (Blocks * BlockSizeOfDiscSet(DiscSet)) / (1024.0*1024.0); - - // Format string -#ifdef WIN32 - sprintf(string, "%I64d (%.2fMb)", Blocks, mb); -#else - sprintf(string, "%lld (%.2fMb)", Blocks, mb); -#endif - - return string; + return FormatUsageBar(Blocks, Blocks * BlockSizeOfDiscSet(DiscSet), + MaxBlocks * BlockSizeOfDiscSet(DiscSet), + sMachineReadableOutput); } int64_t SizeStringToBlocks(const char *string, int DiscSet) @@ -118,7 +115,7 @@ int64_t SizeStringToBlocks(const char *string, int DiscSet) default: BOX_FATAL(string << " has an invalid units specifier " - "(use B for blocks, M for Mb, G for Gb, eg 2Gb)"); + "(use B for blocks, M for MB, G for GB, eg 2GB)"); exit(1); break; } @@ -211,7 +208,9 @@ int SetLimit(Configuration &rConfig, const std::string &rUsername, int32_t ID, c int AccountInfo(Configuration &rConfig, int32_t ID) { // Load in the account database - std::auto_ptr db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); + std::auto_ptr db( + BackupStoreAccountDatabase::Read( + rConfig.GetKeyValue("AccountDatabase").c_str())); // Exists? if(!db->EntryExists(ID)) @@ -226,18 +225,34 @@ int AccountInfo(Configuration &rConfig, int32_t ID) std::string rootDir; int discSet; acc.GetAccountRoot(ID, rootDir, discSet); - std::auto_ptr info(BackupStoreInfo::Load(ID, rootDir, discSet, true /* ReadOnly */)); + std::auto_ptr info(BackupStoreInfo::Load(ID, + rootDir, discSet, true /* ReadOnly */)); // Then print out lots of info - printf(" Account ID: %08x\n", ID); - printf(" Last object ID: %lld\n", info->GetLastObjectIDUsed()); - printf(" Blocks used: %s\n", BlockSizeToString(info->GetBlocksUsed(), discSet)); - printf(" Blocks used by old files: %s\n", BlockSizeToString(info->GetBlocksInOldFiles(), discSet)); - printf("Blocks used by deleted files: %s\n", BlockSizeToString(info->GetBlocksInDeletedFiles(), discSet)); - printf(" Blocks used by directories: %s\n", BlockSizeToString(info->GetBlocksInDirectories(), discSet)); - printf(" Block soft limit: %s\n", BlockSizeToString(info->GetBlocksSoftLimit(), discSet)); - printf(" Block hard limit: %s\n", BlockSizeToString(info->GetBlocksHardLimit(), discSet)); - printf(" Client store marker: %lld\n", info->GetClientStoreMarker()); + std::cout << FormatUsageLineStart("Account ID", sMachineReadableOutput) << + BOX_FORMAT_ACCOUNT(ID) << std::endl; + std::cout << FormatUsageLineStart("Last object ID", sMachineReadableOutput) << + BOX_FORMAT_OBJECTID(info->GetLastObjectIDUsed()) << std::endl; + std::cout << FormatUsageLineStart("Used", sMachineReadableOutput) << + BlockSizeToString(info->GetBlocksUsed(), + info->GetBlocksHardLimit(), discSet) << std::endl; + std::cout << FormatUsageLineStart("Old files", sMachineReadableOutput) << + BlockSizeToString(info->GetBlocksInOldFiles(), + info->GetBlocksHardLimit(), discSet) << std::endl; + std::cout << FormatUsageLineStart("Deleted files", sMachineReadableOutput) << + BlockSizeToString(info->GetBlocksInDeletedFiles(), + info->GetBlocksHardLimit(), discSet) << std::endl; + std::cout << FormatUsageLineStart("Directories", sMachineReadableOutput) << + BlockSizeToString(info->GetBlocksInDirectories(), + info->GetBlocksHardLimit(), discSet) << std::endl; + std::cout << FormatUsageLineStart("Soft limit", sMachineReadableOutput) << + BlockSizeToString(info->GetBlocksSoftLimit(), + info->GetBlocksHardLimit(), discSet) << std::endl; + std::cout << FormatUsageLineStart("Hard limit", sMachineReadableOutput) << + BlockSizeToString(info->GetBlocksHardLimit(), + info->GetBlocksHardLimit(), discSet) << std::endl; + std::cout << FormatUsageLineStart("Client store marker", sMachineReadableOutput) << + info->GetLastObjectIDUsed() << std::endl; return 0; } @@ -409,8 +424,32 @@ int CreateAccount(Configuration &rConfig, const std::string &rUsername, int32_t void PrintUsageAndExit() { - printf("Usage: bbstoreaccounts [-c config_file] action account_id [args]\nAccount ID is integer specified in hex\n"); - exit(1); + printf( +"Usage: bbstoreaccounts [-c config_file] action account_id [args]\n" +"Account ID is integer specified in hex\n" +"\n" +"Commands (and arguments):\n" +" create \n" +" Creates the specified account number (in hex with no 0x) on the\n" +" specified raidfile disc set number (see raidfile.conf for valid\n" +" set numbers) with the specified soft and hard limits (in blocks\n" +" if suffixed with B, MB with M, GB with G)\n" +" info [-m] \n" +" Prints information about the specified account including number\n" +" of blocks used. The -m option enable machine-readable output.\n" +" setlimit \n" +" Changes the limits of the account as specified. Numbers are\n" +" interpreted as for the 'create' command (suffixed with B, M or G)\n" +" delete [yes]\n" +" Deletes the specified account. Prompts for confirmation unless\n" +" the optional 'yes' parameter is provided.\n" +" check [fix] [quiet]\n" +" Checks the specified account for errors. If the 'fix' option is\n" +" provided, any errors discovered that can be fixed automatically\n" +" will be fixed. If the 'quiet' option is provided, less output is\n" +" produced.\n" + ); + exit(2); } int main(int argc, const char *argv[]) @@ -420,6 +459,8 @@ int main(int argc, const char *argv[]) MAINHELPER_START + Logging::SetProgramName("bbstoreaccounts"); + // Filename for configuration file? std::string configFilename; @@ -431,7 +472,7 @@ int main(int argc, const char *argv[]) // See if there's another entry on the command line int c; - while((c = getopt(argc, (char * const *)argv, "c:")) != -1) + while((c = getopt(argc, (char * const *)argv, "c:m")) != -1) { switch(c) { @@ -440,6 +481,11 @@ int main(int argc, const char *argv[]) configFilename = optarg; break; + case 'm': + // enable machine readable output + sMachineReadableOutput = true; + break; + case '?': default: PrintUsageAndExit(); diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/bin/bbstored/BBStoreDHousekeeping.cpp index 16a1432a..1c1767ca 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/bin/bbstored/BBStoreDHousekeeping.cpp @@ -46,6 +46,12 @@ void BackupStoreDaemon::HousekeepingProcess() { RunHousekeepingIfNeeded(); + // Stop early? + if(StopRun()) + { + break; + } + // Calculate how long should wait before doing the next // housekeeping run int64_t timeNow = GetCurrentBoxTime(); @@ -155,7 +161,11 @@ void BackupStoreDaemon::RunHousekeepingIfNeeded() void BackupStoreDaemon::OnIdle() { - #ifdef WIN32 + if (!IsSingleProcess()) + { + return; + } + if (!mHousekeepingInited) { HousekeepingInit(); @@ -163,7 +173,6 @@ void BackupStoreDaemon::OnIdle() } RunHousekeepingIfNeeded(); - #endif } // -------------------------------------------------------------------------- @@ -193,7 +202,8 @@ bool BackupStoreDaemon::CheckForInterProcessMsg(int AccountNum, int MaximumWaitT std::string line; if(mInterProcessComms.GetLine(line, false /* no pre-processing */, MaximumWaitTime)) { - TRACE1("Housekeeping received command '%s' over interprocess comms\n", line.c_str()); + BOX_TRACE("Housekeeping received command '" << line << + "' over interprocess comms"); int account = 0; diff --git a/bin/bbstored/BackupCommands.cpp b/bin/bbstored/BackupCommands.cpp index bca52c04..38cda234 100644 --- a/bin/bbstored/BackupCommands.cpp +++ b/bin/bbstored/BackupCommands.cpp @@ -13,25 +13,25 @@ #include #include "autogen_BackupProtocolServer.h" +#include "autogen_RaidFileException.h" #include "BackupConstants.h" -#include "BackupContext.h" -#include "CollectInBufferStream.h" +#include "BackupStoreContext.h" +#include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "StreamableMemBlock.h" -#include "BackupStoreConstants.h" -#include "RaidFileController.h" #include "BackupStoreInfo.h" -#include "RaidFileController.h" +#include "BufferedStream.h" +#include "CollectInBufferStream.h" #include "FileStream.h" #include "InvisibleTempFileStream.h" -#include "BufferedStream.h" +#include "RaidFileController.h" +#include "StreamableMemBlock.h" #include "MemLeakFindOn.h" #define CHECK_PHASE(phase) \ - if(rContext.GetPhase() != BackupContext::phase) \ + if(rContext.GetPhase() != BackupStoreContext::phase) \ { \ return std::auto_ptr(new BackupProtocolServerError( \ BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_NotInRightProtocolPhase)); \ @@ -47,12 +47,12 @@ // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerVersion::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerVersion::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Return the current version, or an error if the requested version isn't allowed // Created: 2003/08/20 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerVersion::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerVersion::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Version) @@ -64,7 +64,7 @@ std::auto_ptr BackupProtocolServerVersion::DoCommand(BackupProto } // Mark the next phase - rContext.SetPhase(BackupContext::Phase_Login); + rContext.SetPhase(BackupStoreContext::Phase_Login); // Return our version return std::auto_ptr(new BackupProtocolServerVersion(BACKUP_STORE_SERVER_VERSION)); @@ -73,12 +73,12 @@ std::auto_ptr BackupProtocolServerVersion::DoCommand(BackupProto // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerLogin::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerLogin::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Return the current version, or an error if the requested version isn't allowed // Created: 2003/08/20 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Login) @@ -131,7 +131,7 @@ std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtoco int64_t clientStoreMarker = rContext.GetClientStoreMarker(); // Mark the next phase - rContext.SetPhase(BackupContext::Phase_Commands); + rContext.SetPhase(BackupStoreContext::Phase_Commands); // Log login BOX_NOTICE("Login from Client ID " << @@ -151,12 +151,12 @@ std::auto_ptr BackupProtocolServerLogin::DoCommand(BackupProtoco // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerFinished::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerFinished::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Marks end of conversation (Protocol framework handles this) // Created: 2003/08/20 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerFinished::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { BOX_NOTICE("Session finished for Client ID " << BOX_FORMAT_ACCOUNT(rContext.GetClientID())); @@ -172,47 +172,72 @@ std::auto_ptr BackupProtocolServerFinished::DoCommand(BackupProt // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerListDirectory::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerListDirectory::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Command to list a directory // Created: 2003/09/02 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerListDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerListDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) - // Ask the context for a directory - const BackupStoreDirectory &rdir(rContext.GetDirectory(mObjectID)); - // Store the listing to a stream std::auto_ptr stream(new CollectInBufferStream); - rdir.WriteToStream(*stream, mFlagsMustBeSet, mFlagsNotToBeSet, mSendAttributes, - false /* never send dependency info to the client */); + + try + { + // Ask the context for a directory + const BackupStoreDirectory &rdir( + rContext.GetDirectory(mObjectID)); + rdir.WriteToStream(*stream, mFlagsMustBeSet, + mFlagsNotToBeSet, mSendAttributes, + false /* never send dependency info to the client */); + } + catch (RaidFileException &e) + { + if (e.GetSubType() == RaidFileException::RaidFileDoesntExist) + { + return std::auto_ptr( + new BackupProtocolServerError( + BackupProtocolServerError::ErrorType, + BackupProtocolServerError::Err_DoesNotExist)); + } + throw; + } + stream->SetForReading(); // Get the protocol to send the stream rProtocol.SendStreamAfterCommand(stream.release()); - return std::auto_ptr(new BackupProtocolServerSuccess(mObjectID)); + return std::auto_ptr( + new BackupProtocolServerSuccess(mObjectID)); } // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerStoreFile::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerStoreFile::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Command to store a file on the server // Created: 2003/09/02 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerStoreFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerStoreFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION + + std::auto_ptr hookResult = + rContext.StartCommandHook(*this); + if(hookResult.get()) + { + return hookResult; + } // Check that the diff from file actually exists, if it's specified if(mDiffFromFileID != 0) { - if(!rContext.ObjectExists(mDiffFromFileID, BackupContext::ObjectExists_File)) + if(!rContext.ObjectExists(mDiffFromFileID, BackupStoreContext::ObjectExists_File)) { return std::auto_ptr(new BackupProtocolServerError( BackupProtocolServerError::ErrorType, BackupProtocolServerError::Err_DiffFromFileDoesNotExist)); @@ -257,12 +282,12 @@ std::auto_ptr BackupProtocolServerStoreFile::DoCommand(BackupPro // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetObject::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerGetObject::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Command to get an arbitary object from the server // Created: 2003/09/03 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetObject::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -285,13 +310,13 @@ std::auto_ptr BackupProtocolServerGetObject::DoCommand(BackupPro // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetFile::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerGetFile::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Command to get an file object from the server -- may have to do a bit of // work to get the object. // Created: 2003/09/03 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -457,12 +482,12 @@ std::auto_ptr BackupProtocolServerGetFile::DoCommand(BackupProto // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerCreateDirectory::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerCreateDirectory::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Create directory command // Created: 2003/09/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerCreateDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerCreateDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -500,12 +525,12 @@ std::auto_ptr BackupProtocolServerCreateDirectory::DoCommand(Bac // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerChangeDirAttributes::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerChangeDirAttributes::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Change attributes on directory // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerChangeDirAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerChangeDirAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -528,12 +553,12 @@ std::auto_ptr BackupProtocolServerChangeDirAttributes::DoCommand // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerSetReplacementFileAttributes::DoCommand(Protocol &, BackupContext &) +// Name: BackupProtocolServerSetReplacementFileAttributes::DoCommand(Protocol &, BackupStoreContext &) // Purpose: Change attributes on directory // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerSetReplacementFileAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerSetReplacementFileAttributes::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -563,12 +588,12 @@ std::auto_ptr BackupProtocolServerSetReplacementFileAttributes:: // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Delete a file // Created: 2003/10/21 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerDeleteFile::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -585,12 +610,36 @@ std::auto_ptr BackupProtocolServerDeleteFile::DoCommand(BackupPr // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerUndeleteFile::DoCommand( +// BackupProtocolServer &, BackupStoreContext &) +// Purpose: Undelete a file +// Created: 2008-09-12 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupProtocolServerUndeleteFile::DoCommand( + BackupProtocolServer &rProtocol, BackupStoreContext &rContext) +{ + CHECK_PHASE(Phase_Commands) + CHECK_WRITEABLE_SESSION + + // Context handles this + bool result = rContext.UndeleteFile(mObjectID, mInDirectory); + + // return the object ID or zero for not found + return std::auto_ptr( + new BackupProtocolServerSuccess(result ? mObjectID : 0)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Delete a directory // Created: 2003/10/21 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerDeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -613,12 +662,12 @@ std::auto_ptr BackupProtocolServerDeleteDirectory::DoCommand(Bac // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Undelete a directory // Created: 23/11/03 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerUndeleteDirectory::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -640,12 +689,12 @@ std::auto_ptr BackupProtocolServerUndeleteDirectory::DoCommand(B // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Command to set the client's store marker // Created: 2003/10/29 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerSetClientStoreMarker::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -661,12 +710,12 @@ std::auto_ptr BackupProtocolServerSetClientStoreMarker::DoComman // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Command to move an object from one directory to another // Created: 2003/11/12 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerMoveObject::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -704,12 +753,12 @@ std::auto_ptr BackupProtocolServerMoveObject::DoCommand(BackupPr // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Command to find the name of an object // Created: 12/11/03 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetObjectName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -730,7 +779,7 @@ std::auto_ptr BackupProtocolServerGetObjectName::DoCommand(Backu do { // Check the directory really exists - if(!rContext.ObjectExists(dirID, BackupContext::ObjectExists_Directory)) + if(!rContext.ObjectExists(dirID, BackupStoreContext::ObjectExists_Directory)) { return std::auto_ptr(new BackupProtocolServerObjectName(BackupProtocolServerObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0)); } @@ -795,12 +844,12 @@ std::auto_ptr BackupProtocolServerGetObjectName::DoCommand(Backu // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Get the block index from a file, by ID // Created: 19/1/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetBlockIndexByID::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -821,12 +870,12 @@ std::auto_ptr BackupProtocolServerGetBlockIndexByID::DoCommand(B // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Get the block index from a file, by name within a directory // Created: 19/1/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetBlockIndexByName::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -873,12 +922,12 @@ std::auto_ptr BackupProtocolServerGetBlockIndexByName::DoCommand // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Return the amount of disc space used // Created: 19/4/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetAccountUsage::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) @@ -904,12 +953,12 @@ std::auto_ptr BackupProtocolServerGetAccountUsage::DoCommand(Bac // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &, BackupContext &) +// Name: BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &, BackupStoreContext &) // Purpose: Return the amount of disc space used // Created: 19/4/04 // // -------------------------------------------------------------------------- -std::auto_ptr BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &rProtocol, BackupContext &rContext) +std::auto_ptr BackupProtocolServerGetIsAlive::DoCommand(BackupProtocolServer &rProtocol, BackupStoreContext &rContext) { CHECK_PHASE(Phase_Commands) diff --git a/bin/bbstored/BackupConstants.h b/bin/bbstored/BackupConstants.h index 515b3bcd..19d06a15 100644 --- a/bin/bbstored/BackupConstants.h +++ b/bin/bbstored/BackupConstants.h @@ -10,8 +10,6 @@ #ifndef BACKUPCONSTANTS__H #define BACKUPCONSTANTS__H -#define BACKUP_STORE_DEFAULT_ACCOUNT_DATABASE_FILE "/etc/box/backupstoreaccounts" - // 15 minutes to timeout (milliseconds) #define BACKUP_STORE_TIMEOUT (15*60*1000) diff --git a/bin/bbstored/BackupContext.cpp b/bin/bbstored/BackupContext.cpp deleted file mode 100644 index 16388099..00000000 --- a/bin/bbstored/BackupContext.cpp +++ /dev/null @@ -1,1661 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupContext.cpp -// Purpose: Context for backup store server -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include - -#include "BackupContext.h" -#include "RaidFileWrite.h" -#include "RaidFileRead.h" -#include "BackupStoreDirectory.h" -#include "BackupStoreException.h" -#include "BackupStoreInfo.h" -#include "BackupConstants.h" -#include "BackupStoreFile.h" -#include "BackupStoreObjectMagic.h" -#include "StoreStructure.h" -#include "BackupStoreDaemon.h" -#include "RaidFileController.h" -#include "FileStream.h" -#include "InvisibleTempFileStream.h" -#include "BufferedStream.h" - -#include "MemLeakFindOn.h" - - -// Maximum number of directories to keep in the cache -// When the cache is bigger than this, everything gets -// deleted. -#ifdef NDEBUG - #define MAX_CACHE_SIZE 32 -#else - #define MAX_CACHE_SIZE 2 -#endif - -// Allow the housekeeping process 4 seconds to release an account -#define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT 4 - -// Maximum amount of store info updates before it's actually saved to disc. -#define STORE_INFO_SAVE_DELAY 96 - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::BackupContext() -// Purpose: Constructor -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- -BackupContext::BackupContext(int32_t ClientID, BackupStoreDaemon &rDaemon) - : mClientID(ClientID), - mrDaemon(rDaemon), - mProtocolPhase(Phase_START), - mClientHasAccount(false), - mStoreDiscSet(-1), - mReadOnly(true), - mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY) -{ -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::~BackupContext() -// Purpose: Destructor -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- -BackupContext::~BackupContext() -{ - // Delete the objects in the cache - for(std::map::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) - { - delete (i->second); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::CleanUp() -// Purpose: Clean up after a connection -// Created: 16/12/03 -// -// -------------------------------------------------------------------------- -void BackupContext::CleanUp() -{ - // Make sure the store info is saved, if it has been loaded, isn't read only and has been modified - if(mpStoreInfo.get() && !(mpStoreInfo->IsReadOnly()) && mpStoreInfo->IsModified()) - { - mpStoreInfo->Save(); - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::ReceivedFinishCommand() -// Purpose: Called when the finish command is received by the protocol -// Created: 16/12/03 -// -// -------------------------------------------------------------------------- -void BackupContext::ReceivedFinishCommand() -{ - if(!mReadOnly && mpStoreInfo.get()) - { - // Save the store info, not delayed - SaveStoreInfo(false); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::AttemptToGetWriteLock() -// Purpose: Attempt to get a write lock for the store, and if so, unset the read only flags -// Created: 2003/09/02 -// -// -------------------------------------------------------------------------- -bool BackupContext::AttemptToGetWriteLock() -{ - // Make the filename of the write lock file - std::string writeLockFile; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); - - // Request the lock - bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - - if(!gotLock) - { - // The housekeeping process might have the thing open -- ask it to stop - char msg[256]; - int msgLen = sprintf(msg, "r%x\n", mClientID); - // Send message - mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen); - - // Then try again a few times - int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; - do - { - ::sleep(1 /* second */); - --tries; - gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - - } while(!gotLock && tries > 0); - } - - if(gotLock) - { - // Got the lock, mark as not read only - mReadOnly = false; - } - - return gotLock; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::LoadStoreInfo() -// Purpose: Load the store info from disc -// Created: 2003/09/03 -// -// -------------------------------------------------------------------------- -void BackupContext::LoadStoreInfo() -{ - if(mpStoreInfo.get() != 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) - } - - // Load it up! - std::auto_ptr i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly)); - - // Check it - if(i->GetAccountID() != mClientID) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) - } - - // Keep the pointer to it - mpStoreInfo = i; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::SaveStoreInfo(bool) -// Purpose: Potentially delayed saving of the store info -// Created: 16/12/03 -// -// -------------------------------------------------------------------------- -void BackupContext::SaveStoreInfo(bool AllowDelay) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Can delay saving it a little while? - if(AllowDelay) - { - --mSaveStoreInfoDelay; - if(mSaveStoreInfoDelay > 0) - { - return; - } - } - - // Want to save now - mpStoreInfo->Save(); - - // Set count for next delay - mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::MakeObjectFilename(int64_t, std::string &, bool) -// Purpose: Create the filename of an object in the store, optionally creating the -// containing directory if it doesn't already exist. -// Created: 2003/09/02 -// -// -------------------------------------------------------------------------- -void BackupContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) -{ - // Delegate to utility function - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::GetDirectoryInternal(int64_t) -// Purpose: Return a reference to a directory. Valid only until the -// next time a function which affects directories is called. -// Mainly this funciton, and creation of files. -// Private version of this, which returns non-const directories. -// Created: 2003/09/02 -// -// -------------------------------------------------------------------------- -BackupStoreDirectory &BackupContext::GetDirectoryInternal(int64_t ObjectID) -{ - // Get the filename - std::string filename; - MakeObjectFilename(ObjectID, filename); - - // Already in cache? - std::map::iterator item(mDirectoryCache.find(ObjectID)); - if(item != mDirectoryCache.end()) - { - // Check the revision ID of the file -- does it need refreshing? - int64_t revID = 0; - if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID)) - { - THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) - } - - if(revID == item->second->GetRevisionID()) - { - // Looks good... return the cached object - return *(item->second); - } - - // Delete this cached object - delete item->second; - mDirectoryCache.erase(item); - } - - // Need to load it up - - // First check to see if the cache is too big - if(mDirectoryCache.size() > MAX_CACHE_SIZE) - { - // Very simple. Just delete everything! - for(std::map::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) - { - delete (i->second); - } - mDirectoryCache.clear(); - } - - // Get a RaidFileRead to read it - int64_t revID = 0; - std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID)); - ASSERT(revID != 0); - - // New directory object - std::auto_ptr dir(new BackupStoreDirectory); - - // Read it from the stream, then set it's revision ID - BufferedStream buf(*objectFile); - dir->ReadFromStream(buf, IOStream::TimeOutInfinite); - dir->SetRevisionID(revID); - - // Make sure the size of the directory is available for writing the dir back - int64_t dirSize = objectFile->GetDiscUsageInBlocks(); - ASSERT(dirSize > 0); - dir->SetUserInfo1_SizeInBlocks(dirSize); - - // Store in cache - BackupStoreDirectory *pdir = dir.release(); - try - { - mDirectoryCache[ObjectID] = pdir; - } - catch(...) - { - delete pdir; - throw; - } - - // Return it - return *pdir; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::AllocateObjectID() -// Purpose: Allocate a new object ID, tolerant of failures to save store info -// Created: 16/12/03 -// -// -------------------------------------------------------------------------- -int64_t BackupContext::AllocateObjectID() -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - // Given that the store info may not be saved for STORE_INFO_SAVE_DELAY - // times after it has been updated, this is a reasonable number of times - // to try for finding an unused ID. - // (Sizes used in the store info are fixed by the housekeeping process) - int retryLimit = (STORE_INFO_SAVE_DELAY * 2); - - while(retryLimit > 0) - { - // Attempt to allocate an ID from the store - int64_t id = mpStoreInfo->AllocateObjectID(); - - // Generate filename - std::string filename; - MakeObjectFilename(id, filename); - // Check it doesn't exist - if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) - { - // Success! - return id; - } - - // Decrement retry count, and try again - --retryLimit; - - // Mark that the store info should be saved as soon as possible - mSaveStoreInfoDelay = 0; - - TRACE1("When allocating object ID, found that %lld is already in use\n", id); - } - - THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation) -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) -// Purpose: Add a file to the store, from a given stream, into a specified directory. -// Returns object ID of the new file. -// Created: 2003/09/03 -// -// -------------------------------------------------------------------------- -int64_t BackupContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, - int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, - bool MarkFileWithSameNameAsOldVersions) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // This is going to be a bit complex to make sure it copes OK - // with things going wrong. - // The only thing which isn't safe is incrementing the object ID - // and keeping the blocks used entirely accurate -- but these - // aren't big problems if they go horribly wrong. The sizes will - // be corrected the next time the account has a housekeeping run, - // and the object ID allocation code is tolerant of missed IDs. - // (the info is written lazily, so these are necessary) - - // Get the directory we want to modify - BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - - // Allocate the next ID - int64_t id = AllocateObjectID(); - - // Stream the file to disc - std::string fn; - MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); - int64_t blocksUsed = 0; - RaidFileWrite *ppreviousVerStoreFile = 0; - bool reversedDiffIsCompletelyDifferent = false; - int64_t oldVersionNewBlocksUsed = 0; - try - { - RaidFileWrite storeFile(mStoreDiscSet, fn); - storeFile.Open(false /* no overwriting */); - int64_t spaceAdjustFromDiff = 0; // size adjustment from use of patch in old file - - // Diff or full file? - if(DiffFromFileID == 0) - { - // A full file, just store to disc - if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT)) - { - THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) - } - } - else - { - // Check that the diffed from ID actually exists in the directory - if(dir.FindEntryByID(DiffFromFileID) == 0) - { - THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) - } - - // Diff file, needs to be recreated. - // Choose a temporary filename. - std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", - 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); - - try - { - // Open it twice -#ifdef WIN32 - InvisibleTempFileStream diff(tempFn.c_str(), - O_RDWR | O_CREAT | O_BINARY); - InvisibleTempFileStream diff2(tempFn.c_str(), - O_RDWR | O_BINARY); -#else - FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); - FileStream diff2(tempFn.c_str(), O_RDONLY); - - // Unlink it immediately, so it definitely goes away - if(::unlink(tempFn.c_str()) != 0) - { - THROW_EXCEPTION(CommonException, OSFileError); - } -#endif - - // Stream the incoming diff to this temporary file - if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) - { - THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) - } - - // Verify the diff - diff.Seek(0, IOStream::SeekType_Absolute); - if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) - { - THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) - } - - // Seek to beginning of diff file - diff.Seek(0, IOStream::SeekType_Absolute); - - // Filename of the old version - std::string oldVersionFilename; - MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); - - // Reassemble that diff -- open previous file, and combine the patch and file - std::auto_ptr from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); - BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); - - // Then... reverse the patch back (open the from file again, and create a write file to overwrite it) - std::auto_ptr from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); - ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename); - ppreviousVerStoreFile->Open(true /* allow overwriting */); - from->Seek(0, IOStream::SeekType_Absolute); - diff.Seek(0, IOStream::SeekType_Absolute); - BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, - DiffFromFileID, &reversedDiffIsCompletelyDifferent); - - // Store disc space used - oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); - - // And make a space adjustment for the size calculation - spaceAdjustFromDiff = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed; - - // Everything cleans up here... - } - catch(...) - { - // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway - ::unlink(tempFn.c_str()); - throw; - } - } - - // Get the blocks used - blocksUsed = storeFile.GetDiscUsageInBlocks(); - - // Exceeds the hard limit? - if((mpStoreInfo->GetBlocksUsed() + blocksUsed - spaceAdjustFromDiff) > mpStoreInfo->GetBlocksHardLimit()) - { - THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) - // The store file will be deleted automatically by the RaidFile object - } - - // Commit the file - storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - } - catch(...) - { - // Delete any previous version store file - if(ppreviousVerStoreFile != 0) - { - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } - - throw; - } - - // Verify the file -- only necessary for non-diffed versions - // NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because - // in the non-diffed code path it's never allocated. - if(DiffFromFileID == 0) - { - std::auto_ptr checkFile(RaidFileRead::Open(mStoreDiscSet, fn)); - if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile)) - { - // Error! Delete the file - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); - - // Exception - THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) - } - } - - // Modify the directory -- first make all files with the same name - // marked as an old version - int64_t blocksInOldFiles = 0; - try - { - if(MarkFileWithSameNameAsOldVersions) - { - BackupStoreDirectory::Iterator i(dir); - - BackupStoreDirectory::Entry *e = 0; - while((e = i.Next()) != 0) - { - // First, check it's not an old version (cheaper comparison) - if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) - { - // Compare name - if(e->GetName() == rFilename) - { - // Check that it's definately not an old version - ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0); - // Set old version flag - e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion); - // Can safely do this, because we know we won't be here if it's already - // an old version - blocksInOldFiles += e->GetSizeInBlocks(); - } - } - } - } - - // Then the new entry - BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, - ModificationTime, id, blocksUsed, BackupStoreDirectory::Entry::Flags_File, AttributesHash); - - // Adjust for the patch back stuff? - if(DiffFromFileID != 0) - { - // Get old version entry - BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID); - ASSERT(poldEntry != 0); - - // Adjust dependency info of file? - if(!reversedDiffIsCompletelyDifferent) - { - poldEntry->SetDependsNewer(id); - pnewEntry->SetDependsOlder(DiffFromFileID); - } - - // Adjust size of old entry - int64_t oldSize = poldEntry->GetSizeInBlocks(); - poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); - - // And adjust blocks used count, for later adjustment - blocksUsed += (oldVersionNewBlocksUsed - oldSize); - blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize); - } - - // Write the directory back to disc - SaveDirectory(dir, InDirectory); - - // Commit the old version's new patched version, now that the directory safely reflects - // the state of the files on disc. - if(ppreviousVerStoreFile != 0) - { - ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } - } - catch(...) - { - // Back out on adding that file - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); - - // Remove this entry from the cache - RemoveDirectoryFromCache(InDirectory); - - // Delete any previous version store file - if(ppreviousVerStoreFile != 0) - { - delete ppreviousVerStoreFile; - ppreviousVerStoreFile = 0; - } - - // Don't worry about the incremented number in the store info - throw; - } - - // Check logic - ASSERT(ppreviousVerStoreFile == 0); - - // Modify the store info - mpStoreInfo->ChangeBlocksUsed(blocksUsed); - mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); - - // Save the store info -- can cope if this exceptions because infomation - // will be rebuilt by housekeeping, and ID allocation can recover. - SaveStoreInfo(); - - // Return the ID to the caller - return id; -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) -// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. -// Created: 2003/10/21 -// -// -------------------------------------------------------------------------- -bool BackupContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut) -{ - // Essential checks! - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Find the directory the file is in (will exception if it fails) - BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - - // Setup flags - bool fileExisted = false; - bool madeChanges = false; - rObjectIDOut = 0; // not found - - // Count of deleted blocks - int64_t blocksDel = 0; - - try - { - // Iterate through directory, only looking at files which haven't been deleted - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *e = 0; - while((e = i.Next(BackupStoreDirectory::Entry::Flags_File, - BackupStoreDirectory::Entry::Flags_Deleted)) != 0) - { - // Compare name - if(e->GetName() == rFilename) - { - // Check that it's definately not already deleted - ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0); - // Set deleted flag - e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); - // Mark as made a change - madeChanges = true; - // Can safely do this, because we know we won't be here if it's already - // an old version - blocksDel += e->GetSizeInBlocks(); - // Is this the last version? - if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) - { - // Yes. It's been found. - rObjectIDOut = e->GetObjectID(); - fileExisted = true; - } - } - } - - // Save changes? - if(madeChanges) - { - // Save the directory back - SaveDirectory(dir, InDirectory); - - // Modify the store info, and write - mpStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); - - // Maybe postponed save of store info - SaveStoreInfo(); - } - } - catch(...) - { - RemoveDirectoryFromCache(InDirectory); - throw; - } - - - return fileExisted; -} - - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::RemoveDirectoryFromCache(int64_t) -// Purpose: Remove directory from cache -// Created: 2003/09/04 -// -// -------------------------------------------------------------------------- -void BackupContext::RemoveDirectoryFromCache(int64_t ObjectID) -{ - std::map::iterator item(mDirectoryCache.find(ObjectID)); - if(item != mDirectoryCache.end()) - { - // Delete this cached object - delete item->second; - // Erase the entry form the map - mDirectoryCache.erase(item); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::SaveDirectory(BackupStoreDirectory &, int64_t) -// Purpose: Save directory back to disc, update time in cache -// Created: 2003/09/04 -// -// -------------------------------------------------------------------------- -void BackupContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(rDir.GetObjectID() != ObjectID) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - - try - { - // Write to disc, adjust size in store info - std::string dirfn; - MakeObjectFilename(ObjectID, dirfn); - { - RaidFileWrite writeDir(mStoreDiscSet, dirfn); - writeDir.Open(true /* allow overwriting */); - rDir.WriteToStream(writeDir); - - // get the disc usage (must do this before commiting it) - int64_t dirSize = writeDir.GetDiscUsageInBlocks(); - - // Commit directory - writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - - // Make sure the size of the directory is available for writing the dir back - ASSERT(dirSize > 0); - int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); - mpStoreInfo->ChangeBlocksUsed(sizeAdjustment); - mpStoreInfo->ChangeBlocksInDirectories(sizeAdjustment); - // Update size stored in directory - rDir.SetUserInfo1_SizeInBlocks(dirSize); - } - // Refresh revision ID in cache - { - int64_t revid = 0; - if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid)) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } - rDir.SetRevisionID(revid); - } - } - catch(...) - { - // Remove it from the cache if anything went wrong - RemoveDirectoryFromCache(ObjectID); - throw; - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &) -// Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely. -// Created: 2003/09/04 -// -// -------------------------------------------------------------------------- -int64_t BackupContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Flags as not already existing - rAlreadyExists = false; - - // Get the directory we want to modify - BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - - // Scan the directory for the name (only looking for directories which already exist) - { - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, - BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) // Ignore deleted and old directories - { - if(en->GetName() == rFilename) - { - // Already exists - rAlreadyExists = true; - return en->GetObjectID(); - } - } - } - - // Allocate the next ID - int64_t id = AllocateObjectID(); - - // Create a blank directory with the given attributes on disc - std::string fn; - MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); - { - BackupStoreDirectory emptyDir(id, InDirectory); - // add the atttribues - emptyDir.SetAttributes(Attributes, AttributesModTime); - - // Write... - RaidFileWrite dirFile(mStoreDiscSet, fn); - dirFile.Open(false /* no overwriting */); - emptyDir.WriteToStream(dirFile); - // Get disc usage, before it's commited - int64_t dirSize = dirFile.GetDiscUsageInBlocks(); - // Commit the file - dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - - // Make sure the size of the directory is added to the usage counts in the info - ASSERT(dirSize > 0); - mpStoreInfo->ChangeBlocksUsed(dirSize); - mpStoreInfo->ChangeBlocksInDirectories(dirSize); - // Not added to cache, so don't set the size in the directory - } - - // Then add it into the directory - try - { - dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); - SaveDirectory(dir, InDirectory); - } - catch(...) - { - // Back out on adding that directory - RaidFileWrite del(mStoreDiscSet, fn); - del.Delete(); - - // Remove this entry from the cache - RemoveDirectoryFromCache(InDirectory); - - // Don't worry about the incremented number in the store info - throw; - } - - // Save the store info (may be postponed) - SaveStoreInfo(); - - // tell caller what the ID was - return id; -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &, bool) -// Purpose: Recusively deletes a directory (or undeletes if Undelete = true) -// Created: 2003/10/21 -// -// -------------------------------------------------------------------------- -void BackupContext::DeleteDirectory(int64_t ObjectID, bool Undelete) -{ - // Essential checks! - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Containing directory - int64_t InDirectory = 0; - - // Count of blocks deleted - int64_t blocksDeleted = 0; - - try - { - // Get the directory that's to be deleted - { - // In block, because dir may not be valid after the delete directory call - BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - - // Store the directory it's in for later - InDirectory = dir.GetContainerID(); - - // Depth first delete of contents - DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete); - } - - // Remove the entry from the directory it's in - ASSERT(InDirectory != 0); - BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory)); - - BackupStoreDirectory::Iterator i(parentDir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), - Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) - { - if(en->GetObjectID() == ObjectID) - { - // This is the one to delete - if(Undelete) - { - en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); - } - else - { - en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); - } - - // Save it - SaveDirectory(parentDir, InDirectory); - - // Done - break; - } - } - - // Update blocks deleted count - mpStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted)); - - // Save store info, may be postponed - SaveStoreInfo(); - } - catch(...) - { - RemoveDirectoryFromCache(InDirectory); - throw; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::DeleteDirectoryRecurse(BackupStoreDirectory &, int64_t) -// Purpose: Private. Deletes a directory depth-first recusively. -// Created: 2003/10/21 -// -// -------------------------------------------------------------------------- -void BackupContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete) -{ - try - { - // Does things carefully to avoid using a directory in the cache after recursive call - // because it may have been deleted. - - // Do sub directories - { - // Get the directory... - BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - - // Then scan it for directories - std::vector subDirs; - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - if(Undelete) - { - while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, // deleted dirs - BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING)) != 0) - { - // Store the directory ID. - subDirs.push_back(en->GetObjectID()); - } - } - else - { - while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir, // dirs only - BackupStoreDirectory::Entry::Flags_Deleted)) != 0) // but not deleted ones - { - // Store the directory ID. - subDirs.push_back(en->GetObjectID()); - } - } - - // Done with the directory for now. Recurse to sub directories - for(std::vector::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i) - { - DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete); - } - } - - // Then, delete the files. Will need to load the directory again because it might have - // been removed from the cache. - { - // Get the directory... - BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - - // Changes made? - bool changesMade = false; - - // Run through files - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - - while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), - Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) - { - // Add/remove the deleted flags - if(Undelete) - { - en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); - } - else - { - en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); - } - - // Keep count of the deleted blocks - if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) - { - rBlocksDeletedOut += en->GetSizeInBlocks(); - } - - // Did something - changesMade = true; - } - - // Save the directory - if(changesMade) - { - SaveDirectory(dir, ObjectID); - } - } - } - catch(...) - { - RemoveDirectoryFromCache(ObjectID); - throw; - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::ChangeDirAttributes(int64_t, const StreamableMemBlock &, int64_t) -// Purpose: Change the attributes of a directory -// Created: 2003/09/06 -// -// -------------------------------------------------------------------------- -void BackupContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - try - { - // Get the directory we want to modify - BackupStoreDirectory &dir(GetDirectoryInternal(Directory)); - - // Set attributes - dir.SetAttributes(Attributes, AttributesModTime); - - // Save back - SaveDirectory(dir, Directory); - } - catch(...) - { - RemoveDirectoryFromCache(Directory); - throw; - } -} - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::ChangeFileAttributes(int64_t, int64_t, const StreamableMemBlock &, int64_t) -// Purpose: Sets the attributes on a directory entry. Returns true if the object existed, false if it didn't. -// Created: 2003/09/06 -// -// -------------------------------------------------------------------------- -bool BackupContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - try - { - // Get the directory we want to modify - BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - - // Find the file entry - BackupStoreDirectory::Entry *en = 0; - // Iterate through current versions of files, only - BackupStoreDirectory::Iterator i(dir); - while((en = i.Next( - BackupStoreDirectory::Entry::Flags_File, - BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion) - ) != 0) - { - if(en->GetName() == rFilename) - { - // Set attributes - en->SetAttributes(Attributes, AttributesHash); - - // Tell caller the object ID - rObjectIDOut = en->GetObjectID(); - - // Done - break; - } - } - if(en == 0) - { - // Didn't find it - return false; - } - - // Save back - SaveDirectory(dir, InDirectory); - } - catch(...) - { - RemoveDirectoryFromCache(InDirectory); - throw; - } - - // Changed, everything OK - return true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::ObjectExists(int64_t) -// Purpose: Test to see if an object of this ID exists in the store -// Created: 2003/09/03 -// -// -------------------------------------------------------------------------- -bool BackupContext::ObjectExists(int64_t ObjectID, int MustBe) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - // Note that we need to allow object IDs a little bit greater than the last one in the store info, - // because the store info may not have got saved in an error condition. Max greater ID is - // STORE_INFO_SAVE_DELAY in this case, *2 to be safe. - if(ObjectID <= 0 || ObjectID > (mpStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2))) - { - // Obviously bad object ID - return false; - } - - // Test to see if it exists on the disc - std::string filename; - MakeObjectFilename(ObjectID, filename); - if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) - { - // RaidFile reports no file there - return false; - } - - // Do we need to be more specific? - if(MustBe != ObjectExists_Anything) - { - // Open the file - std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); - - // Read the first integer - u_int32_t magic; - if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */)) - { - // Failed to get any bytes, must have failed - return false; - } - -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - if(MustBe == ObjectExists_File && ntohl(magic) == OBJECTMAGIC_FILE_MAGIC_VALUE_V0) - { - // Old version detected - return true; - } -#endif - - // Right one? - u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; - - // Check - if(ntohl(magic) != requiredMagic) - { - return false; - } - - // File is implicitly closed - } - - return true; -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::OpenObject(int64_t) -// Purpose: Opens an object -// Created: 2003/09/03 -// -// -------------------------------------------------------------------------- -std::auto_ptr BackupContext::OpenObject(int64_t ObjectID) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - // Attempt to open the file - std::string fn; - MakeObjectFilename(ObjectID, fn); - return std::auto_ptr(RaidFileRead::Open(mStoreDiscSet, fn).release()); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::GetClientStoreMarker() -// Purpose: Retrieve the client store marker -// Created: 2003/10/29 -// -// -------------------------------------------------------------------------- -int64_t BackupContext::GetClientStoreMarker() -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - return mpStoreInfo->GetClientStoreMarker(); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::GetStoreDiscUsageInfo(int64_t &, int64_t &, int64_t &) -// Purpose: Get disc usage info from store info -// Created: 1/1/04 -// -// -------------------------------------------------------------------------- -void BackupContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - rBlocksUsed = mpStoreInfo->GetBlocksUsed(); - rBlocksSoftLimit = mpStoreInfo->GetBlocksSoftLimit(); - rBlocksHardLimit = mpStoreInfo->GetBlocksHardLimit(); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::HardLimitExceeded() -// Purpose: Returns true if the hard limit has been exceeded -// Created: 1/1/04 -// -// -------------------------------------------------------------------------- -bool BackupContext::HardLimitExceeded() -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - return mpStoreInfo->GetBlocksUsed() > mpStoreInfo->GetBlocksHardLimit(); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::SetClientStoreMarker(int64_t) -// Purpose: Sets the client store marker, and commits it to disc -// Created: 2003/10/29 -// -// -------------------------------------------------------------------------- -void BackupContext::SetClientStoreMarker(int64_t ClientStoreMarker) -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - mpStoreInfo->SetClientStoreMarker(ClientStoreMarker); - SaveStoreInfo(false /* don't delay saving this */); -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::MoveObject(int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) -// Purpose: Move an object (and all objects with the same name) from one directory to another -// Created: 12/11/03 -// -// -------------------------------------------------------------------------- -void BackupContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) -{ - if(mReadOnly) - { - THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) - } - - // Should deleted files be excluded when checking for the existance of objects with the target name? - int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject) - ?(BackupStoreDirectory::Entry::Flags_Deleted) - :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); - - // Special case if the directories are the same... - if(MoveFromDirectory == MoveToDirectory) - { - try - { - // Get the first directory - BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory)); - - // Find the file entry - BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); - - // Error if not found - if(en == 0) - { - THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) - } - - // Check the new name doens't already exist (optionally ignoring deleted files) - { - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *c = 0; - while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) - { - if(c->GetName() == rNewFilename) - { - THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) - } - } - } - - // Need to get all the entries with the same name? - if(MoveAllWithSameName) - { - // Iterate through the directory, copying all with matching names - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *c = 0; - while((c = i.Next()) != 0) - { - if(c->GetName() == en->GetName()) - { - // Rename this one - c->SetName(rNewFilename); - } - } - } - else - { - // Just copy this one - en->SetName(rNewFilename); - } - - // Save the directory back - SaveDirectory(dir, MoveFromDirectory); - } - catch(...) - { - RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same - throw; - } - - return; - } - - // Got to be careful how this is written, as we can't guarentte that if we have two - // directories open, the first won't be deleted as the second is opened. (cache) - - // List of entries to move - std::vector moving; - - // list of directory IDs which need to have containing dir id changed - std::vector dirsToChangeContainingID; - - try - { - // First of all, get copies of the entries to move to the to directory. - - { - // Get the first directory - BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - - // Find the file entry - BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID); - - // Error if not found - if(en == 0) - { - THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) - } - - // Need to get all the entries with the same name? - if(MoveAllWithSameName) - { - // Iterate through the directory, copying all with matching names - BackupStoreDirectory::Iterator i(from); - BackupStoreDirectory::Entry *c = 0; - while((c = i.Next()) != 0) - { - if(c->GetName() == en->GetName()) - { - // Copy - moving.push_back(new BackupStoreDirectory::Entry(*c)); - - // Check for containing directory correction - if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID()); - } - } - ASSERT(!moving.empty()); - } - else - { - // Just copy this one - moving.push_back(new BackupStoreDirectory::Entry(*en)); - - // Check for containing directory correction - if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID()); - } - } - - // Secondly, insert them into the to directory, and save it - - { - // To directory - BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - - // Check the new name doens't already exist - { - BackupStoreDirectory::Iterator i(to); - BackupStoreDirectory::Entry *c = 0; - while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) - { - if(c->GetName() == rNewFilename) - { - THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) - } - } - } - - // Copy the entries into it, changing the name as we go - for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) - { - BackupStoreDirectory::Entry *en = (*i); - en->SetName(rNewFilename); - to.AddEntry(*en); // adds copy - } - - // Save back - SaveDirectory(to, MoveToDirectory); - } - - // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory - try - { - // Get directory - BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - - // Delete each one - for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) - { - from.DeleteEntry((*i)->GetObjectID()); - } - - // Save back - SaveDirectory(from, MoveFromDirectory); - } - catch(...) - { - // UNDO modification to To directory - - // Get directory - BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - - // Delete each one - for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) - { - to.DeleteEntry((*i)->GetObjectID()); - } - - // Save back - SaveDirectory(to, MoveToDirectory); - - // Throw the error - throw; - } - - // Finally... for all the directories we moved, modify their containing directory ID - for(std::vector::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) - { - // Load the directory - BackupStoreDirectory &change(GetDirectoryInternal(*i)); - - // Modify containing dir ID - change.SetContainerID(MoveToDirectory); - - // Save it back - SaveDirectory(change, *i); - } - } - catch(...) - { - // Make sure directories aren't in the cache, as they may have been modified - RemoveDirectoryFromCache(MoveToDirectory); - RemoveDirectoryFromCache(MoveFromDirectory); - for(std::vector::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) - { - RemoveDirectoryFromCache(*i); - } - - while(!moving.empty()) - { - delete moving.back(); - moving.pop_back(); - } - throw; - } - - // Clean up - while(!moving.empty()) - { - delete moving.back(); - moving.pop_back(); - } -} - - - -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupContext::GetBackupStoreInfo() -// Purpose: Return the backup store info object, exception if it isn't loaded -// Created: 19/4/04 -// -// -------------------------------------------------------------------------- -const BackupStoreInfo &BackupContext::GetBackupStoreInfo() const -{ - if(mpStoreInfo.get() == 0) - { - THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) - } - - return *(mpStoreInfo.get()); -} - - diff --git a/bin/bbstored/BackupContext.h b/bin/bbstored/BackupContext.h deleted file mode 100644 index 18f2f25c..00000000 --- a/bin/bbstored/BackupContext.h +++ /dev/null @@ -1,149 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: BackupContext.h -// Purpose: Context for backup store server -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- - -#ifndef BACKUPCONTEXT__H -#define BACKUPCONTEXT__H - -#include -#include -#include - -#include "NamedLock.h" -#include "Utils.h" - -class BackupStoreDirectory; -class BackupStoreFilename; -class BackupStoreDaemon; -class BackupStoreInfo; -class IOStream; -class StreamableMemBlock; - -// -------------------------------------------------------------------------- -// -// Class -// Name: BackupContext -// Purpose: Context for backup store server -// Created: 2003/08/20 -// -// -------------------------------------------------------------------------- -class BackupContext -{ -public: - BackupContext(int32_t ClientID, BackupStoreDaemon &rDaemon); - ~BackupContext(); -private: - BackupContext(const BackupContext &rToCopy); -public: - - void ReceivedFinishCommand(); - void CleanUp(); - - int32_t GetClientID() {return mClientID;} - - enum - { - Phase_START = 0, - Phase_Version = 0, - Phase_Login = 1, - Phase_Commands = 2 - }; - - int GetPhase() const {return mProtocolPhase;} - void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;} - - // Read only locking - bool SessionIsReadOnly() {return mReadOnly;} - bool AttemptToGetWriteLock(); - - void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;} - bool GetClientHasAccount() const {return mClientHasAccount;} - const std::string &GetStoreRoot() const {return mStoreRoot;} - int GetStoreDiscSet() const {return mStoreDiscSet;} - - // Store info - void LoadStoreInfo(); - void SaveStoreInfo(bool AllowDelay = true); - const BackupStoreInfo &GetBackupStoreInfo() const; - - // Client marker - int64_t GetClientStoreMarker(); - void SetClientStoreMarker(int64_t ClientStoreMarker); - - // Usage information - void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit); - bool HardLimitExceeded(); - - // Reading directories - // -------------------------------------------------------------------------- - // - // Function - // Name: BackupContext::GetDirectory(int64_t) - // Purpose: Return a reference to a directory. Valid only until the - // next time a function which affects directories is called. - // Mainly this funciton, and creation of files. - // Created: 2003/09/02 - // - // -------------------------------------------------------------------------- - const BackupStoreDirectory &GetDirectory(int64_t ObjectID) - { - // External callers aren't allowed to change it -- this function - // merely turns the the returned directory const. - return GetDirectoryInternal(ObjectID); - } - - // Manipulating files/directories - int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions); - int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists); - void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime); - bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut); - bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut); - void DeleteDirectory(int64_t ObjectID, bool Undelete = false); - void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject); - - // Manipulating objects - enum - { - ObjectExists_Anything = 0, - ObjectExists_File = 1, - ObjectExists_Directory = 2 - }; - bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything); - std::auto_ptr OpenObject(int64_t ObjectID); - - // Info - int32_t GetClientID() const {return mClientID;} - -private: - void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); - BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID); - void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID); - void RemoveDirectoryFromCache(int64_t ObjectID); - void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); - int64_t AllocateObjectID(); - -private: - int32_t mClientID; - BackupStoreDaemon &mrDaemon; - int mProtocolPhase; - bool mClientHasAccount; - std::string mStoreRoot; // has final directory separator - int mStoreDiscSet; - bool mReadOnly; - NamedLock mWriteLock; - int mSaveStoreInfoDelay; // how many times to delay saving the store info - - // Store info - std::auto_ptr mpStoreInfo; - - // Directory cache - std::map mDirectoryCache; -}; - -#endif // BACKUPCONTEXT__H - diff --git a/bin/bbstored/BackupStoreContext.cpp b/bin/bbstored/BackupStoreContext.cpp new file mode 100644 index 00000000..990be05d --- /dev/null +++ b/bin/bbstored/BackupStoreContext.cpp @@ -0,0 +1,1752 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContext.cpp +// Purpose: Context for backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "BackupStoreContext.h" +#include "RaidFileWrite.h" +#include "RaidFileRead.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreException.h" +#include "BackupStoreInfo.h" +#include "BackupConstants.h" +#include "BackupStoreFile.h" +#include "BackupStoreObjectMagic.h" +#include "StoreStructure.h" +#include "BackupStoreDaemon.h" +#include "RaidFileController.h" +#include "FileStream.h" +#include "InvisibleTempFileStream.h" +#include "BufferedStream.h" + +#include "MemLeakFindOn.h" + + +// Maximum number of directories to keep in the cache +// When the cache is bigger than this, everything gets +// deleted. +#ifdef BOX_RELEASE_BUILD + #define MAX_CACHE_SIZE 32 +#else + #define MAX_CACHE_SIZE 2 +#endif + +// Allow the housekeeping process 4 seconds to release an account +#define MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT 4 + +// Maximum amount of store info updates before it's actually saved to disc. +#define STORE_INFO_SAVE_DELAY 96 + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::BackupStoreContext() +// Purpose: Constructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreContext::BackupStoreContext(int32_t ClientID, + HousekeepingInterface &rDaemon) + : mClientID(ClientID), + mrDaemon(rDaemon), + mProtocolPhase(Phase_START), + mClientHasAccount(false), + mStoreDiscSet(-1), + mReadOnly(true), + mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), + mpTestHook(NULL) +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::~BackupStoreContext() +// Purpose: Destructor +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +BackupStoreContext::~BackupStoreContext() +{ + // Delete the objects in the cache + for(std::map::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + { + delete (i->second); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::CleanUp() +// Purpose: Clean up after a connection +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::CleanUp() +{ + // Make sure the store info is saved, if it has been loaded, isn't read only and has been modified + if(mpStoreInfo.get() && !(mpStoreInfo->IsReadOnly()) && mpStoreInfo->IsModified()) + { + mpStoreInfo->Save(); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ReceivedFinishCommand() +// Purpose: Called when the finish command is received by the protocol +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::ReceivedFinishCommand() +{ + if(!mReadOnly && mpStoreInfo.get()) + { + // Save the store info, not delayed + SaveStoreInfo(false); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AttemptToGetWriteLock() +// Purpose: Attempt to get a write lock for the store, and if so, unset the read only flags +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::AttemptToGetWriteLock() +{ + // Make the filename of the write lock file + std::string writeLockFile; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); + + // Request the lock + bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + + if(!gotLock) + { + // The housekeeping process might have the thing open -- ask it to stop + char msg[256]; + int msgLen = sprintf(msg, "r%x\n", mClientID); + // Send message + mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen); + + // Then try again a few times + int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; + do + { + ::sleep(1 /* second */); + --tries; + gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); + + } while(!gotLock && tries > 0); + } + + if(gotLock) + { + // Got the lock, mark as not read only + mReadOnly = false; + } + + return gotLock; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::LoadStoreInfo() +// Purpose: Load the store info from disc +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::LoadStoreInfo() +{ + if(mpStoreInfo.get() != 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) + } + + // Load it up! + std::auto_ptr i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly)); + + // Check it + if(i->GetAccountID() != mClientID) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) + } + + // Keep the pointer to it + mpStoreInfo = i; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::SaveStoreInfo(bool) +// Purpose: Potentially delayed saving of the store info +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::SaveStoreInfo(bool AllowDelay) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Can delay saving it a little while? + if(AllowDelay) + { + --mSaveStoreInfoDelay; + if(mSaveStoreInfoDelay > 0) + { + return; + } + } + + // Want to save now + mpStoreInfo->Save(); + + // Set count for next delay + mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::MakeObjectFilename(int64_t, std::string &, bool) +// Purpose: Create the filename of an object in the store, optionally creating the +// containing directory if it doesn't already exist. +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) +{ + // Delegate to utility function + StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetDirectoryInternal(int64_t) +// Purpose: Return a reference to a directory. Valid only until the +// next time a function which affects directories is called. +// Mainly this funciton, and creation of files. +// Private version of this, which returns non-const directories. +// Created: 2003/09/02 +// +// -------------------------------------------------------------------------- +BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) +{ + // Get the filename + std::string filename; + MakeObjectFilename(ObjectID, filename); + + // Already in cache? + std::map::iterator item(mDirectoryCache.find(ObjectID)); + if(item != mDirectoryCache.end()) + { + // Check the revision ID of the file -- does it need refreshing? + int64_t revID = 0; + if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID)) + { + THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) + } + + if(revID == item->second->GetRevisionID()) + { + // Looks good... return the cached object + BOX_TRACE("Returning object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache, modtime = " << revID); + return *(item->second); + } + + BOX_TRACE("Refreshing object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " in cache, modtime changed from " << + item->second->GetRevisionID() << " to " << revID); + + // Delete this cached object + delete item->second; + mDirectoryCache.erase(item); + } + + // Need to load it up + + // First check to see if the cache is too big + if(mDirectoryCache.size() > MAX_CACHE_SIZE) + { + // Very simple. Just delete everything! + for(std::map::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + { + delete (i->second); + } + mDirectoryCache.clear(); + } + + // Get a RaidFileRead to read it + int64_t revID = 0; + std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID)); + ASSERT(revID != 0); + + // New directory object + std::auto_ptr dir(new BackupStoreDirectory); + + // Read it from the stream, then set it's revision ID + BufferedStream buf(*objectFile); + dir->ReadFromStream(buf, IOStream::TimeOutInfinite); + dir->SetRevisionID(revID); + + // Make sure the size of the directory is available for writing the dir back + int64_t dirSize = objectFile->GetDiscUsageInBlocks(); + ASSERT(dirSize > 0); + dir->SetUserInfo1_SizeInBlocks(dirSize); + + // Store in cache + BackupStoreDirectory *pdir = dir.release(); + try + { + mDirectoryCache[ObjectID] = pdir; + } + catch(...) + { + delete pdir; + throw; + } + + // Return it + return *pdir; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AllocateObjectID() +// Purpose: Allocate a new object ID, tolerant of failures to save store info +// Created: 16/12/03 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::AllocateObjectID() +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + // Given that the store info may not be saved for STORE_INFO_SAVE_DELAY + // times after it has been updated, this is a reasonable number of times + // to try for finding an unused ID. + // (Sizes used in the store info are fixed by the housekeeping process) + int retryLimit = (STORE_INFO_SAVE_DELAY * 2); + + while(retryLimit > 0) + { + // Attempt to allocate an ID from the store + int64_t id = mpStoreInfo->AllocateObjectID(); + + // Generate filename + std::string filename; + MakeObjectFilename(id, filename); + // Check it doesn't exist + if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + { + // Success! + return id; + } + + // Decrement retry count, and try again + --retryLimit; + + // Mark that the store info should be saved as soon as possible + mSaveStoreInfoDelay = 0; + + BOX_WARNING("When allocating object ID, found that " << + BOX_FORMAT_OBJECTID(id) << " is already in use"); + } + + THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AddFile(IOStream &, int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) +// Purpose: Add a file to the store, from a given stream, into a specified directory. +// Returns object ID of the new file. +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, + int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, + bool MarkFileWithSameNameAsOldVersions) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // This is going to be a bit complex to make sure it copes OK + // with things going wrong. + // The only thing which isn't safe is incrementing the object ID + // and keeping the blocks used entirely accurate -- but these + // aren't big problems if they go horribly wrong. The sizes will + // be corrected the next time the account has a housekeeping run, + // and the object ID allocation code is tolerant of missed IDs. + // (the info is written lazily, so these are necessary) + + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Allocate the next ID + int64_t id = AllocateObjectID(); + + // Stream the file to disc + std::string fn; + MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); + int64_t blocksUsed = 0; + RaidFileWrite *ppreviousVerStoreFile = 0; + bool reversedDiffIsCompletelyDifferent = false; + int64_t oldVersionNewBlocksUsed = 0; + try + { + RaidFileWrite storeFile(mStoreDiscSet, fn); + storeFile.Open(false /* no overwriting */); + int64_t spaceAdjustFromDiff = 0; // size adjustment from use of patch in old file + + // Diff or full file? + if(DiffFromFileID == 0) + { + // A full file, just store to disc + if(!rFile.CopyStreamTo(storeFile, BACKUP_STORE_TIMEOUT)) + { + THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) + } + } + else + { + // Check that the diffed from ID actually exists in the directory + if(dir.FindEntryByID(DiffFromFileID) == 0) + { + THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) + } + + // Diff file, needs to be recreated. + // Choose a temporary filename. + std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", + 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); + + try + { + // Open it twice +#ifdef WIN32 + InvisibleTempFileStream diff(tempFn.c_str(), + O_RDWR | O_CREAT | O_BINARY); + InvisibleTempFileStream diff2(tempFn.c_str(), + O_RDWR | O_BINARY); +#else + FileStream diff(tempFn.c_str(), O_RDWR | O_CREAT | O_EXCL); + FileStream diff2(tempFn.c_str(), O_RDONLY); + + // Unlink it immediately, so it definitely goes away + if(::unlink(tempFn.c_str()) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError); + } +#endif + + // Stream the incoming diff to this temporary file + if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) + { + THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) + } + + // Verify the diff + diff.Seek(0, IOStream::SeekType_Absolute); + if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) + { + THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) + } + + // Seek to beginning of diff file + diff.Seek(0, IOStream::SeekType_Absolute); + + // Filename of the old version + std::string oldVersionFilename; + MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); + + // Reassemble that diff -- open previous file, and combine the patch and file + std::auto_ptr from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); + BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); + + // Then... reverse the patch back (open the from file again, and create a write file to overwrite it) + std::auto_ptr from2(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); + ppreviousVerStoreFile = new RaidFileWrite(mStoreDiscSet, oldVersionFilename); + ppreviousVerStoreFile->Open(true /* allow overwriting */); + from->Seek(0, IOStream::SeekType_Absolute); + diff.Seek(0, IOStream::SeekType_Absolute); + BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, + DiffFromFileID, &reversedDiffIsCompletelyDifferent); + + // Store disc space used + oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); + + // And make a space adjustment for the size calculation + spaceAdjustFromDiff = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed; + + // Everything cleans up here... + } + catch(...) + { + // Be very paranoid about deleting this temp file -- we could only leave a zero byte file anyway + ::unlink(tempFn.c_str()); + throw; + } + } + + // Get the blocks used + blocksUsed = storeFile.GetDiscUsageInBlocks(); + + // Exceeds the hard limit? + if((mpStoreInfo->GetBlocksUsed() + blocksUsed - spaceAdjustFromDiff) > mpStoreInfo->GetBlocksHardLimit()) + { + THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) + // The store file will be deleted automatically by the RaidFile object + } + + // Commit the file + storeFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + } + catch(...) + { + // Delete any previous version store file + if(ppreviousVerStoreFile != 0) + { + delete ppreviousVerStoreFile; + ppreviousVerStoreFile = 0; + } + + throw; + } + + // Verify the file -- only necessary for non-diffed versions + // NOTE: No need to catch exceptions and delete ppreviousVerStoreFile, because + // in the non-diffed code path it's never allocated. + if(DiffFromFileID == 0) + { + std::auto_ptr checkFile(RaidFileRead::Open(mStoreDiscSet, fn)); + if(!BackupStoreFile::VerifyEncodedFileFormat(*checkFile)) + { + // Error! Delete the file + RaidFileWrite del(mStoreDiscSet, fn); + del.Delete(); + + // Exception + THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) + } + } + + // Modify the directory -- first make all files with the same name + // marked as an old version + int64_t blocksInOldFiles = 0; + try + { + if(MarkFileWithSameNameAsOldVersions) + { + BackupStoreDirectory::Iterator i(dir); + + BackupStoreDirectory::Entry *e = 0; + while((e = i.Next()) != 0) + { + // First, check it's not an old version (cheaper comparison) + if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) + { + // Compare name + if(e->GetName() == rFilename) + { + // Check that it's definately not an old version + ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0); + // Set old version flag + e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion); + // Can safely do this, because we know we won't be here if it's already + // an old version + blocksInOldFiles += e->GetSizeInBlocks(); + } + } + } + } + + // Then the new entry + BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, + ModificationTime, id, blocksUsed, BackupStoreDirectory::Entry::Flags_File, AttributesHash); + + // Adjust for the patch back stuff? + if(DiffFromFileID != 0) + { + // Get old version entry + BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID); + ASSERT(poldEntry != 0); + + // Adjust dependency info of file? + if(!reversedDiffIsCompletelyDifferent) + { + poldEntry->SetDependsNewer(id); + pnewEntry->SetDependsOlder(DiffFromFileID); + } + + // Adjust size of old entry + int64_t oldSize = poldEntry->GetSizeInBlocks(); + poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); + + // And adjust blocks used count, for later adjustment + blocksUsed += (oldVersionNewBlocksUsed - oldSize); + blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize); + } + + // Write the directory back to disc + SaveDirectory(dir, InDirectory); + + // Commit the old version's new patched version, now that the directory safely reflects + // the state of the files on disc. + if(ppreviousVerStoreFile != 0) + { + ppreviousVerStoreFile->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + delete ppreviousVerStoreFile; + ppreviousVerStoreFile = 0; + } + } + catch(...) + { + // Back out on adding that file + RaidFileWrite del(mStoreDiscSet, fn); + del.Delete(); + + // Remove this entry from the cache + RemoveDirectoryFromCache(InDirectory); + + // Delete any previous version store file + if(ppreviousVerStoreFile != 0) + { + delete ppreviousVerStoreFile; + ppreviousVerStoreFile = 0; + } + + // Don't worry about the incremented number in the store info + throw; + } + + // Check logic + ASSERT(ppreviousVerStoreFile == 0); + + // Modify the store info + mpStoreInfo->ChangeBlocksUsed(blocksUsed); + mpStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); + + // Save the store info -- can cope if this exceptions because infomation + // will be rebuilt by housekeeping, and ID allocation can recover. + SaveStoreInfo(); + + // Return the ID to the caller + return id; +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) +// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut) +{ + // Essential checks! + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Find the directory the file is in (will exception if it fails) + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Setup flags + bool fileExisted = false; + bool madeChanges = false; + rObjectIDOut = 0; // not found + + // Count of deleted blocks + int64_t blocksDel = 0; + + try + { + // Iterate through directory, only looking at files which haven't been deleted + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *e = 0; + while((e = i.Next(BackupStoreDirectory::Entry::Flags_File, + BackupStoreDirectory::Entry::Flags_Deleted)) != 0) + { + // Compare name + if(e->GetName() == rFilename) + { + // Check that it's definately not already deleted + ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0); + // Set deleted flag + e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); + // Mark as made a change + madeChanges = true; + // Can safely do this, because we know we won't be here if it's already + // an old version + blocksDel += e->GetSizeInBlocks(); + // Is this the last version? + if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) + { + // Yes. It's been found. + rObjectIDOut = e->GetObjectID(); + fileExisted = true; + } + } + } + + // Save changes? + if(madeChanges) + { + // Save the directory back + SaveDirectory(dir, InDirectory); + + // Modify the store info, and write + mpStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); + + // Maybe postponed save of store info + SaveStoreInfo(); + } + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } + + return fileExisted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) +// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory) +{ + // Essential checks! + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Find the directory the file is in (will exception if it fails) + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Setup flags + bool fileExisted = false; + bool madeChanges = false; + + // Count of deleted blocks + int64_t blocksDel = 0; + + try + { + // Iterate through directory, only looking at files which have been deleted + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *e = 0; + while((e = i.Next(BackupStoreDirectory::Entry::Flags_File | + BackupStoreDirectory::Entry::Flags_Deleted, 0)) != 0) + { + // Compare name + if(e->GetObjectID() == ObjectID) + { + // Check that it's definitely already deleted + ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0); + // Clear deleted flag + e->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); + // Mark as made a change + madeChanges = true; + blocksDel -= e->GetSizeInBlocks(); + + // Is this the last version? + if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) + { + // Yes. It's been found. + fileExisted = true; + } + } + } + + // Save changes? + if(madeChanges) + { + // Save the directory back + SaveDirectory(dir, InDirectory); + + // Modify the store info, and write + mpStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); + + // Maybe postponed save of store info + SaveStoreInfo(); + } + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } + + return fileExisted; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::RemoveDirectoryFromCache(int64_t) +// Purpose: Remove directory from cache +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID) +{ + std::map::iterator item(mDirectoryCache.find(ObjectID)); + if(item != mDirectoryCache.end()) + { + // Delete this cached object + delete item->second; + // Erase the entry form the map + mDirectoryCache.erase(item); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &, int64_t) +// Purpose: Save directory back to disc, update time in cache +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(rDir.GetObjectID() != ObjectID) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + try + { + // Write to disc, adjust size in store info + std::string dirfn; + MakeObjectFilename(ObjectID, dirfn); + { + RaidFileWrite writeDir(mStoreDiscSet, dirfn); + writeDir.Open(true /* allow overwriting */); + rDir.WriteToStream(writeDir); + + // get the disc usage (must do this before commiting it) + int64_t dirSize = writeDir.GetDiscUsageInBlocks(); + + // Commit directory + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + + // Make sure the size of the directory is available for writing the dir back + ASSERT(dirSize > 0); + int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); + mpStoreInfo->ChangeBlocksUsed(sizeAdjustment); + mpStoreInfo->ChangeBlocksInDirectories(sizeAdjustment); + // Update size stored in directory + rDir.SetUserInfo1_SizeInBlocks(dirSize); + } + // Refresh revision ID in cache + { + int64_t revid = 0; + if(!RaidFileRead::FileExists(mStoreDiscSet, dirfn, &revid)) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + rDir.SetRevisionID(revid); + } + } + catch(...) + { + // Remove it from the cache if anything went wrong + RemoveDirectoryFromCache(ObjectID); + throw; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::AddDirectory(int64_t, const BackupStoreFilename &, bool &) +// Purpose: Creates a directory (or just returns the ID of an existing one). rAlreadyExists set appropraitely. +// Created: 2003/09/04 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Flags as not already existing + rAlreadyExists = false; + + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Scan the directory for the name (only looking for directories which already exist) + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, + BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) // Ignore deleted and old directories + { + if(en->GetName() == rFilename) + { + // Already exists + rAlreadyExists = true; + return en->GetObjectID(); + } + } + } + + // Allocate the next ID + int64_t id = AllocateObjectID(); + + // Create a blank directory with the given attributes on disc + std::string fn; + MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); + { + BackupStoreDirectory emptyDir(id, InDirectory); + // add the atttribues + emptyDir.SetAttributes(Attributes, AttributesModTime); + + // Write... + RaidFileWrite dirFile(mStoreDiscSet, fn); + dirFile.Open(false /* no overwriting */); + emptyDir.WriteToStream(dirFile); + // Get disc usage, before it's commited + int64_t dirSize = dirFile.GetDiscUsageInBlocks(); + // Commit the file + dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + + // Make sure the size of the directory is added to the usage counts in the info + ASSERT(dirSize > 0); + mpStoreInfo->ChangeBlocksUsed(dirSize); + mpStoreInfo->ChangeBlocksInDirectories(dirSize); + // Not added to cache, so don't set the size in the directory + } + + // Then add it into the directory + try + { + dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); + SaveDirectory(dir, InDirectory); + } + catch(...) + { + // Back out on adding that directory + RaidFileWrite del(mStoreDiscSet, fn); + del.Delete(); + + // Remove this entry from the cache + RemoveDirectoryFromCache(InDirectory); + + // Don't worry about the incremented number in the store info + throw; + } + + // Save the store info (may be postponed) + SaveStoreInfo(); + + // tell caller what the ID was + return id; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &, bool) +// Purpose: Recusively deletes a directory (or undeletes if Undelete = true) +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) +{ + // Essential checks! + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Containing directory + int64_t InDirectory = 0; + + // Count of blocks deleted + int64_t blocksDeleted = 0; + + try + { + // Get the directory that's to be deleted + { + // In block, because dir may not be valid after the delete directory call + BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); + + // Store the directory it's in for later + InDirectory = dir.GetContainerID(); + + // Depth first delete of contents + DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete); + } + + // Remove the entry from the directory it's in + ASSERT(InDirectory != 0); + BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory)); + + BackupStoreDirectory::Iterator i(parentDir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), + Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) + { + if(en->GetObjectID() == ObjectID) + { + // This is the one to delete + if(Undelete) + { + en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + else + { + en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + + // Save it + SaveDirectory(parentDir, InDirectory); + + // Done + break; + } + } + + // Update blocks deleted count + mpStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted)); + + // Save store info, may be postponed + SaveStoreInfo(); + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::DeleteDirectoryRecurse(BackupStoreDirectory &, int64_t) +// Purpose: Private. Deletes a directory depth-first recusively. +// Created: 2003/10/21 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete) +{ + try + { + // Does things carefully to avoid using a directory in the cache after recursive call + // because it may have been deleted. + + // Do sub directories + { + // Get the directory... + BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); + + // Then scan it for directories + std::vector subDirs; + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + if(Undelete) + { + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_Deleted, // deleted dirs + BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING)) != 0) + { + // Store the directory ID. + subDirs.push_back(en->GetObjectID()); + } + } + else + { + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir, // dirs only + BackupStoreDirectory::Entry::Flags_Deleted)) != 0) // but not deleted ones + { + // Store the directory ID. + subDirs.push_back(en->GetObjectID()); + } + } + + // Done with the directory for now. Recurse to sub directories + for(std::vector::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i) + { + DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete); + } + } + + // Then, delete the files. Will need to load the directory again because it might have + // been removed from the cache. + { + // Get the directory... + BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); + + // Changes made? + bool changesMade = false; + + // Run through files + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + + while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), + Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) + { + // Add/remove the deleted flags + if(Undelete) + { + en->RemoveFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + else + { + en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); + } + + // Keep count of the deleted blocks + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) + { + rBlocksDeletedOut += en->GetSizeInBlocks(); + } + + // Did something + changesMade = true; + } + + // Save the directory + if(changesMade) + { + SaveDirectory(dir, ObjectID); + } + } + } + catch(...) + { + RemoveDirectoryFromCache(ObjectID); + throw; + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ChangeDirAttributes(int64_t, const StreamableMemBlock &, int64_t) +// Purpose: Change the attributes of a directory +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + try + { + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(Directory)); + + // Set attributes + dir.SetAttributes(Attributes, AttributesModTime); + + // Save back + SaveDirectory(dir, Directory); + } + catch(...) + { + RemoveDirectoryFromCache(Directory); + throw; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ChangeFileAttributes(int64_t, int64_t, const StreamableMemBlock &, int64_t) +// Purpose: Sets the attributes on a directory entry. Returns true if the object existed, false if it didn't. +// Created: 2003/09/06 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + try + { + // Get the directory we want to modify + BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); + + // Find the file entry + BackupStoreDirectory::Entry *en = 0; + // Iterate through current versions of files, only + BackupStoreDirectory::Iterator i(dir); + while((en = i.Next( + BackupStoreDirectory::Entry::Flags_File, + BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion) + ) != 0) + { + if(en->GetName() == rFilename) + { + // Set attributes + en->SetAttributes(Attributes, AttributesHash); + + // Tell caller the object ID + rObjectIDOut = en->GetObjectID(); + + // Done + break; + } + } + if(en == 0) + { + // Didn't find it + return false; + } + + // Save back + SaveDirectory(dir, InDirectory); + } + catch(...) + { + RemoveDirectoryFromCache(InDirectory); + throw; + } + + // Changed, everything OK + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::ObjectExists(int64_t) +// Purpose: Test to see if an object of this ID exists in the store +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + // Note that we need to allow object IDs a little bit greater than the last one in the store info, + // because the store info may not have got saved in an error condition. Max greater ID is + // STORE_INFO_SAVE_DELAY in this case, *2 to be safe. + if(ObjectID <= 0 || ObjectID > (mpStoreInfo->GetLastObjectIDUsed() + (STORE_INFO_SAVE_DELAY * 2))) + { + // Obviously bad object ID + return false; + } + + // Test to see if it exists on the disc + std::string filename; + MakeObjectFilename(ObjectID, filename); + if(!RaidFileRead::FileExists(mStoreDiscSet, filename)) + { + // RaidFile reports no file there + return false; + } + + // Do we need to be more specific? + if(MustBe != ObjectExists_Anything) + { + // Open the file + std::auto_ptr objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); + + // Read the first integer + u_int32_t magic; + if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */)) + { + // Failed to get any bytes, must have failed + return false; + } + +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + if(MustBe == ObjectExists_File && ntohl(magic) == OBJECTMAGIC_FILE_MAGIC_VALUE_V0) + { + // Old version detected + return true; + } +#endif + + // Right one? + u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; + + // Check + if(ntohl(magic) != requiredMagic) + { + return false; + } + + // File is implicitly closed + } + + return true; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::OpenObject(int64_t) +// Purpose: Opens an object +// Created: 2003/09/03 +// +// -------------------------------------------------------------------------- +std::auto_ptr BackupStoreContext::OpenObject(int64_t ObjectID) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + // Attempt to open the file + std::string fn; + MakeObjectFilename(ObjectID, fn); + return std::auto_ptr(RaidFileRead::Open(mStoreDiscSet, fn).release()); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetClientStoreMarker() +// Purpose: Retrieve the client store marker +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +int64_t BackupStoreContext::GetClientStoreMarker() +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + return mpStoreInfo->GetClientStoreMarker(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetStoreDiscUsageInfo(int64_t &, int64_t &, int64_t &) +// Purpose: Get disc usage info from store info +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + rBlocksUsed = mpStoreInfo->GetBlocksUsed(); + rBlocksSoftLimit = mpStoreInfo->GetBlocksSoftLimit(); + rBlocksHardLimit = mpStoreInfo->GetBlocksHardLimit(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::HardLimitExceeded() +// Purpose: Returns true if the hard limit has been exceeded +// Created: 1/1/04 +// +// -------------------------------------------------------------------------- +bool BackupStoreContext::HardLimitExceeded() +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + return mpStoreInfo->GetBlocksUsed() > mpStoreInfo->GetBlocksHardLimit(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::SetClientStoreMarker(int64_t) +// Purpose: Sets the client store marker, and commits it to disc +// Created: 2003/10/29 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + mpStoreInfo->SetClientStoreMarker(ClientStoreMarker); + SaveStoreInfo(false /* don't delay saving this */); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::MoveObject(int64_t, int64_t, int64_t, const BackupStoreFilename &, bool) +// Purpose: Move an object (and all objects with the same name) from one directory to another +// Created: 12/11/03 +// +// -------------------------------------------------------------------------- +void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) + } + + // Should deleted files be excluded when checking for the existance of objects with the target name? + int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject) + ?(BackupStoreDirectory::Entry::Flags_Deleted) + :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); + + // Special case if the directories are the same... + if(MoveFromDirectory == MoveToDirectory) + { + try + { + // Get the first directory + BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory)); + + // Find the file entry + BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); + + // Error if not found + if(en == 0) + { + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) + } + + // Check the new name doens't already exist (optionally ignoring deleted files) + { + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) + { + if(c->GetName() == rNewFilename) + { + THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) + } + } + } + + // Need to get all the entries with the same name? + if(MoveAllWithSameName) + { + // Iterate through the directory, copying all with matching names + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next()) != 0) + { + if(c->GetName() == en->GetName()) + { + // Rename this one + c->SetName(rNewFilename); + } + } + } + else + { + // Just copy this one + en->SetName(rNewFilename); + } + + // Save the directory back + SaveDirectory(dir, MoveFromDirectory); + } + catch(...) + { + RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same + throw; + } + + return; + } + + // Got to be careful how this is written, as we can't guarentte that if we have two + // directories open, the first won't be deleted as the second is opened. (cache) + + // List of entries to move + std::vector moving; + + // list of directory IDs which need to have containing dir id changed + std::vector dirsToChangeContainingID; + + try + { + // First of all, get copies of the entries to move to the to directory. + + { + // Get the first directory + BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); + + // Find the file entry + BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID); + + // Error if not found + if(en == 0) + { + THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) + } + + // Need to get all the entries with the same name? + if(MoveAllWithSameName) + { + // Iterate through the directory, copying all with matching names + BackupStoreDirectory::Iterator i(from); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next()) != 0) + { + if(c->GetName() == en->GetName()) + { + // Copy + moving.push_back(new BackupStoreDirectory::Entry(*c)); + + // Check for containing directory correction + if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID()); + } + } + ASSERT(!moving.empty()); + } + else + { + // Just copy this one + moving.push_back(new BackupStoreDirectory::Entry(*en)); + + // Check for containing directory correction + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID()); + } + } + + // Secondly, insert them into the to directory, and save it + + { + // To directory + BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); + + // Check the new name doens't already exist + { + BackupStoreDirectory::Iterator i(to); + BackupStoreDirectory::Entry *c = 0; + while((c = i.Next(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, targetSearchExcludeFlags)) != 0) + { + if(c->GetName() == rNewFilename) + { + THROW_EXCEPTION(BackupStoreException, NameAlreadyExistsInDirectory) + } + } + } + + // Copy the entries into it, changing the name as we go + for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) + { + BackupStoreDirectory::Entry *en = (*i); + en->SetName(rNewFilename); + to.AddEntry(*en); // adds copy + } + + // Save back + SaveDirectory(to, MoveToDirectory); + } + + // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory + try + { + // Get directory + BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); + + // Delete each one + for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) + { + from.DeleteEntry((*i)->GetObjectID()); + } + + // Save back + SaveDirectory(from, MoveFromDirectory); + } + catch(...) + { + // UNDO modification to To directory + + // Get directory + BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); + + // Delete each one + for(std::vector::iterator i(moving.begin()); i != moving.end(); ++i) + { + to.DeleteEntry((*i)->GetObjectID()); + } + + // Save back + SaveDirectory(to, MoveToDirectory); + + // Throw the error + throw; + } + + // Finally... for all the directories we moved, modify their containing directory ID + for(std::vector::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) + { + // Load the directory + BackupStoreDirectory &change(GetDirectoryInternal(*i)); + + // Modify containing dir ID + change.SetContainerID(MoveToDirectory); + + // Save it back + SaveDirectory(change, *i); + } + } + catch(...) + { + // Make sure directories aren't in the cache, as they may have been modified + RemoveDirectoryFromCache(MoveToDirectory); + RemoveDirectoryFromCache(MoveFromDirectory); + for(std::vector::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) + { + RemoveDirectoryFromCache(*i); + } + + while(!moving.empty()) + { + delete moving.back(); + moving.pop_back(); + } + throw; + } + + // Clean up + while(!moving.empty()) + { + delete moving.back(); + moving.pop_back(); + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreContext::GetBackupStoreInfo() +// Purpose: Return the backup store info object, exception if it isn't loaded +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +const BackupStoreInfo &BackupStoreContext::GetBackupStoreInfo() const +{ + if(mpStoreInfo.get() == 0) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) + } + + return *(mpStoreInfo.get()); +} + + diff --git a/bin/bbstored/BackupStoreContext.h b/bin/bbstored/BackupStoreContext.h new file mode 100644 index 00000000..4cfdb601 --- /dev/null +++ b/bin/bbstored/BackupStoreContext.h @@ -0,0 +1,183 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupStoreContext.h +// Purpose: Context for backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPCONTEXT__H +#define BACKUPCONTEXT__H + +#include +#include +#include + +#include "NamedLock.h" +#include "ProtocolObject.h" +#include "Utils.h" + +class BackupStoreDirectory; +class BackupStoreFilename; +class BackupStoreDaemon; +class BackupStoreInfo; +class IOStream; +class BackupProtocolObject; +class StreamableMemBlock; + +class HousekeepingInterface +{ + public: + virtual ~HousekeepingInterface() { } + virtual void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) = 0; +}; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupStoreContext +// Purpose: Context for backup store server +// Created: 2003/08/20 +// +// -------------------------------------------------------------------------- +class BackupStoreContext +{ +public: + BackupStoreContext(int32_t ClientID, HousekeepingInterface &rDaemon); + ~BackupStoreContext(); +private: + BackupStoreContext(const BackupStoreContext &rToCopy); +public: + + void ReceivedFinishCommand(); + void CleanUp(); + + int32_t GetClientID() {return mClientID;} + + enum + { + Phase_START = 0, + Phase_Version = 0, + Phase_Login = 1, + Phase_Commands = 2 + }; + + int GetPhase() const {return mProtocolPhase;} + void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;} + + // Read only locking + bool SessionIsReadOnly() {return mReadOnly;} + bool AttemptToGetWriteLock(); + + void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;} + bool GetClientHasAccount() const {return mClientHasAccount;} + const std::string &GetStoreRoot() const {return mStoreRoot;} + int GetStoreDiscSet() const {return mStoreDiscSet;} + + // Store info + void LoadStoreInfo(); + void SaveStoreInfo(bool AllowDelay = true); + const BackupStoreInfo &GetBackupStoreInfo() const; + + // Client marker + int64_t GetClientStoreMarker(); + void SetClientStoreMarker(int64_t ClientStoreMarker); + + // Usage information + void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit); + bool HardLimitExceeded(); + + // Reading directories + // -------------------------------------------------------------------------- + // + // Function + // Name: BackupStoreContext::GetDirectory(int64_t) + // Purpose: Return a reference to a directory. Valid only until the + // next time a function which affects directories is called. + // Mainly this funciton, and creation of files. + // Created: 2003/09/02 + // + // -------------------------------------------------------------------------- + const BackupStoreDirectory &GetDirectory(int64_t ObjectID) + { + // External callers aren't allowed to change it -- this function + // merely turns the the returned directory const. + return GetDirectoryInternal(ObjectID); + } + + // Manipulating files/directories + int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions); + int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists); + void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime); + bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut); + bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut); + bool UndeleteFile(int64_t ObjectID, int64_t InDirectory); + void DeleteDirectory(int64_t ObjectID, bool Undelete = false); + void MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject); + + // Manipulating objects + enum + { + ObjectExists_Anything = 0, + ObjectExists_File = 1, + ObjectExists_Directory = 2 + }; + bool ObjectExists(int64_t ObjectID, int MustBe = ObjectExists_Anything); + std::auto_ptr OpenObject(int64_t ObjectID); + + // Info + int32_t GetClientID() const {return mClientID;} + +private: + void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); + BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID); + void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID); + void RemoveDirectoryFromCache(int64_t ObjectID); + void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); + int64_t AllocateObjectID(); + +private: + int32_t mClientID; + HousekeepingInterface &mrDaemon; + int mProtocolPhase; + bool mClientHasAccount; + std::string mStoreRoot; // has final directory separator + int mStoreDiscSet; + bool mReadOnly; + NamedLock mWriteLock; + int mSaveStoreInfoDelay; // how many times to delay saving the store info + + // Store info + std::auto_ptr mpStoreInfo; + + // Directory cache + std::map mDirectoryCache; + +public: + class TestHook + { + public: + virtual std::auto_ptr StartCommand(BackupProtocolObject& + rCommand) = 0; + virtual ~TestHook() { } + }; + void SetTestHook(TestHook& rTestHook) + { + mpTestHook = &rTestHook; + } + std::auto_ptr StartCommandHook(BackupProtocolObject& rCommand) + { + if(mpTestHook) + { + return mpTestHook->StartCommand(rCommand); + } + return std::auto_ptr(); + } + +private: + TestHook* mpTestHook; +}; + +#endif // BACKUPCONTEXT__H + diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/bin/bbstored/BackupStoreDaemon.cpp index 28e28176..4de0a078 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/bin/bbstored/BackupStoreDaemon.cpp @@ -17,7 +17,7 @@ #include #endif -#include "BackupContext.h" +#include "BackupStoreContext.h" #include "BackupStoreDaemon.h" #include "BackupStoreConfigVerify.h" #include "autogen_BackupProtocolServer.h" @@ -43,7 +43,8 @@ BackupStoreDaemon::BackupStoreDaemon() mHaveForkedHousekeeping(false), mIsHousekeepingProcess(false), mHousekeepingInited(false), - mInterProcessComms(mInterProcessCommsSocket) + mInterProcessComms(mInterProcessCommsSocket), + mpTestHook(NULL) { } @@ -171,11 +172,12 @@ void BackupStoreDaemon::Run() const Configuration &config(GetConfiguration()); mExtendedLogging = config.GetKeyValueBool("ExtendedLogging"); -#ifdef WIN32 - // Housekeeping runs synchronously on Win32 -#else - // Fork off housekeeping daemon -- must only do this the first time Run() is called - if(!mHaveForkedHousekeeping) + // Fork off housekeeping daemon -- must only do this the first + // time Run() is called. Housekeeping runs synchronously on Win32 + // because IsSingleProcess() is always true + +#ifndef WIN32 + if(!IsSingleProcess() && !mHaveForkedHousekeeping) { // Open a socket pair for communication int sv[2] = {-1,-1}; @@ -205,7 +207,9 @@ void BackupStoreDaemon::Run() // Log that housekeeping started BOX_INFO("Housekeeping process started"); // Ignore term and hup - // Parent will handle these and alert the child via the socket, don't want to randomly die + // Parent will handle these and alert the + // child via the socket, don't want to + // randomly die! ::signal(SIGHUP, SIG_IGN); ::signal(SIGTERM, SIG_IGN); } @@ -317,10 +321,18 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) } // Make ps listings clearer - SetProcessTitle("client %08x", id); + std::ostringstream tag; + tag << "client=" << BOX_FORMAT_ACCOUNT(id); + SetProcessTitle(tag.str().c_str()); + Logging::Tagger tagWithClientID(tag.str()); // Create a context, using this ID - BackupContext context(id, *this); + BackupStoreContext context(id, *this); + + if (mpTestHook) + { + context.SetTestHook(*mpTestHook); + } // See if the client has an account? if(mpAccounts && mpAccounts->AccountExists(id)) @@ -352,7 +364,7 @@ void BackupStoreDaemon::LogConnectionStats(const char *commonName, const SocketStreamTLS &s) { // Log the amount of data transferred - BOX_INFO("Connection statistics for " << commonName << ":" + BOX_NOTICE("Connection statistics for " << commonName << ":" " IN=" << s.GetBytesRead() << " OUT=" << s.GetBytesWritten() << " TOTAL=" << (s.GetBytesRead() + s.GetBytesWritten())); diff --git a/bin/bbstored/BackupStoreDaemon.h b/bin/bbstored/BackupStoreDaemon.h index 387a1d5b..a5d216f4 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/bin/bbstored/BackupStoreDaemon.h @@ -13,6 +13,7 @@ #include "ServerTLS.h" #include "BoxPortsAndFiles.h" #include "BackupConstants.h" +#include "BackupStoreContext.h" #include "IOStreamGetLine.h" class BackupStoreAccounts; @@ -27,7 +28,8 @@ class HousekeepStoreAccount; // Created: 2003/08/20 // // -------------------------------------------------------------------------- -class BackupStoreDaemon : public ServerTLS +class BackupStoreDaemon : public ServerTLS, + HousekeepingInterface { friend class HousekeepStoreAccount; @@ -38,7 +40,7 @@ private: BackupStoreDaemon(const BackupStoreDaemon &rToCopy); public: - // For BackupContext to communicate with housekeeping process + // For BackupStoreContext to communicate with housekeeping process void SendMessageToHousekeepingProcess(const void *Msg, int MsgLen) { #ifndef WIN32 @@ -81,6 +83,15 @@ private: void HousekeepingInit(); void RunHousekeepingIfNeeded(); int64_t mLastHousekeepingRun; + +public: + void SetTestHook(BackupStoreContext::TestHook& rTestHook) + { + mpTestHook = &rTestHook; + } + +private: + BackupStoreContext::TestHook* mpTestHook; }; diff --git a/bin/bbstored/HousekeepStoreAccount.cpp b/bin/bbstored/HousekeepStoreAccount.cpp index 9f4239e7..dbb9b544 100644 --- a/bin/bbstored/HousekeepStoreAccount.cpp +++ b/bin/bbstored/HousekeepStoreAccount.cpp @@ -85,16 +85,19 @@ void HousekeepStoreAccount::DoHousekeeping() { // Attempt to lock the account std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFilename); + StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, + writeLockFilename); NamedLock writeLock; - if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */)) + if(!writeLock.TryAndGetLock(writeLockFilename.c_str(), + 0600 /* restrictive file permissions */)) { // Couldn't lock the account -- just stop now return; } // Load the store info to find necessary info for the housekeeping - std::auto_ptr info(BackupStoreInfo::Load(mAccountID, mStoreRoot, mStoreDiscSet, false /* Read/Write */)); + std::auto_ptr info(BackupStoreInfo::Load(mAccountID, + mStoreRoot, mStoreDiscSet, false /* Read/Write */)); // Calculate how much should be deleted mDeletionSizeTarget = info->GetBlocksUsed() - info->GetBlocksSoftLimit(); @@ -104,14 +107,18 @@ void HousekeepStoreAccount::DoHousekeeping() } // Scan the directory for potential things to delete - // This will also remove elegiable items marked with RemoveASAP + // This will also remove eligible items marked with RemoveASAP bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID); - // If scan directory stopped for some reason, probably parent instructed to teminate, stop now. + // If scan directory stopped for some reason, probably parent + // instructed to terminate, stop now. if(!continueHousekeeping) { - // If any files were marked "delete now", then update the size of the store. - if(mBlocksUsedDelta != 0 || mBlocksInOldFilesDelta != 0 || mBlocksInDeletedFilesDelta != 0) + // If any files were marked "delete now", then update + // the size of the store. + if(mBlocksUsedDelta != 0 || + mBlocksInOldFilesDelta != 0 || + mBlocksInDeletedFilesDelta != 0) { info->ChangeBlocksUsed(mBlocksUsedDelta); info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); @@ -124,8 +131,8 @@ void HousekeepStoreAccount::DoHousekeeping() return; } - // Log any difference in opinion between the values recorded in the store info, and - // the values just calculated for space usage. + // Log any difference in opinion between the values recorded in + // the store info, and the values just calculated for space usage. // BLOCK { int64_t used = info->GetBlocksUsed(); @@ -133,9 +140,12 @@ void HousekeepStoreAccount::DoHousekeeping() int64_t usedDeleted = info->GetBlocksInDeletedFiles(); int64_t usedDirectories = info->GetBlocksInDirectories(); - // If the counts were wrong, taking into account RemoveASAP items deleted, log a message - if((used + mBlocksUsedDelta) != mBlocksUsed || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles - || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles || usedDirectories != mBlocksInDirectories) + // If the counts were wrong, taking into account RemoveASAP + // items deleted, log a message + if((used + mBlocksUsedDelta) != mBlocksUsed + || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles + || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles + || usedDirectories != mBlocksInDirectories) { // Log this BOX_ERROR("Housekeeping on account " << @@ -153,18 +163,25 @@ void HousekeepStoreAccount::DoHousekeeping() } // If the current values don't match, store them - if(used != mBlocksUsed || usedOld != mBlocksInOldFiles - || usedDeleted != mBlocksInDeletedFiles || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta)) + if(used != mBlocksUsed + || usedOld != mBlocksInOldFiles + || usedDeleted != mBlocksInDeletedFiles + || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta)) { // Set corrected values in store info - info->CorrectAllUsedValues(mBlocksUsed, mBlocksInOldFiles, mBlocksInDeletedFiles, mBlocksInDirectories + mBlocksInDirectoriesDelta); + info->CorrectAllUsedValues(mBlocksUsed, + mBlocksInOldFiles, mBlocksInDeletedFiles, + mBlocksInDirectories + mBlocksInDirectoriesDelta); info->Save(); } } - // Reset the delta counts for files, as they will include RemoveASAP flagged files deleted - // during the initial scan. - int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta; // keep for reporting + // Reset the delta counts for files, as they will include + // RemoveASAP flagged files deleted during the initial scan. + + // keep for reporting + int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta; + mBlocksUsedDelta = 0; mBlocksInOldFilesDelta = 0; mBlocksInDeletedFilesDelta = 0; @@ -172,7 +189,8 @@ void HousekeepStoreAccount::DoHousekeeping() // Go and delete items from the accounts bool deleteInterrupted = DeleteFiles(); - // If that wasn't interrupted, remove any empty directories which are also marked as deleted in their containing directory + // If that wasn't interrupted, remove any empty directories which + // are also marked as deleted in their containing directory if(!deleteInterrupted) { deleteInterrupted = DeleteEmptyDirectories(); @@ -190,8 +208,9 @@ void HousekeepStoreAccount::DoHousekeeping() (deleteInterrupted?" and was interrupted":"")); } - // Make sure the delta's won't cause problems if the counts are really wrong, and - // it wasn't fixed because the store was updated during the scan. + // Make sure the delta's won't cause problems if the counts are + // really wrong, and it wasn't fixed because the store was + // updated during the scan. if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) { mBlocksUsedDelta = (0 - info->GetBlocksUsed()); @@ -218,7 +237,8 @@ void HousekeepStoreAccount::DoHousekeeping() // Save the store info back info->Save(); - // Explicity release the lock (would happen automatically on going out of scope, included for code clarity) + // Explicity release the lock (would happen automatically on + // going out of scope, included for code clarity) writeLock.ReleaseLock(); } @@ -243,8 +263,9 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // // Function // Name: HousekeepStoreAccount::ScanDirectory(int64_t) -// Purpose: Private. Scan a directory for potenitally deleteable items, and -// add them to the list. Returns true if the scan should continue. +// Purpose: Private. Scan a directory for potentially deleteable +// items, and add them to the list. Returns true if the +// scan should continue. // Created: 11/12/03 // // -------------------------------------------------------------------------- @@ -253,9 +274,12 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) #ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) { - mCountUntilNextInterprocessMsgCheck = POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; + mCountUntilNextInterprocessMsgCheck = + POLL_INTERPROCESS_MSG_CHECK_FREQUENCY; + // Check for having to stop - if(mrDaemon.CheckForInterProcessMsg(mAccountID)) // include account ID here as the specified account is locked + // Include account ID here as the specified account is locked + if(mrDaemon.CheckForInterProcessMsg(mAccountID)) { // Need to abort now return false; @@ -268,7 +292,8 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) MakeObjectFilename(ObjectID, objectFilename); // Open it. - std::auto_ptr dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename)); + std::auto_ptr dirStream(RaidFileRead::Open(mStoreDiscSet, + objectFilename)); // Add the size of the directory on disc to the size being calculated int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); @@ -290,8 +315,8 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // BLOCK { - // Remove any files which are marked for removal as soon as they become old - // or deleted. + // Remove any files which are marked for removal as soon + // as they become old or deleted. bool deletedSomething = false; do { @@ -324,7 +349,8 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Add files to the list of potential deletions // map to count the distance from the mark - std::map, int32_t> markVersionAges; + typedef std::pair version_t; + std::map markVersionAges; // map of pair (filename, mark number) -> version age // NOTE: use a reverse iterator to allow the distance from mark stuff to work @@ -342,7 +368,10 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Work out ages of this version from the last mark int32_t enVersionAge = 0; - std::map, int32_t>::iterator enVersionAgeI(markVersionAges.find(std::pair(en->GetName(), en->GetMarkNumber()))); + std::map::iterator enVersionAgeI( + markVersionAges.find( + version_t(en->GetName().GetEncodedFilename(), + en->GetMarkNumber()))); if(enVersionAgeI != markVersionAges.end()) { enVersionAge = enVersionAgeI->second + 1; @@ -350,7 +379,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } else { - markVersionAges[std::pair(en->GetName(), en->GetMarkNumber())] = enVersionAge; + markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge; } // enVersionAge is now the age of this version. @@ -364,6 +393,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) d.mSizeInBlocks = en->GetSizeInBlocks(); d.mMarkNumber = en->GetMarkNumber(); d.mVersionAgeWithinMark = enVersionAge; + d.mIsFlagDeleted = (enFlags & + BackupStoreDirectory::Entry::Flags_Deleted) + ? true : false; // Add it to the list mPotentialDeletions.insert(d); @@ -541,6 +573,10 @@ bool HousekeepStoreAccount::DeleteFiles() // Delete the file DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig); + BOX_INFO("Housekeeping removed " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); // Stop if the deletion target has been matched or exceeded // (checking here rather than at the beginning will tend to reduce the @@ -786,99 +822,115 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() continue; } - // Load up the directory to potentially delete - std::string dirFilename; - BackupStoreDirectory dir; - int64_t dirSizeInBlocks = 0; - { - MakeObjectFilename(*i, dirFilename); - // Check it actually exists (just in case it gets added twice to the list) - if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename)) - { - // doesn't exist, next! - continue; - } - // load - std::auto_ptr dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename)); - dirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); - dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); - } + DeleteEmptyDirectory(*i, toExamine); + } - // Make sure this directory is actually empty - if(dir.GetNumberOfEntries() != 0) - { - // Not actually empty, try next one - continue; - } + // Remove contents of empty directories + mEmptyDirectories.clear(); + // Swap in new, so it's examined next time round + mEmptyDirectories.swap(toExamine); + } + + // Not interrupted + return false; +} - // Candiate for deletion... open containing directory - std::string containingDirFilename; - BackupStoreDirectory containingDir; - int64_t containingDirSizeInBlocksOrig = 0; - { - MakeObjectFilename(dir.GetContainerID(), containingDirFilename); - std::auto_ptr containingDirStream(RaidFileRead::Open(mStoreDiscSet, containingDirFilename)); - containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks(); - containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite); - } +void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, + std::vector& rToExamine) +{ + // Load up the directory to potentially delete + std::string dirFilename; + BackupStoreDirectory dir; + int64_t dirSizeInBlocks = 0; - // Find the entry - BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID()); - if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)) - { - // Should be deleted - containingDir.DeleteEntry(dir.GetObjectID()); + // BLOCK + { + MakeObjectFilename(dirId, dirFilename); + // Check it actually exists (just in case it gets + // added twice to the list) + if(!RaidFileRead::FileExists(mStoreDiscSet, dirFilename)) + { + // doesn't exist, next! + return; + } + // load + std::auto_ptr dirStream( + RaidFileRead::Open(mStoreDiscSet, dirFilename)); + dirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); + dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + } - // Is the containing dir now a candidate for deletion? - if(containingDir.GetNumberOfEntries() == 0) - { - toExamine.push_back(containingDir.GetObjectID()); - } + // Make sure this directory is actually empty + if(dir.GetNumberOfEntries() != 0) + { + // Not actually empty, try next one + return; + } - // Write revised parent directory - RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename); - writeDir.Open(true /* allow overwriting */); - containingDir.WriteToStream(writeDir); + // Candidate for deletion... open containing directory + std::string containingDirFilename; + BackupStoreDirectory containingDir; + int64_t containingDirSizeInBlocksOrig = 0; + { + MakeObjectFilename(dir.GetContainerID(), containingDirFilename); + std::auto_ptr containingDirStream( + RaidFileRead::Open(mStoreDiscSet, + containingDirFilename)); + containingDirSizeInBlocksOrig = + containingDirStream->GetDiscUsageInBlocks(); + containingDir.ReadFromStream(*containingDirStream, + IOStream::TimeOutInfinite); + } - // get the disc usage (must do this before commiting it) - int64_t dirSize = writeDir.GetDiscUsageInBlocks(); + // Find the entry + BackupStoreDirectory::Entry *pdirentry = + containingDir.FindEntryByID(dir.GetObjectID()); + if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)) + { + // Should be deleted + containingDir.DeleteEntry(dir.GetObjectID()); - // Commit directory - writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + // Is the containing dir now a candidate for deletion? + if(containingDir.GetNumberOfEntries() == 0) + { + rToExamine.push_back(containingDir.GetObjectID()); + } - // adjust usage counts for this directory - if(dirSize > 0) - { - int64_t adjust = dirSize - containingDirSizeInBlocksOrig; - mBlocksUsedDelta += adjust; - mBlocksInDirectoriesDelta += adjust; - } + // Write revised parent directory + RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename); + writeDir.Open(true /* allow overwriting */); + containingDir.WriteToStream(writeDir); - // Delete the directory itself - { - RaidFileWrite del(mStoreDiscSet, dirFilename); - del.Delete(); - } + // get the disc usage (must do this before commiting it) + int64_t dirSize = writeDir.GetDiscUsageInBlocks(); - // And adjust usage counts for the directory that's just been deleted - mBlocksUsedDelta -= dirSizeInBlocks; - mBlocksInDirectoriesDelta -= dirSizeInBlocks; + // Commit directory + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - // Update count - ++mEmptyDirectoriesDeleted; - } + // adjust usage counts for this directory + if(dirSize > 0) + { + int64_t adjust = dirSize - containingDirSizeInBlocksOrig; + mBlocksUsedDelta += adjust; + mBlocksInDirectoriesDelta += adjust; } - // Remove contents of empty directories - mEmptyDirectories.clear(); - // Swap in new, so it's examined next time round - mEmptyDirectories.swap(toExamine); - } - - // Not interrupted - return false; -} + // Delete the directory itself + { + RaidFileWrite del(mStoreDiscSet, dirFilename); + del.Delete(); + } + BOX_INFO("Housekeeping removed empty deleted dir " << + BOX_FORMAT_OBJECTID(dirId)); + // And adjust usage counts for the directory that's + // just been deleted + mBlocksUsedDelta -= dirSizeInBlocks; + mBlocksInDirectoriesDelta -= dirSizeInBlocks; + // Update count + ++mEmptyDirectoriesDeleted; + } +} diff --git a/bin/bbstored/HousekeepStoreAccount.h b/bin/bbstored/HousekeepStoreAccount.h index 6c8f251d..5c2a9885 100644 --- a/bin/bbstored/HousekeepStoreAccount.h +++ b/bin/bbstored/HousekeepStoreAccount.h @@ -42,6 +42,8 @@ private: bool ScanDirectory(int64_t ObjectID); bool DeleteFiles(); bool DeleteEmptyDirectories(); + void DeleteEmptyDirectory(int64_t dirId, + std::vector& rToExamine); void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks); private: @@ -52,6 +54,7 @@ private: int64_t mSizeInBlocks; int32_t mMarkNumber; int32_t mVersionAgeWithinMark; // 0 == current, 1 latest old version, etc + bool mIsFlagDeleted; // false for files flagged "Old" } DelEn; struct DelEnCompare diff --git a/bin/bbstored/backupprotocol.txt b/bin/bbstored/backupprotocol.txt index 62d837ff..3eca514a 100644 --- a/bin/bbstored/backupprotocol.txt +++ b/bin/bbstored/backupprotocol.txt @@ -4,7 +4,7 @@ Name Backup IdentString Box-Backup:v=C -ServerContextClass BackupContext BackupContext.h +ServerContextClass BackupStoreContext BackupStoreContext.h ClientType Filename BackupStoreFilenameClear BackupStoreFilenameClear.h ServerType Filename BackupStoreFilename BackupStoreFilename.h @@ -204,6 +204,12 @@ GetBlockIndexByName 35 Command(Success) # stream of the block index follows the reply if found ID != 0 +UndeleteFile 36 Command(Success) + int64 InDirectory + int64 ObjectID + # will return 0 if the object couldn't be found in the specified directory + + # ------------------------------------------------------------------------------------- # Information commands # ------------------------------------------------------------------------------------- diff --git a/bin/bbstored/bbstored-config.in b/bin/bbstored/bbstored-config.in index 33cfb39a..5ad96d50 100755 --- a/bin/bbstored/bbstored-config.in +++ b/bin/bbstored/bbstored-config.in @@ -196,7 +196,7 @@ TimeBetweenHousekeeping = 900 Server { - PidFile = @localstatedir_expanded@/bbstored.pid + PidFile = @localstatedir_expanded@/run/bbstored.pid User = $username ListenAddresses = inet:$server CertificateFile = $certificate @@ -234,7 +234,7 @@ What you need to do now... 4) Create accounts with bbstoreaccounts 5) Start the backup store daemon with the command - @bindir_expanded@/bbstored$daemon_args + @sbindir_expanded@/bbstored$daemon_args in /etc/rc.local, or your local equivalent. =================================================================== diff --git a/bin/bbstored/bbstored.cpp b/bin/bbstored/bbstored.cpp index 54858dd4..21a9e5f1 100644 --- a/bin/bbstored/bbstored.cpp +++ b/bin/bbstored/bbstored.cpp @@ -18,7 +18,7 @@ int main(int argc, const char *argv[]) { MAINHELPER_START - Logging::SetProgramName("Box Backup (bbstored)"); + Logging::SetProgramName("bbstored"); Logging::ToConsole(true); Logging::ToSyslog (true); diff --git a/cleanupforcvs.pl b/cleanupforcvs.pl index 270ea525..3379a7ad 100755 --- a/cleanupforcvs.pl +++ b/cleanupforcvs.pl @@ -10,7 +10,7 @@ my $cleaned = 1; my $dist_archives_exist = 0; my @bad_h; -open EVERYTHING,'find . |' or die "Can't open find for file listing"; +open EVERYTHING,'find . -type d \( -name docs \) -prune -o -type f |' or die "Can't open find for file listing"; my %exclude_from_memtest_checks = ('PollEmulator.cpp'=>1,'DebugMemLeakFinder.cpp'=>1,'MemLeakFinder.h'=>1,'MemLeakFindOn.h'=>1,'MemLeakFindOff.h'=>1,'Box.h'=>1); @@ -178,7 +178,7 @@ sub ask_about_delete { print $_,"\n"; } - print "Delete these ",$#$del_r + 1, " $name?"; + print "Delete these ",$#$del_r + 1, " $name? "; my $in = ; chomp $in; if($in eq 'yes') diff --git a/configure.ac b/configure.ac index 60caf9d5..a6e8c812 100644 --- a/configure.ac +++ b/configure.ac @@ -2,14 +2,10 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.59) -AC_INIT([Box Backup], 0.11, [boxbackup@fluffy.co.uk]) +AC_INIT([Box Backup], 0.11, [boxbackup@boxbackup.org]) AC_CONFIG_SRCDIR([lib/common/Box.h]) AC_CONFIG_HEADERS([lib/common/BoxConfig.h]) -# override default sysconfdir, for backwards compatibility -AC_SUBST([sysconfdir], ['/etc'])dnl -AC_SUBST([localstatedir], ['/var/run'])dnl - touch install-sh AC_CANONICAL_SYSTEM test -s install-sh || rm install-sh @@ -24,11 +20,16 @@ if test "x$ac_cv_cxx_exceptions" != "xyes" || \ test "x$ac_cv_cxx_namespaces" != "xyes"; then AC_MSG_ERROR([[basic compile checks failed, the C++ compiler is broken]]) fi + if test "x$GXX" = "xyes"; then # Use -Wall if we have gcc. This gives better warnings AC_SUBST([CXXFLAGS_STRICT], ['-Wall -Wundef']) - # Use -rdynamic if we have gcc. This is needed for backtrace - AC_SUBST([LDADD_RDYNAMIC], ['-rdynamic']) + + # Use -rdynamic if we have gcc, but not mingw. This is needed for backtrace + case $target_os in + mingw*) ;; + *) AC_SUBST([LDADD_RDYNAMIC], ['-rdynamic']) ;; + esac fi AC_PATH_PROG([PERL], [perl], [AC_MSG_ERROR([[perl executable was not found]])]) @@ -46,11 +47,18 @@ AC_SUBST([TARGET_PERL]) AC_DEFINE_UNQUOTED([PERL_EXECUTABLE], ["$TARGET_PERL"], [Location of the perl executable]) -AC_CHECK_PROGS([AR], [ar], +AC_CHECK_TOOL([AR], [ar], [AC_MSG_ERROR([[cannot find ar executable]])]) -AC_CHECK_PROGS([RANLIB], [ranlib], +AC_CHECK_TOOL([RANLIB], [ranlib], [AC_MSG_ERROR([[cannot find ranlib executable]])]) +case $target_os in +mingw*) + AC_CHECK_TOOL([WINDRES], [windres], + [AC_MSG_ERROR([[cannot find windres executable]])]) + ;; +esac + ### Checks for libraries. case $target_os in @@ -62,6 +70,7 @@ winnt) ;; ;; esac +AC_CHECK_HEADER([zlib.h],, [AC_MSG_ERROR([[cannot find zlib.h]])]) AC_CHECK_LIB([z], [zlibVersion],, [AC_MSG_ERROR([[cannot find zlib]])]) VL_LIB_READLINE([have_libreadline=yes], [have_libreadline=no]) @@ -112,7 +121,7 @@ esac AC_HEADER_STDC AC_HEADER_SYS_WAIT AC_CHECK_HEADERS([dlfcn.h execinfo.h getopt.h process.h pwd.h signal.h]) -AC_CHECK_HEADERS([syslog.h time.h]) +AC_CHECK_HEADERS([syslog.h time.h cxxabi.h]) AC_CHECK_HEADERS([netinet/in.h]) AC_CHECK_HEADERS([sys/param.h sys/socket.h sys/time.h sys/types.h sys/wait.h]) AC_CHECK_HEADERS([sys/uio.h sys/xattr.h]) @@ -126,6 +135,7 @@ else fi if test "$have_pcreposix_h" = "yes"; then + AC_DEFINE([PCRE_STATIC], [1], [Box Backup always uses static PCRE]) AC_SEARCH_LIBS([regcomp], ["pcreposix -lpcre"],,[have_pcreposix_h=no_regcomp]) fi @@ -150,7 +160,6 @@ AC_CHECK_TYPES([uint8_t, uint16_t, uint32_t, uint64_t]) AC_HEADER_STDBOOL AC_C_CONST AC_C_BIGENDIAN -AX_CHECK_NONALIGNED_ACCESS AC_TYPE_UID_T AC_TYPE_MODE_T AC_TYPE_OFF_T @@ -159,6 +168,8 @@ AC_TYPE_SIZE_T AC_CHECK_MEMBERS([struct stat.st_flags]) AC_CHECK_MEMBERS([struct stat.st_mtimespec]) +AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec]) +AC_CHECK_MEMBERS([struct stat.st_atimensec]) AC_CHECK_MEMBERS([struct sockaddr_in.sin_len],,, [[ #include #include @@ -170,6 +181,10 @@ AC_CHECK_DECLS([INFTIM],,, [[#include ]]) AC_CHECK_DECLS([SO_PEERCRED],,, [[#include ]]) AC_CHECK_DECLS([O_BINARY],,,) +# Solaris provides getpeerucred() instead of getpeereid() or SO_PEERCRED +AC_CHECK_HEADERS([ucred.h]) +AC_CHECK_FUNCS([getpeerucred]) + AC_CHECK_DECLS([optreset],,, [[#include ]]) AC_CHECK_DECLS([dirfd],,, [[ @@ -207,13 +222,15 @@ AC_FUNC_CLOSEDIR_VOID AC_FUNC_ERROR_AT_LINE AC_TYPE_SIGNAL AC_FUNC_STAT -AC_CHECK_FUNCS([getpeereid lchown setproctitle getpid gettimeofday]) +AC_CHECK_FUNCS([getpeereid lchown setproctitle getpid gettimeofday waitpid]) + # NetBSD implements kqueue too differently for us to get it fixed by 0.10 # TODO: Remove this when NetBSD kqueue implementation is working netbsd_hack=`echo $target_os | sed 's/netbsd.*/netbsd/'` if test "$netbsd_hack" != "netbsd"; then AC_CHECK_FUNCS([kqueue]) fi + AX_FUNC_SYSCALL AX_CHECK_SYSCALL_LSEEK AC_CHECK_FUNCS([listxattr llistxattr getxattr lgetxattr setxattr lsetxattr]) @@ -224,14 +241,16 @@ AC_CHECK_DECLS([XATTR_NOFOLLOW],,, [[#include ]]) ## Check for large file support active. AC_SYS_LARGEFILE has already worked ## out how to enable it if necessary, we just use this to report to the user -AC_CACHE_CHECK([if we have large file support enabled], [have_large_file_support], - [AC_RUN_IFELSE([AC_LANG_PROGRAM([[$ac_includes_default]], [[ +AC_CACHE_CHECK([if we have large file support enabled], [box_cv_have_large_file_support], + [AC_TRY_RUN([AC_LANG_PROGRAM([[$ac_includes_default]], [[ return sizeof(off_t)==4; ]])], - [have_large_file_support=yes], [have_large_file_support=no] + [box_cv_have_large_file_support=yes], + [box_cv_have_large_file_support=no], + [box_cv_have_large_file_support=no # safe for cross-compile] )]) -if test "x$have_large_file_support" = "xyes"; then +if test "x$box_cv_have_large_file_support" = "xyes"; then AC_DEFINE([HAVE_LARGE_FILE_SUPPORT], [1], [Define to 1 if large files are supported]) fi @@ -271,23 +290,56 @@ if test "x$enable_static_bin" = "xyes"; then LIBS="-Wl,-Bstatic $LIBS -Wl,-Bdynamic" fi +# override default sysconfdir, for backwards compatibility +test "$sysconfdir" = '${prefix}/etc' && sysconfdir=/etc +test "$localstatedir" = '${prefix}/var' && localstatedir=/var + ## Kludge to allow makeparcels.pl to use bindir. This is not a good long term ## solution because it prevents use of "make exec_prefix=/some/dir" saved_prefix=$prefix saved_exec_prefix=$exec_prefix test "x$prefix" = xNONE && prefix=$ac_default_prefix test "x$exec_prefix" = xNONE && exec_prefix='${prefix}' -bindir_expanded=` eval "echo $bindir"` -bindir_expanded=` eval "echo $bindir_expanded"` -sysconfdir_expanded=` eval "echo $sysconfdir"` -localstatedir_expanded=`eval "echo $localstatedir"` +eval bindir_expanded=` eval "echo $bindir"` +eval sbindir_expanded=` eval "echo $sbindir"` +eval sysconfdir_expanded=` eval "echo $sysconfdir"` +eval localstatedir_expanded=`eval "echo $localstatedir"` prefix=$saved_prefix exec_prefix=$saved_exec_prefix -AC_SUBST([bindir_expanded sysconfdir_expanded localstatedir_expanded]) +AC_SUBST([bindir_expanded]) +AC_SUBST([sbindir_expanded]) +AC_SUBST([sysconfdir_expanded]) +AC_SUBST([localstatedir_expanded]) + +## Figure out the client parcel directory and substitute it +build_dir=`dirname $0` +build_dir=`cd $build_dir && pwd` +client_parcel_dir=`$PERL infrastructure/parcelpath.pl backup-client $target_os` + +if test "$build_os" = "cygwin"; then + client_parcel_dir=`cygpath -wa $client_parcel_dir | sed -e 's|\\\|/|g'` + build_dir=` cygpath -wa $build_dir | sed -e 's|\\\|/|g'` +fi + +AC_SUBST([client_parcel_dir]) +AC_SUBST([build_dir]) +## Figure out version and substitute it in +box_version=`$PERL infrastructure/printversion.pl` +AC_SUBST([box_version]) ### Output files -AC_CONFIG_FILES([infrastructure/BoxPlatform.pm]) +AC_CONFIG_FILES([infrastructure/BoxPlatform.pm + contrib/mac_osx/org.boxbackup.bbackupd.plist + contrib/mac_osx/org.boxbackup.bbstored.plist + contrib/solaris/bbackupd-manifest.xml + contrib/solaris/bbstored-manifest.xml + lib/common/BoxPortsAndFiles.h + test/bbackupd/testfiles/bbackupd.conf + test/bbackupd/testfiles/bbackupd-exclude.conf + test/bbackupd/testfiles/bbackupd-snapshot.conf + test/bbackupd/testfiles/bbackupd-symlink.conf + ]) AX_CONFIG_SCRIPTS([bin/bbackupd/bbackupd-config bin/bbackupquery/makedocumentation.pl bin/bbstored/bbstored-certs @@ -298,20 +350,17 @@ AX_CONFIG_SCRIPTS([bin/bbackupd/bbackupd-config contrib/redhat/bbstored contrib/suse/bbackupd contrib/suse/bbstored - contrib/solaris/bbackupd-manifest.xml - contrib/solaris/bbstored-manifest.xml contrib/solaris/bbackupd-smf-method contrib/solaris/bbstored-smf-method + contrib/windows/installer/boxbackup.mpi infrastructure/makebuildenv.pl infrastructure/makeparcels.pl infrastructure/makedistribution.pl - lib/common/BoxPortsAndFiles.h lib/common/makeexception.pl lib/raidfile/raidfile-config lib/server/makeprotocol.pl runtest.pl test/backupstorefix/testfiles/testbackupstorefix.pl - test/bbackupd/testfiles/bbackupd.conf test/bbackupd/testfiles/extcheck1.pl test/bbackupd/testfiles/extcheck2.pl test/bbackupd/testfiles/notifyscript.pl @@ -334,7 +383,7 @@ without these features, but will work better where they are present. Refer to the documentation for more information on each feature. Regular expressions: $have_regex_support -Large files: $have_large_file_support +Large files: $box_cv_have_large_file_support Berkeley DB: $ax_path_bdb_ok Readline: $have_libreadline Extended attributes: $ac_cv_header_sys_xattr_h @@ -343,7 +392,7 @@ EOC ### Warnings at end for visibility -if test "x$gcc_3_plus" != "xyes" && test "x$malloc_workaround" != "xyes"; then +if test "x$box_cv_gcc_3_plus" != "xyes" && test "x$box_cv_malloc_workaround" != "xyes"; then echo AC_MSG_WARN([[the implementation of the C++ STL on this platform may have a flaw which causes it to apparently leak memory, and this flaw cannot be diff --git a/contrib/bbadmin/accounts.cgi b/contrib/bbadmin/accounts.cgi new file mode 100755 index 00000000..d68b82c6 --- /dev/null +++ b/contrib/bbadmin/accounts.cgi @@ -0,0 +1,580 @@ +#!/usr/bin/perl + +# Box Backup web management interface (c) Chris Wilson, 2008 +# +# LICENSE: The Box Backup license applies to this code, with the following +# additional conditions: +# +# If you make any changes to this code, except for changes to existing +# variables in the Configuration section below, you must publish the changes +# under the same license, whether or not you distribute copies of the +# changed version. +# +# If you use any of this code in a derivative work, you must publish the +# source code of the derivative work under the same or compatible license, +# whether or not you distribute copies of the derivative work. +# +# The terms of the Box Backup license may be viewed here: +# https://www.boxbackup.org/license.html +# +# If you require access to the code under a different license, this may +# be negotiated with the copyright holder. + +use strict; +use warnings; + +# Variables which you may need to change to match your installation +# Changes to existing variables are NOT required to be published. + +my $box_dir = "/etc/box"; +my $bbstored_dir = "$box_dir/bbstored"; +my $ca_dir = "/mnt/backup/boxbackup/ca"; + +# You should not need to change these unless you have a non-standard installation + +my $bbstored_conf_file = "$box_dir/bbstored.conf"; +my $bbstoreaccounts = "/usr/local/sbin/bbstoreaccounts"; +my $accounts_db_file = undef; +# my $accounts_db_file = "/etc/box/bbstored/accounts.txt"; +my $raidfile_conf_file = undef; +# my $raidfile_conf_file = "/etc/box/raidfile.conf"; +my $sign_period = '5000'; + +# install Perl module with: +# perl -MCPAN -e 'install Config::Scoped' +# perl -MCPAN -e 'force install P/PT/PTHOMSEN/BoxBackup/BBConfig-0.03.tar.gz' +# perl -MCPAN -e 'install Convert::ASN1' +# download http://search.cpan.org/CPAN/authors/id/L/LE/LEO/Convert-X509-0.1.tar.gz, +# unpack, and move the Convert folder to /usr/lib/perl5/site_perl/X.X.X + +# Check that SSL is being used. +# DO NOT DISABLE THIS unless you really know what you're doing! +die "This script requires an SSL web server" unless $ENV{HTTPS}; + +# Check that the script is protected by basic authentication. +# DO NOT DISABLE THIS unless you really know what you're doing! +die "This script requires HTTP Authentication" unless $ENV{REMOTE_USER}; + +# You should not need to change anything below this line. +# If you do, you must publish your changes to comply with the license. + +use BoxBackup::Config::Accounts; +use BoxBackup::Config::DiskSets; +use CGI::Carp qw(fatalsToBrowser); +use CGI::Pretty; +use Config::Scoped; +use Convert::X509::Request; +use English; +use Fcntl; +use File::Temp; +use URI; +use URI::QueryParam; + +sub check_access($$) +{ + my ($file,$desc) = @_; + unless (-r $file) + { + open FILE, "< $file" and die "should have failed"; + die "Failed to access $desc ($file): $!"; + } +} + +sub check_executable($$) +{ + my ($file,$desc) = @_; + unless (-x $file) + { + open FILE, "< $file" and die "should have failed"; + die "$desc is not executable ($file): $!"; + } +} + + +my $cgi = new CGI; + +if (my $download = $cgi->param("download")) +{ + my ($filename, $acct_no); + + if ($download eq "cert") + { + $acct_no = $cgi->param("account"); + $acct_no =~ tr/0-9a-fA-F//cd; + $filename = "$acct_no-cert.pem"; + } + elsif ($download eq "cacert") + { + $filename = "serverCA.pem"; + } + else + { + die "No such download method $download"; + } + + print $cgi->header(-type => "text/plain", + -"content-disposition" => "attachment; filename=$filename"); + + my $send_file; + + if ($download eq "cert") + { + $send_file = "$ca_dir/clients/$filename"; + } + elsif ($download eq "cacert") + { + $send_file = "$ca_dir/roots/serverCA.pem"; + } + + die "File does not exist: $send_file" + unless -f $send_file; + die "File is not readable by user " . getpwuid($UID) . + ": $send_file" unless -r $send_file; + + open SENDFILE, "< $send_file" or die "Failed to open file " . + "$send_file: $!"; + while (my $line = ) + { + print $line; + } + close SENDFILE; + exit 0; +} + +print $cgi->header(), $cgi->start_html(-title=>"Box Backup Certificates", + -style=>'bb.css'); +print $cgi->h1("Box Backup Certificates"); + +check_access($bbstored_conf_file, "BBStoreD configuration file"); + +my $bbstored_conf = Config::Scoped->new(file => $bbstored_conf_file)->parse(); + +$accounts_db_file ||= $bbstored_conf->{'Server'}{'AccountDatabase'}; +die "Missing AccountDatabase in $bbstored_conf_file" unless $accounts_db_file; +check_access($accounts_db_file, "Accounts Database"); + +$raidfile_conf_file ||= $bbstored_conf->{'Server'}{'RaidFileConf'}; +die "Missing RaidFileConf in $bbstored_conf_file" unless $raidfile_conf_file; +check_access($raidfile_conf_file, "RaidFile configuration file"); + +my $accounts_db = BoxBackup::Config::Accounts->new($accounts_db_file); + +check_executable($bbstoreaccounts, "bbstoreaccounts program"); + +sub error($) +{ + my ($message) = @_; + unless ($message =~ /^p($message); + } + print $cgi->div({-class=>"error"}, $message); + return 0; +} + +sub url +{ + my $cgi = shift @_; + my %params = @_; + my $uri = URI->new($cgi->url(-absolute=>1)); + foreach my $param (keys %params) + { + $uri->query_param($param, $params{$param}); + } + return $uri; +} + +sub create_account($) +{ + my ($cgi) = @_; + + my $upload = $cgi->upload('req'); + unless ($upload) + { + return error("Please attach a certificate request file."); + } + + my $tempfile = File::Temp->new("bbaccount-certreq-XXXXXX.pem"); + my $csr_data = ""; + + while (my $line = <$upload>) + { + print $tempfile $line; + $csr_data .= $line; + } + + my @accounts = $accounts_db->getAccountIDs(); + my $new_acc_no = $cgi->param('account'); + if (not $new_acc_no) + { + return error("Please enter an account number."); + } + + foreach my $account_no (@accounts) + { + if ($account_no == $new_acc_no) + { + return error("The account number $new_acc_no " . + "already exists, please use one which " . + "does not."); + } + } + + my $req = Convert::X509::Request->new($csr_data); + my $cn; + foreach my $part ($req->subject) + { + if ($part =~ /^cn=(.*)/i) + { + $cn = $1; + last; + } + } + + unless ($cn) + { + return error("The certificate request does not include a " . + "common name, which should be BACKUP-$new_acc_no."); + } + + unless ($cn eq "BACKUP-$new_acc_no") + { + return error("The certificate request includes the wrong " . + "common name. Expected " . + "BACKUP-$new_acc_no but found " . + "$cn."); + } + + my $out_cert_dir = "$ca_dir/clients"; + unless (-w $out_cert_dir) + { + return error("Cannot write to certificate directory " . + "$out_cert_dir as user " . + "" . getpwuid($UID) . "."); + } + + my $out_cert = "$out_cert_dir/$new_acc_no-cert.pem"; + if (-f $out_cert and not -w $out_cert) + { + return error("The certificate file $out_cert " . + "exists and is not writable as user " . + "$out_cert_dir as user " . + "" . getpwuid($UID) . "."); + } + + my $client_ca_cert_file = "$ca_dir/roots/clientCA.pem"; + unless (-r $client_ca_cert_file) + { + return error("The client CA certificate file " . + "$client_ca_cert_file " . + "is not readable by user " . + "" . getpwuid($UID) . "."); + } + + my $client_ca_key_file = "$ca_dir/keys/clientRootKey.pem"; + unless (-r $client_ca_key_file) + { + return error("The client CA key file " . + "$client_ca_key_file " . + "is not readable by user " . + "" . getpwuid($UID) . "."); + } + + my $serial_file = "$ca_dir/roots/clientCA.srl"; + unless (-w $serial_file) + { + return error("The certificate serial number file " . + "$serial_file " . + "is not writable by user " . + "" . getpwuid($UID) . "."); + } + + my $outputfile = File::Temp->new("bbaccounts-openssl-output-XXXXXX"); + + if (system("openssl x509 -req -in $tempfile -sha1 " . + "-extensions usr_crt " . + "-CA $client_ca_cert_file " . + "-CAkey $client_ca_key_file " . + "-out $out_cert -days $sign_period " . + ">$outputfile 2>&1") != 0) + { + open ERR, "< $outputfile" or die "$outputfile: $!"; + my $errors = join("", ); + close ERR; + return error($cgi->p("Failed to sign certificate:") . + $cgi->pre($errors)); + } + + my $cert_uri = url($cgi, download => "cert", account => $new_acc_no); + my $ca_uri = url($cgi, download => "cacert"); + + print $cgi->div({-class=>"success"}, + $cgi->p("Account created. Please download the following " . + "files:") . + $cgi->ul( + $cgi->li($cgi->a({href=>$cert_uri}, + "Client Certificate")), + $cgi->li($cgi->a({href=>$ca_uri}, + "CA Certificate")) + ) + ); + + return 1; +} + +if ($cgi->param("create")) +{ + print $cgi->h2("Account Creation"); + create_account($cgi); +} + +print $cgi->h2("Accounts"); +print $cgi->start_table({-border=>0, -class=>"numbers"}); + +print $cgi->Tr( + $cgi->th("Account"), + $cgi->th('Used'), $cgi->th('%'), + $cgi->th('Old files'), $cgi->th('%'), + $cgi->th('Deleted files'), $cgi->th('%'), + $cgi->th('Directories'), $cgi->th('%'), + $cgi->th('Soft limit'), $cgi->th('%'), + $cgi->th('Hard limit'), + $cgi->th('Actions') + ); + +sub human_format($) +{ + my ($kb) = @_; + die "bad format in value: expected number followed by kB, " . + "found '$kb'" unless $kb =~ /^(\d+) (kB)$/; + + my $value = $1; + my $units = $2; + + if ($value > 1024) + { + $value /= 1024; + $units = "MB"; + } + + if ($value > 1024) + { + $value /= 1024; + $units = "GB"; + } + + $value = sprintf("%.1f", $value); + return "$value $units"; +} + +sub bbstoreaccounts_format($) +{ + my ($kb) = @_; + die unless $kb =~ /^(\d+) (kB)$/; + + my $value = $1; + my $units = "K"; + + unless ($value % 1024) + { + $value /= 1024; + $units = "M"; + } + + unless ($value % 1024) + { + $value /= 1024; + $units = "G"; + } + + return "$value$units"; +} + +sub get_account_info($) +{ + my ($account) = @_; + + open BBSA, "$bbstoreaccounts -c $bbstored_conf_file -m info $account |" + or die "Failed to get account info for $account: $!"; + + my $account_info = {}; + + while (my $line = ) + { + unless ($line =~ m/([^:]*): (.*)/) + { + die "Bad format in bbstoreaccounts info output " . + "for account $account: '$line'"; + } + + my $name = $1; + my $value = $2; + + if ($value =~ /(.*), (.*)/) + { + $account_info->{$name} = [$1, $2]; + } + else + { + $account_info->{$name} = $value; + } + } + + return $account_info; +} + +sub format_account_info($) +{ + my ($values) = @_; + my $kb = $values->[0]; + my $pc = $values->[1]; + return $cgi->td(human_format($kb)), $cgi->td($values->[1]); +} + +my %account_numbers; + +my @accounts = $accounts_db->getAccountIDs(); +foreach my $i (@accounts) +{ + die "Duplicate account number $i" if $account_numbers{hex($i)}; + $account_numbers{hex($i)} = 1; + + # Find out what account is on what diskset. + my $disk = $accounts_db->getDisk($i); + + # store limits + my $account_info = get_account_info($i); + + print $cgi->Tr( + $cgi->td($i), + format_account_info($account_info->{'Used'}), + format_account_info($account_info->{'Old files'}), + format_account_info($account_info->{'Deleted files'}), + format_account_info($account_info->{'Directories'}), + format_account_info($account_info->{'Soft limit'}), + $cgi->td(human_format($account_info->{'Hard limit'}[0])), + $cgi->td($cgi->a({-href=>url($cgi, account=>$i)}, + "Edit")) + ); +} + +print $cgi->end_table(); + +my $account_no = $cgi->param("account"); +$account_no =~ tr/0-9a-fA-F//cd; + +if (not $cgi->param("showcreate")) +{ + print $cgi->start_form, + $cgi->submit(-name=>"showcreate", + -value=>"Create Account"), + $cgi->end_form(); +} + +if ($account_no) +{ + print $cgi->h2("Edit Account"); + my $account_info = get_account_info($account_no); + $cgi->param("account", $account_no); + $cgi->param("soft_limit", + bbstoreaccounts_format($account_info->{'Soft limit'}[0])); + $cgi->param("hard_limit", + bbstoreaccounts_format($account_info->{'Hard limit'}[0])); +} +elsif ($cgi->param("showcreate")) +{ + print $cgi->h2("Create Account"); +} + +if ($account_no or $cgi->param("showcreate")) +{ + my $new_account_no = 1; + while ($account_numbers{$new_account_no}) + { + $new_account_no++; + } + + my $disksets_conf = BoxBackup::Config::DiskSets->new($raidfile_conf_file); + my @disk_names = $disksets_conf->getListofDisks(); + my @disk_numbers; + my %disk_labels; + + foreach my $name (@disk_names) + { + my $num = $disksets_conf->getParamVal($name, "SetNumber"); + push @disk_numbers, $num; + $disk_labels{$num} = $name; + } + + print $cgi->start_multipart_form(), + $cgi->start_table(); + + if ($account_no) + { + print $cgi->Tr( + $cgi->th("Account Number"), + $cgi->td($account_no . + $cgi->hidden("account", $account_no)) + ); + } + else + { + print $cgi->Tr( + $cgi->th("Account Number"), + $account_no ? $account_no : + $cgi->td($cgi->textfield(-name => "account", + -default => sprintf("%x", $new_account_no))), + ); + } + + if (not $account_no) + { + print $cgi->Tr( + $cgi->th("Disk Set"), + $cgi->td($cgi->popup_menu(-name => "disk_set", + -values => \@disk_numbers, + -labels => \%disk_labels)) + ); + } + + print $cgi->Tr( + $cgi->th("Soft Limit"), + $cgi->td($cgi->textfield(-name => "soft_limit", + -default => "10G")) + ), + $cgi->Tr( + $cgi->th("Hard Limit"), + $cgi->td($cgi->textfield(-name => "hard_limit", + -default => "20G")) + ), + $cgi->Tr( + $cgi->th("Certificate Request"), + $cgi->td($cgi->filefield({ + -name => "req", + -default => "*.crt" + })) + ); + + if ($account_no) + { + print $cgi->Tr( + $cgi->th(), + $cgi->td($cgi->submit(-name => "update", + -value => "Update Account")) + ); + } + else + { + print $cgi->Tr( + $cgi->th(), + $cgi->td($cgi->submit(-name => "create", + -value => "Create Account")) + ); + } + + print $cgi->end_table(), $cgi->end_form(); +} + +print $cgi->end_html; + +exit 0; diff --git a/contrib/bbadmin/apache.conf b/contrib/bbadmin/apache.conf new file mode 100644 index 00000000..e22668ab --- /dev/null +++ b/contrib/bbadmin/apache.conf @@ -0,0 +1,14 @@ +Alias /bbadmin /var/www/localhost/bbadmin + + + AuthType basic + AuthName "Box Backup Web Management Interface" + AuthUserFile /etc/apache2/bbadmin.cgi.htpasswd + Require valid-user + + Allow from all + + Options ExecCGI + AddHandler cgi-script .cgi + DirectoryIndex accounts.cgi + diff --git a/contrib/bbadmin/bb.css b/contrib/bbadmin/bb.css new file mode 100644 index 00000000..76d48e93 --- /dev/null +++ b/contrib/bbadmin/bb.css @@ -0,0 +1,70 @@ +body +{ + background: #edeef3; +} + +table +{ + border-spacing: 0px; +} + +h1, th +{ + background: #e4e6ed; +} + +h1 +{ + border-top: 1px solid #c4c4d5; + border-bottom: 1px solid #fff; +} + +td, th +{ + border-top: 1px solid #fff; + border-bottom: 1px solid #c4c4d5; + margin: 0px; + padding: 0.2em 0.5em; +} + +th +{ + text-align: left; +} + +table.numbers td +{ + text-align: right; +} + +div.error, div.success +{ + margin: 1em; +} + +div.error>*, div.success>* +{ + margin: 0.5em; +} + +div.error +{ + background: #fdd; + border: 2px solid #c00; +} + +div.success +{ + background: #dfd; + border: 2px solid #0c0; +} + +h2, table, p +{ + margin: 0.5em; +} + +form +{ + margin: 0.5em 0; +} diff --git a/contrib/bbreporter/LICENSE b/contrib/bbreporter/LICENSE new file mode 100644 index 00000000..94a9ed02 --- /dev/null +++ b/contrib/bbreporter/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/contrib/bbreporter/bbreporter.py b/contrib/bbreporter/bbreporter.py new file mode 100755 index 00000000..9c8253f1 --- /dev/null +++ b/contrib/bbreporter/bbreporter.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python +# BoxBackupReporter - Simple script to report on backups that have been +# performed using BoxBackup. +# +# Copyright: (C) 2007 Three A IT Limited +# Author: Kenny Millington +# +# Credit: This script is based on the ideas of BoxReport.pl by Matt Brown of +# Three A IT Support Limited. +# +################################################################################ +# !! Important !! +# To make use of this script you need to run the boxbackup client with the -v +# commandline option and set LogAllFileAccess = yes in your bbackupd.conf file. +# +# Notes on lazy mode: +# If reporting on lazy mode backups you absolutely must ensure that +# logrotate (or similar) rotates the log files at the same rate at +# which you run this reporting script or you will report on the same +# backup sessions on each execution. +# +# Notes on --rotate and log rotation in general: +# The use-case for --rotate that I imagine is that you'll add a line like the +# following into your syslog.conf file:- +# +# local6.* -/var/log/box +# +# Then specifying --rotate to this script will make it rotate the logs +# each time you report on the backup so that you don't risk a backup session +# being spread across two log files (e.g. syslog and syslog.0). +# +# NB: To do this you'll need to prevent logrotate/syslog from rotating your +# /var/log/box file. On Debian based distros you'll need to edit two files. +# +# First: /etc/cron.daily/sysklogd, find the following line and make the +# the required change: +# Change: for LOG in `syslogd-listfiles` +# To: for LOG in `syslogd-listfiles -s box` +# +# Second: /etc/cron.weekly/sysklogd, find the following line and make the +# the required change: +# Change: for LOG in `syslogd-listfiles --weekly` +# To: for LOG in `syslogd-listfiles --weekly -s box` +# +# Alternatively, if suitable just ensure the backups stop before the +# /etc/cron.daily/sysklogd file runs (usually 6:25am) and report on it +# before the files get rotated. (If going for this option I'd just use +# the main syslog file instead of creating a separate log file for box +# backup since you know for a fact the syslog will get rotated daily.) +# +################################################################################ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# If sendmail is not in one of these paths, add the path. +SENDMAIL_PATHS = ["/usr/sbin/", "/usr/bin/", "/bin/" , "/sbin/"] + +# The name of the sendmail binary, you probably won't need to change this. +SENDMAIL_BIN = "sendmail" + +# Number of files to rotate around +ROTATE_COUNT = 7 + +# Import the required libraries +import sys, os, re, getopt, shutil, gzip + +class BoxBackupReporter: + class BoxBackupReporterError(Exception): + pass + + def __init__(self, config_file="/etc/box/bbackupd.conf", + log_file="/var/log/syslog", email_to=None, + email_from="report@boxbackup", rotate=False, + verbose=False, stats=False, sort=False, debug=False): + + # Config options + self.config_file = config_file + self.log_file = log_file + self.email_to = email_to + self.email_from = email_from + self.rotate_log_file = rotate + self.verbose_report = verbose + self.usage_stats = stats + self.sort_files = sort + self.debug = debug + + # Regex's + self.re_automatic_backup = re.compile(" *AutomaticBackup *= *no", re.I) + self.re_syslog = re.compile("(\S+) +(\S+) +([\d:]+) +(\S+) +([^:]+): +"+ + "(?:[A-Z]+:)? *([^:]+): *(.*)") + + # Initialise report + self.reset() + + def _debug(self, msg): + if self.debug: + sys.stderr.write("[bbreporter.py Debug]: %s\n" % msg) + + def reset(self): + # Reset report data to default values + self.hostname = "" + self.patched_files = [] + self.synced_files = [] + self.uploaded_files = [] + self.warnings = [] + self.errors = [] + self.stats = None + self.start_datetime = "Unknown" + self.end_datetime = "Unfinished" + self.report = "No report generated" + + def run(self): + try: + self._determine_operating_mode() + + if self.lazy_mode: + self._debug("Operating in LAZY MODE.") + else: + self._debug("Operating in SNAPSHOT MODE.") + + except IOError: + raise BoxBackupReporter.BoxBackupReporterError("Error: "+\ + "Config file \"%s\" could not be read." % self.config_file) + + try: + self._parse_syslog() + except IOError: + raise BoxBackupReporter.BoxBackupReporterError("Error: "+\ + "Log file \"%s\" could not be read." % self.log_file) + + self._parse_stats() + self._generate_report() + + def deliver(self): + # If we're not e-mailing the report then just dump it to stdout + # and return. + if self.email_to is None: + print self.report + # Now that we've delivered the report it's time to rotate the logs + # if we're requested to do so. + self._rotate_log() + return + + # Locate the sendmail binary + sendmail = self._locate_sendmail() + if(sendmail is None): + raise BoxBackupReporter.BoxBackupReporterError("Error: "+\ + "Could not find sendmail binary - Unable to send e-mail!") + + + # Set the subject based on whether we think we failed or not. + # (suffice it to say I consider getting an error and backing up + # no files a failure or indeed not finding a start time in the logs). + subject = "BoxBackup Reporter (%s) - " % self.hostname + if self.start_datetime == "Unknown" or\ + (len(self.patched_files) == 0 and len(self.synced_files) == 0 and\ + len(self.uploaded_files) == 0): + subject = subject + "FAILED" + else: + subject = subject + "SUCCESS" + + if len(self.errors) > 0: + subject = subject + " (with errors)" + + # Prepare the e-mail message. + mail = [] + mail.append("To: " + self.email_to) + mail.append("From: " + self.email_from) + mail.append("Subject: " + subject) + mail.append("") + mail.append(self.report) + + # Send the mail. + p = os.popen(sendmail + " -t", "w") + p.write("\r\n".join(mail)) + p.close() + + # Now that we've delivered the report it's time to rotate the logs + # if we're requested to do so. + self._rotate_log() + + def _determine_operating_mode(self): + # Scan the config file and determine if we're running in lazy or + # snapshot mode. + cfh = open(self.config_file) + + for line in cfh: + if not line.startswith("#"): + if self.re_automatic_backup.match(line): + self.lazy_mode = False + cfh.close() + return + + self.lazy_mode = True + cfh.close() + + def _parse_syslog(self): + lfh = open(self.log_file) + + patched_files = {} + uploaded_files = {} + synced_files = {} + + for line in lfh: + # Only run the regex if we find a box backup entry. + if line.find("Box Backup") > -1 or line.find("bbackupd") > -1: + raw_data = self.re_syslog.findall(line) + try: + data = raw_data[0] + except IndexError: + # If the regex didn't match it's not a message that we're + # interested in so move to the next line. + continue + + # Set the hostname, it shouldn't change in a log file + self.hostname = data[3] + + # If we find the backup-start event then set the start_datetime. + if data[6].find("backup-start") > -1: + # If we're not in lazy mode or the start_datetime hasn't + # been set then reset the data and set it. + # + # If we're in lazy mode and encounter a second backup-start + # we don't want to change the start_datetime likewise if + # we're not in lazy mode we do want to and we want to reset + # so we only capture the most recent session. + if not self.lazy_mode or self.start_datetime == "Unknown": + self._debug("Reset start dtime with old time: %s." % + self.start_datetime) + + # Reset ourselves + self.reset() + + # Reset our temporary variables which we store + # the files in. + patched_files = {} + uploaded_files = {} + synced_files = {} + + self.start_datetime = data[1]+" "+data[0]+ " "+data[2] + self._debug("Reset start dtime with new time %s." % + self.start_datetime) + + # If we find the backup-finish event then set the end_datetime. + elif data[6].find("backup-finish") > -1: + self.end_datetime = data[1] + " " + data[0] + " " + data[2] + self._debug("Set end dtime: %s" % self.end_datetime) + + # Only log the events if we have our start time. + elif self.start_datetime != "Unknown": + # We found a patch event, add the file to the patched_files. + if data[5] == "Uploading patch to file": + patched_files[data[6]] = "" + + # We found an upload event, add to uploaded files. + elif data[5] == "Uploading complete file": + uploaded_files[data[6]] = "" + + # We found another upload event. + elif data[5] == "Uploaded file": + uploaded_files[data[6]] = "" + + # We found a sync event, add the file to the synced_files. + elif data[5] == "Synchronised file": + synced_files[data[6]] = "" + + # We found a warning, add the warning to the warnings. + elif data[5] == "WARNING": + self.warnings.append(data[6]) + + # We found an error, add the error to the errors. + elif data[5] == "ERROR": + self.errors.append(data[6]) + + + self.patched_files = patched_files.keys() + self.uploaded_files = uploaded_files.keys() + self.synced_files = synced_files.keys() + + # There's no point running the sort functions if we're not going + # to display the resultant lists. + if self.sort_files and self.verbose_report: + self.patched_files.sort() + self.uploaded_files.sort() + + + lfh.close() + + def _parse_stats(self): + if(not self.usage_stats): + return + + # Grab the stats from bbackupquery + sfh = os.popen("bbackupquery usage quit", "r") + raw_stats = sfh.read() + sfh.close() + + # Parse the stats + stats_re = re.compile("commands.[\n ]*\n(.*)\n+", re.S) + stats = stats_re.findall(raw_stats) + + try: + self.stats = stats[0] + except IndexError: + self.stats = "Unable to retrieve usage information." + + def _generate_report(self): + if self.start_datetime == "Unknown": + self.report = "No report data has been found." + return + + total_files = len(self.patched_files) + len(self.uploaded_files) + + report = [] + report.append("--------------------------------------------------") + report.append("Report Title : Box Backup - Backup Statistics") + report.append("Report Period : %s - %s" % (self.start_datetime, + self.end_datetime)) + report.append("--------------------------------------------------") + report.append("") + report.append("This is your box backup report, in summary:") + report.append("") + report.append("%d file(s) have been backed up." % total_files) + report.append("%d file(s) were uploaded." % len(self.uploaded_files)) + report.append("%d file(s) were patched." % len(self.patched_files)) + report.append("%d file(s) were synchronised." % len(self.synced_files)) + + report.append("") + report.append("%d warning(s) occurred." % len(self.warnings)) + report.append("%d error(s) occurred." % len(self.errors)) + report.append("") + report.append("") + + # If we asked for the backup stats and they're available + # show them. + if(self.stats is not None and self.stats != ""): + report.append("Your backup usage information follows:") + report.append("") + report.append(self.stats) + report.append("") + report.append("") + + # List the files if we've been asked for a verbose report. + if(self.verbose_report): + if len(self.uploaded_files) > 0: + report.append("Uploaded Files (%d)" % len(self.uploaded_files)) + report.append("---------------------") + for file in self.uploaded_files: + report.append(file) + report.append("") + report.append("") + + if len(self.patched_files) > 0: + report.append("Patched Files (%d)" % len(self.patched_files)) + report.append("---------------------") + for file in self.patched_files: + report.append(file) + report.append("") + report.append("") + + # Always output the warnings/errors. + if len(self.warnings) > 0: + report.append("Warnings (%d)" % len(self.warnings)) + report.append("---------------------") + for warning in self.warnings: + report.append(warning) + report.append("") + report.append("") + + if len(self.errors) > 0: + report.append("Errors (%d)" % len(self.errors)) + report.append("---------------------") + for error in self.errors: + report.append(error) + report.append("") + report.append("") + + self.report = "\r\n".join(report) + + def _locate_sendmail(self): + for path in SENDMAIL_PATHS: + sendmail = os.path.join(path, SENDMAIL_BIN) + if os.path.isfile(sendmail): + return sendmail + + return None + + def _rotate_log(self): + # If we're not configured to rotate then abort. + if(not self.rotate_log_file): + return + + # So we have these files to possibly account for while we process the + # rotation:- + # self.log_file, self.log_file.0, self.log_file.1.gz, self.log_file.2.gz + # self.log_file.3.gz....self.log_file.(ROTATE_COUNT-1).gz + # + # Algorithm:- + # * Delete last file. + # * Work backwards moving 5->6, 4->5, 3->4, etc... but stop at .0 + # * For .0 move it to .1 then gzip it. + # * Move self.log_file to .0 + # * Done. + + # If it exists, remove the oldest file. + if(os.path.isfile(self.log_file + ".%d.gz" % (ROTATE_COUNT - 1))): + os.unlink(self.log_file + ".%d.gz" % (ROTATE_COUNT - 1)) + + # Copy through the other gzipped log files. + for i in range(ROTATE_COUNT - 1, 1, -1): + src_file = self.log_file + ".%d.gz" % (i - 1) + dst_file = self.log_file + ".%d.gz" % i + + # If the source file exists move/rename it. + if(os.path.isfile(src_file)): + shutil.move(src_file, dst_file) + + # Now we need to handle the .0 -> .1.gz case. + if(os.path.isfile(self.log_file + ".0")): + # Move .0 to .1 + shutil.move(self.log_file + ".0", self.log_file + ".1") + + # gzip the file. + fh = open(self.log_file + ".1", "r") + zfh = gzip.GzipFile(self.log_file + ".1.gz", "w") + zfh.write(fh.read()) + zfh.flush() + zfh.close() + fh.close() + + # If gzip worked remove the original .1 file. + if(os.path.isfile(self.log_file + ".1.gz")): + os.unlink(self.log_file + ".1") + + # Finally move the current logfile to .0 + shutil.move(self.log_file, self.log_file + ".0") + + +def stderr(text): + sys.stderr.write("%s\n" % text) + +def usage(): + stderr("Usage: %s [OPTIONS]\n" % sys.argv[0]) + stderr("Valid Options:-") + stderr(" --logfile=LOGFILE\t\t\tSpecify the logfile to process,\n"+\ + "\t\t\t\t\tdefault: /var/log/syslog\n") + + stderr(" --configfile=CONFIGFILE\t\tSpecify the bbackupd config file,\n "+\ + "\t\t\t\t\tdefault: /etc/box/bbackupd.conf\n") + + stderr(" --email-to=user@example.com\t\tSpecify the e-mail address(es)\n"+\ + "\t\t\t\t\tto send the report to, default is to\n"+\ + "\t\t\t\t\tdisplay the report on the console.\n") + + stderr(" --email-from=user@example.com\t\tSpecify the e-mail address(es)"+\ + "\n\t\t\t\t\tto set the From: address to,\n "+\ + "\t\t\t\t\tdefault: report@boxbackup\n") + + stderr(" --stats\t\t\t\tIncludes the usage stats retrieved from \n"+\ + "\t\t\t\t\t'bbackupquery usage' in the report.\n") + + stderr(" --sort\t\t\t\tSorts the file lists in verbose mode.\n") + + stderr(" --debug\t\t\t\tEnables debug output.\n") + + stderr(" --verbose\t\t\t\tList every file that was backed up to\n"+\ + "\t\t\t\t\tthe server, default is to just display\n"+\ + "\t\t\t\t\tthe summary.\n") + + stderr(" --rotate\t\t\t\tRotates the log files like logrotate\n"+\ + "\t\t\t\t\twould, see the comments for a use-case.\n") + +def main(): + # The defaults + logfile = "/var/log/syslog" + configfile = "/etc/box/bbackupd.conf" + email_to = None + email_from = "report@boxbackup" + rotate = False + verbose = False + stats = False + sort = False + debug = False + # Parse the options + try: + opts, args = getopt.getopt(sys.argv[1:], "dosrvhl:c:t:f:", + ["help", "logfile=", "configfile=","email-to=", + "email-from=","rotate","verbose","stats","sort", + "debug"]) + except getopt.GetoptError: + usage() + return + + for opt, arg in opts: + if(opt in ("--logfile","-l")): + logfile = arg + elif(opt in ("--configfile", "-c")): + configfile = arg + elif(opt in ("--email-to", "-t")): + email_to = arg + elif(opt in ("--email-from", "-f")): + email_from = arg + elif(opt in ("--rotate", "-r")): + rotate = True + elif(opt in ("--verbose", "-v")): + verbose = True + elif(opt in ("--stats", "-s")): + stats = True + elif(opt in ("--sort", "-o")): + sort = True + elif(opt in ("--debug", "-d")): + debug = True + elif(opt in ("--help", "-h")): + usage() + return + + # Run the reporter + bbr = BoxBackupReporter(configfile, logfile, email_to, email_from, + rotate, verbose, stats, sort, debug) + try: + bbr.run() + bbr.deliver() + except BoxBackupReporter.BoxBackupReporterError, error_msg: + print error_msg + +if __name__ == "__main__": + main() diff --git a/contrib/debian/bbackupd.in b/contrib/debian/bbackupd.in index 78d4f4ee..bb7536b5 100644 --- a/contrib/debian/bbackupd.in +++ b/contrib/debian/bbackupd.in @@ -1,46 +1,59 @@ #! /bin/sh # Start and stop the Box Backup client daemon. +# Originally by James Stark, modified by Chris Wilson and James O'Gorman +# For support, visit http://www.boxbackup.org/trac/wiki/MailingLists -BBACKUPD=@bindir_expanded@/bbackupd -CONFIG=@sysconfdir_expanded@/box/bbackupd.conf -PIDFILE=@localstatedir_expanded@/bbackupd.pid +NAME=bbackupd +LONGNAME="Box Backup Client daemon" +BINARY=@sbindir_expanded@/$NAME +CONFIG=@sysconfdir_expanded@/box/$NAME.conf +PIDFILE=@localstatedir_expanded@/bbackupd/$NAME.pid -test -x $BBACKUPD || exit 0 +test -x $BINARY || exit 0 test -f $CONFIG || exit 0 +start_stop() { + start-stop-daemon --quiet --exec $BINARY --pidfile $PIDFILE "$@" +} + +start_stop_verbose() { + if start_stop "$@"; then + echo "." + else + echo " failed!" + exit 1 + fi +} + case $1 in start) - echo -n "Starting Box Backup Client daemon: bbackupd" - start-stop-daemon --start --quiet --exec $BBACKUPD > /dev/null - echo "." + echo -n "Starting $LONGNAME: $NAME" + start_stop_verbose --start ;; - + stop) - echo -n "Stopping Box Backup Client daemon: bbackupd" - start-stop-daemon --stop --quiet \ - --pidfile $PIDFILE --exec $BBACKUPD - echo "." + echo -n "Stopping $LONGNAME: $NAME" + start_stop_verbose --stop ;; reload|force-reload) - echo -n "Reloading Box Backup Client configuration" - start-stop-daemon --stop --signal 1 --quiet --oknodo \ - --pidfile $PIDFILE --exec $BBACKUPD - echo "." + echo -n "Reloading $LONGNAME configuration" + start_stop_verbose --stop --signal 1 ;; - + restart) - echo -n "Restarting Box Backup Client daemon: bbackupd" - start-stop-daemon --stop --quiet --pidfile $PIDFILE \ - --exec $BBACKUPD - start-stop-daemon --start --quiet \ - --exec $BBACKUPD > /dev/null - echo "." + echo -n "Restarting $LONGNAME: $NAME" + if start_stop --stop --retry 5 && start_stop --start; then + echo "." + else + echo " failed!" + exit 1 + fi ;; + *) - echo "Usage: /etc/init.d/bbackupd {start|stop|reload|force-reload|restart}" - exit 1 + echo "Usage: $0 {start|stop|reload|force-reload|restart}" esac exit 0 diff --git a/contrib/debian/bbstored.in b/contrib/debian/bbstored.in index da6a50a0..48fc47f1 100644 --- a/contrib/debian/bbstored.in +++ b/contrib/debian/bbstored.in @@ -1,46 +1,59 @@ #! /bin/sh # Start and stop the Box Backup server daemon. +# Originally by James Stark, modified by Chris Wilson and James O'Gorman +# For support, visit http://www.boxbackup.org/trac/wiki/MailingLists -BBSTORED=@bindir_expanded@/bbstored -CONFIG=@sysconfdir_expanded@/box/bbstored.conf -PIDFILE=@localstatedir_expanded@/bbstored.pid +NAME=bbstored +LONGNAME="Box Backup Server daemon" +BINARY=@sbindir_expanded@/$NAME +CONFIG=@sysconfdir_expanded@/box/$NAME.conf +PIDFILE=@localstatedir_expanded@/run/$NAME.pid -test -x $BBACKUPD || exit 0 +test -x $BINARY || exit 0 test -f $CONFIG || exit 0 +start_stop() { + start-stop-daemon --quiet --exec $BINARY --pidfile $PIDFILE "$@" +} + +start_stop_verbose() { + if start_stop "$@"; then + echo "." + else + echo " failed!" + exit 1 + fi +} + case $1 in start) - echo -n "Starting Box Backup Server daemon: bbstored" - start-stop-daemon --start --quiet --exec $BBSTORED > /dev/null - echo "." + echo -n "Starting $LONGNAME: $NAME" + start_stop_verbose --start ;; stop) - echo -n "Stopping Box Backup Server daemon: bbstored" - start-stop-daemon --stop --quiet \ - --pidfile $PIDFILE --exec $BBSTORED - echo "." + echo -n "Stopping $LONGNAME: $NAME" + start_stop_verbose --stop ;; reload|force-reload) - echo -n "Reloading Box Backup Server configuration" - start-stop-daemon --stop --signal 1 --quiet --oknodo \ - --pidfile $PIDFILE --exec $BBSTORED - echo "." + echo -n "Reloading $LONGNAME configuration" + start_stop_verbose --stop --signal 1 ;; restart) - echo -n "Restarting Box Backup Server daemon: bbstored" - start-stop-daemon --stop --quiet --pidfile $PIDFILE \ - --exec $BBSTORED - start-stop-daemon --start --quiet \ - --exec $BBSTORED > /dev/null - echo "." + echo -n "Restarting $LONGNAME: $NAME" + if start_stop --stop --retry 5 && start_stop --start; then + echo "." + else + echo " failed!" + exit 1 + fi ;; *) - echo "Usage: /etc/init.d/bbstored {start|stop|reload|force-reload|restart}" + echo "Usage: $0 {start|stop|reload|force-reload|restart}" esac exit 0 diff --git a/contrib/mac_osx/org.boxbackup.bbackupd.plist.in b/contrib/mac_osx/org.boxbackup.bbackupd.plist.in new file mode 100644 index 00000000..803deece --- /dev/null +++ b/contrib/mac_osx/org.boxbackup.bbackupd.plist.in @@ -0,0 +1,20 @@ + + + + + Label + org.boxbackup.bbackupd + RunAtLoad + + ProgramArguments + + @prefix@/sbin/bbackupd + -F + @prefix@/etc/boxbackup/bbackupd.conf + + LowPriorityIO + + Nice + 1 + + diff --git a/contrib/mac_osx/org.boxbackup.bbstored.plist.in b/contrib/mac_osx/org.boxbackup.bbstored.plist.in new file mode 100644 index 00000000..bfe8c88c --- /dev/null +++ b/contrib/mac_osx/org.boxbackup.bbstored.plist.in @@ -0,0 +1,21 @@ + + + + + Label + org.boxbackup.bbstored + RunAtLoad + + ProgramArguments + + @prefix@/sbin/bbstored + -F + @prefix@/etc/boxbackup/bbackupd.conf + + + LowPriorityIO + + Nice + 1 + + diff --git a/contrib/redhat/bbackupd.in b/contrib/redhat/bbackupd.in index e8ecdc68..2f51137a 100644 --- a/contrib/redhat/bbackupd.in +++ b/contrib/redhat/bbackupd.in @@ -23,7 +23,7 @@ prog="bbackupd" start() { echo -n $"Starting $prog: " - daemon $prog + daemon @sbindir_expanded@/$prog RETVAL=$? echo [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog @@ -32,7 +32,7 @@ start() { stop() { echo -n $"Stopping $prog: " - killproc $prog + killproc @sbindir_expanded@/$prog RETVAL=$? echo [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog @@ -40,7 +40,7 @@ stop() { } rhstatus() { - status $prog + status @sbindir_expanded@/$prog } restart() { @@ -50,7 +50,7 @@ restart() { reload() { echo -n $"Reloading $prog configuration: " - killproc $prog -HUP + killproc @sbindir_expanded@/$prog -HUP retval=$? echo return $RETVAL diff --git a/contrib/redhat/bbstored.in b/contrib/redhat/bbstored.in index c7675df5..755a8569 100644 --- a/contrib/redhat/bbstored.in +++ b/contrib/redhat/bbstored.in @@ -23,7 +23,7 @@ prog="bbstored" start() { echo -n $"Starting $prog: " - daemon $prog + daemon @sbindir_expanded@/$prog RETVAL=$? echo [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog @@ -32,7 +32,7 @@ start() { stop() { echo -n $"Stopping $prog: " - killproc $prog + killproc @sbindir_expanded@/$prog RETVAL=$? echo [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog @@ -40,7 +40,7 @@ stop() { } rhstatus() { - status $prog + status @sbindir_expanded@/$prog } restart() { @@ -50,7 +50,7 @@ restart() { reload() { echo -n $"Reloading $prog configuration: " - killproc $prog -HUP + killproc @sbindir_expanded@/$prog -HUP retval=$? echo return $RETVAL diff --git a/contrib/rpm/boxbackup.spec b/contrib/rpm/boxbackup.spec index a5b287ff..9c494159 100644 --- a/contrib/rpm/boxbackup.spec +++ b/contrib/rpm/boxbackup.spec @@ -17,7 +17,7 @@ # Detect distribution. So far we only special-case SUSE. If you need to make # any distro specific changes to get the package building on your system -# please email them to boxbackup-dev@fluffy.co.uk +# please email them to boxbackup-dev@boxbackup.org #%define is_fc %(test -e %{_sysconfdir}/fedora-release && echo 1 || echo 0) #%define is_mdk %(test -e %{_sysconfdir}/mandrake-release && echo 1 || echo 0) #%define is_rh %(test -e %{_sysconfdir}/redhat-release && echo 1 || echo 0) @@ -39,8 +39,8 @@ Version: ###DISTRIBUTION-VERSION-NUMBER### Release: 1 License: BSD Group: Applications/Archiving -Packager: Martin Ebourne -URL: http://www.fluffy.co.uk/boxbackup/ +Packager: boxbackup-dev@boxbackup.org +URL: http://www.boxbackup.org/ Source0: %{ident}.tgz Requires: openssl >= 0.9.7a BuildRoot: %{_tmppath}/%{ident}-%{release}-root @@ -96,6 +96,7 @@ make rm -rf $RPM_BUILD_ROOT mkdir -p $RPM_BUILD_ROOT%{_docdir}/%{ident} +mkdir -p $RPM_BUILD_ROOT%{_docdir}/%{ident}/bbreporter mkdir -p $RPM_BUILD_ROOT%{_bindir} mkdir -p $RPM_BUILD_ROOT%{_sbindir} mkdir -p $RPM_BUILD_ROOT%{init_dir} @@ -121,6 +122,11 @@ install -m 644 %{distribution_dir}LINUX.txt \ install -m 644 %{distribution_dir}THANKS.txt \ $RPM_BUILD_ROOT%{_docdir}/%{ident} +install -m 644 contrib/bbreporter/LICENSE \ + $RPM_BUILD_ROOT%{_docdir}/%{ident}/bbreporter +install -m 755 contrib/bbreporter/bbreporter.py \ + $RPM_BUILD_ROOT%{_docdir}/%{ident}/bbreporter + # Client touch $RPM_BUILD_ROOT%{_sysconfdir}/box/bbackupd.conf install -m 755 contrib/%{dist}/bbackupd $RPM_BUILD_ROOT%{init_dir} @@ -143,7 +149,7 @@ ln -s ../../%{init_dir}/bbstored $RPM_BUILD_ROOT%{_sbindir}/rcbbstored %define server_dir parcels/%{ident}-backup-server-linux-gnu install %{server_dir}/bbstored $RPM_BUILD_ROOT%{_sbindir} install %{server_dir}/bbstoreaccounts $RPM_BUILD_ROOT%{_sbindir} -install %{server_dir}/bbstored-certs $RPM_BUILD_ROOT%{_bindir} +install %{server_dir}/bbstored-certs $RPM_BUILD_ROOT%{_sbindir} install %{server_dir}/bbstored-config $RPM_BUILD_ROOT%{_sbindir} install %{server_dir}/raidfile-config $RPM_BUILD_ROOT%{_sbindir} @@ -213,11 +219,15 @@ rm -rf $RPM_BUILD_ROOT %config %ghost %{_sysconfdir}/box/raidfile.conf %{_sbindir}/bbstored %{_sbindir}/bbstoreaccounts -%{_bindir}/bbstored-certs +%{_sbindir}/bbstored-certs %{_sbindir}/bbstored-config %{_sbindir}/raidfile-config +%doc %{_docdir}/%{ident}/bbreporter %changelog +* Thu May 29 2008 Martin Ebourne +- Fix paths to bbreporter files + * Sat Jan 13 2006 Chris Wilson - Support building from an unofficial tarball (from svn) by changing %{distribution_dir} at the top. diff --git a/contrib/solaris/bbackupd-smf-method.in b/contrib/solaris/bbackupd-smf-method.in index 2c839961..0ff610bf 100755 --- a/contrib/solaris/bbackupd-smf-method.in +++ b/contrib/solaris/bbackupd-smf-method.in @@ -5,7 +5,7 @@ case $1 in # SMF arguments (start and restart [really "refresh"]) 'start') - @bindir_expanded@/bbackupd + @sbindir_expanded@/bbackupd ;; 'restart') diff --git a/contrib/solaris/bbstored-smf-method.in b/contrib/solaris/bbstored-smf-method.in index 0ea25e40..dbdb3e69 100755 --- a/contrib/solaris/bbstored-smf-method.in +++ b/contrib/solaris/bbstored-smf-method.in @@ -4,7 +4,7 @@ case $1 in # SMF arguments (start and restart [really "refresh"]) 'start') - @bindir_expanded@/bbstored + @sbindir_expanded@/bbstored ;; 'restart') diff --git a/contrib/suse/bbackupd.in b/contrib/suse/bbackupd.in index d3a5659e..77fd40ba 100644 --- a/contrib/suse/bbackupd.in +++ b/contrib/suse/bbackupd.in @@ -28,7 +28,7 @@ ### END INIT INFO # Check for missing binaries (stale symlinks should not happen) -BBACKUPD_BIN=@bindir_expanded@/bbackupd +BBACKUPD_BIN=@sbindir_expanded@/bbackupd if [ ! -x $BBACKUPD_BIN ] ; then echo "$BBACKUPD_BIN not installed" exit 5 @@ -91,7 +91,7 @@ case "$1" in probe) test @sysconfdir_expanded@/box/bbackupd.conf \ - -nt @localstatedir_expanded@/bbackupd.pid \ + -nt @localstatedir_expanded@/bbackupd/bbackupd.pid \ && echo reload ;; diff --git a/contrib/suse/bbstored.in b/contrib/suse/bbstored.in index e8c74278..e84edfd1 100644 --- a/contrib/suse/bbstored.in +++ b/contrib/suse/bbstored.in @@ -29,7 +29,7 @@ # # Check for missing binaries (stale symlinks should not happen) -BBSTORED_BIN=@bindir_expanded@/bbstored +BBSTORED_BIN=@sbindir_expanded@/bbstored if [ ! -x $BBSTORED_BIN ] ; then echo "$BBSTORED_BIN not installed" exit 5 @@ -92,7 +92,7 @@ case "$1" in probe) test @sysconfdir_expanded@/box/bbstored.conf \ - -nt @localstatedir_expanded@/bbstored.pid && echo reload + -nt @localstatedir_expanded@/run/bbstored.pid && echo reload ;; *) diff --git a/contrib/windows/installer/boxbackup.mpi.in b/contrib/windows/installer/boxbackup.mpi.in new file mode 100755 index 00000000..12f6d62d --- /dev/null +++ b/contrib/windows/installer/boxbackup.mpi.in @@ -0,0 +1,3392 @@ +array set info { +AccountNo +10005005 + +AllowLanguageSelection +No + +AppName +<%BrandName%> + +ApplicationID +E10C6FD9-E524-28BD-B0AB3588F16C + +ApplicationURL +http://www.boxbackup.org/ + +AutoFileGroups +No + +AutoRefreshFiles +Yes + +BBVersionNo +@box_version@ + +BrandName +{Box Backup} + +BuildFailureAction +{Fail (recommended)} + +CancelledInstallAction +{Rollback and Stop} + +CleanupCancelledInstall +Yes + +CommandLineFailureAction +{Fail (recommended)} + +Company +{Tebuco, Inc. and Ben Summers and Contributors} + +CompressionLevel +6 + +CompressionMethod +zlib + +ConfigFileName +{<%InstallDir%>\bbackupd.conf} + +ConfigFileTemplate +{<%InstallDir%>\templates\template.conf} + +Copyright +{2003-2008 Tebuco, Inc. and Ben Summers and Contributors} + +CreateDesktopShortcut +No + +CreateQuickLaunchShortcut +No + +DefaultDirectoryLocation +{} + +DefaultLanguage +English + +EncryptedKeyFilePassword +Enter_EncryptedKeys_Password_Here + +Ext +.exe + +ExtractSolidArchivesOnStartup +No + +Icon +{} + +Image +@build_dir@/docs/html/images/bblogo.png + +IncludeDebugging +Yes + +InstallDirSuffix +<%ShortAppName%> + +InstallPassword +{} + +InstallVersion +@box_version@ + +Language,de +No + +Language,en +Yes + +Language,es +No + +Language,fr +No + +Language,hu +Yes + +Language,it +Yes + +Language,nl +Yes + +Language,pl +No + +Language,pt_br +No + +Language,ru +Yes + +LaunchApplication +No + +PackageDescription +{<%BrandName%> Backup Service} + +PackageLicense +{} + +PackageMaintainer +{Tebuco, Inc. and Ben Summers and Contributors} + +PackageName +<%ShortAppName%> + +PackagePackager +{Tebuco, Inc. and Ben Summers and Contributors} + +PackageRelease +<%PatchVersion%> + +PackageSummary +{} + +PackageVersion +<%MajorVersion%>.<%MinorVersion%> + +PreserveFileAttributes +Yes + +PreserveFilePermissions +Yes + +ProjectID +140B9882-3327-FEA8-13415A62FBB2 + +ProjectVersion +1.2.9.0 + +SaveOnlyToplevelDirs +No + +ScriptExt +.bat + +ServiceExeName +bbackupd.exe + +ServiceName +<%BrandName%> + +ShortAppName +<%BrandName%> + +SkipUnusedFileGroups +Yes + +SystemLanguage +en_us + +Theme +Modern_Wizard + +ThemeDir +Modern_Wizard + +ThemeVersion +1 + +UpgradeApplicationID +{} + +UserInfoAcctNo +<%AccountNo%> + +UserInfoCompany +{} + +UserInfoEmail +{} + +UserInfoName +{} + +UserInfoPhone +{} + +Version +@box_version@ + +ViewReadme +No + +WizardHeight +365 + +WizardWidth +500 + +} + +array set ::InstallJammer::InstallCommandLineOptions { +D +{{} Prefix No No {} {set the value of an option in the installer}} + +S +{InstallMode Switch No No Silent {run the installer in silent mode}} + +T +{Testing Switch Yes No {} {run installer without installing any files}} + +Y +{InstallMode Switch No No Default {accept all defaults and run the installer}} + +debug +{Debugging Switch Yes No {} {run installer in debug mode}} + +debugconsole +{ShowConsole Switch Yes No {} {run installer with a debug console open}} + +mode +{InstallMode Choice No No {Console Default Silent Standard} {set the mode to run the installer in}} + +prefix +{InstallDir String No No {} {set the installation directory}} + +test +{Testing Switch Yes No {} {run installer without installing any files}} + +} +array set ::InstallJammer::UninstallCommandLineOptions { +S +{InstallMode Switch No No Silent {run the uninstaller in silent mode}} + +Y +{InstallMode Switch No No Default {accept all defaults and run the uninstaller}} + +debugconsole +{ShowConsole Switch Yes No {} {run uninstaller with a debug console open}} + +mode +{UninstallMode Choice No No {Console Silent Standard} {set the mode to run the uninstaller in}} + +test +{Testing Switch Yes No {} {run uninstaller without uninstalling any files}} + +} +FileGroup ::481451CC-F49C-D389-8645076F595B -setup Install -active Yes -platforms {Windows MacOS-X} -name {Program Files} -parent FileGroups +File ::B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 -filemethod {Update files with more recent dates} -type dir -directory <%InstallDir%> -name /home/petjal/doc/teb/cli/bu/installer/win/2.2 -location @client_parcel_dir@ -parent 481451CC-F49C-D389-8645076F595B +File ::CDDED10B-2747-DD07-5F9D-42A7FD7BB7E6 -name LICENSE.txt -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::D6E262BC-8A84-B6DB-794B-8FDC8AECB079 -name mgwz.dll -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::E56A0360-7D7F-D99E-E9A4-3C20BC4C2B99 -name mingwm10.dll -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::47602DF7-8463-AB89-E13F-11983610CAA2 -type dir -name tools -location @build_dir@/contrib/windows/installer/tools -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::F7F61231-C340-5CD5-686B-01F521994B0C -name InstallService.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::68DAE474-165D-81FE-1396-FDD2E6081B41 -name KillBackupProcess.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::2436C940-3332-13AA-7613-8EE67C35CE9B -name ReloadConfig.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::336DDAC3-F3BA-1117-73D4-11DFEF9E98AB -name RemoveService.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::0C15AE46-0FF3-3B7F-FC55-D91EF279DBD3 -name RestartService.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::58D97EDE-58F2-15D7-7113-BEE3047F0782 -name StartService.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::BE7CDB16-D3FE-30FA-2153-7C0509CD5E78 -name StopService.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::73BD5859-FB38-71F8-24BD-BDCF871F9FD3 -name Sync.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::67B3838F-4EF7-2C1C-2E86-78DB8ADD6682 -name ShowUsage.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::2646A97C-C0D9-A29C-E145-C5C371F44938 -name QueryOutputAll.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::2F41E3D2-DA4D-2FCB-B3D5-F04032D17A63 -name QueryOutputCurrent.bat -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::C6430BDD-8A07-B80E-FC0C-426C16EB4187 -name RemoteControl.exe -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::5EADAB59-F559-44CB-A3EE-9A56D4EF17C8 -type dir -name .svn -active 0 -parent 47602DF7-8463-AB89-E13F-11983610CAA2 +File ::31429CC4-525E-4E30-9328-4774AFA9F619 -name entries -active 0 -parent 5EADAB59-F559-44CB-A3EE-9A56D4EF17C8 +File ::A27BEFB6-1421-4030-8F11-F04316BCE57C -name format -active 0 -parent 5EADAB59-F559-44CB-A3EE-9A56D4EF17C8 +File ::C99700CE-1035-498C-9A96-B60835652077 -type dir -name prop-base -active 0 -parent 5EADAB59-F559-44CB-A3EE-9A56D4EF17C8 +File ::6FF9DFDE-4BB7-4319-AC85-BF1E88731C1C -name InstallService.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::6AAE8600-5CFC-4240-A47F-5CFCF4E571C6 -name KillBackupProcess.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::834C33D3-4B53-4B2D-9380-A05420AEE75F -name QueryOutputAll.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::854AD23A-5DDE-44C4-900C-34F5845E6747 -name QueryOutputCurrent.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::9A587DE7-B17C-4CDF-B92C-0D267E30E361 -name ReloadConfig.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::197AAEFA-C62E-4E79-890F-C2E4C8440239 -name RemoteControl.exe.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::912E0F50-53DD-4483-A4F4-CA69944A5959 -name RemoveService.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::BDD695BF-9C7E-4F06-BBCE-EC89536DCF27 -name RestartService.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::61615EE4-BF66-40E7-B89A-E6A50B92AF93 -name ShowUsage.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::A6F7C6B7-9759-4B86-9388-4A42E6F7C5C3 -name StartService.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::45D3E7F5-277B-4E52-81BA-ED6D2BB441D7 -name StopService.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::58966972-8387-4D14-A06E-15AA176633A3 -name Sync.bat.svn-base -active 0 -parent C99700CE-1035-498C-9A96-B60835652077 +File ::2A77AF5B-4761-45B5-A543-6328A7F0F39B -type dir -name props -active 0 -parent 5EADAB59-F559-44CB-A3EE-9A56D4EF17C8 +File ::BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 -type dir -name text-base -active 0 -parent 5EADAB59-F559-44CB-A3EE-9A56D4EF17C8 +File ::D972D6B2-40E5-40B3-BC06-66B8B7F51B04 -name InstallService.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::2F14E4F3-5331-4AC5-93F7-C4748970C7F4 -name KillBackupProcess.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::9F3663B2-8BAA-4EAE-B606-53D5C922E703 -name QueryOutputAll.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::BE9BD4FB-DF44-4F4B-BB55-15285A8566BA -name QueryOutputCurrent.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::3B4E0BF4-7FDD-4903-8D43-76C43F38C6C4 -name ReloadConfig.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::038CEEA0-3F21-48F6-B109-BBE1EF7D3E96 -name RemoteControl.exe.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::0C177E04-DF2D-43AB-8A42-9E7A389AB1D1 -name RemoveService.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::9BE16F40-9D1D-4C84-843D-955A44069040 -name RestartService.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::4E503A3C-EB42-4870-9849-D508A3D9BAB7 -name ShowUsage.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::131B7D8B-1BEB-456C-8F05-386C2EAFBEAE -name StartService.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::D3D0AFC1-CB6C-42D4-8C05-21898505DA40 -name StopService.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::C89F78B2-25A7-432B-9D92-DC2AB636CFB5 -name Sync.bat.svn-base -active 0 -parent BF74F2C1-3CE7-4875-9B52-CD0F527E01C7 +File ::90695C82-0000-4F6A-8FE7-0ABDEAA17CAE -type dir -name tmp -active 0 -parent 5EADAB59-F559-44CB-A3EE-9A56D4EF17C8 +File ::7DFF0EE4-7298-4C8C-A5BC-56BBDD81CFC8 -type dir -name prop-base -active 0 -parent 90695C82-0000-4F6A-8FE7-0ABDEAA17CAE +File ::4C60E473-119E-4B0B-9B01-56240F24D9D5 -type dir -name props -active 0 -parent 90695C82-0000-4F6A-8FE7-0ABDEAA17CAE +File ::E1E25ACC-487B-4191-B8CF-9E7C8C88EA09 -type dir -name text-base -active 0 -parent 90695C82-0000-4F6A-8FE7-0ABDEAA17CAE +File ::E7258732-3D21-4E89-AA41-24AA8B8EBF29 -name all-wcprops -active 0 -parent 5EADAB59-F559-44CB-A3EE-9A56D4EF17C8 +File ::9CEBA2A0-C68B-48BA-944E-2E8EE9A35D97 -name bbackupctl.exe -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::497483A6-7264-4361-86F2-F2703F719908 -name bbackupd-config -active 0 -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::204358FC-610F-47DF-8928-1D0E39921700 -targetfilename templates/original.conf -name bbackupd.conf -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::98C376E2-A6C3-4B6C-BBD4-F70CAC2E6A7B -name bbackupd.exe -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::5C3EAB34-7CD4-4DF3-9DEB-0FC23A6F5812 -name bbackupquery.exe -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::7633DBC3-EACA-4F9B-9A87-AD3AF0EC298E -name installer.iss -active 0 -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::D3CF86E1-CAFF-4342-8730-463F96EACC39 -targetfilename templates/NotifySysAdmin.original.vbs -name NotifySysAdmin.vbs -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +File ::A702625F-29C7-4CA0-A8F8-E50DBF5C541B -name uninstall.exe -active 0 -parent B9F58CFC-EE7A-BEE4-62CB-2C10665095A2 +Component ::4A9C852B-647E-EED5-5482FFBCC2AF -setup Install -active Yes -platforms {Windows MacOS-X} -name {Default Component} -parent Components +SetupType ::8202CECC-54A0-9B6C-D24D111BA52E -setup Install -active Yes -platforms {Windows MacOS-X} -name Typical -parent SetupTypes + +InstallComponent AE3BD5B4-35DE-4240-B79914D43E56 -setup Install -type pane -title {Welcome Screen} -component Welcome -active No -parent StandardInstall +InstallComponent 2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8 -setup Install -type pane -conditions 4EE35849-FAD7-170B-0E45-FA30636467B1 -title {Install Password} -component InstallPassword -command insert -active No -parent StandardInstall +Condition 4EE35849-FAD7-170B-0E45-FA30636467B1 -active Yes -parent 2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8 -title {Password Test Condition} -component PasswordTestCondition -TreeObject::id 4EE35849-FAD7-170B-0E45-FA30636467B1 +InstallComponent B3B99E2D-C368-A921-B7BC-A71EBDE3AD4D -setup Install -type action -title {Set Install Password} -component SetInstallPassword -active Yes -parent 2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8 +InstallComponent 1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E -setup Install -type pane -title {User Information} -component UserInformation -active Yes -parent StandardInstall +InstallComponent 9013E862-8E81-5290-64F9-D8BCD13EC7E5 -setup Install -type pane -title {User Information Phone Email} -component UserInformation -active Yes -parent StandardInstall +InstallComponent F8FD4BD6-F1DF-3F8D-B857-98310E4B1143 -setup Install -type pane -title {User Information Account No} -component UserInformation -active Yes -parent StandardInstall +InstallComponent 58E1119F-639E-17C9-5D3898F385AA -setup Install -type pane -conditions 84DA7F05-9FB7-CC36-9EC98F8A6826 -title {Select Destination} -component SelectDestination -command insert -active Yes -parent StandardInstall +Condition 84DA7F05-9FB7-CC36-9EC98F8A6826 -active Yes -parent 58E1119F-639E-17C9-5D3898F385AA -title {File Permission Condition} -component FilePermissionCondition -TreeObject::id 84DA7F05-9FB7-CC36-9EC98F8A6826 +InstallComponent 0FDBA082-90AB-808C-478A-A13E7C525336 -setup Install -type action -title BackupLocationNumber -component ExecuteScript -active Yes -parent 58E1119F-639E-17C9-5D3898F385AA +InstallComponent 0047FF40-0139-2A59-AAC0-A44D46D6F5CC -setup Install -type action -title BackupLocationName -component ExecuteScript -active No -parent 58E1119F-639E-17C9-5D3898F385AA +InstallComponent 2BB06B72-DE53-2319-B1B8-351CDCBA2008 -setup Install -type action -title AddBackupLocation -component ExecuteScript -active Yes -parent 58E1119F-639E-17C9-5D3898F385AA +InstallComponent B506E7DA-E7C4-4D42-8C03-FD27BA16D078 -setup Install -type pane -title {License Agreement} -component License -active Yes -parent StandardInstall +InstallComponent B93D2216-1DDB-484C-A9AC-D6C18ED7DE23 -setup Install -type action -conditions {6D9D1ABC-7146-443F-9EE9-205D5CA6C830 79DAC913-A33D-4ED6-9BAE-B3A2053C0F2C} -title {Modify Widget} -component ModifyWidget -command insert -active Yes -parent B506E7DA-E7C4-4D42-8C03-FD27BA16D078 +Condition 6D9D1ABC-7146-443F-9EE9-205D5CA6C830 -active Yes -parent B93D2216-1DDB-484C-A9AC-D6C18ED7DE23 -title {String Is Condition} -component StringIsCondition -TreeObject::id 6D9D1ABC-7146-443F-9EE9-205D5CA6C830 +Condition 79DAC913-A33D-4ED6-9BAE-B3A2053C0F2C -active Yes -parent B93D2216-1DDB-484C-A9AC-D6C18ED7DE23 -title {String Is Condition} -component StringIsCondition -TreeObject::id 79DAC913-A33D-4ED6-9BAE-B3A2053C0F2C +InstallComponent 37E627F2-E04B-AEF2-D566C017A4D6 -setup Install -type pane -title {Copying Files} -component CopyFiles -active Yes -parent StandardInstall +InstallComponent 3CFFF099-6122-46DD-9CE4-F5819434AC53 -setup Install -type action -title {Stop running service} -component ExecuteExternalProgram -active Yes -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent FB697A88-2842-468E-9776-85E84B009340 -setup Install -type action -title {Remove installed service} -component ExecuteExternalProgram -active No -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent 41CDE776-2667-5CEB-312A-FC4C33A83E7F -setup Install -type action -title {Backup File} -component BackupFile -active Yes -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent 0D93323D-779D-44A8-1E0614E5285D -setup Install -type action -title {Disable Buttons} -component ModifyWidget -active Yes -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent 5CA3EA16-E37C-AABE-E576C4636EB0 -setup Install -type action -title {Execute Action} -component ExecuteAction -active Yes -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent F5F21749-8B3A-49C6-9138-9C4D6D703D26 -setup Install -type action -title {Unpack Keys} -component ExecuteExternalProgram -active No -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent FDF68FD6-BEA8-4A74-867D-5139F4D9E793 -setup Install -type action -title Wait -component Wait -active No -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent E56ADFF4-C15E-AEDB-A599-C468AF72C4BB -setup Install -type action -title {Copy File NotifySysAdmin} -component CopyFile -active Yes -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent D9F88AC1-3D2D-F6DB-871E-3A0E016770B1 -setup Install -type action -title {Copy File config} -component CopyFile -active Yes -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent 5F2C1F1C-B9F7-1642-59D9-A18318C1D70B -setup Install -type action -title {Replace Text In File} -component ReplaceTextInFile -active Yes -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent 2EC82FBD-8294-A3E4-7F39-1CBA0582FA64 -setup Install -type action -title {Write Text To File} -component WriteTextToFile -active Yes -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent 28E76C8B-2605-4739-9FFE-9C2880C17E59 -setup Install -type action -title {Edit config file} -component ExecuteExternalProgram -active No -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent 52F0A238-57E1-A578-2CE4DA177B32 -setup Install -type action -title {Move Forward} -component MoveForward -active Yes -parent 37E627F2-E04B-AEF2-D566C017A4D6 +InstallComponent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 -setup Install -type pane -title SetBackupLocations -component CustomBlankPane2 -command reorder -active Yes -parent StandardInstall +InstallComponent 614C45B2-7515-780C-E444-7F165CF02DD7 -setup Install -type action -title {Execute Script} -component ExecuteScript -active No -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent A5B32DA1-B2FE-C1FA-6057-FBC3059EF076 -setup Install -type action -title {Execute Script} -component ExecuteScript -active Yes -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent F9E38720-6ABA-8B99-2471-496902E4CBC2 -setup Install -type action -title {Execute Script} -component ExecuteScript -active No -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent 362B6D6A-11BC-83CE-AFF6-410D8FBCF54D -setup Install -type action -title {Execute Script} -component ExecuteScript -active No -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent 2E2963BD-DDBD-738D-A910-B7F3F04946F9 -setup Install -type action -title ShowAddAnotherValue -component AddWidget -active Yes -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent 93AA298C-B64E-5683-14D2-7B86F7DEFD2C -setup Install -type action -title BackupLocationName -component ExecuteScript -active No -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent 3FDB57ED-598D-8A4E-CEF7-D90833305558 -setup Install -type action -title {Backup Directory} -component AddWidget -active Yes -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent B927A5AF-4DFE-82A3-DCA8-35FA4D91EC5A -setup Install -type action -title BackupLocationShortName -component AddWidget -active Yes -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent 855DE408-060E-3D35-08B5-1D9AB05C2865 -setup Install -type action -title Exclusions -component AddWidget -active Yes -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent 9892B25C-689B-5B8F-F0C9-B14FF6ACC40C -setup Install -type action -title {Execute Script} -component ExecuteScript -active No -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent 8419AAAD-5860-F73E-8D11-4D1BDA4D7D37 -setup Install -type action -title AddAnother -component AddWidget -active Yes -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent C7762473-273F-E3CA-17E3-65789B14CDB0 -setup Install -type action -title {Write Text To File} -component WriteTextToFile -active Yes -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent D7FBBEBB-2186-5674-BA87-BB7151859D4E -setup Install -type action -title BackupLocationNumber -component ExecuteScript -active Yes -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +InstallComponent 49E80443-62DB-1C10-392D-1091AEA5ED88 -setup Install -type action -conditions EB532611-5F30-3C24-66EB-F3826D9054FD -title {Move to Pane} -component MoveToPane -command insert -active Yes -parent 3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 +Condition EB532611-5F30-3C24-66EB-F3826D9054FD -active Yes -parent 49E80443-62DB-1C10-392D-1091AEA5ED88 -title {String Is Condition} -component StringIsCondition -TreeObject::id EB532611-5F30-3C24-66EB-F3826D9054FD +InstallComponent 9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266 -setup Install -type pane -title {Click Next to Continue} -component CustomBlankPane2 -active Yes -parent StandardInstall +InstallComponent DDBBD8A9-13D7-9509-9202-419E989F60A9 -setup Install -type action -title {Add Widget} -component AddWidget -active No -parent 9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266 +InstallComponent 8E095096-F018-A880-429D-A2177A9B70EA -setup Install -type action -title {Add Widget} -component AddWidget -active No -parent 9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266 +InstallComponent 88A50FD5-480F-19A5-DA74-C915EB0A9765 -setup Install -type action -conditions 5EE78EF7-37CA-D440-3DB5-09136CD566B3 -title {Move to Pane} -component MoveToPane -command insert -active No -parent 9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266 +Condition 5EE78EF7-37CA-D440-3DB5-09136CD566B3 -active Yes -parent 88A50FD5-480F-19A5-DA74-C915EB0A9765 -title {String Is Condition} -component StringIsCondition -TreeObject::id 5EE78EF7-37CA-D440-3DB5-09136CD566B3 +InstallComponent 908CE221-5A3D-0A78-24A1-E7C91EBE38D4 -setup Install -type pane -title {Next-Build Config} -component CustomBlankPane2 -active No -parent StandardInstall +InstallComponent DA33B826-E633-A845-4646-76DFA78B907B -setup Install -type pane -title {Custom Blank Pane 2} -component CustomBlankPane2 -active Yes -parent StandardInstall +InstallComponent 6FEE2889-0338-1D49-60BF-1471F465AB26 -setup Install -type action -title {Write Text To File} -component WriteTextToFile -active Yes -parent DA33B826-E633-A845-4646-76DFA78B907B +InstallComponent 73DD4D07-B1DC-BA38-2B12-07EB24A7F0C8 -setup Install -type action -title {Copy File} -component CopyFile -active Yes -parent DA33B826-E633-A845-4646-76DFA78B907B +InstallComponent D23DD94C-E517-7F34-FD59-802CB18AB887 -setup Install -type action -title {Adjust Line Feeds} -component AdjustLineFeeds -active Yes -parent DA33B826-E633-A845-4646-76DFA78B907B +InstallComponent 7D8E1902-2BC4-80D8-2C18771E7C22 -setup Install -type action -title {Installing service} -component ExecuteExternalProgram -active Yes -parent DA33B826-E633-A845-4646-76DFA78B907B +InstallComponent 1C14291C-0971-4283-92E9-3808401303F5 -setup Install -type action -title {Starting service} -component ExecuteExternalProgram -active No -parent DA33B826-E633-A845-4646-76DFA78B907B +InstallComponent 6C323815-B9AB-FA94-4F5D152EBC51 -setup Install -type pane -title {Setup Complete} -component SetupComplete -active Yes -parent StandardInstall +InstallComponent 574198A7-7322-2F5E-02EF185D965C -setup Install -type pane -title {Copying Files} -component CopyFiles -active Yes -parent DefaultInstall +InstallComponent 8A761DBD-0640-D98C-9B3AD7672A8F -setup Install -type action -title {Disable Buttons} -component ModifyWidget -active Yes -parent 574198A7-7322-2F5E-02EF185D965C +InstallComponent 6E70FB1F-6A43-6C23-3242E965A0D0 -setup Install -type action -title {Execute Action} -component ExecuteAction -active Yes -parent 574198A7-7322-2F5E-02EF185D965C +InstallComponent 8E1A5944-5AF5-5906-16D395E386D8 -setup Install -type action -title {Move Forward} -component MoveForward -active Yes -parent 574198A7-7322-2F5E-02EF185D965C +InstallComponent 1F0926EE-6884-1330-B4A1DB11C1BF -setup Install -type pane -title {Setup Complete} -component SetupComplete -active Yes -parent DefaultInstall +InstallComponent 3B6E2E7C-1A26-27F1-D578E383B128 -setup Install -type action -conditions {13BD88FE-CD71-5AC7-E99C10B6CB28 E02368C5-95B5-03A7-3282740037B0} -title {View Readme Checkbutton} -component AddWidget -command insert -active Yes -parent 1F0926EE-6884-1330-B4A1DB11C1BF +Condition 13BD88FE-CD71-5AC7-E99C10B6CB28 -active Yes -parent 3B6E2E7C-1A26-27F1-D578E383B128 -title {File Exists Condition} -component FileExistsCondition -TreeObject::id 13BD88FE-CD71-5AC7-E99C10B6CB28 +Condition E02368C5-95B5-03A7-3282740037B0 -active Yes -parent 3B6E2E7C-1A26-27F1-D578E383B128 -title {String Is Condition} -component StringIsCondition -TreeObject::id E02368C5-95B5-03A7-3282740037B0 +InstallComponent CFFA27AF-A641-E41C-B4A0E3BB3CBB -setup Install -type action -conditions {592F46AE-8CEE-01F3-0BA7EBDCA4F4 793D8178-0F51-7F07-BC5886586D3C} -title {Launch Application Checkbutton} -component AddWidget -command insert -active Yes -parent 1F0926EE-6884-1330-B4A1DB11C1BF +Condition 592F46AE-8CEE-01F3-0BA7EBDCA4F4 -active Yes -parent CFFA27AF-A641-E41C-B4A0E3BB3CBB -title {File Exists Condition} -component FileExistsCondition -TreeObject::id 592F46AE-8CEE-01F3-0BA7EBDCA4F4 +Condition 793D8178-0F51-7F07-BC5886586D3C -active Yes -parent CFFA27AF-A641-E41C-B4A0E3BB3CBB -title {String Is Condition} -component StringIsCondition -TreeObject::id 793D8178-0F51-7F07-BC5886586D3C +InstallComponent 16D53E40-546B-54C3-088B1B5E3BBB -setup Install -type action -conditions {4E643D8A-CA31-018D-57D7053C2CE8 B39C0455-D1B6-7DDC-E2717F83463E} -title {Desktop Shortcut Checkbutton} -component AddWidget -command insert -active Yes -parent 1F0926EE-6884-1330-B4A1DB11C1BF +Condition 4E643D8A-CA31-018D-57D7053C2CE8 -active Yes -parent 16D53E40-546B-54C3-088B1B5E3BBB -title {File Exists Condition} -component FileExistsCondition -TreeObject::id 4E643D8A-CA31-018D-57D7053C2CE8 +Condition B39C0455-D1B6-7DDC-E2717F83463E -active Yes -parent 16D53E40-546B-54C3-088B1B5E3BBB -title {String Is Condition} -component StringIsCondition -TreeObject::id B39C0455-D1B6-7DDC-E2717F83463E +InstallComponent 937C3FDD-FB28-98BD-3DAB276E59ED -setup Install -type action -conditions {6B966959-05D9-DB32-8D9C4AD2A3DF 748D673B-DFE6-5F74-329903ACE4DB 3379F80B-36D6-73DC-6FC1D6223A26} -title {Quick Launch Shortcut Checkbutton} -component AddWidget -command insert -active Yes -parent 1F0926EE-6884-1330-B4A1DB11C1BF +Condition 6B966959-05D9-DB32-8D9C4AD2A3DF -active Yes -parent 937C3FDD-FB28-98BD-3DAB276E59ED -title {Platform Condition} -component PlatformCondition -TreeObject::id 6B966959-05D9-DB32-8D9C4AD2A3DF +Condition 748D673B-DFE6-5F74-329903ACE4DB -active Yes -parent 937C3FDD-FB28-98BD-3DAB276E59ED -title {File Exists Condition} -component FileExistsCondition -TreeObject::id 748D673B-DFE6-5F74-329903ACE4DB +Condition 3379F80B-36D6-73DC-6FC1D6223A26 -active Yes -parent 937C3FDD-FB28-98BD-3DAB276E59ED -title {String Is Condition} -component StringIsCondition -TreeObject::id 3379F80B-36D6-73DC-6FC1D6223A26 +InstallComponent 3FE82C17-A3E2-4A57-A563-F80818B00B81 -setup Install -type action -title {Console Ask Yes Or No} -component ConsoleAskYesOrNo -active Yes -parent ConsoleInstall +InstallComponent 56EE5149-6AA2-4E0C-8841-F66A2EF9276E -setup Install -type action -conditions 241BBFCE-4EB1-432F-94DD-69D444DDB6C0 -title Exit -component Exit -command insert -active Yes -parent ConsoleInstall +Condition 241BBFCE-4EB1-432F-94DD-69D444DDB6C0 -active Yes -parent 56EE5149-6AA2-4E0C-8841-F66A2EF9276E -title {String Is Condition} -component StringIsCondition -TreeObject::id 241BBFCE-4EB1-432F-94DD-69D444DDB6C0 +InstallComponent 0C12D2D3-AEBC-42FE-A73A-0815EFB10DA5 -setup Install -type action -conditions BC4EA5FD-50BD-4D6E-953F-5E3EDB957360 -title {Console Get User Input} -component ConsoleGetUserInput -command insert -active Yes -parent ConsoleInstall +Condition BC4EA5FD-50BD-4D6E-953F-5E3EDB957360 -active Yes -parent 0C12D2D3-AEBC-42FE-A73A-0815EFB10DA5 -title {File Permission Condition} -component FilePermissionCondition -TreeObject::id BC4EA5FD-50BD-4D6E-953F-5E3EDB957360 +InstallComponent B002A311-F8E7-41DE-B039-521391924E5B -setup Install -type action -title {Console Message} -component ConsoleMessage -active Yes -parent ConsoleInstall +InstallComponent D4FC6EB5-DDEE-4E4A-B8E1-D4B588A7928B -setup Install -type action -title {Execute Action} -component ExecuteAction -active Yes -parent ConsoleInstall +InstallComponent 2BF07B5A-9B06-4C1E-810D-5B5E9303D2C6 -setup Install -type action -title {Console Message} -component ConsoleMessage -active Yes -parent ConsoleInstall +InstallComponent 6B4CB3C2-4799-4C9F-BA8E-1EE47C4606E1 -setup Install -type action -title Exit -component Exit -active Yes -parent ConsoleInstall +InstallComponent D8F0AA0F-AD79-C566-15CC508F503B -setup Install -type action -title {Execute Action} -component ExecuteAction -active Yes -parent SilentInstall +InstallComponent 175CBE81-9EBE-1E21-A91479BEEFAE -setup Install -type action -title Exit -component Exit -active Yes -parent SilentInstall +InstallComponent A1DD1DC2-85D7-9BC6-998AC3D4A3A9 -setup Install -type actiongroup -title {Startup Actions} -active Yes -parent ActionGroupsInstall +InstallComponent 1F9E8CB8-02C1-0416-1F7445B4147F -setup Install -type action -conditions {3D0D1898-4C65-3E66-F82F56581E87 32F5B0AF-EB83-7A03-D8FAE1ECE473} -title Exit -component Exit -command insert -active Yes -parent A1DD1DC2-85D7-9BC6-998AC3D4A3A9 +Condition 3D0D1898-4C65-3E66-F82F56581E87 -active Yes -parent 1F9E8CB8-02C1-0416-1F7445B4147F -title {String Is Condition} -component StringIsCondition -TreeObject::id 3D0D1898-4C65-3E66-F82F56581E87 +Condition 32F5B0AF-EB83-7A03-D8FAE1ECE473 -active Yes -parent 1F9E8CB8-02C1-0416-1F7445B4147F -title {Ask Yes or No} -component AskYesOrNo -TreeObject::id 32F5B0AF-EB83-7A03-D8FAE1ECE473 +InstallComponent 32DC8FB1-A04B-71AA-EC18496D4BD0 -setup Install -type action -title {Create Install Panes} -component CreateInstallPanes -active Yes -parent A1DD1DC2-85D7-9BC6-998AC3D4A3A9 +InstallComponent 198905FB-9FAC-23DE-7422D25B8ECA -setup Install -type actiongroup -title {Install Actions} -active Yes -parent ActionGroupsInstall +InstallComponent 4D4A7BF0-7CCE-46E6-BDE5222F82D7 -setup Install -type action -title {Install Selected Files} -component InstallSelectedFiles -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent 53588803-6B41-D9FC-A385906A5106 -setup Install -type action -title {Install Uninstaller} -component InstallUninstaller -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent 73EA65C1-3BE3-B190-55C3E99F6269 -setup Install -type action -conditions 4EF787E3-0643-DE46-15E64BAF0816 -title {Windows Uninstall Registry} -component AddWindowsUninstallEntry -command insert -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +Condition 4EF787E3-0643-DE46-15E64BAF0816 -active Yes -parent 73EA65C1-3BE3-B190-55C3E99F6269 -title {Platform Condition} -component PlatformCondition -TreeObject::id 4EF787E3-0643-DE46-15E64BAF0816 +InstallComponent 39B2B666-78D8-75E6-6EA071594D34 -setup Install -type action -conditions 18C00430-D6B1-151F-307762B3A045 -title {Uninstall Shortcut} -component InstallWindowsShortcut -command insert -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +Condition 18C00430-D6B1-151F-307762B3A045 -active Yes -parent 39B2B666-78D8-75E6-6EA071594D34 -title {Platform Condition} -component PlatformCondition -TreeObject::id 18C00430-D6B1-151F-307762B3A045 +InstallComponent 6652193C-5D4B-44B6-ABC6-D6E96D89E5DC -setup Install -type action -title {Install Program Folder Shortcut} -component InstallProgramFolderShortcut -active No -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent 9D101299-B80C-441B-8685-6E3AC61808E8 -setup Install -type action -title {RemoteControl Shortcut} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent B01CBBB2-6A78-CA53-9ED9-C3C4CFC9239E -setup Install -type action -title {stopservice Shortcut} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent DE800F1C-CB1A-E1CE-AEB8-B0A6DB4818E7 -setup Install -type action -title {Install Backup Service} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent 25AA533E-02FC-47D9-9273-25266B8FA1F9 -setup Install -type action -title {Remove Backup Service} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent CDD84DE3-C970-458F-9162-1A3CE0AA716B -setup Install -type action -title {startservice Shortcut} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent B5DFEC63-92A9-4686-909E-0CE78A7069D6 -setup Install -type action -title {restartservice Shortcut} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent C0452595-F3EB-43AD-BCA2-661437584636 -setup Install -type action -title {editconfig Shortcut} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent 1AF5CD58-65C0-49CB-9A9D-994816CF414E -setup Install -type action -title {QueryUpload Shortcut} -component InstallProgramFolderShortcut -active No -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent 1681CF85-A5D2-4D73-A3FC-52B2A6A1847D -setup Install -type action -title {killbackupprocess Shortcut} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent D8B8A9BF-5F2E-4236-A63E-5A8C5FFA8968 -setup Install -type action -title {reloadconfig Shortcut} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent 6F61CDA8-30C9-454F-82A3-9987E1203079 -setup Install -type action -title {sync Shortcut} -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent 9A663209-495B-ED16-09BE-457B61148022 -setup Install -type action -title QueryCurrent -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent C0AF7C05-A31A-8376-BCB9-BA8B3A666252 -setup Install -type action -title SafeQueryAll -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent 32B08FB1-99DF-234E-8BAF-333E80AAC9F5 -setup Install -type action -title Usage -component InstallProgramFolderShortcut -active Yes -parent 198905FB-9FAC-23DE-7422D25B8ECA +InstallComponent FEFD090D-C133-BC95-B3564F693CD3 -setup Install -type actiongroup -title {Finish Actions} -active Yes -parent ActionGroupsInstall +InstallComponent DECC120D-6904-7F17-45A49184A5A3 -setup Install -type action -conditions {E44CFF46-6302-C518-B9C30D2E43F7 B0AA6839-AAB6-A602-C0E4ECA2E4FF} -title {Install Desktop Shortcut} -component InstallDesktopShortcut -command insert -active No -parent FEFD090D-C133-BC95-B3564F693CD3 +Condition E44CFF46-6302-C518-B9C30D2E43F7 -active Yes -parent DECC120D-6904-7F17-45A49184A5A3 -title {String Is Condition} -component StringIsCondition -TreeObject::id E44CFF46-6302-C518-B9C30D2E43F7 +Condition B0AA6839-AAB6-A602-C0E4ECA2E4FF -active Yes -parent DECC120D-6904-7F17-45A49184A5A3 -title {File Exists Condition} -component FileExistsCondition -TreeObject::id B0AA6839-AAB6-A602-C0E4ECA2E4FF +InstallComponent 7B770A07-A785-5215-956FA82CF14E -setup Install -type action -conditions {6F94698F-0839-3ABF-0CF2DF05A4C8 738DD098-7E3B-BC89-875CDB93CBE2 8C866252-8760-9B08-FE569C25B60D} -title {Install Quick Launch Shortcut} -component InstallWindowsShortcut -command insert -active No -parent FEFD090D-C133-BC95-B3564F693CD3 +Condition 6F94698F-0839-3ABF-0CF2DF05A4C8 -active Yes -parent 7B770A07-A785-5215-956FA82CF14E -title {String Is Condition} -component StringIsCondition -TreeObject::id 6F94698F-0839-3ABF-0CF2DF05A4C8 +Condition 738DD098-7E3B-BC89-875CDB93CBE2 -active Yes -parent 7B770A07-A785-5215-956FA82CF14E -title {Platform Condition} -component PlatformCondition -TreeObject::id 738DD098-7E3B-BC89-875CDB93CBE2 +Condition 8C866252-8760-9B08-FE569C25B60D -active Yes -parent 7B770A07-A785-5215-956FA82CF14E -title {File Exists Condition} -component FileExistsCondition -TreeObject::id 8C866252-8760-9B08-FE569C25B60D +InstallComponent C105AAAE-7C16-2C9E-769FE4535B60 -setup Install -type action -conditions {2583A547-11DE-1C27-B6D04B023CC0 A6E1B027-A1B4-5848-4F868D028D00 0357FAE9-FCFD-26D8-6541D810CD61} -title {View Readme Window} -component TextWindow -command insert -active No -parent FEFD090D-C133-BC95-B3564F693CD3 +Condition 2583A547-11DE-1C27-B6D04B023CC0 -active Yes -parent C105AAAE-7C16-2C9E-769FE4535B60 -title {String Is Condition} -component StringIsCondition -TreeObject::id 2583A547-11DE-1C27-B6D04B023CC0 +Condition A6E1B027-A1B4-5848-4F868D028D00 -active Yes -parent C105AAAE-7C16-2C9E-769FE4535B60 -title {String Is Condition} -component StringIsCondition -TreeObject::id A6E1B027-A1B4-5848-4F868D028D00 +Condition 0357FAE9-FCFD-26D8-6541D810CD61 -active Yes -parent C105AAAE-7C16-2C9E-769FE4535B60 -title {File Exists Condition} -component FileExistsCondition -TreeObject::id 0357FAE9-FCFD-26D8-6541D810CD61 +InstallComponent C33D74B2-26FA-16F5-433A10C6A747 -setup Install -type action -conditions {CC4337CC-F3B5-757C-DFCF5D1D365A 795EE61F-6C0D-4A8B-93E02AA3894A 1528F4F0-145C-A48D-A8526DBB6289} -title {Launch Application} -component ExecuteExternalProgram -command insert -active No -parent FEFD090D-C133-BC95-B3564F693CD3 +Condition CC4337CC-F3B5-757C-DFCF5D1D365A -active Yes -parent C33D74B2-26FA-16F5-433A10C6A747 -title {String Is Condition} -component StringIsCondition -TreeObject::id CC4337CC-F3B5-757C-DFCF5D1D365A +Condition 795EE61F-6C0D-4A8B-93E02AA3894A -active Yes -parent C33D74B2-26FA-16F5-433A10C6A747 -title {String Is Condition} -component StringIsCondition -TreeObject::id 795EE61F-6C0D-4A8B-93E02AA3894A +Condition 1528F4F0-145C-A48D-A8526DBB6289 -active Yes -parent C33D74B2-26FA-16F5-433A10C6A747 -title {File Exists Condition} -component FileExistsCondition -TreeObject::id 1528F4F0-145C-A48D-A8526DBB6289 +InstallComponent E23AC50D-7CFB-800E-A99C6F4068F8 -setup Install -type actiongroup -title {Cancel Actions} -active Yes -parent ActionGroupsInstall +InstallComponent 3B8CDC8E-1239-D2E9-DF4CA6B1756D -setup Uninstall -type pane -title Uninstall -component Uninstall -active Yes -parent StandardUninstall +InstallComponent 19ADBDDB-1690-4A57-913E32A026C4 -setup Uninstall -type action -title {Modify Widget} -component ModifyWidget -active Yes -parent 3B8CDC8E-1239-D2E9-DF4CA6B1756D +InstallComponent 7A983CD8-302C-4942-BE59-525C5B5FA2F2 -setup Uninstall -type action -title {Stop Backup Process} -component ExecuteExternalProgram -active Yes -parent 3B8CDC8E-1239-D2E9-DF4CA6B1756D +InstallComponent E4DEA723-FC78-45D7-BAB1-A3E4C4C96EA1 -setup Uninstall -type action -title {Stop Service} -component ExecuteExternalProgram -active Yes -parent 3B8CDC8E-1239-D2E9-DF4CA6B1756D +InstallComponent B4D31D1E-ADB1-DE8F-18EB7294DDA8 -setup Uninstall -type action -title {Remove Service} -component ExecuteExternalProgram -active Yes -parent 3B8CDC8E-1239-D2E9-DF4CA6B1756D +InstallComponent D55BA4AF-E73B-60D1-E26F79175227 -setup Uninstall -type action -title {Execute Action} -component ExecuteAction -active Yes -parent 3B8CDC8E-1239-D2E9-DF4CA6B1756D +InstallComponent 69FD7409-5E2A-143B-DABD1C3B1E67 -setup Uninstall -type action -conditions {96A68CAC-9ED7-806C-086B104720FD E161F216-E597-B340-C1A71C476E2C} -title {Uninstall Leftover Files} -component UninstallLeftoverFiles -command insert -active Yes -parent 3B8CDC8E-1239-D2E9-DF4CA6B1756D +Condition 96A68CAC-9ED7-806C-086B104720FD -active Yes -parent 69FD7409-5E2A-143B-DABD1C3B1E67 -title {String Is Condition} -component StringIsCondition -TreeObject::id 96A68CAC-9ED7-806C-086B104720FD +Condition E161F216-E597-B340-C1A71C476E2C -active Yes -parent 69FD7409-5E2A-143B-DABD1C3B1E67 -title {Ask Yes or No} -component AskYesOrNo -TreeObject::id E161F216-E597-B340-C1A71C476E2C +InstallComponent 05060263-E852-87AB-8D0F2954CAA6 -setup Uninstall -type action -title {Move Forward} -component MoveForward -active Yes -parent 3B8CDC8E-1239-D2E9-DF4CA6B1756D +InstallComponent 41D3E165-C263-5F80-0FEEC0AEE47A -setup Uninstall -type pane -conditions EB2B31A1-C111-3582-0C8A5656692A -title {Uninstall Details} -component UninstallDetails -command insert -active Yes -parent StandardUninstall +Condition EB2B31A1-C111-3582-0C8A5656692A -active Yes -parent 41D3E165-C263-5F80-0FEEC0AEE47A -title {String Is Condition} -component StringIsCondition -TreeObject::id EB2B31A1-C111-3582-0C8A5656692A +InstallComponent 3D33AA8C-0037-204B-39A339FD38BD -setup Uninstall -type pane -title {Uninstall Complete} -component UninstallComplete -active Yes -parent StandardUninstall +InstallComponent 49E59F91-27F7-46D1-A1C1-19865C2392D3 -setup Uninstall -type action -title {Console Ask Yes Or No} -component ConsoleAskYesOrNo -active Yes -parent ConsoleUninstall +InstallComponent ADA6EB2F-8820-4366-BBEF-ED1335B7F828 -setup Uninstall -type action -conditions 87DE6D78-81E1-495B-A214-B3FF3E7E5614 -title Exit -component Exit -command insert -active Yes -parent ConsoleUninstall +Condition 87DE6D78-81E1-495B-A214-B3FF3E7E5614 -active Yes -parent ADA6EB2F-8820-4366-BBEF-ED1335B7F828 -title {String Is Condition} -component StringIsCondition -TreeObject::id 87DE6D78-81E1-495B-A214-B3FF3E7E5614 +InstallComponent B4ED4636-22D8-41DC-9E3D-BD1E1CAD2174 -setup Uninstall -type action -title {Console Message} -component ConsoleMessage -active Yes -parent ConsoleUninstall +InstallComponent 3C7130B3-3206-403D-B09E-59D4A758FBAD -setup Uninstall -type action -title {Execute Action} -component ExecuteAction -active Yes -parent ConsoleUninstall +InstallComponent 20CBDBEA-2217-457B-8D98-D692C4F591E9 -setup Uninstall -type action -title {Console Message} -component ConsoleMessage -active Yes -parent ConsoleUninstall +InstallComponent 7F85263E-CAE2-46BA-AAC0-6B89D20FD2DE -setup Uninstall -type action -title Exit -component Exit -active Yes -parent ConsoleUninstall +InstallComponent 17D8BA8E-5992-AA5C-F5ECB73A3433 -setup Uninstall -type action -title {Execute Action} -component ExecuteAction -active Yes -parent SilentUninstall +InstallComponent D3D73C76-D9D3-07DA-63D4163A44BE -setup Uninstall -type action -title Exit -component Exit -active Yes -parent SilentUninstall +InstallComponent 848844B5-6103-9343-8B731B0BE4E0 -setup Uninstall -type actiongroup -title {Startup Actions} -active Yes -parent ActionGroupsUninstall +InstallComponent 97ACF525-C075-8635-E019202A83D8 -setup Uninstall -type action -conditions {DFFF91A9-2CA5-6ABE-8474D814AF88 4ACB0B47-42B3-2B3A-BFE9AA4EC707} -title Exit -component Exit -command insert -active Yes -parent 848844B5-6103-9343-8B731B0BE4E0 +Condition DFFF91A9-2CA5-6ABE-8474D814AF88 -active Yes -parent 97ACF525-C075-8635-E019202A83D8 -title {String Is Condition} -component StringIsCondition -TreeObject::id DFFF91A9-2CA5-6ABE-8474D814AF88 +Condition 4ACB0B47-42B3-2B3A-BFE9AA4EC707 -active Yes -parent 97ACF525-C075-8635-E019202A83D8 -title {Ask Yes or No} -component AskYesOrNo -TreeObject::id 4ACB0B47-42B3-2B3A-BFE9AA4EC707 +InstallComponent F4024A3E-9A6D-2726-5E0CFFA93054 -setup Uninstall -type actiongroup -title {Uninstall Actions} -active Yes -parent ActionGroupsUninstall +InstallComponent 39D7394E-04E9-CA70-0034DB830BFE -setup Uninstall -type action -title {Uninstall Selected Files} -component UninstallSelectedFiles -active Yes -parent F4024A3E-9A6D-2726-5E0CFFA93054 +InstallComponent 39270FD8-932E-6132-7EF795ED9B93 -setup Uninstall -type actiongroup -title {Finish Actions} -active Yes -parent ActionGroupsUninstall +InstallComponent 905DA2E9-988C-2F27-BB1F5F274AC9 -setup Uninstall -type actiongroup -title {Cancel Actions} -active Yes -parent ActionGroupsUninstall + +array set Properties { +0047FF40-0139-2A59-AAC0-A44D46D6F5CC,Active +No + +0047FF40-0139-2A59-AAC0-A44D46D6F5CC,Comment +{set BackupLocationName "BackupLocation_${BackupLocationNumber}"} + +0047FF40-0139-2A59-AAC0-A44D46D6F5CC,Conditions +{0 conditions} + +0047FF40-0139-2A59-AAC0-A44D46D6F5CC,ExecuteAction +{Before Next Pane is Displayed} + +0047FF40-0139-2A59-AAC0-A44D46D6F5CC,ResultVirtualText +BackupLocationName + +0047FF40-0139-2A59-AAC0-A44D46D6F5CC,TclScript +{set BackupLocationName "BackupLocation_${BackupLocationNumber}"} + +0357FAE9-FCFD-26D8-6541D810CD61,CheckCondition +{Before Action is Executed} + +0357FAE9-FCFD-26D8-6541D810CD61,Filename +<%ProgramReadme%> + +05060263-E852-87AB-8D0F2954CAA6,Conditions +{0 conditions} + +0C12D2D3-AEBC-42FE-A73A-0815EFB10DA5,Prompt +<%ConsoleSelectDestinationText%> + +0C12D2D3-AEBC-42FE-A73A-0815EFB10DA5,VirtualText +InstallDir + +0D93323D-779D-44A8-1E0614E5285D,Conditions +{0 conditions} + +0D93323D-779D-44A8-1E0614E5285D,State +disabled + +0D93323D-779D-44A8-1E0614E5285D,Widget +{Back Button;Next Button} + +0FDBA082-90AB-808C-478A-A13E7C525336,Conditions +{0 conditions} + +0FDBA082-90AB-808C-478A-A13E7C525336,ExecuteAction +{Before Next Pane is Displayed} + +0FDBA082-90AB-808C-478A-A13E7C525336,ResultVirtualText +BackupLocationNumber + +0FDBA082-90AB-808C-478A-A13E7C525336,TclScript +{set BackupLocationNumber 1} + +13BD88FE-CD71-5AC7-E99C10B6CB28,CheckCondition +{Before Action is Executed} + +13BD88FE-CD71-5AC7-E99C10B6CB28,Filename +<%ProgramReadme%> + +1528F4F0-145C-A48D-A8526DBB6289,CheckCondition +{Before Action is Executed} + +1528F4F0-145C-A48D-A8526DBB6289,Filename +<%ProgramExecutable%> + +1681CF85-A5D2-4D73-A3FC-52B2A6A1847D,Alias +{Stop Backup Windows Process} + +1681CF85-A5D2-4D73-A3FC-52B2A6A1847D,Conditions +{0 conditions} + +1681CF85-A5D2-4D73-A3FC-52B2A6A1847D,FileName +<%ShortAppName%>-program-killbackupprocess + +1681CF85-A5D2-4D73-A3FC-52B2A6A1847D,ShortcutName +{Stop backup process} + +1681CF85-A5D2-4D73-A3FC-52B2A6A1847D,TargetFileName +<%InstallDir%>/tools/KillBackupProcess.bat + +1681CF85-A5D2-4D73-A3FC-52B2A6A1847D,WorkingDirectory +<%InstallDir%> + +16D53E40-546B-54C3-088B1B5E3BBB,Background +white + +16D53E40-546B-54C3-088B1B5E3BBB,Conditions +{2 conditions} + +16D53E40-546B-54C3-088B1B5E3BBB,Text,subst +1 + +16D53E40-546B-54C3-088B1B5E3BBB,Type +checkbutton + +16D53E40-546B-54C3-088B1B5E3BBB,VirtualText +CreateDesktopShortcut + +16D53E40-546B-54C3-088B1B5E3BBB,X +185 + +16D53E40-546B-54C3-088B1B5E3BBB,Y +180 + +175CBE81-9EBE-1E21-A91479BEEFAE,ExitType +Finish + +17D8BA8E-5992-AA5C-F5ECB73A3433,Action +{Uninstall Actions} + +17D8BA8E-5992-AA5C-F5ECB73A3433,Conditions +{0 conditions} + +18C00430-D6B1-151F-307762B3A045,CheckCondition +{Before Action is Executed} + +18C00430-D6B1-151F-307762B3A045,Platform +Windows + +198905FB-9FAC-23DE-7422D25B8ECA,Alias +{Install Actions} + +198905FB-9FAC-23DE-7422D25B8ECA,Conditions +{0 conditions} + +19ADBDDB-1690-4A57-913E32A026C4,Conditions +{0 conditions} + +19ADBDDB-1690-4A57-913E32A026C4,State +disabled + +19ADBDDB-1690-4A57-913E32A026C4,Widget +{NextButton; CancelButton} + +1AF5CD58-65C0-49CB-9A9D-994816CF414E,Active +No + +1AF5CD58-65C0-49CB-9A9D-994816CF414E,Alias +{Upload File Listing} + +1AF5CD58-65C0-49CB-9A9D-994816CF414E,Comment +{Upload list of backed up files for Tech Support purposes. May be blocked by firewalls.} + +1AF5CD58-65C0-49CB-9A9D-994816CF414E,Conditions +{0 conditions} + +1AF5CD58-65C0-49CB-9A9D-994816CF414E,FileName +<%ShortAppName%>-program-TebucoSafeQuerypload + +1AF5CD58-65C0-49CB-9A9D-994816CF414E,ShortcutName +{Upload Filelisting to TebucoSafe for review} + +1AF5CD58-65C0-49CB-9A9D-994816CF414E,TargetFileName +<%InstallDir%>/tools/TebucoSafeQueryUpload.bat + +1AF5CD58-65C0-49CB-9A9D-994816CF414E,WorkingDirectory +<%InstallDir%> + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,Active +Yes + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,BackButton,subst +1 + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,CancelButton,subst +1 + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,Caption,subst +1 + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,CompanyLabel,subst +0 + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,Conditions +{0 conditions} + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,Message,subst +1 + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,NextButton,subst +1 + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,Subtitle,subst +1 + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,Title,subst +1 + +1BEFB82C-C073-73D4-CFCE-F5DE7A674D9E,UserNameLabel,subst +0 + +1C14291C-0971-4283-92E9-3808401303F5,Active +No + +1C14291C-0971-4283-92E9-3808401303F5,Comment +{Don't start it yet, need to install keys by hand.} + +1C14291C-0971-4283-92E9-3808401303F5,Conditions +{0 conditions} + +1C14291C-0971-4283-92E9-3808401303F5,ProgramCommandLine +{net start <%ServiceName%>} + +1C14291C-0971-4283-92E9-3808401303F5,WorkingDirectory +<%InstallDir%> + +1F0926EE-6884-1330-B4A1DB11C1BF,BackButton,subst +1 + +1F0926EE-6884-1330-B4A1DB11C1BF,CancelButton,subst +1 + +1F0926EE-6884-1330-B4A1DB11C1BF,Caption,subst +1 + +1F0926EE-6884-1330-B4A1DB11C1BF,Message,subst +1 + +1F0926EE-6884-1330-B4A1DB11C1BF,NextButton,subst +1 + +1F9E8CB8-02C1-0416-1F7445B4147F,Comment +{Ask the user if they want to proceed with the install.} + +1F9E8CB8-02C1-0416-1F7445B4147F,Conditions +{2 conditions} + +20CBDBEA-2217-457B-8D98-D692C4F591E9,Message,subst +1 + +241BBFCE-4EB1-432F-94DD-69D444DDB6C0,CheckCondition +{Before Action is Executed} + +241BBFCE-4EB1-432F-94DD-69D444DDB6C0,Operator +false + +241BBFCE-4EB1-432F-94DD-69D444DDB6C0,String +<%Answer%> + +2583A547-11DE-1C27-B6D04B023CC0,CheckCondition +{Before Action is Executed} + +2583A547-11DE-1C27-B6D04B023CC0,Operator +false + +2583A547-11DE-1C27-B6D04B023CC0,String +<%SilentMode%> + +25AA533E-02FC-47D9-9273-25266B8FA1F9,Alias +{Remove Backup Service} + +25AA533E-02FC-47D9-9273-25266B8FA1F9,Comment +{Remove the Backup Windows Service} + +25AA533E-02FC-47D9-9273-25266B8FA1F9,Conditions +{0 conditions} + +25AA533E-02FC-47D9-9273-25266B8FA1F9,FileName +<%ShortAppName%>-program-removeService + +25AA533E-02FC-47D9-9273-25266B8FA1F9,ShortcutName +{Remove Service} + +25AA533E-02FC-47D9-9273-25266B8FA1F9,TargetFileName +<%InstallDir%>/tools/RemoveService.bat + +25AA533E-02FC-47D9-9273-25266B8FA1F9,WorkingDirectory +<%InstallDir%> + +28E76C8B-2605-4739-9FFE-9C2880C17E59,Active +No + +28E76C8B-2605-4739-9FFE-9C2880C17E59,Conditions +{0 conditions} + +28E76C8B-2605-4739-9FFE-9C2880C17E59,ProgramCommandLine +{notepad <%ConfigFileName%>} + +28E76C8B-2605-4739-9FFE-9C2880C17E59,WorkingDirectory +<%InstallDir%> + +2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8,BackButton,subst +1 + +2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8,CancelButton,subst +1 + +2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8,Caption,subst +1 + +2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8,Conditions +{1 condition} + +2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8,Message,subst +1 + +2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8,NextButton,subst +1 + +2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8,Subtitle,subst +1 + +2AC89879-6E9D-3D4E-F28E-5985EEBFAAA8,Title,subst +1 + +2BB06B72-DE53-2319-B1B8-351CDCBA2008,Conditions +{0 conditions} + +2BB06B72-DE53-2319-B1B8-351CDCBA2008,ExecuteAction +{Before Next Pane is Displayed} + +2BB06B72-DE53-2319-B1B8-351CDCBA2008,ResultVirtualText +AddBackupLocation + +2BB06B72-DE53-2319-B1B8-351CDCBA2008,TclScript +{set AddBackupLocation no} + +2BF07B5A-9B06-4C1E-810D-5B5E9303D2C6,Message,subst +1 + +2E2963BD-DDBD-738D-A910-B7F3F04946F9,Conditions +{0 conditions} + +2E2963BD-DDBD-738D-A910-B7F3F04946F9,Text,subst +1 + +2E2963BD-DDBD-738D-A910-B7F3F04946F9,Value +<%AddBackupLocation%> + +2E2963BD-DDBD-738D-A910-B7F3F04946F9,X +400 + +2E2963BD-DDBD-738D-A910-B7F3F04946F9,Y +70 + +2EC82FBD-8294-A3E4-7F39-1CBA0582FA64,AppendNewline +No + +2EC82FBD-8294-A3E4-7F39-1CBA0582FA64,Comment +{.conf doesn't exist yet} + +2EC82FBD-8294-A3E4-7F39-1CBA0582FA64,Conditions +{0 conditions} + +2EC82FBD-8294-A3E4-7F39-1CBA0582FA64,FileOpenAction +{Append to file} + +2EC82FBD-8294-A3E4-7F39-1CBA0582FA64,Files +<%ConfigFileTemplate%> + +2EC82FBD-8294-A3E4-7F39-1CBA0582FA64,TextToWrite,subst +1 + +32B08FB1-99DF-234E-8BAF-333E80AAC9F5,Conditions +{0 conditions} + +32B08FB1-99DF-234E-8BAF-333E80AAC9F5,FileName +<%ShortAppName%>-program-Usage + +32B08FB1-99DF-234E-8BAF-333E80AAC9F5,ShortcutName +Usage + +32B08FB1-99DF-234E-8BAF-333E80AAC9F5,TargetFileName +<%InstallDir%>/tools/ShowUsage.bat + +32B08FB1-99DF-234E-8BAF-333E80AAC9F5,WorkingDirectory +<%InstallDir%> + +32DC8FB1-A04B-71AA-EC18496D4BD0,Conditions +{0 conditions} + +32F5B0AF-EB83-7A03-D8FAE1ECE473,CheckCondition +{Before Action is Executed} + +32F5B0AF-EB83-7A03-D8FAE1ECE473,Message,subst +1 + +32F5B0AF-EB83-7A03-D8FAE1ECE473,Title,subst +1 + +32F5B0AF-EB83-7A03-D8FAE1ECE473,TrueValue +No + +3379F80B-36D6-73DC-6FC1D6223A26,CheckCondition +{Before Action is Executed} + +3379F80B-36D6-73DC-6FC1D6223A26,Operator +false + +3379F80B-36D6-73DC-6FC1D6223A26,String +<%InstallStopped%> + +362B6D6A-11BC-83CE-AFF6-410D8FBCF54D,Active +No + +362B6D6A-11BC-83CE-AFF6-410D8FBCF54D,Conditions +{0 conditions} + +362B6D6A-11BC-83CE-AFF6-410D8FBCF54D,ResultVirtualText +BackupLocationExclusions + +362B6D6A-11BC-83CE-AFF6-410D8FBCF54D,TclScript +{set BackupLocationExclusions ""} + +37E627F2-E04B-AEF2-D566C017A4D6,BackButton,subst +1 + +37E627F2-E04B-AEF2-D566C017A4D6,CancelButton,subst +1 + +37E627F2-E04B-AEF2-D566C017A4D6,Caption,subst +1 + +37E627F2-E04B-AEF2-D566C017A4D6,Conditions +{0 conditions} + +37E627F2-E04B-AEF2-D566C017A4D6,FileLabel,subst +1 + +37E627F2-E04B-AEF2-D566C017A4D6,Message,subst +1 + +37E627F2-E04B-AEF2-D566C017A4D6,NextButton,subst +1 + +37E627F2-E04B-AEF2-D566C017A4D6,ProgressValue,subst +1 + +37E627F2-E04B-AEF2-D566C017A4D6,Subtitle,subst +1 + +37E627F2-E04B-AEF2-D566C017A4D6,Title,subst +1 + +39270FD8-932E-6132-7EF795ED9B93,Alias +{Finish Actions} + +39270FD8-932E-6132-7EF795ED9B93,Conditions +{0 conditions} + +39B2B666-78D8-75E6-6EA071594D34,Conditions +{1 condition} + +39B2B666-78D8-75E6-6EA071594D34,ShortcutName +{Uninstall <%BrandName%>} + +39B2B666-78D8-75E6-6EA071594D34,TargetFileName +<%Uninstaller%> + +39B2B666-78D8-75E6-6EA071594D34,WorkingDirectory +<%InstallDir%> + +39D7394E-04E9-CA70-0034DB830BFE,Conditions +{0 conditions} + +3B6E2E7C-1A26-27F1-D578E383B128,Background +white + +3B6E2E7C-1A26-27F1-D578E383B128,Conditions +{2 conditions} + +3B6E2E7C-1A26-27F1-D578E383B128,Text,subst +1 + +3B6E2E7C-1A26-27F1-D578E383B128,Type +checkbutton + +3B6E2E7C-1A26-27F1-D578E383B128,VirtualText +ViewReadme + +3B6E2E7C-1A26-27F1-D578E383B128,X +185 + +3B6E2E7C-1A26-27F1-D578E383B128,Y +140 + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,BackButton,subst +1 + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,CancelButton,subst +1 + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,Caption,subst +1 + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,Conditions +{0 conditions} + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,FileValue,subst +1 + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,Message,subst +1 + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,NextButton,subst +1 + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,ProgressValue,subst +1 + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,Subtitle,subst +1 + +3B8CDC8E-1239-D2E9-DF4CA6B1756D,Title,subst +1 + +3C7130B3-3206-403D-B09E-59D4A758FBAD,Action +{Uninstall Actions} + +3CFFF099-6122-46DD-9CE4-F5819434AC53,Conditions +{0 conditions} + +3CFFF099-6122-46DD-9CE4-F5819434AC53,IgnoreErrors +Yes + +3CFFF099-6122-46DD-9CE4-F5819434AC53,ProgramCommandLine +{net stop <%ServiceName%>} + +3CFFF099-6122-46DD-9CE4-F5819434AC53,ProgressiveOutputWidget +Message + +3CFFF099-6122-46DD-9CE4-F5819434AC53,WorkingDirectory +<%Temp%> + +3D0D1898-4C65-3E66-F82F56581E87,CheckCondition +{Before Action is Executed} + +3D0D1898-4C65-3E66-F82F56581E87,Operator +false + +3D0D1898-4C65-3E66-F82F56581E87,String +<%SilentMode%> + +3D33AA8C-0037-204B-39A339FD38BD,BackButton,subst +1 + +3D33AA8C-0037-204B-39A339FD38BD,CancelButton,subst +1 + +3D33AA8C-0037-204B-39A339FD38BD,Caption,subst +1 + +3D33AA8C-0037-204B-39A339FD38BD,Conditions +{0 conditions} + +3D33AA8C-0037-204B-39A339FD38BD,Message,subst +1 + +3D33AA8C-0037-204B-39A339FD38BD,NextButton,subst +1 + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Active +Yes + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Alias +SetBackupLocations + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,BackButton,subst +1 + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,CancelButton,subst +1 + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Caption,subst +1 + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Conditions +{0 conditions} + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Message,subst +1 + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,NextButton,subst +1 + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Subtitle,subst +1 + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Title,subst +1 + +3FDB57ED-598D-8A4E-CEF7-D90833305558,Conditions +{0 conditions} + +3FDB57ED-598D-8A4E-CEF7-D90833305558,LabelSide +left + +3FDB57ED-598D-8A4E-CEF7-D90833305558,Text,subst +1 + +3FDB57ED-598D-8A4E-CEF7-D90833305558,Type +{browse entry} + +3FDB57ED-598D-8A4E-CEF7-D90833305558,VirtualText +BackupLocationPath + +3FDB57ED-598D-8A4E-CEF7-D90833305558,Y +70 + +3FE82C17-A3E2-4A57-A563-F80818B00B81,Default +Yes + +3FE82C17-A3E2-4A57-A563-F80818B00B81,Prompt +<%InstallStartupText%> + +41CDE776-2667-5CEB-312A-FC4C33A83E7F,Conditions +{0 conditions} + +41CDE776-2667-5CEB-312A-FC4C33A83E7F,Files +{*/*.conf;*/*.txt;*/*.pem;*/*.raw;*/*.exe;*/*.bat;*/*.dll} + +41CDE776-2667-5CEB-312A-FC4C33A83E7F,RenameFiles +Yes + +41D3E165-C263-5F80-0FEEC0AEE47A,BackButton,subst +1 + +41D3E165-C263-5F80-0FEEC0AEE47A,CancelButton,subst +1 + +41D3E165-C263-5F80-0FEEC0AEE47A,Caption,subst +1 + +41D3E165-C263-5F80-0FEEC0AEE47A,Conditions +{1 condition} + +41D3E165-C263-5F80-0FEEC0AEE47A,Message,subst +1 + +41D3E165-C263-5F80-0FEEC0AEE47A,NextButton,subst +1 + +41D3E165-C263-5F80-0FEEC0AEE47A,Subtitle,subst +1 + +41D3E165-C263-5F80-0FEEC0AEE47A,Text,subst +1 + +41D3E165-C263-5F80-0FEEC0AEE47A,Title,subst +1 + +481451CC-F49C-D389-8645076F595B,Destination +<%InstallDir%> + +481451CC-F49C-D389-8645076F595B,FileSize +15288767 + +481451CC-F49C-D389-8645076F595B,Name +{Program Files} + +49E59F91-27F7-46D1-A1C1-19865C2392D3,Default +Yes + +49E59F91-27F7-46D1-A1C1-19865C2392D3,Prompt +<%UninstallStartupText%> + +49E80443-62DB-1C10-392D-1091AEA5ED88,Conditions +{1 condition} + +49E80443-62DB-1C10-392D-1091AEA5ED88,ExecuteAction +{Before Next Pane is Displayed} + +49E80443-62DB-1C10-392D-1091AEA5ED88,Pane +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 + +4A9C852B-647E-EED5-5482FFBCC2AF,Description,subst +1 + +4A9C852B-647E-EED5-5482FFBCC2AF,DisplayName,subst +1 + +4A9C852B-647E-EED5-5482FFBCC2AF,FileGroups +481451CC-F49C-D389-8645076F595B + +4A9C852B-647E-EED5-5482FFBCC2AF,Name +{Default Component} + +4A9C852B-647E-EED5-5482FFBCC2AF,RequiredComponent +Yes + +4ACB0B47-42B3-2B3A-BFE9AA4EC707,CheckCondition +{Before Action is Executed} + +4ACB0B47-42B3-2B3A-BFE9AA4EC707,Message,subst +1 + +4ACB0B47-42B3-2B3A-BFE9AA4EC707,Title,subst +1 + +4ACB0B47-42B3-2B3A-BFE9AA4EC707,TrueValue +No + +4D4A7BF0-7CCE-46E6-BDE5222F82D7,Conditions +{0 conditions} + +4D4A7BF0-7CCE-46E6-BDE5222F82D7,UpdateFilePercentage +Yes + +4D4A7BF0-7CCE-46E6-BDE5222F82D7,UpdateFileText +Yes + +4E643D8A-CA31-018D-57D7053C2CE8,CheckCondition +{Before Action is Executed} + +4E643D8A-CA31-018D-57D7053C2CE8,Filename +<%ProgramExecutable%> + +4EE35849-FAD7-170B-0E45-FA30636467B1,CheckCondition +{Before Next Pane is Displayed} + +4EE35849-FAD7-170B-0E45-FA30636467B1,EncryptedPassword +<%InstallPasswordEncrypted%> + +4EE35849-FAD7-170B-0E45-FA30636467B1,FailureFocus +{Password Entry} + +4EE35849-FAD7-170B-0E45-FA30636467B1,FailureMessage +<%PasswordIncorrectText%> + +4EE35849-FAD7-170B-0E45-FA30636467B1,UnencryptedPassword +<%InstallPassword%> + +4EF787E3-0643-DE46-15E64BAF0816,CheckCondition +{Before Action is Executed} + +4EF787E3-0643-DE46-15E64BAF0816,Platform +Windows + +52F0A238-57E1-A578-2CE4DA177B32,Conditions +{0 conditions} + +53588803-6B41-D9FC-A385906A5106,Conditions +{0 conditions} + +574198A7-7322-2F5E-02EF185D965C,BackButton,subst +1 + +574198A7-7322-2F5E-02EF185D965C,CancelButton,subst +1 + +574198A7-7322-2F5E-02EF185D965C,Caption,subst +1 + +574198A7-7322-2F5E-02EF185D965C,Conditions +{0 conditions} + +574198A7-7322-2F5E-02EF185D965C,FileLabel,subst +1 + +574198A7-7322-2F5E-02EF185D965C,Message,subst +1 + +574198A7-7322-2F5E-02EF185D965C,NextButton,subst +1 + +574198A7-7322-2F5E-02EF185D965C,ProgressValue,subst +1 + +574198A7-7322-2F5E-02EF185D965C,Subtitle,subst +1 + +574198A7-7322-2F5E-02EF185D965C,Title,subst +1 + +58E1119F-639E-17C9-5D3898F385AA,BackButton,subst +1 + +58E1119F-639E-17C9-5D3898F385AA,BrowseButton,subst +1 + +58E1119F-639E-17C9-5D3898F385AA,BrowseText,subst +1 + +58E1119F-639E-17C9-5D3898F385AA,CancelButton,subst +1 + +58E1119F-639E-17C9-5D3898F385AA,Caption,subst +1 + +58E1119F-639E-17C9-5D3898F385AA,Conditions +{1 condition} + +58E1119F-639E-17C9-5D3898F385AA,DestinationLabel,subst +0 + +58E1119F-639E-17C9-5D3898F385AA,Message,subst +1 + +58E1119F-639E-17C9-5D3898F385AA,NextButton,subst +1 + +58E1119F-639E-17C9-5D3898F385AA,Subtitle,subst +1 + +58E1119F-639E-17C9-5D3898F385AA,Title,subst +1 + +592F46AE-8CEE-01F3-0BA7EBDCA4F4,CheckCondition +{Before Action is Executed} + +592F46AE-8CEE-01F3-0BA7EBDCA4F4,Filename +<%ProgramExecutable%> + +5CA3EA16-E37C-AABE-E576C4636EB0,Action +{Install Actions} + +5CA3EA16-E37C-AABE-E576C4636EB0,Conditions +{0 conditions} + +5EE78EF7-37CA-D440-3DB5-09136CD566B3,CheckCondition +{Before Action is Executed} + +5EE78EF7-37CA-D440-3DB5-09136CD566B3,String +<%AddBackupLocation%> + +5F2C1F1C-B9F7-1642-59D9-A18318C1D70B,Conditions +{0 conditions} + +5F2C1F1C-B9F7-1642-59D9-A18318C1D70B,Files +{<%ConfigFileTemplate%>;*/*.bat;<%InstallDir%>/*.vbs} + +5F2C1F1C-B9F7-1642-59D9-A18318C1D70B,LineFeed +Windows + +5F2C1F1C-B9F7-1642-59D9-A18318C1D70B,StringMap +{"@@CUSTOMERCOMPANY@@" <%UserInfoCompany%> +"@@BRANDNAME@@" <%BrandName%> +"@@SERVICENAME@@" <%ServiceName%> +"@@CUSTOMERUSERNAME@@" <%UserInfoName%> +"@@CUSTOMERUSERPHONE@@" <%UserInfoPhone%> +"@@CUSTOMERUSEREMAIL@@" <%UserInfoEmail%> +"@@ACCTNO@@" <%UserInfoAcctNo%> +"@@INSTALLDIR@@" <%InstallDir%>} + +614C45B2-7515-780C-E444-7F165CF02DD7,Active +No + +614C45B2-7515-780C-E444-7F165CF02DD7,Conditions +{0 conditions} + +614C45B2-7515-780C-E444-7F165CF02DD7,ResultVirtualText +BackupLocationShortName + +614C45B2-7515-780C-E444-7F165CF02DD7,TclScript +{set BackupLocationShortName ""} + +6652193C-5D4B-44B6-ABC6-D6E96D89E5DC,Active +No + +6652193C-5D4B-44B6-ABC6-D6E96D89E5DC,Comment +{PJ removed. Is this the one at the top leve?} + +6652193C-5D4B-44B6-ABC6-D6E96D89E5DC,Conditions +{0 conditions} + +69FD7409-5E2A-143B-DABD1C3B1E67,Conditions +{2 conditions} + +6B4CB3C2-4799-4C9F-BA8E-1EE47C4606E1,ExitType +Finish + +6B966959-05D9-DB32-8D9C4AD2A3DF,CheckCondition +{Before Action is Executed} + +6B966959-05D9-DB32-8D9C4AD2A3DF,Platform +Windows + +6C323815-B9AB-FA94-4F5D152EBC51,BackButton,subst +1 + +6C323815-B9AB-FA94-4F5D152EBC51,CancelButton,subst +1 + +6C323815-B9AB-FA94-4F5D152EBC51,Caption,subst +1 + +6C323815-B9AB-FA94-4F5D152EBC51,Conditions +{0 conditions} + +6C323815-B9AB-FA94-4F5D152EBC51,Message,subst +1 + +6C323815-B9AB-FA94-4F5D152EBC51,NextButton,subst +1 + +6D9D1ABC-7146-443F-9EE9-205D5CA6C830,CheckCondition +{Before Action is Executed} + +6D9D1ABC-7146-443F-9EE9-205D5CA6C830,String +{<%Property <%CurrentPane%> UserMustAcceptLicense%>} + +6E70FB1F-6A43-6C23-3242E965A0D0,Action +{Install Actions} + +6E70FB1F-6A43-6C23-3242E965A0D0,Conditions +{0 conditions} + +6F61CDA8-30C9-454F-82A3-9987E1203079,Alias +{Start a sync now.} + +6F61CDA8-30C9-454F-82A3-9987E1203079,Conditions +{0 conditions} + +6F61CDA8-30C9-454F-82A3-9987E1203079,FileName +<%ShortAppName%>-program-sync + +6F61CDA8-30C9-454F-82A3-9987E1203079,ShortcutName +{Sync now} + +6F61CDA8-30C9-454F-82A3-9987E1203079,TargetFileName +<%InstallDir%>/tools/Sync.bat + +6F61CDA8-30C9-454F-82A3-9987E1203079,WorkingDirectory +<%InstallDir%> + +6F94698F-0839-3ABF-0CF2DF05A4C8,CheckCondition +{Before Action is Executed} + +6F94698F-0839-3ABF-0CF2DF05A4C8,String +<%CreateQuickLaunchShortcut%> + +6FEE2889-0338-1D49-60BF-1471F465AB26,AppendNewline +No + +6FEE2889-0338-1D49-60BF-1471F465AB26,Comment +{Closing final BackupLocations bracket} + +6FEE2889-0338-1D49-60BF-1471F465AB26,Conditions +{0 conditions} + +6FEE2889-0338-1D49-60BF-1471F465AB26,FileOpenAction +{Append to file} + +6FEE2889-0338-1D49-60BF-1471F465AB26,Files +<%ConfigFileTemplate%> + +6FEE2889-0338-1D49-60BF-1471F465AB26,LineFeed +Windows + +6FEE2889-0338-1D49-60BF-1471F465AB26,TextToWrite,subst +1 + +738DD098-7E3B-BC89-875CDB93CBE2,CheckCondition +{Before Action is Executed} + +738DD098-7E3B-BC89-875CDB93CBE2,Platform +Windows + +73DD4D07-B1DC-BA38-2B12-07EB24A7F0C8,Conditions +{0 conditions} + +73DD4D07-B1DC-BA38-2B12-07EB24A7F0C8,Destination +<%ConfigFileName%> + +73DD4D07-B1DC-BA38-2B12-07EB24A7F0C8,Source +<%ConfigFileTemplate%> + +73EA65C1-3BE3-B190-55C3E99F6269,Conditions +{1 condition} + +748D673B-DFE6-5F74-329903ACE4DB,CheckCondition +{Before Action is Executed} + +748D673B-DFE6-5F74-329903ACE4DB,Filename +<%ProgramExecutable%> + +793D8178-0F51-7F07-BC5886586D3C,CheckCondition +{Before Action is Executed} + +793D8178-0F51-7F07-BC5886586D3C,Operator +false + +793D8178-0F51-7F07-BC5886586D3C,String +<%InstallStopped%> + +795EE61F-6C0D-4A8B-93E02AA3894A,CheckCondition +{Before Action is Executed} + +795EE61F-6C0D-4A8B-93E02AA3894A,String +<%LaunchApplication%> + +79DAC913-A33D-4ED6-9BAE-B3A2053C0F2C,CheckCondition +{Before Action is Executed} + +79DAC913-A33D-4ED6-9BAE-B3A2053C0F2C,Operator +false + +79DAC913-A33D-4ED6-9BAE-B3A2053C0F2C,String +<%LicenseAccepted%> + +7A983CD8-302C-4942-BE59-525C5B5FA2F2,Conditions +{0 conditions} + +7A983CD8-302C-4942-BE59-525C5B5FA2F2,ProgramCommandLine +{ServiceControl terminate} + +7A983CD8-302C-4942-BE59-525C5B5FA2F2,WorkingDirectory +<%InstallDir%> + +7B770A07-A785-5215-956FA82CF14E,Active +No + +7B770A07-A785-5215-956FA82CF14E,Conditions +{3 conditions} + +7B770A07-A785-5215-956FA82CF14E,ShortcutDirectory +<%QUICK_LAUNCH%> + +7B770A07-A785-5215-956FA82CF14E,ShortcutName +<%BrandName%> + +7B770A07-A785-5215-956FA82CF14E,TargetFileName +<%ProgramExecutable%> + +7B770A07-A785-5215-956FA82CF14E,WorkingDirectory +<%InstallDir%> + +7D8E1902-2BC4-80D8-2C18771E7C22,Conditions +{0 conditions} + +7D8E1902-2BC4-80D8-2C18771E7C22,ProgramCommandLine +{<%ServiceExeName%> -i -S <%ServiceName%> -c "<%ConfigFileName%>"} + +7D8E1902-2BC4-80D8-2C18771E7C22,ProgressiveOutputWidget +Message + +7D8E1902-2BC4-80D8-2C18771E7C22,ShowProgressiveOutput +Yes + +7D8E1902-2BC4-80D8-2C18771E7C22,WorkingDirectory +<%InstallDir%> + +7F85263E-CAE2-46BA-AAC0-6B89D20FD2DE,ExitType +Finish + +8202CECC-54A0-9B6C-D24D111BA52E,Components +4A9C852B-647E-EED5-5482FFBCC2AF + +8202CECC-54A0-9B6C-D24D111BA52E,Description,subst +1 + +8202CECC-54A0-9B6C-D24D111BA52E,DisplayName,subst +1 + +8202CECC-54A0-9B6C-D24D111BA52E,Name +Typical + +8419AAAD-5860-F73E-8D11-4D1BDA4D7D37,Checked +No + +8419AAAD-5860-F73E-8D11-4D1BDA4D7D37,Conditions +{0 conditions} + +8419AAAD-5860-F73E-8D11-4D1BDA4D7D37,Text,subst +1 + +8419AAAD-5860-F73E-8D11-4D1BDA4D7D37,Type +checkbutton + +8419AAAD-5860-F73E-8D11-4D1BDA4D7D37,Value +Yes + +8419AAAD-5860-F73E-8D11-4D1BDA4D7D37,VirtualText +AddBackupLocation + +8419AAAD-5860-F73E-8D11-4D1BDA4D7D37,Y +250 + +848844B5-6103-9343-8B731B0BE4E0,Alias +{Startup Actions} + +848844B5-6103-9343-8B731B0BE4E0,Conditions +{0 conditions} + +84DA7F05-9FB7-CC36-9EC98F8A6826,CheckCondition +{Before Next Pane is Displayed} + +84DA7F05-9FB7-CC36-9EC98F8A6826,FailureMessage +<%DirectoryPermissionText%> + +84DA7F05-9FB7-CC36-9EC98F8A6826,Filename +<%InstallDir%> + +84DA7F05-9FB7-CC36-9EC98F8A6826,Permission +{can create} + +855DE408-060E-3D35-08B5-1D9AB05C2865,Conditions +{0 conditions} + +855DE408-060E-3D35-08B5-1D9AB05C2865,Height +100 + +855DE408-060E-3D35-08B5-1D9AB05C2865,Text,subst +1 + +855DE408-060E-3D35-08B5-1D9AB05C2865,Type +text + +855DE408-060E-3D35-08B5-1D9AB05C2865,VirtualText +BackupLocationExclusions + +855DE408-060E-3D35-08B5-1D9AB05C2865,Y +130 + +87DE6D78-81E1-495B-A214-B3FF3E7E5614,CheckCondition +{Before Action is Executed} + +87DE6D78-81E1-495B-A214-B3FF3E7E5614,Operator +false + +87DE6D78-81E1-495B-A214-B3FF3E7E5614,String +<%Answer%> + +88A50FD5-480F-19A5-DA74-C915EB0A9765,Active +No + +88A50FD5-480F-19A5-DA74-C915EB0A9765,Conditions +{1 condition} + +88A50FD5-480F-19A5-DA74-C915EB0A9765,ExecuteAction +{After Pane is Finished} + +88A50FD5-480F-19A5-DA74-C915EB0A9765,Pane +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7 + +8A761DBD-0640-D98C-9B3AD7672A8F,Conditions +{0 conditions} + +8A761DBD-0640-D98C-9B3AD7672A8F,State +disabled + +8A761DBD-0640-D98C-9B3AD7672A8F,Widget +{Back Button;Next Button} + +8C866252-8760-9B08-FE569C25B60D,CheckCondition +{Before Action is Executed} + +8C866252-8760-9B08-FE569C25B60D,Filename +<%ProgramExecutable%> + +8E095096-F018-A880-429D-A2177A9B70EA,Active +No + +8E095096-F018-A880-429D-A2177A9B70EA,Conditions +{0 conditions} + +8E095096-F018-A880-429D-A2177A9B70EA,Text,subst +1 + +8E095096-F018-A880-429D-A2177A9B70EA,X +50 + +8E095096-F018-A880-429D-A2177A9B70EA,Y +150 + +8E1A5944-5AF5-5906-16D395E386D8,Conditions +{0 conditions} + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,Active +Yes + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,BackButton,subst +1 + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,CancelButton,subst +1 + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,Caption,subst +1 + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,CompanyLabel,subst +1 + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,Conditions +{0 conditions} + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,Message,subst +1 + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,NextButton,subst +1 + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,Subtitle,subst +1 + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,Title,subst +1 + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,UserNameLabel,subst +1 + +905DA2E9-988C-2F27-BB1F5F274AC9,Alias +{Cancel Actions} + +905DA2E9-988C-2F27-BB1F5F274AC9,Conditions +{0 conditions} + +908CE221-5A3D-0A78-24A1-E7C91EBE38D4,BackButton,subst +1 + +908CE221-5A3D-0A78-24A1-E7C91EBE38D4,CancelButton,subst +1 + +908CE221-5A3D-0A78-24A1-E7C91EBE38D4,Caption,subst +1 + +908CE221-5A3D-0A78-24A1-E7C91EBE38D4,Conditions +{0 conditions} + +908CE221-5A3D-0A78-24A1-E7C91EBE38D4,Message,subst +1 + +908CE221-5A3D-0A78-24A1-E7C91EBE38D4,NextButton,subst +1 + +908CE221-5A3D-0A78-24A1-E7C91EBE38D4,Subtitle,subst +1 + +908CE221-5A3D-0A78-24A1-E7C91EBE38D4,Title,subst +1 + +937C3FDD-FB28-98BD-3DAB276E59ED,Background +white + +937C3FDD-FB28-98BD-3DAB276E59ED,Conditions +{3 conditions} + +937C3FDD-FB28-98BD-3DAB276E59ED,Text,subst +1 + +937C3FDD-FB28-98BD-3DAB276E59ED,Type +checkbutton + +937C3FDD-FB28-98BD-3DAB276E59ED,VirtualText +CreateQuickLaunchShortcut + +937C3FDD-FB28-98BD-3DAB276E59ED,X +185 + +937C3FDD-FB28-98BD-3DAB276E59ED,Y +200 + +93AA298C-B64E-5683-14D2-7B86F7DEFD2C,Active +No + +93AA298C-B64E-5683-14D2-7B86F7DEFD2C,Comment +{set BackupLocationName "BackupLocation_${BackupLocationNumber}"} + +93AA298C-B64E-5683-14D2-7B86F7DEFD2C,Conditions +{0 conditions} + +93AA298C-B64E-5683-14D2-7B86F7DEFD2C,ResultVirtualText +BackupLocationName + +93AA298C-B64E-5683-14D2-7B86F7DEFD2C,TclScript +{set BackupLocationName "BackupLocation_${BackupLocationNumber}"} + +96A68CAC-9ED7-806C-086B104720FD,CheckCondition +{Before Action is Executed} + +96A68CAC-9ED7-806C-086B104720FD,String +<%ErrorsOccurred%> + +97ACF525-C075-8635-E019202A83D8,Comment +{Ask the user if they want to proceed with the uninstall.} + +9892B25C-689B-5B8F-F0C9-B14FF6ACC40C,Active +No + +9892B25C-689B-5B8F-F0C9-B14FF6ACC40C,Conditions +{0 conditions} + +9892B25C-689B-5B8F-F0C9-B14FF6ACC40C,ResultVirtualText +AddBackupLocation + +9892B25C-689B-5B8F-F0C9-B14FF6ACC40C,TclScript +{set AddBackupLocation no} + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Active +Yes + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,BackButton,subst +1 + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,CancelButton,subst +1 + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Caption,subst +1 + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Conditions +{0 conditions} + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Message,subst +1 + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,NextButton,subst +1 + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Subtitle,subst +1 + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Title,subst +1 + +9A663209-495B-ED16-09BE-457B61148022,Conditions +{0 conditions} + +9A663209-495B-ED16-09BE-457B61148022,FileName +<%ShortAppName%>-program-QueryCurrent + +9A663209-495B-ED16-09BE-457B61148022,ShortcutName +{Query Current Only} + +9A663209-495B-ED16-09BE-457B61148022,TargetFileName +<%InstallDir%>/tools/QueryOutputCurrent.bat + +9A663209-495B-ED16-09BE-457B61148022,WorkingDirectory +<%InstallDir%> + +9D101299-B80C-441B-8685-6E3AC61808E8,Alias +{Remote Control} + +9D101299-B80C-441B-8685-6E3AC61808E8,Comment +{Get tech support via remote control} + +9D101299-B80C-441B-8685-6E3AC61808E8,Conditions +{0 conditions} + +9D101299-B80C-441B-8685-6E3AC61808E8,FileName +<%ShortAppName%>-program-RemoteControl + +9D101299-B80C-441B-8685-6E3AC61808E8,ShortcutName +RemoteControl + +9D101299-B80C-441B-8685-6E3AC61808E8,TargetFileName +<%InstallDir%>/tools/RemoteControl.exe + +9D101299-B80C-441B-8685-6E3AC61808E8,WorkingDirectory +<%InstallDir%> + +A1DD1DC2-85D7-9BC6-998AC3D4A3A9,Alias +{Startup Actions} + +A1DD1DC2-85D7-9BC6-998AC3D4A3A9,Conditions +{0 conditions} + +A5B32DA1-B2FE-C1FA-6057-FBC3059EF076,Conditions +{0 conditions} + +A5B32DA1-B2FE-C1FA-6057-FBC3059EF076,ResultVirtualText +AddBackupLocation + +A5B32DA1-B2FE-C1FA-6057-FBC3059EF076,TclScript +{set AddBackupLocation no } + +A6E1B027-A1B4-5848-4F868D028D00,CheckCondition +{Before Action is Executed} + +A6E1B027-A1B4-5848-4F868D028D00,String +<%ViewReadme%> + +ADA6EB2F-8820-4366-BBEF-ED1335B7F828,Conditions +{1 condition} + +AE3BD5B4-35DE-4240-B79914D43E56,Active +No + +AE3BD5B4-35DE-4240-B79914D43E56,BackButton,subst +1 + +AE3BD5B4-35DE-4240-B79914D43E56,CancelButton,subst +1 + +AE3BD5B4-35DE-4240-B79914D43E56,Caption,subst +1 + +AE3BD5B4-35DE-4240-B79914D43E56,Conditions +{0 conditions} + +AE3BD5B4-35DE-4240-B79914D43E56,Message,subst +1 + +AE3BD5B4-35DE-4240-B79914D43E56,NextButton,subst +1 + +AIX-ppc,Active +No + +AIX-ppc,DefaultDirectoryPermission +0755 + +AIX-ppc,DefaultFilePermission +0755 + +AIX-ppc,Executable +<%AppName%>-<%Version%>-<%Platform%>-Install<%Ext%> + +AIX-ppc,FallBackToConsole +Yes + +AIX-ppc,InstallDir +<%Home%>/<%ShortAppName%> + +AIX-ppc,InstallMode +Standard + +AIX-ppc,InstallType +Typical + +AIX-ppc,ProgramExecutable +{} + +AIX-ppc,ProgramFolderAllUsers +No + +AIX-ppc,ProgramFolderName +<%AppName%> + +AIX-ppc,ProgramLicense +<%InstallDir%>/LICENSE.txt + +AIX-ppc,ProgramName +{} + +AIX-ppc,ProgramReadme +<%InstallDir%>/README.txt + +AIX-ppc,PromptForRoot +Yes + +AIX-ppc,RequireRoot +No + +AIX-ppc,RootInstallDir +/usr/local/<%ShortAppName%> + +B002A311-F8E7-41DE-B039-521391924E5B,Message,subst +1 + +B01CBBB2-6A78-CA53-9ED9-C3C4CFC9239E,Alias +{Stop Backup Service} + +B01CBBB2-6A78-CA53-9ED9-C3C4CFC9239E,Comment +{Stop the Backup Windows Service} + +B01CBBB2-6A78-CA53-9ED9-C3C4CFC9239E,Conditions +{0 conditions} + +B01CBBB2-6A78-CA53-9ED9-C3C4CFC9239E,FileName +<%ShortAppName%>-program-stopservice + +B01CBBB2-6A78-CA53-9ED9-C3C4CFC9239E,ShortcutName +{Stop Service} + +B01CBBB2-6A78-CA53-9ED9-C3C4CFC9239E,TargetFileName +<%InstallDir%>/tools/StopService.bat + +B01CBBB2-6A78-CA53-9ED9-C3C4CFC9239E,WorkingDirectory +<%InstallDir%> + +B0AA6839-AAB6-A602-C0E4ECA2E4FF,CheckCondition +{Before Action is Executed} + +B0AA6839-AAB6-A602-C0E4ECA2E4FF,Filename +<%ProgramExecutable%> + +B39C0455-D1B6-7DDC-E2717F83463E,CheckCondition +{Before Action is Executed} + +B39C0455-D1B6-7DDC-E2717F83463E,Operator +false + +B39C0455-D1B6-7DDC-E2717F83463E,String +<%InstallStopped%> + +B3B99E2D-C368-A921-B7BC-A71EBDE3AD4D,Conditions +{0 conditions} + +B3B99E2D-C368-A921-B7BC-A71EBDE3AD4D,ExecuteAction +{Before Next Pane is Displayed} + +B3B99E2D-C368-A921-B7BC-A71EBDE3AD4D,Password +<%InstallPassword%> + +B4D31D1E-ADB1-DE8F-18EB7294DDA8,Conditions +{0 conditions} + +B4D31D1E-ADB1-DE8F-18EB7294DDA8,ProgramCommandLine +{<%ServiceExeName%> -r -S <%ServiceName%>} + +B4D31D1E-ADB1-DE8F-18EB7294DDA8,WorkingDirectory +<%InstallDir%> + +B4ED4636-22D8-41DC-9E3D-BD1E1CAD2174,Message,subst +1 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,AcceptRadiobutton,subst +0 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,Active +Yes + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,BackButton,subst +1 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,CancelButton,subst +1 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,Caption,subst +1 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,Conditions +{0 conditions} + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,DeclineRadiobutton,subst +0 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,Message,subst +1 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,NextButton,subst +1 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,Subtitle,subst +1 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,Text,subst +1 + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,Title,subst +1 + +B5DFEC63-92A9-4686-909E-0CE78A7069D6,Alias +{Restart Backup Service} + +B5DFEC63-92A9-4686-909E-0CE78A7069D6,Comment +{Stop and restart the Backup Windows Service} + +B5DFEC63-92A9-4686-909E-0CE78A7069D6,Conditions +{0 conditions} + +B5DFEC63-92A9-4686-909E-0CE78A7069D6,FileName +<%ShortAppName%>-program-restartservice + +B5DFEC63-92A9-4686-909E-0CE78A7069D6,ShortcutName +{Restart Service} + +B5DFEC63-92A9-4686-909E-0CE78A7069D6,TargetFileName +{<%InstallDir%>\tools\RestartService.bat} + +B5DFEC63-92A9-4686-909E-0CE78A7069D6,WorkingDirectory +<%InstallDir%> + +B927A5AF-4DFE-82A3-DCA8-35FA4D91EC5A,Conditions +{0 conditions} + +B927A5AF-4DFE-82A3-DCA8-35FA4D91EC5A,LabelSide +left + +B927A5AF-4DFE-82A3-DCA8-35FA4D91EC5A,Text,subst +1 + +B927A5AF-4DFE-82A3-DCA8-35FA4D91EC5A,Type +entry + +B927A5AF-4DFE-82A3-DCA8-35FA4D91EC5A,VirtualText +BackupLocationShortName + +B927A5AF-4DFE-82A3-DCA8-35FA4D91EC5A,Y +100 + +B93D2216-1DDB-484C-A9AC-D6C18ED7DE23,Conditions +{2 conditions} + +B93D2216-1DDB-484C-A9AC-D6C18ED7DE23,State +disabled + +B93D2216-1DDB-484C-A9AC-D6C18ED7DE23,Widget +NextButton + +BC4EA5FD-50BD-4D6E-953F-5E3EDB957360,CheckCondition +{Before Next Action is Executed} + +BC4EA5FD-50BD-4D6E-953F-5E3EDB957360,FailureMessage +<%DirectoryPermissionText%> + +BC4EA5FD-50BD-4D6E-953F-5E3EDB957360,Filename +<%InstallDir%> + +BC4EA5FD-50BD-4D6E-953F-5E3EDB957360,Permission +{can create} + +C0452595-F3EB-43AD-BCA2-661437584636,Alias +{Modify Backup Configuration} + +C0452595-F3EB-43AD-BCA2-661437584636,Comment +{Modify your Backup Configuration} + +C0452595-F3EB-43AD-BCA2-661437584636,Conditions +{0 conditions} + +C0452595-F3EB-43AD-BCA2-661437584636,FileName +<%ShortAppName%>-program-editconfig + +C0452595-F3EB-43AD-BCA2-661437584636,ShortcutName +{Edit Config File} + +C0452595-F3EB-43AD-BCA2-661437584636,TargetFileName +<%InstallDir%>/tools/EditConfig.bat + +C0452595-F3EB-43AD-BCA2-661437584636,WorkingDirectory +<%InstallDir%> + +C0AF7C05-A31A-8376-BCB9-BA8B3A666252,Conditions +{0 conditions} + +C0AF7C05-A31A-8376-BCB9-BA8B3A666252,FileName +<%ShortAppName%>-program-QueryAll + +C0AF7C05-A31A-8376-BCB9-BA8B3A666252,ShortcutName +{Query All} + +C0AF7C05-A31A-8376-BCB9-BA8B3A666252,TargetFileName +<%InstallDir%>/tools/QueryOutputAll.bat + +C0AF7C05-A31A-8376-BCB9-BA8B3A666252,WorkingDirectory +<%InstallDir%> + +C105AAAE-7C16-2C9E-769FE4535B60,Active +No + +C105AAAE-7C16-2C9E-769FE4535B60,Caption,subst +1 + +C105AAAE-7C16-2C9E-769FE4535B60,CloseButton,subst +1 + +C105AAAE-7C16-2C9E-769FE4535B60,Conditions +{3 conditions} + +C105AAAE-7C16-2C9E-769FE4535B60,Message,subst +1 + +C105AAAE-7C16-2C9E-769FE4535B60,TextFile +<%ProgramReadme%> + +C105AAAE-7C16-2C9E-769FE4535B60,Title,subst +1 + +C33D74B2-26FA-16F5-433A10C6A747,Active +No + +C33D74B2-26FA-16F5-433A10C6A747,Conditions +{3 conditions} + +C33D74B2-26FA-16F5-433A10C6A747,ProgramCommandLine +<%ProgramExecutable%> + +C33D74B2-26FA-16F5-433A10C6A747,WaitForProgram +No + +C33D74B2-26FA-16F5-433A10C6A747,WorkingDirectory +<%InstallDir%> + +C7762473-273F-E3CA-17E3-65789B14CDB0,Conditions +{0 conditions} + +C7762473-273F-E3CA-17E3-65789B14CDB0,ExecuteAction +{Before Next Pane is Displayed} + +C7762473-273F-E3CA-17E3-65789B14CDB0,FileOpenAction +{Append to file} + +C7762473-273F-E3CA-17E3-65789B14CDB0,Files +<%ConfigFileTemplate%> + +C7762473-273F-E3CA-17E3-65789B14CDB0,TextToWrite,subst +1 + +CC4337CC-F3B5-757C-DFCF5D1D365A,CheckCondition +{Before Action is Executed} + +CC4337CC-F3B5-757C-DFCF5D1D365A,Operator +false + +CC4337CC-F3B5-757C-DFCF5D1D365A,String +<%SilentMode%> + +CDD84DE3-C970-458F-9162-1A3CE0AA716B,Alias +{Start Backup Service} + +CDD84DE3-C970-458F-9162-1A3CE0AA716B,Comment +{Start Backup Windows Service} + +CDD84DE3-C970-458F-9162-1A3CE0AA716B,Conditions +{0 conditions} + +CDD84DE3-C970-458F-9162-1A3CE0AA716B,FileName +<%ShortAppName%>-program-startservice + +CDD84DE3-C970-458F-9162-1A3CE0AA716B,ShortcutName +{Start Service} + +CDD84DE3-C970-458F-9162-1A3CE0AA716B,TargetFileName +{<%InstallDir%>\tools\StartService.bat} + +CDD84DE3-C970-458F-9162-1A3CE0AA716B,WorkingDirectory +<%InstallDir%> + +CFFA27AF-A641-E41C-B4A0E3BB3CBB,Background +white + +CFFA27AF-A641-E41C-B4A0E3BB3CBB,Conditions +{2 conditions} + +CFFA27AF-A641-E41C-B4A0E3BB3CBB,Text,subst +1 + +CFFA27AF-A641-E41C-B4A0E3BB3CBB,Type +checkbutton + +CFFA27AF-A641-E41C-B4A0E3BB3CBB,VirtualText +LaunchApplication + +CFFA27AF-A641-E41C-B4A0E3BB3CBB,X +185 + +CFFA27AF-A641-E41C-B4A0E3BB3CBB,Y +160 + +D23DD94C-E517-7F34-FD59-802CB18AB887,Comment +{Need to do before starting anything else.} + +D23DD94C-E517-7F34-FD59-802CB18AB887,Conditions +{0 conditions} + +D23DD94C-E517-7F34-FD59-802CB18AB887,Files +{*/*.conf;*/*.txt;*/*.bat} + +D23DD94C-E517-7F34-FD59-802CB18AB887,LineFeed +Windows + +D3D73C76-D9D3-07DA-63D4163A44BE,Conditions +{0 conditions} + +D3D73C76-D9D3-07DA-63D4163A44BE,ExitType +Finish + +D4FC6EB5-DDEE-4E4A-B8E1-D4B588A7928B,Action +{Install Actions} + +D55BA4AF-E73B-60D1-E26F79175227,Action +{Uninstall Actions} + +D55BA4AF-E73B-60D1-E26F79175227,Conditions +{0 conditions} + +D7FBBEBB-2186-5674-BA87-BB7151859D4E,Conditions +{0 conditions} + +D7FBBEBB-2186-5674-BA87-BB7151859D4E,ResultVirtualText +BackupLocationNumber + +D7FBBEBB-2186-5674-BA87-BB7151859D4E,TclScript +{incr BackupLocationNumber} + +D8B8A9BF-5F2E-4236-A63E-5A8C5FFA8968,Alias +{Reload Configuration File, after editing it.} + +D8B8A9BF-5F2E-4236-A63E-5A8C5FFA8968,Conditions +{0 conditions} + +D8B8A9BF-5F2E-4236-A63E-5A8C5FFA8968,FileName +<%ShortAppName%>-program-reloadconfig + +D8B8A9BF-5F2E-4236-A63E-5A8C5FFA8968,ShortcutName +{Reload configuration file} + +D8B8A9BF-5F2E-4236-A63E-5A8C5FFA8968,TargetFileName +<%InstallDir%>/tools/ReloadConfig.bat + +D8B8A9BF-5F2E-4236-A63E-5A8C5FFA8968,WorkingDirectory +<%InstallDir%> + +D8F0AA0F-AD79-C566-15CC508F503B,Action +{Install Actions} + +D8F0AA0F-AD79-C566-15CC508F503B,Conditions +{0 conditions} + +D9F88AC1-3D2D-F6DB-871E-3A0E016770B1,Conditions +{0 conditions} + +D9F88AC1-3D2D-F6DB-871E-3A0E016770B1,Destination +<%ConfigFileTemplate%> + +D9F88AC1-3D2D-F6DB-871E-3A0E016770B1,Source +<%InstallDir%>/templates/original.conf + +DA33B826-E633-A845-4646-76DFA78B907B,Active +Yes + +DA33B826-E633-A845-4646-76DFA78B907B,BackButton,subst +1 + +DA33B826-E633-A845-4646-76DFA78B907B,CancelButton,subst +1 + +DA33B826-E633-A845-4646-76DFA78B907B,Caption,subst +1 + +DA33B826-E633-A845-4646-76DFA78B907B,Conditions +{0 conditions} + +DA33B826-E633-A845-4646-76DFA78B907B,Message,subst +1 + +DA33B826-E633-A845-4646-76DFA78B907B,NextButton,subst +1 + +DA33B826-E633-A845-4646-76DFA78B907B,Subtitle,subst +1 + +DA33B826-E633-A845-4646-76DFA78B907B,Title,subst +1 + +DDBBD8A9-13D7-9509-9202-419E989F60A9,Active +No + +DDBBD8A9-13D7-9509-9202-419E989F60A9,Checked +No + +DDBBD8A9-13D7-9509-9202-419E989F60A9,Conditions +{0 conditions} + +DDBBD8A9-13D7-9509-9202-419E989F60A9,Text,subst +1 + +DDBBD8A9-13D7-9509-9202-419E989F60A9,Type +checkbutton + +DDBBD8A9-13D7-9509-9202-419E989F60A9,VirtualText +AddBackupLocation + +DDBBD8A9-13D7-9509-9202-419E989F60A9,X +50 + +DDBBD8A9-13D7-9509-9202-419E989F60A9,Y +200 + +DE800F1C-CB1A-E1CE-AEB8-B0A6DB4818E7,Alias +{Install Backup Service} + +DE800F1C-CB1A-E1CE-AEB8-B0A6DB4818E7,Comment +{Install the Backup Windows Service} + +DE800F1C-CB1A-E1CE-AEB8-B0A6DB4818E7,Conditions +{0 conditions} + +DE800F1C-CB1A-E1CE-AEB8-B0A6DB4818E7,FileName +<%ShortAppName%>-program-installService + +DE800F1C-CB1A-E1CE-AEB8-B0A6DB4818E7,ShortcutName +{Install Service} + +DE800F1C-CB1A-E1CE-AEB8-B0A6DB4818E7,TargetFileName +<%InstallDir%>/tools/InstallService.bat + +DE800F1C-CB1A-E1CE-AEB8-B0A6DB4818E7,WorkingDirectory +<%InstallDir%> + +DECC120D-6904-7F17-45A49184A5A3,Active +No + +DECC120D-6904-7F17-45A49184A5A3,Conditions +{2 conditions} + +DECC120D-6904-7F17-45A49184A5A3,ShortcutName +<%AppName%> + +DECC120D-6904-7F17-45A49184A5A3,TargetFileName +<%ProgramExecutable%> + +DECC120D-6904-7F17-45A49184A5A3,WorkingDirectory +<%InstallDir%> + +DFFF91A9-2CA5-6ABE-8474D814AF88,CheckCondition +{Before Action is Executed} + +DFFF91A9-2CA5-6ABE-8474D814AF88,Operator +false + +DFFF91A9-2CA5-6ABE-8474D814AF88,String +<%SilentMode%> + +E02368C5-95B5-03A7-3282740037B0,CheckCondition +{Before Action is Executed} + +E02368C5-95B5-03A7-3282740037B0,Operator +false + +E02368C5-95B5-03A7-3282740037B0,String +<%InstallStopped%> + +E161F216-E597-B340-C1A71C476E2C,CheckCondition +{Before Action is Executed} + +E161F216-E597-B340-C1A71C476E2C,Message,subst +1 + +E161F216-E597-B340-C1A71C476E2C,Title,subst +1 + +E23AC50D-7CFB-800E-A99C6F4068F8,Alias +{Cancel Actions} + +E23AC50D-7CFB-800E-A99C6F4068F8,Conditions +{0 conditions} + +E44CFF46-6302-C518-B9C30D2E43F7,CheckCondition +{Before Action is Executed} + +E44CFF46-6302-C518-B9C30D2E43F7,String +<%CreateDesktopShortcut%> + +E4DEA723-FC78-45D7-BAB1-A3E4C4C96EA1,Conditions +{0 conditions} + +E4DEA723-FC78-45D7-BAB1-A3E4C4C96EA1,ProgramCommandLine +{net stop <%ServiceName%>} + +E56ADFF4-C15E-AEDB-A599-C468AF72C4BB,Conditions +{0 conditions} + +E56ADFF4-C15E-AEDB-A599-C468AF72C4BB,Destination +{<%InstallDir%>\templates\NotifySysAdmin.original.vbs} + +E56ADFF4-C15E-AEDB-A599-C468AF72C4BB,Source +{<%InstallDir%>\templates\NotifySysAdmin.template.vbs} + +EB2B31A1-C111-3582-0C8A5656692A,String +<%ErrorsOccurred%> + +EB532611-5F30-3C24-66EB-F3826D9054FD,CheckCondition +{Before Action is Executed} + +EB532611-5F30-3C24-66EB-F3826D9054FD,String +<%AddBackupLocation%> + +F4024A3E-9A6D-2726-5E0CFFA93054,Alias +{Uninstall Actions} + +F4024A3E-9A6D-2726-5E0CFFA93054,Conditions +{0 conditions} + +F5F21749-8B3A-49C6-9138-9C4D6D703D26,Active +No + +F5F21749-8B3A-49C6-9138-9C4D6D703D26,Conditions +{0 conditions} + +F5F21749-8B3A-49C6-9138-9C4D6D703D26,ProgramCommandLine +{cmd /k tools/7za.exe encrypted_keys.exe -p<%InstallPassword%>} + +F5F21749-8B3A-49C6-9138-9C4D6D703D26,ProgressiveOutputWidget +Message + +F5F21749-8B3A-49C6-9138-9C4D6D703D26,ShowProgressiveOutput +Yes + +F5F21749-8B3A-49C6-9138-9C4D6D703D26,WorkingDirectory +<%InstallDir%> + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,Active +Yes + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,BackButton,subst +1 + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,CancelButton,subst +1 + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,Caption,subst +1 + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,CompanyLabel,subst +0 + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,Conditions +{0 conditions} + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,Message,subst +1 + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,NextButton,subst +1 + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,Subtitle,subst +1 + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,Title,subst +1 + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,UserNameLabel,subst +1 + +F9E38720-6ABA-8B99-2471-496902E4CBC2,Active +No + +F9E38720-6ABA-8B99-2471-496902E4CBC2,Conditions +{0 conditions} + +F9E38720-6ABA-8B99-2471-496902E4CBC2,ResultVirtualText +BackupLocationPath + +F9E38720-6ABA-8B99-2471-496902E4CBC2,TclScript +{set BackupLocationPath "" } + +FB697A88-2842-468E-9776-85E84B009340,Active +No + +FB697A88-2842-468E-9776-85E84B009340,Conditions +{0 conditions} + +FB697A88-2842-468E-9776-85E84B009340,IgnoreErrors +Yes + +FB697A88-2842-468E-9776-85E84B009340,ProgramCommandLine +{"<%InstallDir%>\<%ServiceExeName%> -r -S <%ServiceName%>} + +FB697A88-2842-468E-9776-85E84B009340,WorkingDirectory +<%Temp%> + +FDF68FD6-BEA8-4A74-867D-5139F4D9E793,Active +No + +FDF68FD6-BEA8-4A74-867D-5139F4D9E793,Conditions +{0 conditions} + +FDF68FD6-BEA8-4A74-867D-5139F4D9E793,WaitTime +2000 + +FEFD090D-C133-BC95-B3564F693CD3,Alias +{Finish Actions} + +FEFD090D-C133-BC95-B3564F693CD3,Conditions +{0 conditions} + +FreeBSD-4-x86,Active +No + +FreeBSD-4-x86,DefaultDirectoryPermission +0755 + +FreeBSD-4-x86,DefaultFilePermission +0755 + +FreeBSD-4-x86,Executable +<%AppName%>-<%Version%>-<%Platform%>-Install<%Ext%> + +FreeBSD-4-x86,FallBackToConsole +Yes + +FreeBSD-4-x86,InstallDir +<%Home%>/<%ShortAppName%> + +FreeBSD-4-x86,InstallMode +Standard + +FreeBSD-4-x86,InstallType +Typical + +FreeBSD-4-x86,ProgramExecutable +{} + +FreeBSD-4-x86,ProgramFolderAllUsers +No + +FreeBSD-4-x86,ProgramFolderName +<%AppName%> + +FreeBSD-4-x86,ProgramLicense +<%InstallDir%>/LICENSE.txt + +FreeBSD-4-x86,ProgramName +{} + +FreeBSD-4-x86,ProgramReadme +<%InstallDir%>/README.txt + +FreeBSD-4-x86,PromptForRoot +Yes + +FreeBSD-4-x86,RequireRoot +No + +FreeBSD-4-x86,RootInstallDir +/usr/local/<%ShortAppName%> + +FreeBSD-x86,Active +No + +FreeBSD-x86,DefaultDirectoryPermission +0755 + +FreeBSD-x86,DefaultFilePermission +0755 + +FreeBSD-x86,Executable +<%AppName%>-<%Version%>-<%Platform%>-Install<%Ext%> + +FreeBSD-x86,FallBackToConsole +Yes + +FreeBSD-x86,InstallDir +<%Home%>/<%ShortAppName%> + +FreeBSD-x86,InstallMode +Standard + +FreeBSD-x86,InstallType +Typical + +FreeBSD-x86,ProgramExecutable +{} + +FreeBSD-x86,ProgramFolderAllUsers +No + +FreeBSD-x86,ProgramFolderName +<%AppName%> + +FreeBSD-x86,ProgramLicense +<%InstallDir%>/LICENSE.txt + +FreeBSD-x86,ProgramName +{} + +FreeBSD-x86,ProgramReadme +<%InstallDir%>/README.txt + +FreeBSD-x86,PromptForRoot +Yes + +FreeBSD-x86,RequireRoot +No + +FreeBSD-x86,RootInstallDir +/usr/local/<%ShortAppName%> + +HPUX-hppa,Active +No + +HPUX-hppa,DefaultDirectoryPermission +0755 + +HPUX-hppa,DefaultFilePermission +0755 + +HPUX-hppa,Executable +<%AppName%>-<%Version%>-<%Platform%>-Install<%Ext%> + +HPUX-hppa,FallBackToConsole +Yes + +HPUX-hppa,InstallDir +<%Home%>/<%ShortAppName%> + +HPUX-hppa,InstallMode +Standard + +HPUX-hppa,InstallType +Typical + +HPUX-hppa,ProgramExecutable +{} + +HPUX-hppa,ProgramFolderAllUsers +No + +HPUX-hppa,ProgramFolderName +<%AppName%> + +HPUX-hppa,ProgramLicense +<%InstallDir%>/LICENSE.txt + +HPUX-hppa,ProgramName +{} + +HPUX-hppa,ProgramReadme +<%InstallDir%>/README.txt + +HPUX-hppa,PromptForRoot +Yes + +HPUX-hppa,RequireRoot +No + +HPUX-hppa,RootInstallDir +/usr/local/<%ShortAppName%> + +Linux-x86,Active +No + +Linux-x86,BuildType +dynamic + +Linux-x86,DefaultDirectoryPermission +00755 + +Linux-x86,DefaultFilePermission +00755 + +Linux-x86,Executable +<%AppName%>-<%Version%>-<%Platform%>-Install<%Ext%> + +Linux-x86,FallBackToConsole +Yes + +Linux-x86,InstallDir +<%Home%>/<%ShortAppName%> + +Linux-x86,InstallMode +Standard + +Linux-x86,InstallType +Typical + +Linux-x86,ProgramExecutable +<%InstallDir%>/TebucoSafe + +Linux-x86,ProgramFolderAllUsers +No + +Linux-x86,ProgramFolderName +<%AppName%> + +Linux-x86,ProgramLicense +<%InstallDir%>/LICENSE.txt + +Linux-x86,ProgramName +{} + +Linux-x86,ProgramReadme +<%InstallDir%>/README.txt + +Linux-x86,PromptForRoot +Yes + +Linux-x86,RequireRoot +No + +Linux-x86,RootInstallDir +/usr/local/<%ShortAppName%> + +Solaris-sparc,Active +No + +Solaris-sparc,DefaultDirectoryPermission +0755 + +Solaris-sparc,DefaultFilePermission +0755 + +Solaris-sparc,Executable +<%AppName%>-<%Version%>-<%Platform%>-Install<%Ext%> + +Solaris-sparc,FallBackToConsole +Yes + +Solaris-sparc,InstallDir +<%Home%>/<%ShortAppName%> + +Solaris-sparc,InstallMode +Standard + +Solaris-sparc,InstallType +Typical + +Solaris-sparc,ProgramExecutable +{} + +Solaris-sparc,ProgramFolderAllUsers +No + +Solaris-sparc,ProgramFolderName +<%AppName%> + +Solaris-sparc,ProgramLicense +<%InstallDir%>/LICENSE.txt + +Solaris-sparc,ProgramName +{} + +Solaris-sparc,ProgramReadme +<%InstallDir%>/README.txt + +Solaris-sparc,PromptForRoot +Yes + +Solaris-sparc,RequireRoot +No + +Solaris-sparc,RootInstallDir +/usr/local/<%ShortAppName%> + +TarArchive,Active +No + +TarArchive,CompressionLevel +6 + +TarArchive,DefaultDirectoryPermission +0755 + +TarArchive,DefaultFilePermission +0755 + +TarArchive,Executable +<%AppName%>-<%Version%>-<%Platform%>-Install<%Ext%> + +TarArchive,FallBackToConsole +Yes + +TarArchive,InstallDir +<%Home%>/<%ShortAppName%> + +TarArchive,InstallMode +Standard + +TarArchive,InstallType +Typical + +TarArchive,OutputFileName +<%ShortAppName%>-<%Version%>.tar.gz + +TarArchive,ProgramExecutable +{} + +TarArchive,ProgramFolderAllUsers +No + +TarArchive,ProgramFolderName +<%AppName%> + +TarArchive,ProgramLicense +<%InstallDir%>/LICENSE.txt + +TarArchive,ProgramName +{} + +TarArchive,ProgramReadme +<%InstallDir%>/README.txt + +TarArchive,PromptForRoot +Yes + +TarArchive,RequireRoot +No + +TarArchive,RootInstallDir +/usr/local/<%ShortAppName%> + +TarArchive,VirtualTextMap +{<%InstallDir%> <%ShortAppName%>} + +Windows,Active +Yes + +Windows,BuildType +{} + +Windows,Executable +installer.exe + +Windows,IncludeTWAPI +No + +Windows,InstallDir +{C:\Program Files\<%BrandName%>} + +Windows,InstallMode +Standard + +Windows,InstallType +Typical + +Windows,ProgramExecutable +{} + +Windows,ProgramFolderAllUsers +No + +Windows,ProgramFolderName +<%BrandName%> + +Windows,ProgramLicense +{<%InstallDir%>\LICENSE.txt} + +Windows,ProgramName +{} + +Windows,ProgramReadme +{} + +Windows,WindowsIcon +{} + +ZipArchive,Active +No + +ZipArchive,CompressionLevel +6 + +ZipArchive,DefaultDirectoryPermission +0755 + +ZipArchive,DefaultFilePermission +0755 + +ZipArchive,Executable +<%AppName%>-<%Version%>-<%Platform%>-Install<%Ext%> + +ZipArchive,FallBackToConsole +Yes + +ZipArchive,InstallDir +<%Home%>/<%ShortAppName%> + +ZipArchive,InstallMode +Standard + +ZipArchive,InstallType +Typical + +ZipArchive,OutputFileName +<%ShortAppName%>-<%Version%>.zip + +ZipArchive,ProgramExecutable +{} + +ZipArchive,ProgramFolderAllUsers +No + +ZipArchive,ProgramFolderName +<%AppName%> + +ZipArchive,ProgramLicense +<%InstallDir%>/LICENSE.txt + +ZipArchive,ProgramName +{} + +ZipArchive,ProgramReadme +<%InstallDir%>/README.txt + +ZipArchive,PromptForRoot +Yes + +ZipArchive,RequireRoot +No + +ZipArchive,RootInstallDir +/usr/local/<%ShortAppName%> + +ZipArchive,VirtualTextMap +{<%InstallDir%> <%ShortAppName%>} + +} + +::msgcat::mcmset de { +20CBDBEA-2217-457B-8D98-D692C4F591E9,Message +<%UninstallCompleteText%> + +2BF07B5A-9B06-4C1E-810D-5B5E9303D2C6,Message +<%InstallationCompleteText%> + +B002A311-F8E7-41DE-B039-521391924E5B,Message +<%InstallingApplicationText%> + +B4ED4636-22D8-41DC-9E3D-BD1E1CAD2174,Message +<%UninstallingApplicationText%> + +} +::msgcat::mcmset en { +16D53E40-546B-54C3-088B1B5E3BBB,Text +<%CreateDesktopShortcutText%> + +20CBDBEA-2217-457B-8D98-D692C4F591E9,Message +<%UninstallCompleteText%> + +2BF07B5A-9B06-4C1E-810D-5B5E9303D2C6,Message +<%InstallationCompleteText%> + +2E2963BD-DDBD-738D-A910-B7F3F04946F9,Text +{No:<%BackupLocationNumber%> } + +2EC82FBD-8294-A3E4-7F39-1CBA0582FA64,TextToWrite +BackupLocations\n\{\n + +32F5B0AF-EB83-7A03-D8FAE1ECE473,Message +<%InstallStartupText%> + +32F5B0AF-EB83-7A03-D8FAE1ECE473,Title +<%InstallApplicationText%> + +36FF8915-8148-0F1F-27D7239CBFA1,Text +<%ViewReadmeText%> + +3B6E2E7C-1A26-27F1-D578E383B128,Text +<%ViewReadmeText%> + +3D33AA8C-0037-204B-39A339FD38BD,Message +{<%BrandName%> has been removed from your system. Thank you for using the <%BrandName%> Backup Service. If you need further assistance, please contact us at http://<%BrandName%>.com or support@<%BrandName%>.com.} + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Caption +{} + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Message +{} + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Subtitle +{Add a directory to backup, nickname it, and add some exclusions.} + +3FD9BFF3-2F6E-E4FC-2FAE-98F2017916A7,Title +{Backup Locations} + +3FDB57ED-598D-8A4E-CEF7-D90833305558,Text +{Backup Directory} + +4A9C852B-647E-EED5-5482FFBCC2AF,Description +<%ProgramFilesDescription%> + +4ACB0B47-42B3-2B3A-BFE9AA4EC707,Message +<%UninstallStartupText%> + +4ACB0B47-42B3-2B3A-BFE9AA4EC707,Title +<%UninstallApplicationText%> + +58E1119F-639E-17C9-5D3898F385AA,Caption +{Setup will install <%ShortAppName%> in the following folder. + +To install to this folder, click Next. To install to a different folder, click Browse and select another folder.} + +58E1119F-639E-17C9-5D3898F385AA,Subtitle +{Where should <%ShortAppName%> be installed?} + +59395A3B-6116-F93A-84E1-5E079C2CD44B,Caption +{Backup Exclusions} + +59395A3B-6116-F93A-84E1-5E079C2CD44B,Message +{} + +59395A3B-6116-F93A-84E1-5E079C2CD44B,Subtitle +{Enter your backup exclusions for this Backup Location <%BackupLocationName_ShortName%> here.} + +59395A3B-6116-F93A-84E1-5E079C2CD44B,Text +{} + +59395A3B-6116-F93A-84E1-5E079C2CD44B,Title +{Backup Exclusions} + +640DA2B2-6CF3-0873-D7AE-ABCDDE39EFCF,Text +{Put your custom text here. +<%AddBackupLocation%> +<%BackupLocationNumber%> +<%BackupLocationName%>} + +6C323815-B9AB-FA94-4F5D152EBC51,Caption +{Installation Wizard Complete} + +6C323815-B9AB-FA94-4F5D152EBC51,Message +{The Installation Wizard has successfully installed the <%BrandName%> Backup Service. Click Finish to exit the wizard.} + +6CFBBE13-6B70-4B7C-B5EF-0677752D95A8,Caption +{Installing encryption keys...} + +6CFBBE13-6B70-4B7C-B5EF-0677752D95A8,Message +{Click next to install encrypted keys and configuration file (bbackupd.conf)... + +You will be presented with the current configuration file so that you may make any last minute changes...} + +6FEE2889-0338-1D49-60BF-1471F465AB26,TextToWrite +\}\n + +8202CECC-54A0-9B6C-D24D111BA52E,Description +<%TypicalInstallDescription%> + +8419AAAD-5860-F73E-8D11-4D1BDA4D7D37,Text +{Add another backup location?} + +855DE408-060E-3D35-08B5-1D9AB05C2865,Text +Exclusions + +8E095096-F018-A880-429D-A2177A9B70EA,Text +{AddAnother: <%AddBackupLocation%>} + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,Caption +{Please enter your phone number and email address.} + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,CompanyLabel +Email: + +9013E862-8E81-5290-64F9-D8BCD13EC7E5,UserNameLabel +Phone: + +937C3FDD-FB28-98BD-3DAB276E59ED,Text +<%CreateQuickLaunchShortcutText%> + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Caption +{Click Next to continue...} + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Message +{Building configuration file...} + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Subtitle +{} + +9A23D3ED-4D9D-9C57-C2A7-71DE0FFF0266,Title +Continue + +9BAB328D-414B-D351-CA8D-824DF94B9DCA,Text +{Add another backup location after this one?} + +A18C2977-1409-C1FB-892415711F72,Text +<%LaunchApplicationText%> + +AAF2142A-9FC9-4664-DFF2-13B9EB7BA0E1,CompanyLabel +Company: + +AE3BD5B4-35DE-4240-B79914D43E56,Caption +{Welcome to the Installation Wizard for <%BrandName%> Backup Service!} + +AE3BD5B4-35DE-4240-B79914D43E56,Message +{Thank you for installing the <%BrandName%>(SM) Backup Service. + +If you need any assistance, please contact us at http://<%BrandName%>.com or support@<%BrandName%>.com. + +This will install <%BrandName%> version <%Version%> on your computer. + +It is recommended that you close all other applications before continuing. + +Click Next to continue or Cancel to exit Setup. +} + +B002A311-F8E7-41DE-B039-521391924E5B,Message +<%InstallingApplicationText%> + +B4404713-AF4F-4F4B-670F3115517F,Description +<%CustomInstallDescription%> + +B4ED4636-22D8-41DC-9E3D-BD1E1CAD2174,Message +<%UninstallingApplicationText%> + +B506E7DA-E7C4-4D42-8C03-FD27BA16D078,Text +{Box Backup, http://www.fluffy.co.uk/boxbackup +Copyright (c) 2003-2007 Ben Summers and contributors. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. All use of this software and associated advertising materials must display the following acknowledgement: + This product includes software developed by Ben Summers and contributors. + +4. The names of the Authors may not be used to endorse or promote products derived from this software without specific prior written permission. + +[Where legally impermissible the Authors do not disclaim liability for direct physical injury or death caused solely by defects in the software unless it is modified by a third party.] + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE} + +B57F8C91-2439-CFD3-7EB5-57D4EA48D3C6,Caption +{<%BrandName%> will backup the following folder. + +To backup this folder, click Next. To backup a different folder, click Browse and select another folder.} + +B57F8C91-2439-CFD3-7EB5-57D4EA48D3C6,DestinationLabel +{Backup This Folder} + +B57F8C91-2439-CFD3-7EB5-57D4EA48D3C6,Message +{} + +B57F8C91-2439-CFD3-7EB5-57D4EA48D3C6,Subtitle +{What directory should <%BrandName%> backup?} + +B57F8C91-2439-CFD3-7EB5-57D4EA48D3C6,Title +{Choose Backup Location} + +B927A5AF-4DFE-82A3-DCA8-35FA4D91EC5A,Text +{Short Name} + +B9B85EF1-1D76-4BF5-ABB9-092A8DB35851,Caption +{Please enter the agreed-upon password for your encrypted key file...} + +B9B85EF1-1D76-4BF5-ABB9-092A8DB35851,CompanyLabel +Password: + +B9B85EF1-1D76-4BF5-ABB9-092A8DB35851,Subtitle +{Please enter your encrypted key file password.} + +C105AAAE-7C16-2C9E-769FE4535B60,Caption +<%ApplicationReadmeText%> + +C105AAAE-7C16-2C9E-769FE4535B60,Message +{} + +C105AAAE-7C16-2C9E-769FE4535B60,Title +<%ApplicationReadmeText%> + +C7762473-273F-E3CA-17E3-65789B14CDB0,TextToWrite +{<%BackupLocationShortName%> +{ +Path = <%BackupLocationPath%> +<%BackupLocationExclusions%> +} +} + +CB058DBA-C3B7-2F48-D985-BE2F7107A76D,BrowseText +{To continue, click Next. If you would like to select a folder to backup, click Browse.} + +CB058DBA-C3B7-2F48-D985-BE2F7107A76D,Caption +{} + +CB058DBA-C3B7-2F48-D985-BE2F7107A76D,DestinationLabel +{Backup This Folder} + +CB058DBA-C3B7-2F48-D985-BE2F7107A76D,Subtitle +{What directories should <%BrandName%> backup?} + +CB058DBA-C3B7-2F48-D985-BE2F7107A76D,Title +{Choose Backup Location} + +CFFA27AF-A641-E41C-B4A0E3BB3CBB,Text +<%LaunchApplicationText%> + +D4625CA6-9864-D8EF-F252D7B7DC87,Text +<%CreateDesktopShortcutText%> + +D47BE952-79F2-844E-D2E5-8F22044E7A9D,Text +{Account Number:} + +DA33B826-E633-A845-4646-76DFA78B907B,Caption +{Click Next to continue...} + +DA33B826-E633-A845-4646-76DFA78B907B,Message +{Completing configuration file...} + +DA33B826-E633-A845-4646-76DFA78B907B,Subtitle +{} + +DA33B826-E633-A845-4646-76DFA78B907B,Title +Continue + +DDBBD8A9-13D7-9509-9202-419E989F60A9,Text +{Add another Backup Location?} + +E0CADC4E-08A6-E429-3B49-BB8CFB7B097F,Text +{Simple name for this Backup Location (short, no spaces or special characters)} + +E161F216-E597-B340-C1A71C476E2C,Message +<%UninstallLeftoverText%> + +E161F216-E597-B340-C1A71C476E2C,Title +{Uninstall <%BrandName%>} + +EA2C57E8-CCFB-CF3D-92CA-83369EFF1B08,Text +{Add (another) Backup Location after this one?} + +EDE364D6-22C7-5108-D398-26FC24E0A55A,Text +{Enter your Backup Exclusions for this Backup Location...} + +F59AF47D-4136-64F8-82C7-4506BD4327FD,Text +{Add another Backup Location after this one?} + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,Caption +{Please enter your Account Number.} + +F8FD4BD6-F1DF-3F8D-B857-98310E4B1143,UserNameLabel +{Account Number (like 10005004):} + +F98784B1-1965-0F42-6BB0542AE1A9,Caption +{Installing and starting the TebucoSafe Backup Service...} + +F98784B1-1965-0F42-6BB0542AE1A9,Message +{Click Next to install the TebucoSafe Backup Service as an operating system service on your computer (see services.msc), and start up that service. } + +FC678E76-6823-2E55-204CA01C35EF,Text +<%CreateQuickLaunchShortcutText%> + +FF4F6EEA-F4CC-428E-AF33-EB0E88E2147E,Text +{ +Copyright (c) 2003 - 2006 + Ben Summers and contributors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. All use of this software and associated advertising materials must + display the following acknowledgment: + This product includes software developed by Ben Summers. +4. The names of the Authors may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +Where legally impermissible the Authors do not disclaim liability for +direct physical injury or death caused solely by defects in the software +unless it is modified by a third party.] + +THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + + } + +} +::msgcat::mcmset es { +20CBDBEA-2217-457B-8D98-D692C4F591E9,Message +<%UninstallCompleteText%> + +2BF07B5A-9B06-4C1E-810D-5B5E9303D2C6,Message +<%InstallationCompleteText%> + +B002A311-F8E7-41DE-B039-521391924E5B,Message +<%InstallingApplicationText%> + +B4ED4636-22D8-41DC-9E3D-BD1E1CAD2174,Message +<%UninstallingApplicationText%> + +} +::msgcat::mcmset fr { +20CBDBEA-2217-457B-8D98-D692C4F591E9,Message +<%UninstallCompleteText%> + +2BF07B5A-9B06-4C1E-810D-5B5E9303D2C6,Message +<%InstallationCompleteText%> + +B002A311-F8E7-41DE-B039-521391924E5B,Message +<%InstallingApplicationText%> + +B4ED4636-22D8-41DC-9E3D-BD1E1CAD2174,Message +<%UninstallingApplicationText%> + +} +::msgcat::mcmset pl { +20CBDBEA-2217-457B-8D98-D692C4F591E9,Message +<%UninstallCompleteText%> + +2BF07B5A-9B06-4C1E-810D-5B5E9303D2C6,Message +<%InstallationCompleteText%> + +B002A311-F8E7-41DE-B039-521391924E5B,Message +<%InstallingApplicationText%> + +B4ED4636-22D8-41DC-9E3D-BD1E1CAD2174,Message +<%UninstallingApplicationText%> + +} +::msgcat::mcmset pt_br { +20CBDBEA-2217-457B-8D98-D692C4F591E9,Message +<%UninstallCompleteText%> + +2BF07B5A-9B06-4C1E-810D-5B5E9303D2C6,Message +<%InstallationCompleteText%> + +B002A311-F8E7-41DE-B039-521391924E5B,Message +<%InstallingApplicationText%> + +B4ED4636-22D8-41DC-9E3D-BD1E1CAD2174,Message +<%UninstallingApplicationText%> + +} + diff --git a/contrib/windows/installer/tools/InstallService.bat b/contrib/windows/installer/tools/InstallService.bat new file mode 100755 index 00000000..80a342eb --- /dev/null +++ b/contrib/windows/installer/tools/InstallService.bat @@ -0,0 +1,3 @@ +service.exe -i -S GigaLock +echo off +ping 192.168.254.254 -n 5 -w 1000 > nul diff --git a/contrib/windows/installer/tools/KillBackupProcess.bat b/contrib/windows/installer/tools/KillBackupProcess.bat new file mode 100755 index 00000000..416d4a79 --- /dev/null +++ b/contrib/windows/installer/tools/KillBackupProcess.bat @@ -0,0 +1,3 @@ +control.exe terminate +echo off +ping 192.168.254.254 -n 5 -w 1000 > nul diff --git a/contrib/windows/installer/tools/QueryOutputAll.bat b/contrib/windows/installer/tools/QueryOutputAll.bat new file mode 100755 index 00000000..2ab30a72 --- /dev/null +++ b/contrib/windows/installer/tools/QueryOutputAll.bat @@ -0,0 +1,5 @@ +@ECHO OFF +: o=old, d=deleted, s=size info, t=timestamp, r=recursive +set Queryopts=-odstr +::set Queryopts=-str +query.exe "list %Queryopts%" quit > QueryOutputAllResults.txt diff --git a/contrib/windows/installer/tools/QueryOutputCurrent.bat b/contrib/windows/installer/tools/QueryOutputCurrent.bat new file mode 100755 index 00000000..d59ddbf1 --- /dev/null +++ b/contrib/windows/installer/tools/QueryOutputCurrent.bat @@ -0,0 +1,5 @@ +@ECHO OFF +: o=old, d=deleted, s=size info, t=timestamp, r=recursive +::set Queryopts=-odstr +set Queryopts=-str +query.exe "list %Queryopts%" quit > QueryOutputCurrentResults.txt diff --git a/contrib/windows/installer/tools/ReloadConfig.bat b/contrib/windows/installer/tools/ReloadConfig.bat new file mode 100755 index 00000000..5fd44e83 --- /dev/null +++ b/contrib/windows/installer/tools/ReloadConfig.bat @@ -0,0 +1,3 @@ +control.exe reload +echo off +ping 192.168.254.254 -n 8 -w 1000 > nul diff --git a/contrib/windows/installer/tools/RemoteControl.exe b/contrib/windows/installer/tools/RemoteControl.exe new file mode 100755 index 00000000..f7667421 Binary files /dev/null and b/contrib/windows/installer/tools/RemoteControl.exe differ diff --git a/contrib/windows/installer/tools/RemoveService.bat b/contrib/windows/installer/tools/RemoveService.bat new file mode 100755 index 00000000..881ec5b1 --- /dev/null +++ b/contrib/windows/installer/tools/RemoveService.bat @@ -0,0 +1,3 @@ +@@SERVICEEXENAME@ -r -S GigaLock +echo off +ping 192.168.254.254 -n 5 -w 1000 > nul diff --git a/contrib/windows/installer/tools/RestartService.bat b/contrib/windows/installer/tools/RestartService.bat new file mode 100755 index 00000000..77092a69 --- /dev/null +++ b/contrib/windows/installer/tools/RestartService.bat @@ -0,0 +1,5 @@ +net stop GigaLock +ping 192.168.254.254 -n 2 -w 1000 > nul +net start GigaLock +echo off +ping 192.168.254.254 -n 5 -w 1000 > nul diff --git a/contrib/windows/installer/tools/ShowUsage.bat b/contrib/windows/installer/tools/ShowUsage.bat new file mode 100755 index 00000000..e6f69e9f --- /dev/null +++ b/contrib/windows/installer/tools/ShowUsage.bat @@ -0,0 +1,3 @@ +query.exe usage quit +ping 192.168.254.254 -n 10 -w 1000 > nul + diff --git a/contrib/windows/installer/tools/StartService.bat b/contrib/windows/installer/tools/StartService.bat new file mode 100755 index 00000000..1771238f --- /dev/null +++ b/contrib/windows/installer/tools/StartService.bat @@ -0,0 +1,3 @@ +net start GigaLock +echo off +ping 192.168.254.254 -n 5 -w 1000 > nul diff --git a/contrib/windows/installer/tools/StopService.bat b/contrib/windows/installer/tools/StopService.bat new file mode 100755 index 00000000..8e70d68d --- /dev/null +++ b/contrib/windows/installer/tools/StopService.bat @@ -0,0 +1,3 @@ +net stop GigaLock +echo off +ping 192.168.254.254 -n 5 -w 1000 > nul diff --git a/contrib/windows/installer/tools/Sync.bat b/contrib/windows/installer/tools/Sync.bat new file mode 100755 index 00000000..30a04bec --- /dev/null +++ b/contrib/windows/installer/tools/Sync.bat @@ -0,0 +1,3 @@ +control.exe sync +echo off +ping 192.168.254.254 -n 5 -w 1000 > nul diff --git a/distribution/COMMON-MANIFEST.txt b/distribution/COMMON-MANIFEST.txt index f2be35ba..b6f4b6d7 100644 --- a/distribution/COMMON-MANIFEST.txt +++ b/distribution/COMMON-MANIFEST.txt @@ -1,4 +1,5 @@ RUN ./bootstrap +RUN cd docs; make lib/common lib/crypto lib/server @@ -11,25 +12,25 @@ test/basicserver/testfiles test/crypto test/compress test/win32 -docs/common notes -docs/common/lib_common notes/lib_common -docs/common/lib_crypto notes/lib_crypto -docs/common/lib_server notes/lib_server +docs/api-notes +docs/api-notes/common/lib_common +docs/api-notes/common/lib_crypto +docs/api-notes/common/lib_server MKDIR infrastructure infrastructure/buildenv-testmain-template.cpp infrastructure/makebuildenv.pl.in infrastructure/makedistribution.pl.in infrastructure/makeparcels.pl.in +infrastructure/parcelpath.pl +infrastructure/printversion.pl infrastructure/BoxPlatform.pm.in infrastructure/mingw infrastructure/msvc -configure.ac NO-LICENSE config.sub -config.sub NO-LICENSE config.guess -config.guess bootstrap +configure.ac +configure parcels.txt runtest.pl.in NO-LICENSE-IN-DIR infrastructure/m4 -configure diff --git a/distribution/boxbackup/CONTACT.txt b/distribution/boxbackup/CONTACT.txt index 849c1e76..9f12e435 100644 --- a/distribution/boxbackup/CONTACT.txt +++ b/distribution/boxbackup/CONTACT.txt @@ -1,6 +1,6 @@ -http://www.fluffy.co.uk/boxbackup/ +http://www.boxbackup.org/ -Ben Summers -ben@fluffy.co.uk +Ben Summers & contributors +boxbackup@boxbackup.org diff --git a/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt index dcb23cf3..2147e6e3 100644 --- a/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt +++ b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt @@ -19,21 +19,51 @@ test/backupstorepatch test/bbackupd test/bbackupd/testfiles test/backupdiff -docs/raidfile notes -docs/raidfile/lib_raidfile notes/lib_raidfile -docs/backup notes -documentation -documentation/html -documentation/html/images +docs/Makefile +docs/tools +docs/api-notes +docs/api-notes/raidfile/lib_raidfile +docs/htmlguide +docs/htmlguide/adminguide +docs/htmlguide/images +docs/htmlguide/instguide +docs/htmlguide/manpages +docs/man +MKDIR docs/docbook +docs/docbook/ExceptionCodes.xml +docs/docbook/adminguide.xml +docs/docbook/bb-book.xsl +docs/docbook/bb-man.xsl.tmpl +docs/docbook/bb-nochunk-book.xsl +docs/docbook/bbackupctl.xml +docs/docbook/bbackupd-config.xml +docs/docbook/bbackupd.conf.xml +docs/docbook/bbackupd.xml +docs/docbook/bbackupquery.xml +docs/docbook/bbstoreaccounts.xml +docs/docbook/bbstored-certs.xml +docs/docbook/bbstored-config.xml +docs/docbook/bbstored.conf.xml +docs/docbook/bbstored.xml +docs/docbook/instguide.xml +docs/docbook/raidfile-config.xml +docs/docbook/raidfile.conf.xml +docs/docbook/html +docs/docbook/html/images TODO.txt BUGS.txt contrib NO-LICENSE-IN-DIR contrib/debian +NO-LICENSE-IN-DIR contrib/mac_osx NO-LICENSE-IN-DIR contrib/redhat NO-LICENSE-IN-DIR contrib/rpm REPLACE-VERSION-IN contrib/rpm/boxbackup.spec NO-LICENSE-IN-DIR contrib/solaris NO-LICENSE-IN-DIR contrib/suse +NO-LICENSE-IN-DIR contrib/bbreporter +NO-LICENSE-IN-DIR contrib/windows +NO-LICENSE-IN-DIR contrib/windows/installer +NO-LICENSE-IN-DIR contrib/windows/installer/tools infrastructure/msvc NO-LICENSE-IN-DIR infrastructure/msvc/2003 NO-LICENSE-IN-DIR infrastructure/msvc/2005 diff --git a/distribution/boxbackup/DOCUMENTATION.txt b/distribution/boxbackup/DOCUMENTATION.txt index 1cda3a2a..f45f61d5 100644 --- a/distribution/boxbackup/DOCUMENTATION.txt +++ b/distribution/boxbackup/DOCUMENTATION.txt @@ -1,6 +1,6 @@ For compilation and installation instructions, see the web site at - http://www.fluffy.co.uk/boxbackup/ + http://www.boxbackup.org/ diff --git a/distribution/boxbackup/VERSION.txt b/distribution/boxbackup/VERSION.txt index f32a59a7..990d8ebe 100644 --- a/distribution/boxbackup/VERSION.txt +++ b/distribution/boxbackup/VERSION.txt @@ -1,2 +1,2 @@ -0.11rc2 +0.11_USE_SVN_VERSION boxbackup diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..f337fd86 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,129 @@ + + +# Process DocBook to HTML + +DBPROC=xsltproc + +DOCBOOK_DIR = docbook +HTML_DIR = htmlguide +MAN_DIR = man + +BOOKXSL = $(DOCBOOK_DIR)/bb-book.xsl +NOCHUNKBOOKXSL = $(DOCBOOK_DIR)/bb-nochunk-book.xsl +MANXSL = $(DOCBOOK_DIR)/bb-man.xsl + +VPATH = $(DOCBOOK_DIR) +.SUFFIXES: .html .xml .gz .1 .5 .8 + +all: docs + +docs: instguide adminguide manpages + @mkdir -p $(HTML_DIR)/images + @cp $(DOCBOOK_DIR)/html/images/*.png $(HTML_DIR)/images/. + @cp $(DOCBOOK_DIR)/html/*.css $(HTML_DIR)/. + @cp $(DOCBOOK_DIR)/html/*.ico $(HTML_DIR)/. + +adminguide: $(DOCBOOK_DIR)/ExceptionCodes.xml $(HTML_DIR)/adminguide/index.html + +# all sources ($>) is exactly the right args for xsltproc +$(HTML_DIR)/adminguide/index.html: $(BOOKXSL) $(DOCBOOK_DIR)/adminguide.xml + $(DBPROC) -o $(HTML_DIR)/adminguide/ $> + +instguide: $(HTML_DIR)/instguide/index.html + +$(HTML_DIR)/instguide/index.html: $(BOOKXSL) $(DOCBOOK_DIR)/instguide.xml + $(DBPROC) -o $(HTML_DIR)/instguide/ $> + +# $< is empty on BSD make when making this rule, $> has all sources +# $< has the target on GNU make, $> is empty +$(DOCBOOK_DIR)/ExceptionCodes.xml: ../ExceptionCodes.txt + perl tools/generate_except_xml.pl $< $> $@ + +manpages: man-dirs man-nroff man-html + +xslt: $(MANXSL) + +$(MANXSL): $(MANXSL).tmpl + @if [ -f /usr/local/share/xsl/docbook/manpages/docbook.xsl ]; then \ + DOCBOOK=file:///usr/local/share/xsl/docbook/manpages/docbook.xsl; \ + elif [ -f /opt/local/share/xsl/docbook-xsl/manpages/docbook.xsl ]; then \ + DOCBOOK=file:///opt/local/share/xsl/docbook-xsl/manpages/docbook.xsl; \ + elif [ -f /usr/share/sgml/docbook/xsl-stylesheets/manpages/docbook.xsl ]; then \ + DOCBOOK=file:///usr/share/sgml/docbook/xsl-stylesheets/manpages/docbook.xsl; \ + else \ + DOCBOOK=http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl; \ + fi; \ + sed -e "s,%%DOCBOOK%%,$${DOCBOOK}," $(MANXSL).tmpl > $(MANXSL) + +man-dirs: man/.there $(HTML_DIR)/man-html/.there + +$(HTML_DIR)/man-html/.there: + mkdir -p $(HTML_DIR)/man-html + touch $(HTML_DIR)/man-html/.there + +man/.there: + mkdir -p man + touch man/.there + +NROFF_PAGES = bbackupd.8 bbackupd-config.8 bbackupctl.8 bbackupquery.8 \ + bbstored.8 bbstored-config.8 bbstoreaccounts.8 bbstored-certs.8 \ + raidfile-config.8 \ + bbackupd.conf.5 bbstored.conf.5 raidfile.conf.5 + +NROFF_FILES = $(NROFF_PAGES:%=$(MAN_DIR)/%.gz) + +man-nroff: $(NROFF_FILES) + +HTML_FILES_1 = $(NROFF_PAGES:%.5=%.html) +HTML_FILES_2 = $(HTML_FILES_1:%.8=%.html) +HTML_FILES = $(HTML_FILES_2:%=$(HTML_DIR)/man-html/%) + +man-html: $(HTML_FILES) + +# GNU make +$(HTML_DIR)/man-html/%.html: $(DOCBOOK_DIR)/%.xml $(NOCHUNKBOOKXSL) + $(DBPROC) -o $@ $(NOCHUNKBOOKXSL) $< + +# Before running xsltproc to generate manual pages, we need to check +# that $(MANXSL) has been built. We don't want to add it to dependencies, +# because that would cause # the man pages to try to be rebuilt even if +# they already exist if the date of the xslt file changes, and that +# requires xsltproc, which negates the point of precompiling them for +# distribution users. + +# GNU make +$(MAN_DIR)/%.8.gz: $(DOCBOOK_DIR)/%.xml + $(MAKE) xslt + $(DBPROC) -o $(@:.gz=) $(MANXSL) $< + gzip $(@:.gz=) + +# GNU make +$(MAN_DIR)/%.5.gz: $(DOCBOOK_DIR)/%.xml + $(MAKE) xslt + $(DBPROC) -o $(@:.gz=) $(MANXSL) $< + gzip $(@:.gz=) + +# BSD make: the final colon (:) is required to make the .for and .endfor +# lines valid in GNU make. It creates (different) dummy rules in GNU and +# BSD make. Both dummy rules are harmless. + +.for MAN_PAGE in $(NROFF_PAGES) : +$(MAN_DIR)/$(MAN_PAGE).gz: $(DOCBOOK_DIR)/$(MAN_PAGE:R).xml + $(MAKE) xslt + $(DBPROC) -o $(.TARGET:.gz=) $(MANXSL) $> + gzip $(@:.gz=) + +$(HTML_DIR)/man-html/$(MAN_PAGE:R).html: $(DOCBOOK_DIR)/$(MAN_PAGE:R).xml + $(DBPROC) -o $@ $(NOCHUNKBOOKXSL) $> +.endfor : + +dockit: clean docs + tar zcf documentation-kit-0.10.tar.gz $(HTML_DIR)/ + +clean: + rm -f $(HTML_FILES) + rm -f $(NROFF_FILES) + rm -f $(DOCBOOK_DIR)/ExceptionCodes.xml + rm -f documentation-kit-0.10.tar.gz + rm -f $(MANXSL) + diff --git a/docs/api-notes/INDEX.txt b/docs/api-notes/INDEX.txt new file mode 100644 index 00000000..f333ac3b --- /dev/null +++ b/docs/api-notes/INDEX.txt @@ -0,0 +1,61 @@ +TITLE Programmers Notes for Box Backup + +This directory contains the programmers notes for the Box Backup system. They will be of interest only if you want to review or modify the code. + +These notes are intended to be run through a script to produce HTML at some later stage, hence the marks such as 'TITLE'. + + +SUBTITLE Organisation + +The project is split up into several modules. The modules within 'lib' are building blocks for creation of the actual executable programs within 'bin'. 'test' contains unit tests for lib and bin modules. + +The file modules.txt lists the modules which are to be built, and their dependencies. It also allows for platform differences. + + +SUBTITLE Documentation Organisation + +In this directory, the files correspond to modules or areas of interest. Sub directories of the same name contain files documenting specific classes within that module. + + +SUBTITLE Suggested reading order + +* common/lib_common.txt +* common/lib_server.txt +* bin_bbackupd.txt +* backup_encryption.txt +* bin_bbstored.txt +* raidfile/lib_raidfile.txt + +and refer to other sections as required. + + +SUBTITLE Building + +The makefiles are generated by makebuildenv.pl. (The top level makefile is generated by makeparcels.pl, but this is for the end user to use, not a programmer.) + +To build a module, cd to it and type make. If the -DRELEASE option is specified (RELEASE=1 with GNU make) the release version will be built. The object files and exes are placed in a directory structure under 'release' or 'debug'. + +It is intended that a test will be written for everything, so in general make commands will be issued only within the test/* modules. Once it has been built, cd to debug/test/ and run the test with ./t . + + +SUBTITLE Programming style + +The code is written to be easy to write. Ease of programming is the primary concern, as this should lead to fewer bugs. Efficiency improvements can be made later when the system as a whole works. + +Much use is made of the STL. + +There is no common base class. + +All errors are reported using exceptions. + +Some of the boring code is generated by perl scripts from description files. + +There are a lot of modules and classes which can easily be used to build other projects in the future -- there is a lot of "framework" code. + + +SUBTITLE Lots more documentation + +The files are extensively commented. Consider this notes as an overview, and then read the source files for detailed and definitive information. + +Each function and class has a very brief decsription of it's purpose in a standard header, and extensive efforts have been maed to comment the code itself. + diff --git a/docs/api-notes/Win32_Clients.txt b/docs/api-notes/Win32_Clients.txt new file mode 100644 index 00000000..fad6c4af --- /dev/null +++ b/docs/api-notes/Win32_Clients.txt @@ -0,0 +1,13 @@ +The basic client tools now run on Win32 natively. +The port was done by nick@omniis.com. + +* bbackupd +* bbackupquery +* bbackupctl + +Have been ported. bbackupd runs as a NT style service. + +Known limitations: + +* File attributes and permissions are not backed up. + diff --git a/docs/api-notes/backup_encryption.txt b/docs/api-notes/backup_encryption.txt new file mode 100644 index 00000000..36580581 --- /dev/null +++ b/docs/api-notes/backup_encryption.txt @@ -0,0 +1,109 @@ +TITLE Encryption in the backup system + +This document explains how everything is encrypted in the backup system, and points to the various functions which need reviewing to ensure they do actually follow this scheme. + + +SUBTITLE Security objectives + +The crpyto system is designed to keep the following things secret from an attacker who has full access to the server. + +* The names of the files and directories +* The contents of files and directories +* The exact size of files + +Things which are not secret are + +* Directory heirarchy and number of files in each directory +* How the files change over time +* Approximate size of files + + +SUBTITLE Keys + +There are four separate keys used: + +* Filename +* File attributes +* File block index +* File data + +and an additional secret for file attribute hashes. + +The Cipher is Blowfish in CBC mode in most cases, except for the file data. All keys are maximum length 448 bit keys, since the key size only affects the setup time and this is done very infrequently. + +The file data is encrypted with AES in CBC mode, with a 256 bit key (max length). Blowfish is used elsewhere because the larger block size of AES, while more secure, would be terribly space inefficient. Note that Blowfish may also be used when older versions of OpenSSL are in use, and for backwards compatibility with older versions. + +The keys are generated using "openssl rand", and a 1k file of key material is stored in /etc/box/bbackupd. The configuration scripts make this readable only by root. + +Code for review: BackupClientCryptoKeys_Setup() +in lib/backupclient/BackupClientCryptoKeys.cpp + + +SUBTITLE Filenames + +Filenames need to be secret from the attacker, but they need to be compared on the server so it can determine whether or not is it a new version of an old file. + +So, the same Initialisation Vector is used for every single filename, so the same filename encrypted twice will have the same binary representation. + +Filenames use standard PKCS padding implemented by OpenSSL. They are proceeded by two bytes of header which describe the length, and the encoding. + +Code for review: BackupStoreFilenameClear::EncryptClear() +in lib/backupclient/BackupStoreFilenameClear.cpp + + +SUBTITLE File attributes + +These are kept secret as well, since they reveal information. Especially as they contain the target name of symbolic links. + +To encrypt, a random Initialisation Vector is choosen. This is stored first, followed by the attribute data encrypted with PKCS padding. + +Code for review: BackupClientFileAttributes::EncryptAttr() +in lib/backupclient/BackupClientFileAttributes.cpp + + +SUBTITLE File attribute hashes + +To detect and update file attributes efficiently, the file status change time is not used, as this would give suprious results and result in unnecessary updates to the server. Instead, a hash of user id, group id, and mode is used. + +To avoid revealing details about attributes + +1) The filename is added to the hash, so that an attacker cannot determine whether or not two files have identical attributes + +2) A secret is added to the hash, so that an attacker cannot compare attributes between accounts. + +The hash used is the first 64 bits of an MD5 hash. + + +SUBTITLE File block index + +Files are encoded in blocks, so that the rsync algorithm can be used on them. The data is compressed first before encryption. These small blocks don't give the best possible compression, but there is no alternative because the server can't see their contents. + +The file contains a number of blocks, which contain among other things + +* Size of the block when it's not compressed +* MD5 checksum of the block +* RollingChecksum of the block + +We don't want the attacker to know the size, so the first is bad. (Because of compression and padding, there's uncertainty on the size.) + +When the block is only a few bytes long, the latter two reveal it's contents with only a moderate amount of work. So these need to be encrypted. + +In the header of the index, a 64 bit number is chosen. The sensitive parts of the block are then encrypted, without padding, with an Initialisation Vector of this 64 bit number + the block index. + +If a block from an previous file is included in a new version of a file, the same checksum data will be encrypted again, but with a different IV. An eavesdropper will be able to easily find out which data has been re-encrypted, but the plaintext is not revealed. + +Code for review: BackupStoreFileEncodeStream::Read() (IV base choosen about half-way through) +BackupStoreFileEncodeStream::EncodeCurrentBlock() (encrypt index entry) +in lib/backupclient/BackupStoreFileEncodeStream.cpp + + +SUBTITLE File data + +As above, the first is split into chunks and compressed. + +Then, a random initialisation vector is chosen, stored first, followed by the compressed file data encrypted using PKCS padding. + +Code for review: BackupStoreFileEncodeStream::EncodeCurrentBlock() +in lib/backupclient/BackupStoreFileEncodeStream.cpp + + diff --git a/docs/api-notes/bin_bbackupd.txt b/docs/api-notes/bin_bbackupd.txt new file mode 100644 index 00000000..67ad9267 --- /dev/null +++ b/docs/api-notes/bin_bbackupd.txt @@ -0,0 +1,88 @@ +TITLE bin/bbackupd + +The backup client daemon. + +This aims to maintain as little information as possible to record which files have been uploaded to the server, while minimising the amount of queries which have to be made to the server. + + +SUBTITLE Scanning + +The daemon is given a length of time, t, which files over this age should be uploaded to the server. This is to stop recently updated files being uploaded immediately to avoid uploading something repeatedly (on the assumption that if a file has been written, it is likely to be modified again shortly). + +It will scan the files at a configured interval, and connect to the server if it needs to upload files or make queries about files and directories. + +The scan interval is actually varied slightly between each run by adding a random number up to a 64th of the configured time. This is to reduce cyclic patterns of load on the backup servers -- otherwise if all the boxes are turned on at about 9am, every morning at 9am there will be a huge spike in load on the server. + +Each scan chooses a time interval, which ends at the current time - t. This will be from 0 to current time - t on the first run, then the next run takes the start time as the end time of the previous run. The scan is only performed if the difference between the start and end times is greater or equal to t. + +For each configured location, the client scans the directories on disc recursively. + +For each directory + +* If the directory has never been scanned before (in this invocation of the daemon) or the modified time on the directory is not that recorded, the listing on the server is downloaded. + +* For each file, if it's modified time is within the time period, it is uploaded. If the directory has been downloaded, it is compared against that, and only uploaded if it's changed. + +* Find all the new files, and upload them if they lie within the time interval. + +* Recurse to sub directories, creating them on the server if necessary. + +Hence, the first time it runs, it will download and compare the entries on the disc to those on the server, but in future runs it will use the file and directory modification times to work out if there is anything which needs uploading. + +If there aren't any changes, it won't even need to connect to the server. + +There are some extra details which allow this to work reliably, but they are documented in the source. + + +SUBTITLE File attributes + +The backup client will update the file attributes on files as soon as it notices they are changed. It records most of the details from stat(), but only a few can be restored. Attributes will only be considered changed if the user id, group id or mode is changed. Detection is by a 64 bit hash, so detection is strictly speaking probablistic. + + +SUBTITLE Encryption + +All the user data is encrypted. There is a separate file, backup_encryption.txt which describes this, and where in the code to look to verify it works as described. + + +SUBTITLE Tracking files and directories + +Renaming files is a difficult problem under this minimal data scanning scheme, because you don't really know whether a file has been renamed, or another file deleted and new one created. + +The solution is to keep (on disc) a map of inode numbers to server object IDs for all directories and files over a certain user configurable threshold. Then, when a new file is discovered, it is first checked to see if it's in this map. If so, a rename is considered, which will take place if the local object corresponding to the name of the tracked object doesn't exist any more. + +Because of the renaming requirement, deletions of objects from the server are recorded and delayed until the end of the scan. + + +SUBTITLE Running out of space + +If the store server indicates on login to the backup client, it will scan, but not upload anything nor adjust it's internal stored details of the local objects. However, deletions and renames happen. + +This is to allow deletions to still work and reduce the amount of storage space used on the server, in the hope that in the future there will be enough space. + +Just not doing anything would mean that one big file created and then deleted at the wrong time would stall the whole backup process. + + +SUBTITLE BackupDaemon + +This is the daemon class for the backup daemon. It handles setting up of all the objects, and implements calulcation of the time intervals for the scanning. + + +SUBTITLE BackupClientContext + +State information for the scans, including maintaining a connection to the store server if required. + + +SUBTITLE BackupClientDirectoryRecord + +A record of state of a directory on the local filesystem. Containing the recursive scanning function, which is long and entertaining, but very necessary. It contains lots of comments which explain the exact details of what's going on. + + +SUBTITLE BackupClientInodeToIDMap + +A implementation of a map of inode number to object ID on the server. If Berkeley DB is available on the platform, it is stored on disc, otherwise there is an in memory version which isn't so good. + + + + + + diff --git a/docs/api-notes/bin_bbstored.txt b/docs/api-notes/bin_bbstored.txt new file mode 100644 index 00000000..c9c4e229 --- /dev/null +++ b/docs/api-notes/bin_bbstored.txt @@ -0,0 +1,54 @@ +TITLE bin/bbstored + +The backup store daemon. + +Maintains a store of encrypted files, and every so often goes through deleting unnecessary data. + +Uses an implementation of Protocol to communicate with the backup client daemon. See bin/bbstored/backupprotocol.txt for details. + + +SUBTITLE Data storage + +The data is arranged as a set of objects within a RaidFile disc set. Each object has a 64 bit object ID, which is turned into a filename in a mildly complex manner which ensure that directories don't have too many objects in them, but there is a minimal number of nested directories. See StoreStructure::MakeObjectFilename in lib/backupstore/StoreStructure.cpp for more details. + +An object can be a directory or a file. Directories contain files and other directories. + +Files in directories are supersceded by new versions when uploaded, but the old versions are flagged as such. A new version has a different object ID to the old version. + +Every so often, a housekeeping process works out what can be deleted, and deletes unnecessary files to take them below the storage limits set in the store info file. + + +SUBTITLE Note about file storage and downloading + +There's one slight entertainment to file storage, in that the format of the file streamed depends on whether it's being downloaded or uploaded. + +The problem is that it contains an index of all the blocks. For efficiency in managing these blocks, they all need to be in the same place. + +Files are encoded and decoded as they are streamed to and from the server. With encoding, the index is only completely known at the end of the process, so it's sent last, and lives in the filesystem last. + +When it's downloaded, it can't be decoded without knowing the index. So the index is sent first, followed by the data. + + +SUBTITLE BackupContext + +The context of the current connection, and the object which modifies the store. + +Maintains a cache of directories, to avoid reading them continuously, and keeps a track of a BackupStoreInfo object which is written back periodiocally. + + +SUBTITLE BackupStoreDaemon + +A ServerTLS daemon which forks off a separate housekeeping process as it starts up. + +Handling connections is delegated to a Protocol implementation. + + +SUBTITLE BackupCommands.cpp + +Implementation of all the commands. Work which requires writing is handled in the context, read only commands mainly in this file. + + +SUBTITLE HousekeepStoreAccount + +A class which performs housekeeping on a single account. + diff --git a/docs/api-notes/common/lib_common.txt b/docs/api-notes/common/lib_common.txt new file mode 100644 index 00000000..11d7b02d --- /dev/null +++ b/docs/api-notes/common/lib_common.txt @@ -0,0 +1,52 @@ +TITLE lib/common + +This module is the basic building block of the project. It is included implicitly as a dependency of all other modules. + +It provides basic services such as file and stream functionality, platform information, and various bits of utility code which doesn't fit naturally anywhere else but is useful to many other applications. + + +SUBTITLE Box.h + +The main include file for the project. It should be the first file included in every cpp file, followed by any system headers, then other project headers. This order has been chosen so that eventually it can be used as a target for pre-compiled headers. (This will be important for the Windows port) + + +SUBTITLE BoxPlatform.h + +This contains various bits of information on platform differences. The build scripts include a compile command line definition like PLATFORM_OPENBSD, which is then used to select the required options. + +Some of the stuff is not particularly pleasant, but it aims to produce a consistent compilation environment, and to have a abstracted way of setting other defines to tell the other files what things are available on this platform. + +GNU configure is not used, as it would be just another dependency. Although something does have to be done about compilation on Linux, which is just too different on each distribution to be caught by a common configuration here. + + +SUBTITLE Streams + +Much use is made of streams -- files, connections, buffers, all sorts of things are implemented using streams. + +The abstract base class IOStream defines the interface, see the class file for more details. + +Streams are lib/common's basic contribution to the project. A standard interface for handling data, and a number of utility classes which make performing common stream functions easy. + + +SUBTITLE Serialisation + +Note there is no "serialisable" class. Instead, objects which need to be serialised into Streams simply have two functions, ReadFromStream and WriteToStream. These perform as expected. + +As it turns out, there's no real need for a specific serialisation base class. If you do need to be able to serialise arbitary objects, there are two approaches. + +1) If you're serialising an arbitary object, you're going to know roughly what type it is. So it's likely to be a subclass of a known base class... in which case, you simply use virtual functions. + +2) If it really is an arbitary type, then you just write your function which accepts any type of object to serialise as a template. + + +SUBTITLE BoxException + +The exception base class for the project. All exceptions thrown by this code are dervied from BoxException. In general, each module which has unique errors has it's own exception class. + +CommonException errors are thrown throughout the project. + + +SUBTITLE Other bits + +There are some other extra useful bits which are documented only in the source files. + diff --git a/docs/api-notes/common/lib_common/BoxTime.txt b/docs/api-notes/common/lib_common/BoxTime.txt new file mode 100644 index 00000000..18bef5a5 --- /dev/null +++ b/docs/api-notes/common/lib_common/BoxTime.txt @@ -0,0 +1,7 @@ +TITLE BoxTime + +Not strictly a class, but time is presented in the project as 64 bit integers as microseconds since the UNIX Epoch. + +There are a number of utility functions in the BoxTime*.h files, which are useful. + +To get BoxTime values from a struct stat, use the FileModificationTime.h functions. \ No newline at end of file diff --git a/docs/api-notes/common/lib_common/CollectInBufferStream.txt b/docs/api-notes/common/lib_common/CollectInBufferStream.txt new file mode 100644 index 00000000..5f9556a3 --- /dev/null +++ b/docs/api-notes/common/lib_common/CollectInBufferStream.txt @@ -0,0 +1,26 @@ +CLASS CollectInBufferStream + +This class is essentially a buffer. Data is written to it, and stored in memory. Then when all data is written which will be written, it can be read out again. + +Useful for building streams in memory. + + +FUNCTION CollectInBufferStream::SetForReading() + +After you've written all the data, call this to set it to read mode. + + +FUNCTION CollectInBufferStream::Reset() + +Reset the stream to the state where is has no data, and is about to have data written. + + +FUNCTION CollectInBufferStream::GetSize() + +Get the size of the buffered data -- the entire data. Use the interface function BytesLeftToRead() in most cases. + + +FUNCTION CollectInBufferStream::GetBuffer() + +Get the buffer, so you can do bad things with it if you're feeling naughty. + diff --git a/docs/api-notes/common/lib_common/Configuration.txt b/docs/api-notes/common/lib_common/Configuration.txt new file mode 100644 index 00000000..ef5f38f6 --- /dev/null +++ b/docs/api-notes/common/lib_common/Configuration.txt @@ -0,0 +1,102 @@ +CLASS Configuration + +Implements the reading of multi-level configuration files. This is intended to be a generic way of representing configuration implementation. + +Basic validation is performed on files as they are read, specified by a data structure. + +test/common has some examples of usage. + + +SUBTITLE Configuration file format + +The format is simple, a list of Key = Value pairs. For example + +Key1 = Value1 +Key2 = Value2 + +Optionally, multiple values for the same key are allowed. + +Lists of configurations can be nested to any level. These are called Sub-configurations: + +SubConfig +{ + SubKey1 = ValueX + SubKey2 = ValueY +} + + +SUBTITLE Verification + +The verification structure specifies what keys are required, what are optional, and what sub-configurations are expected. Default values can be specified. + +See Configuration.h for the structures. + +RaidFileController::Initialise has a good example of a simple verification layout. + +Wildcards can be used for SubConfigurations, by specifying a name of "*". This allows you to use multiple sections to configure any number of items. + +Each item has a number of flags, which are combined to say whether an item is required, should be an integer or boolean, and rather importantly, whether it's the last item in the list. + +Verification is limited, so you may wish to do more verification the file. There are unimplemented hooks for custom verification functions to be included in the verification definitions. Should be done at some point. + +Boolean keys have possible values "true", "yes", "false", "no" (case insensitive). + + +FUNCTION Configuration::LoadAndVerify() + +Loads the configuration from disc, and verifies it. If there are problems with the verification, it returns some text which can be used to tell the user the problems with the file. These are fairly basic error messages, but do say what should be done to get it to parse properly. + +This returns a Configuration object, which can then be queries for Keys and Values. Sub-configurations are implemented through an interface which returns a reference to another Configuration object. + + + +FUNCTION Configuration::KeyExists() + +Does a specified key exist? + +Use this for optional key values only -- specify them in the verification structure if they are required. + + +FUNCTION Configuration::GetKeyValue() + +Get the value of a key as a string. + +If ConfigTest_MultiValueAllowed is set in the relevant entry in the verify structure, this string may contain multiple values, separated by a single byte with value 0x01 (use Configuration::MultiValueSeparator in code). Use SplitString() defined in Utils.h to split it into components. + +This representation was chosen as multi-values are probably rare, and unlikely to justify writing a nicer (but more memory intensive and complex) solution. + + +FUNCTION Configuration::GetKeyValueInt() + +Get the value of a key as an integer. Make sure the AsInt property is requried in the verification structure. + + +FUNCTION Configuration::GetKeyValueBool() + +Get the value of a key as a boolean value. Make sure the AsBool property is requried in the verification structure. + +Default to "false", should this verification not be performed and an unknown value is specified. + + +FUNCTION Configuration::GetKeyNames() + +Return a list of all the keys in this configuration. + + +FUNCTION Configuration::SubConfigurationExists() + +Does a specified sub-configuration exist? + + +FUNCTION Configuration::GetSubConfiguration() + +Return another Configuration object representing the sub section. + + +FUNCTION Configuration::GetSubConfigurationNames() + +Get a list of all the sub configurations. + +As there isn't a particularly neat way that configurations can be iterated over, mSubConfigurations is public. (BAD!) + + diff --git a/docs/api-notes/common/lib_common/Conversion.txt b/docs/api-notes/common/lib_common/Conversion.txt new file mode 100644 index 00000000..62d1967a --- /dev/null +++ b/docs/api-notes/common/lib_common/Conversion.txt @@ -0,0 +1,14 @@ +TITLE Generic type conversion + +Conversion.h provides generic type conversion. Within the BoxConvert namespace, it defines the templated function + + ToType Convert(FromType From) + +which converts the data type as expected. In general, from and to types have to be specified explicitly. + +Templates rather than overloaded functions are used, firstly to be absolutely explicit about the conversion required, and secondly because overloaded functions can't have differing return types for the same argument type. + +The function is specialised for various types, and the generic version uses C++ type conversion. + +Exceptions may be thrown if the conversion is not possible. These are all of the ConversionException type. + diff --git a/docs/api-notes/common/lib_common/ExcludeList.txt b/docs/api-notes/common/lib_common/ExcludeList.txt new file mode 100644 index 00000000..8a5bf36c --- /dev/null +++ b/docs/api-notes/common/lib_common/ExcludeList.txt @@ -0,0 +1,21 @@ +TITLE ExcludeList + +A class implementing a list of strings which are to be excluded in some way. Entries can be added as strings to be matched exactly, or as regular expressions. + +Multiple entries can be added in a single function call in a manner designed to work with the multi-value entries store in a Configuation object. + + +FUNCTION ExcludeList::AddDefiniteEntries() + +Definite entries are exact strings to match. + + +FUNCTION ExcludeList::AddRegexEntries() + +Regular expressions as defined by POSIX, and supported by the host platform. Will throw an exception if regular expressions are not supported by the platform. + + +FUNCTION ExcludeList::IsExcluded() + +Test a string for being excluded by definite or regex entries. + diff --git a/docs/api-notes/common/lib_common/FdGetLine.txt b/docs/api-notes/common/lib_common/FdGetLine.txt new file mode 100644 index 00000000..d92ff94d --- /dev/null +++ b/docs/api-notes/common/lib_common/FdGetLine.txt @@ -0,0 +1,11 @@ +CLASS FdGetLine + +See IOStreamGetLine, difference is that it works on basic UNIX file descriptors rather than full blown streams. Follows basic interface, except... + + +FUNCTION FdGetLine::GetLine + +Returns a string containing the optionally preprocessed line. + +Do not call if IsEOF if true. + diff --git a/docs/api-notes/common/lib_common/Guards.txt b/docs/api-notes/common/lib_common/Guards.txt new file mode 100644 index 00000000..6174fea6 --- /dev/null +++ b/docs/api-notes/common/lib_common/Guards.txt @@ -0,0 +1,5 @@ +TITLE Guards.h + +This file contains a couple of classes for using file and memory. When the class goes out of scope, the underlying object is closed or freed, respectively. + + diff --git a/docs/api-notes/common/lib_common/IOStream.txt b/docs/api-notes/common/lib_common/IOStream.txt new file mode 100644 index 00000000..09460656 --- /dev/null +++ b/docs/api-notes/common/lib_common/IOStream.txt @@ -0,0 +1,89 @@ +CLASS IOStream + +The base class for streams of data. See IOStream.h for specific details on functions, but the interface is described here. + +Most streams only implement one direction, but some both. + + +SUBTITLE Reading + + +FUNCTION IOStream::Read() + +Reads data from the stream. Returns 0 if there is no data available within the timeout requested. + +Unlike the UNIX API, a return of 0 does not mean that there is no more data to read. Use IOStream::StreamDataLeft() to find this out. + + +FUNCTION IOStream::ReadFullBuffer() + +If you want to read a specific sized block of data from a stream, it is annoying ot use the Read function, because it might return less, so you have to loop. + +This implements that loop. Note that the timeout is the timeout for each individual read -- it might take a lot longer to complete. + +You must check the return value, which is whether or not it managed to get all that data. + + +FUNCTION IOStream::BytesLeftToRead() + +How many bytes are left to read in the stream treated as a whole. May return IOStream::SizeOfStreamUnknown if it is something like a socket which doesn't know how much data is in the stream. + + +FUNCTION IOStream::StreamDataLeft() + +Returns true if more data can be read. Or more specifically, if more data might be able to be read some time in the future. + + +SUBTITLE Writing + + +FUNCTION IOStream::Write() + +Write data to a stream. Writes the entire block, or exceptions. + + +FUNCTION IOStream::WriteAllBuffered() + +Writes any buffered data to the underlying object. + +Call at the end of a series of writes to make sure the data is actually written. (Buffering is not yet implemented anywhere, but it should be soon.) + + +FUNCTION IOStream::StreamClosed() + +Is the stream closed, and writing no longer possible? + + +FUNCTION IOStream::CopyStreamTo() + +A utility function to copy the contents of one stream to another, until the reading stream has no data left. + + +SUBTITLE Other interfaces + +These are slightly nasty interfaces, because they have to match the UNIX API to some extent. + + +FUNCTION IOStream::Close() + +Close the stream -- intended to indicate that nothing more will be written. + +However, also closes reading in most cases. Stream dependent. Probably best just to delete the object and let it sort itself out. + + +FUNCTION IOStream::Seek() + +Seeks within the stream -- mainly for using files within a stream interface, although some of the utility stream classes support it too. + +This is actually a bad interface, because it does not specify whether it applies to reading or writing. This matches the interface provided by files with a single file pointer, but does not map well onto other stream types. + +Should it be changed? Perhaps. But then it means that files would either have to have an inconsitent interface, or the class keep track of two separate file pointers. Which isn't nice. + +So use carefully, and remember whether you're using a file stream, a reading stream, or a writing stream. + + +FUNCTION IOStream::GetPosition() + +Get the current file pointer. Subject to same problems as Seek with regard to semantics. + + diff --git a/docs/api-notes/common/lib_common/IOStreamGetLine.txt b/docs/api-notes/common/lib_common/IOStreamGetLine.txt new file mode 100644 index 00000000..04c56b57 --- /dev/null +++ b/docs/api-notes/common/lib_common/IOStreamGetLine.txt @@ -0,0 +1,29 @@ +CLASS IOStreamGetLine + +This class provides a convenient way to read text from a file, line by line. It also can preprocess the line to remove leading and trailing whitespace and comments. Comments are started by the character # and run to the end of the line. + +Create an instance by passing a reference to a stream into the constructor. + +Note the class does internal buffering, so you can only detach it later if the stream supports seeking backwards. + + +FUNCTION IOStreamGetLine::GetLine() + +Returns true if a line could be retreieved without a read timing out. + + +FUNCTION IOStreamGetLine::IsEOF() + +Whether the end of the stream has been reached. Do not call GetLine if this is true. + + +FUNCTION IOStreamGetLine::GetLineNumber() + +Returns the line number. + + +FUNCTION IOStreamGetLine::DetachFile() + +Detaches the stream from the GetLine class. Will seek backwards to "replace" data it's buffered back in the stream. + + diff --git a/docs/api-notes/common/lib_common/MainHelper.txt b/docs/api-notes/common/lib_common/MainHelper.txt new file mode 100644 index 00000000..eb5b07f0 --- /dev/null +++ b/docs/api-notes/common/lib_common/MainHelper.txt @@ -0,0 +1,4 @@ +TITLE MainHelper.h + +This header contains a couple of macros which add exception handling and reporting to main() functions for executable programs. + diff --git a/docs/api-notes/common/lib_common/WaitForEvent.txt b/docs/api-notes/common/lib_common/WaitForEvent.txt new file mode 100644 index 00000000..0bc55726 --- /dev/null +++ b/docs/api-notes/common/lib_common/WaitForEvent.txt @@ -0,0 +1,16 @@ +CLASS WaitForEvent + +This class implements a way to efficiently wait on one or more system objects, for example sockets and file descriptors. Where kqueue() is available, this is used, otherwise poll(). The poll() implementation is less comprehensive and rather more limited. + +To add a compatible object, call Add(). To remove it, call Remove(). To wait for an object to become ready, call Wait(), which returns a pointer to the first ready object, or 0 for a timeout. + +The timeout is set either in the constructor, or using the SetTimout() method. It is specified in milliseconds. + +The kqueue based implementation will automatically remove objects when they are closed (actually, the OS does this), but the poll implementation requires that Remove() be called. + +The flags passed to Add() or Remove() are passed to the object, and are ignored by this class. + +For an object to be able to be added to the list, it should implement FillInKEvent() and FillInPoll() for kqueue and poll implementations respectively. Use the PLATFORM_KQUEUE_NOT_SUPPORTED define to test which is necessary for the platform. + +It does not have to be derived off any particular class, as it uses templates to be compatible with any class. + diff --git a/docs/api-notes/common/lib_common/xStream.txt b/docs/api-notes/common/lib_common/xStream.txt new file mode 100644 index 00000000..91e9c0ea --- /dev/null +++ b/docs/api-notes/common/lib_common/xStream.txt @@ -0,0 +1,40 @@ +TITLE Various stream types + +Some useful streams are implemented in lib/common. + + +SUBTITLE CollectInBufferStream + +Described in it's own page. + + +SUBTITLE FileStream + +Implements a stream from a file, allowing both reading and writing. + + +SUBTITLE MemBlockStream + +Turns a memory block into a readable stream. + +Can also be constructed using StreamableMemBlock, CollectInBufferStream and MemBlockStream as sources of the buffer. + + +SUBTITLE PartialReadStream + +Create a readable stream which will read a set number of bytes from another stream, and then declare itself as closed. + +Useful for extracting a small chunk of another stream to present to a function which expects to consume all of a stream. + + +SUBTITLE ReadGatherStream + +Create a readable stream out of blocks from many streams -- so various sections of any number of streams are composed into a single stream. + +To use, register each stream with AddComponent, then use the returned 'handle' with AddBlock to add blocks to the composed stream. + +Optionally, the object will take ownership of the streams added, and delete them when itself is deleted. + +See the comments in the function blocks in the cpp file for more info. + + diff --git a/docs/api-notes/common/lib_compress.txt b/docs/api-notes/common/lib_compress.txt new file mode 100644 index 00000000..f3e26f20 --- /dev/null +++ b/docs/api-notes/common/lib_compress.txt @@ -0,0 +1,8 @@ +TITLE lib/compress + +This is a horrid C++ interface to zlib. It is written this way to avoid having to buffer any data, and so be unnecessarily inefficient. But this does end up just being a wrapper to the zlib interface. + +See test/compress for an example of how to use it. But it should be very familiar. + +For an easier interface, use CompressStream. + diff --git a/docs/api-notes/common/lib_compress/CompressStream.txt b/docs/api-notes/common/lib_compress/CompressStream.txt new file mode 100644 index 00000000..eca43eb6 --- /dev/null +++ b/docs/api-notes/common/lib_compress/CompressStream.txt @@ -0,0 +1,27 @@ +TITLE CompressStream + +This implements a compressing stream class, which compresses and decompresses data sent to an underlying stream object. Data is likely to be buffered. + + +WARNING: Use WriteAllBuffered() (after objects which need to be sent in their entirity) and Close() (after you have finished writing to the stream) otherwise the compression buffering will lose the last bit of data for you. + + +The class works as a filter to an existing stream. Ownership can optionally be taken so that the source stream is deleted when this object is deleted. + +It can operate in one or two directions at any time. Data written to the stream is compressed, data read from the stream is decompressed. If a direction is not being (de)compressed, then it can act as a pass-through without touching the data. + +The constructor specifies the actions to be taken: + +CompressStream(IOStream *pStream, bool TakeOwnership, + bool DecompressRead, bool CompressWrite, + bool PassThroughWhenNotCompressed = false); + +pStream - stream to filter +TakeOwnership - if true, delete the stream when this object is deleted. +DecompressRead - reads pass through a decompress filter +CompressWrite - writes pass through a compression filter +PassThroughWhenNotCompressed - if not filtering a direction, pass through the data unaltered, otherwise exception if an attempt is made on that direction. + + +Note that the buffering and compression will result in different behaviour than the underlying stream: data may not be written in one go, and data being read may come out in different sized chunks. + diff --git a/docs/api-notes/common/lib_crypto.txt b/docs/api-notes/common/lib_crypto.txt new file mode 100644 index 00000000..9ddafe69 --- /dev/null +++ b/docs/api-notes/common/lib_crypto.txt @@ -0,0 +1,28 @@ +TITLE lib/crypto + +Provides cryptographic primatives using the OpenSSL implementation. + + +SUBTITLE CipherContexts and CipherDescriptions + +A CipherContext is the interface to encryption and decryption, providing a generic interface. + +It is constructed with a decrypt or encrypt flag, and a CipherDescription. The CipherDescription specifies which cipher is to be used, the key, and all other cipher specific paramteres. + +The CipherContext has support for padding and initialisation vectors. + + +SUBTITLE Random numbers + +An interface to the OpenSSL psuedo random number generater is provided. + + +SUBTITLE Digests + +An interface to the MD5 digest is provided. + + +SUBTITLE RollingChecksum + +The rsync rolling checksum is implemented. The interface is a little tricky to be efficient as the caller must track the position and provide relevant bytes to advance the checksum. + diff --git a/docs/api-notes/common/lib_crypto/CipherContext.txt b/docs/api-notes/common/lib_crypto/CipherContext.txt new file mode 100644 index 00000000..30fb4608 --- /dev/null +++ b/docs/api-notes/common/lib_crypto/CipherContext.txt @@ -0,0 +1,28 @@ +CLASS CipherContext + +Encryption and decryption using OpenSSL EVP interface. + +See the cpp file for more documentation in the function headers, and test/crypto for examples. + +General notes below. + + +SUBTITLE Construction + +Construct with the encryption direction, and a CipherDescription of the cipher and key required. + + +SUBTITLE Encrypting or decrypting + +Begin() and Transform() allow piece by piece transformation of a block. + +TransformBlock() transforms an entire block. + + +SUBTITLE Buffering + +All transforms expect to have enough space in the buffers for their entire output. Because of block boundaries and padding, it is necessary that the output buffer should be bigger than the input buffer. The amount of space depends on the cipher. + +InSizeForOutBufferSize() and MaxOutSizeForInBufferSize() perform these space calculations, returning the maximuim in size for a specified out size, and the reverse, respectively. + + diff --git a/docs/api-notes/common/lib_crypto/RollingChecksum.txt b/docs/api-notes/common/lib_crypto/RollingChecksum.txt new file mode 100644 index 00000000..d871b3f2 --- /dev/null +++ b/docs/api-notes/common/lib_crypto/RollingChecksum.txt @@ -0,0 +1,36 @@ +CLASS RollingChecksum + +Implementing the rsync rolling checksum algorithm. Read it's description first: + +http://samba.anu.edu.au/rsync/tech_report/node3.html + + +SUBTITLE Construction and initial checksum calculation + +The constructor takes a pointer to a block of data and a size, and calculates the checksum of this block. It can now be "rolled forward" to find the checksum of the block of the same size, one byte forward, with minimal calculation. + + +FUNCTION RollingChecksum::GetChecksum() + +Returns the checksum for the current block. + + +FUNCTION RollingChecksum::RollForward() + +This function takes the byte at the start of the current block, and the last byte of the block it's rolling forward to, and moves the checksum on. + +If the block is + + char *pBlock = ; + +with size s, then it should be called with + + RollForward(pBlock[0], pBlock[s]) + +and now GetChecksum will return the checksum of the block (pBlock+1) of size s. + + +FUNCTION RollingChecksum::RollForwardSeveral() + +Similar to RollForward(), but is more efficient for skipping several bytes at once. Takes pointers to the data buffer rather than the actual data values. + diff --git a/docs/api-notes/common/lib_server.txt b/docs/api-notes/common/lib_server.txt new file mode 100644 index 00000000..392f331d --- /dev/null +++ b/docs/api-notes/common/lib_server.txt @@ -0,0 +1,9 @@ +TITLE lib/server + +This provides various classes for implementing client/server applications (and UNIX daemons). + +The stars of the show are ServerTLS and Protocol, which allow client server applications which communicate using TLS (SSL successor) to be built with surprisingly few lines of code. + +All details in the respective class files. + +Look at test/basicserver for examples. diff --git a/docs/api-notes/common/lib_server/Daemon.txt b/docs/api-notes/common/lib_server/Daemon.txt new file mode 100644 index 00000000..6956ec2b --- /dev/null +++ b/docs/api-notes/common/lib_server/Daemon.txt @@ -0,0 +1,96 @@ +CLASS Daemon + +Implement a UNIX daemon which + +* Daemonises (detach from console, etc) +* Sets up signal handlers, and does useful things with them +* Reads configuration files +* Writes PID file +* Opens syslog + +all the usual UNIX basic daemon behaviour. + +The daemon exe takes optional arguments: The first is a configuration file filename which overrides the default. If the second argument is 'SINGLEPROCESS' the daemon will not daemonise. + +The configuration file must have a section like this + +Server +{ + PidFile = /var/run/daemon.pid + User = username +} + +Use DAEMON_VERIFY_SERVER_KEYS (defined in Daemon.h) to include the necessary keys in your configuration file's verify structure. + +The "User" line is optional, and if it is present the Daemon will change user to this username just before it daemonises. Note that unless the directory the PID file is written to has write permission for this user, the PID file will not be deleted on exit. + + +To implement a daemon, derive a class from Daemon, and override Run(). + +Then in your main file, do something like + +int main(int argc, const char *argv[]) +{ + MAINHELPER_START + + BackupDaemon daemon; + return daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); + + MAINHELPER_END +} + +and that's it. Obviously it's worth doing a few more things as well, but that's it. + + +FUNCTION Daemon::Run() + +Override with the main daemon code. It should behave like this + +void SomethingDaemon::Run() +{ + // Read configuration file + + // Do lots of work until StopRun() returns true + while(!StopRun()) + { + ::sleep(10); + } + + // Clean up nicely +} + +which allows the base class to implement the standard start, terminate and -HUP behaviours correctly. + + +FUNCTION Daemon::DaemonName() + +Returns the name of the daemon, for use in syslog. + + +FUNCTION Daemon::DaemonBanner() + +Returns the banner to be displayed on startup, or 0 for no banner. + + +FUNCTION Daemon::GetConfigVerify() + +Returns a configuration verify structure for verifying the config file. Note that this configuration structure should include a sub-configuration called "Server, and have entries defined by DAEMON_VERIFY_SERVER_KEYS. See one of the bin/bbackupd for an example. + + +FUNCTION Daemon::StopRun() + +Returns true when the daemon needs to be terminated or restarted. Use IsReloadConfigWanted() and IsTerminateWanted() to find out which one, if you need to know. + + +FUNCTION Daemon::SetupInInitialProcess() + +Override to perform additional functions in the initial process, before forking and detachment happens. + + +FUNCTION Daemon::EnterChild() + +Called when a child is entered. If you override it, remember to call the base class. + + + + diff --git a/docs/api-notes/common/lib_server/Protocol.txt b/docs/api-notes/common/lib_server/Protocol.txt new file mode 100644 index 00000000..09d3c1f1 --- /dev/null +++ b/docs/api-notes/common/lib_server/Protocol.txt @@ -0,0 +1,120 @@ +CLASS Protocol + +Protocol + +* serialises and deserialises data objects +* sends arbitary streams + +through a bi-directional IOStream object, usually a socket. + +These data objects are auto-generated by a perl script, along with the logic to implement a simple protocol where one end is a client, and the other a server. The client sends a command object (and optional streams) and the server returns a reply object (and optional streams). + +The perl script uses a description file which specifies the data inside these objects (which can be extended to include any type which implements the standard serialisation functions), the required responses to the commands, and whether streams are involved. + +It then implements a server object, which given a stream and a user defined context object, waits for commands, processes them, and sends the results back. All you need to do is implement a DoCommand() function for each command object you define. + +On the client side, a Query() function is implemented which takes objects as parameters, and returns the right type of reply object (or throws an exception if it doesn't get it.) Short cut Query() functions are also implemented which don't require objects to be created manually. + +Thus, implementing a server is as simple as deriving a daemon off ServerStream or ServerTLS, and implementing the Connection() function as + + void TestProtocolServer::Connection(SocketStream &rStream) + { + TestProtocolServer server(rStream); + TestContext context; + server.DoServer(context); + } + +and that's it. TestContext is a user defined class which keeps track of all the state of the connection, and is passed to all the DoCommand() functions, which look like this: + + std::auto_ptr + TestProtocolServerSimple::DoCommand(TestProtocolServer &rProtocol, + TestContext &rContext) + { + return std::auto_ptr + (new TestProtocolServerSimpleReply(mValue+1)); + } + +(taken from test/basicserver) + +The client code looks like this + + SocketStream conn; + conn.Open(Socket::TypeUNIX, "testfiles/srv4.sock"); + + TestProtocolClient protocol(conn); + + // Query + { + std::auto_ptr + reply(protocol.QuerySimple(41)); + TEST_THAT(reply->GetValuePlusOne() == 42); + } + + +Finally, debug logging can be generated which allows a list of all commands and their parameters to be logged to syslog or a file. + + +SUBTITLE Protocol Description File + +This file is passed to the lib/server/makeprotocol.pl script, which generates a h and cpp file to implement the protocol. + +It is in two sections, separated by a 'BEGIN_OBJECTS' on a line of it's own. + +In the top half, the following statements must be made. + +Name + The name of the protocol, used in naming classes. + +IdentString + The idenfitifaction string sent over the IOStream to confirm it it + is talking to another Protocol object speaking the same Protocol. + +ServerContextClass + The user defined context class used for the server, and the header + file it is defined in. + +Additionally, the following optional commands can be made. + +ClientType +ServerType (similarly) + Extends the types used in the objects below. Server and client + can use different types for the same object type. + +ImplementLog (Client|Server) (syslog|file) + Implement command logging for client or server into syslog or a file. + +LogTypeToText (Client|Server) + + For extended types, optionally define how to convert them into printf + elements and the code to run to get the argument. Within the evaluate + parameter, VAR is replaced by the name of the variable to display. + If this is not specified for a given type, OPAQUE is output instead. + + +In the object section, an object is defined by a line + + + +followed by lines beginning with whitespace defining the data transmitted in the object. The type may be list, which specifies a list (implemented as a std::vector) of entries of that type. + +The attributes specify exactly how that object is used in the defined protocol. + +Reply + The object is a reply object, sent from the server to the client. + +Command(Reply-Type) + The object is a command, send from the client to the server, and the server + will send back an object of type Reply-Type. + +IsError(Type-Field,SubType-Field) + The command is an error object, and the two files specify the data member + which describes the error type and sub type. + +EndsConversation + When this command is received, the connection is to be terminated. + (ie a logout command) + +StreamWithCommand + When this command is sent as a command, a stream follows it. + + diff --git a/docs/api-notes/common/lib_server/ServerStream.txt b/docs/api-notes/common/lib_server/ServerStream.txt new file mode 100644 index 00000000..6c5932a0 --- /dev/null +++ b/docs/api-notes/common/lib_server/ServerStream.txt @@ -0,0 +1,29 @@ +CLASS ServerStream + +ServerStream implementes a Daemon which accepts stream connections over sockets, and forks into a child process to handle them. + +To implement a daemon, derive from + + ServerStream + +The type SocketStream specifies that incoming connections should be treated as normal sockets, and SERVER_LISTEN_PORT is the port to listen to (if it's a inet socket, and if not overridden in the config file). The actual addresses (or names) to bind to are specified in the configuration file by the user. + +Make sure SERVERSTREAM_VERIFY_SERVER_KEYS(0) is included in the configuration verification structure in the "Server" sub configuration. 0 could be replaced with a default address, for example "unix:/var/run/server.sock" to specific a default UNIX socket in the filesystem. + +See test/basicserver for a simple example. + +The ListenAddresses key in the Server subconfiguration is a comma separated list of addresses, specified as family:name. Internet sockets are family 'inet', for example 'inet:localhost' (or 'inet:localhost:1080' to specify a port number as well), and unix domain sockets are 'unix', example above. + + +Override Connection to handle the connection. + +Remember to override Daemon functions like the server name, and start it up, just like a generic Daemon. + + +FUNCTION ServerStream::Connection + +This function takes a connected stream as it's argument. It should then proceed to do whatever it needs to do to talk to the client. + +Using IOStreamGetLine or a Protocol class to communicate may be quick ways of implementing this functionality. + + diff --git a/docs/api-notes/common/lib_server/ServerTLS.txt b/docs/api-notes/common/lib_server/ServerTLS.txt new file mode 100644 index 00000000..dbde500f --- /dev/null +++ b/docs/api-notes/common/lib_server/ServerTLS.txt @@ -0,0 +1,6 @@ +CLASS ServerTLS + +Implements a server which uses TLS (SSL) to encrypt and authenticate connections. + +Very similar to ServerStream, except it reads the certificate files for the TLSContext out of the Server sub-configuration to set up a TLSContext ("CertificateFile", "PrivateKeyFile" and "TrustedCAsFile"). Otherwise works exactly the same. + diff --git a/docs/api-notes/common/lib_server/SocketStream.txt b/docs/api-notes/common/lib_server/SocketStream.txt new file mode 100644 index 00000000..82813279 --- /dev/null +++ b/docs/api-notes/common/lib_server/SocketStream.txt @@ -0,0 +1,8 @@ +CLASS SocketStream + +A implementation of IOStream which wraps a socket connection. + +It can either be created by attaching to an existing object, or use the Open() function to open a connection to a named host on a specific port (or a local UNIX socket in the filesystem). + +Follows stream interface. + diff --git a/docs/api-notes/common/lib_server/SocketStreamTLS.txt b/docs/api-notes/common/lib_server/SocketStreamTLS.txt new file mode 100644 index 00000000..ebb3f233 --- /dev/null +++ b/docs/api-notes/common/lib_server/SocketStreamTLS.txt @@ -0,0 +1,11 @@ +CLASS SocketStreamTLS + +An implementation of IOStream which wraps a TLS (SSL) connection over a socket. + +The Open function takes a TLSContext reference which specifies the parameters for the connection. + + +FUNCTION GetPeerCommonName() + +Returns the common name of the certificate presented by the remote end. + diff --git a/docs/api-notes/common/lib_server/TLSContext.txt b/docs/api-notes/common/lib_server/TLSContext.txt new file mode 100644 index 00000000..ff50d3e6 --- /dev/null +++ b/docs/api-notes/common/lib_server/TLSContext.txt @@ -0,0 +1,16 @@ +CLASS TLSContext + +A wrapper over the OpenSSL context object. + +Note: you need to call SSLLib::Initialise at the beginning of your program to use these functions. + + +SUBTITLE Construction + +The constuctor takes the following parameters + +* Boolean for whether this is acting as a server or a client +* The .pem file containing the certificate which will be used +* The .pem file containing the private key for this certificate +* The .pem file containing the certificates which will certify the other end of the connection. + diff --git a/docs/api-notes/common/memory_leaks.txt b/docs/api-notes/common/memory_leaks.txt new file mode 100644 index 00000000..9a9764ea --- /dev/null +++ b/docs/api-notes/common/memory_leaks.txt @@ -0,0 +1,44 @@ +TITLE Memory leak detection + +The build system contains a primative memory leak detection system in debug builds. + +It works by using #defines to replace calls to malloc, free, realloc, new and delete with debug versions which record their use. When a process ends, it dumps a list of all the blocks or objects which were allocated by not freed, and the file and line of the source where they are originally allocated. + +It's not perfect, but should catch most things and work on most platforms. + +If it doesn't work on your platform, define PLATFORM_DISABLE_MEM_LEAK_TESTING in BoxPlatform.h within the relevant section. + + +SUBTITLE Requirements in source + +It does require some extra lines in the source file. The last included file in each .cpp file must be + + #include "MemLeakFindOn.h" + +and if a .h file uses any of these functions, the contents of the .h file should be bounded with + + #include "MemLeakFindOn.h" + + // ... some code, but absolutely no #includes + + #include "MemLeakFindOff.h" + +The cleanupforcvs.pl script checks for correct usage. + + +SUBTITLE Use in tests + +Tests will automatically dump memory leaks and regard them as a failure for anything executing in the main test process. + +To test for memory leaks in programs run from the test, or daemons, use something like + + TestRemoteProcessMemLeaks("bbackupd.memleaks"); + +If memory leak detection is being used, it will check the specified file for memory leak reports (deleting it afterwards). Any leak is an error. + +The Daemon class automactically arranges for the memory leak reports to be written. Other exes should set this up themselves, preferably using the define in MainHelper.h, as the first thing in their main() function. + + MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, name) + +File is the filename to write the report to, conventionally called ".memleaks", and name is the name of the exe. + diff --git a/docs/api-notes/encrypt_rsync.txt b/docs/api-notes/encrypt_rsync.txt new file mode 100644 index 00000000..a5db2df2 --- /dev/null +++ b/docs/api-notes/encrypt_rsync.txt @@ -0,0 +1,66 @@ +TITLE Encrypted rsync algorithm + +The backup system uses a modified version of the rsync algorithm. A description of the plain algorithm can be found here: + + http://samba.anu.edu.au/rsync/tech_report/ + +The algorithm is modified to allow the server side to be encrypted, yet still benefit from the reduced bandwidth usage. For a single file transfer, the result will be only slightly less efficient than plain rsync. For a backup of a large directory, the overall bandwidth may be less due to the way the backup client daemon detects changes. + +This document assumes you have read the rsync document. + +The code is in lib/backupclient/BackupStoreFile*.*. + + +SUBTITLE Blocks + +Each file is broken up into small blocks. These are individually compressed and encrypted, and have an entry in an index which contains, encrypted, it's weak and strong checksums and decoded plaintext size. This is all done on the client. + +Why not just encrypt the file, and use the standard rsync algorithm? + +1) Compression cannot be used, since encryption turns the file into essentially random data. This is not very compressible. + +2) Any modification to the file will result in all data after that in the file having different ciphertext (in any cipher mode we might want to use). Therefore the rsync algorithm will only be able to detect "same" blocks up until the first modification. This significantly reduces the effectiveness of the process. + +Note that blocks are not all the same size. The last block in the file is unlikely to be a full block, and if data is inserted which is not a integral multiple of the block size, odd sized blocks need to be created. This is because the server cannot reassemble the blocks, because the contents are opaque to the server. + + +SUBTITLE Modifed algorithm + +To produce a list of the changes to send the new version, the client requests the block index of the file. This is the same step as requesting the weak and strong checksums from the remote side with rsync. + +The client then decrypts the index, and builds a list of the 8 most used block sizes above a certain threshold size. + +The new version of the file is then scanned in exactly the same way as rsync for these 8 block sizes. If a block is found, then it is added to a list of found blocks, sorted by position in the file. If a block has already been found at that position, then the old entry is only replaced by the new entry if the new entry is a "better" (bigger) match. + +The block size covering the biggest file area is searched first, so that most of the file can be skipped over after the first pass without expensive checksumming. + +A "recipe" is then built from the found list, by trivially discarding overlapping blocks. Each entry consists of a number of bytes of "new" data, a block start number, and a number of blocks from the old file. The data is stored like this as a memory optimisation, assuming that files mostly stay the same rather than having all their blocks reordered. + +The file is then encoded, with new data being sent as blocks of data, and references to blocks in the old file. The new index is built completely, as the checksums and size need to be rencrypted to match their position in the index. + + +SUBTITLE Combination on server + +The "diff" which is sent from the client is assembled into a full file on the server, simply by adding in blocks from the old file where they are specified in the block index. + + +SUBTITLE Storage on server + +Given that the server will in general store several versions of a file, combining old and new files to form a new file is not terribly efficient on storage space. Particularly for large multi-Gb database files. + +An alternative scheme is outlined below, however, it is significantly more complex to implement, and so is not implemented in this version. + +1) In the block index of the files, store the file ID of the file which each block is source from. This allows a single file to reference blocks from many files. + +2) When the file is downloaded, the server combines the blocks from all the files into a new file as it is streamed to the client. (This is not particuarly complicated to do.) + +This all sounds fine, until housekeeping is considered. Old versions need to be deleted, without losing any blocks necessary for future versions. + +Instead of just deleting a file, the server works out which blocks are still required, and rebuilds the file omitting those blocks which aren't required. + +This complicates working out how much space a file will release when it is "deleted", and indeed, adds a whole new level of complexity to the housekeeping process. (And the tests!) + +The directory structure will need an additional flag, "Partial file", which specifies that the entry cannot be built as previous blocks are no longer available. Entries with this flag should never be sent to the client. + + + diff --git a/docs/api-notes/lib_backupclient.txt b/docs/api-notes/lib_backupclient.txt new file mode 100644 index 00000000..3e4a079b --- /dev/null +++ b/docs/api-notes/lib_backupclient.txt @@ -0,0 +1,46 @@ +TITLE lib/backupclient + +Classes used on the store and on the server. + +See documentation in the files for more details. + + +SUBTITLE BackupStoreDirectory + +The directory listing class, containing a number of entries, representing files. + + +SUBTITLE BackupStoreFile + +Handles compressing and encrypting files, and decoding files downloaded from the server. + + +SUBTITLE BackupStoreFilename + +An encrypted filename. + + +SUBTITLE BackupStoreFilenameClear + +Derived from BackupStoreFilename, but with the ability to encrypt and decrypt filenames. Client side only. + + +SUBTITLE BackupClientFileAttributes + +Only used on the client -- the server treats attributes as blocks of opaque data. + +This reads attributes from files on discs, stores them, encrypts them, and applies them to new files. + +Also has a static function to generate filename attribute hashes given a struct stat and the filename. + + +SUBTITLE BackupClientRestore + +Routines to restore files from the server onto the client filesystem. + + +SUBTITLE BackupClientCryptoKeys + +This reads the key material from disc, and sets up the crypto for storing files, attributes and directories. + + diff --git a/docs/api-notes/lib_backupstore.txt b/docs/api-notes/lib_backupstore.txt new file mode 100644 index 00000000..8f24eb7b --- /dev/null +++ b/docs/api-notes/lib_backupstore.txt @@ -0,0 +1,30 @@ +TITLE lib/backupstore + +Classes which are shared amongst the server side store utilities, bbstored and bbstoreaccounts. Note also depends on lib/backupclient, as a lot of code is shared between the client and server. + + +SUBTITLE BackupStoreAccountDatabase + +A simple implementation of an account database. This will be replaced with a more suitable implementation. + + +SUBTITLE BackupStoreAccounts + +An interface to the account database, and knowledge of how to initialise an account on disc. + + +SUBTITLE BackupStoreConfigVerify + +The same configuration file is used by all the utilities. This is the Configuration verification structure for this file. + + +SUBTITLE BackupStoreInfo + +The "header" information about an account, specifying current disc usage, space limits, etc. + + +SUBTITLE StoreStructure + +Functions specifying how the files are laid out on disc in the store. + + diff --git a/docs/api-notes/raidfile/RaidFileRead.txt b/docs/api-notes/raidfile/RaidFileRead.txt new file mode 100644 index 00000000..0a5efbf7 --- /dev/null +++ b/docs/api-notes/raidfile/RaidFileRead.txt @@ -0,0 +1,14 @@ +CLASS RaidFileRead + +Read a raid file. + +IOStream interface, plus a few extras, including reading directories and checking that files exist. + + +FUNCTION RaidFileRead::Open + +Open a given raid file -- returns a pointer to a new RaidFileRead object. + +Note that one of two types could be returned, depending on the representation of the file. + + diff --git a/docs/api-notes/raidfile/RaidFileWrite.txt b/docs/api-notes/raidfile/RaidFileWrite.txt new file mode 100644 index 00000000..89500c37 --- /dev/null +++ b/docs/api-notes/raidfile/RaidFileWrite.txt @@ -0,0 +1,36 @@ +CLASS RaidFileWrite + +Interface to writing raidfiles. + +See IOStream interface. + +Some other useful functions are available, see h and cpp files. + + +FUNCTION RaidFileWrite::RaidFileWrite() + +The constructor takes the disc set number and filename of the file you're interested. + + +FUNCTION RaidFileWrite::Open() + +Open() opens the file for writing, and takes an "allow overwrite" flag. + + +FUNCTION RaidFileWrite::Commit() + +Commmit the file, and make it visible to RaidFileRead. If ConvertToRaidNow == true, it will be converted to raid file representation immediately. + +Setting it to false is not a good idea. Later on, it will tell a daemon to convert it in the background, but for now it simply won't be converted. + + +FUNCTION RaidFileWrite::Discard() + +Abort the creation/update. Equivalent to just deleting the object without calling Commit(). + + +FUNCTION RaidFileWrite::Delete() + +Delete a file -- don't need to Open() it first. + + diff --git a/docs/api-notes/raidfile/lib_raidfile.txt b/docs/api-notes/raidfile/lib_raidfile.txt new file mode 100644 index 00000000..0c4244ce --- /dev/null +++ b/docs/api-notes/raidfile/lib_raidfile.txt @@ -0,0 +1,73 @@ +TITLE lib/raidfile + +Implements RAID type abilities in files in userland, along with simple transaction like semantics for writing and reading files. + +IOStream interfaces are used to read and write files. + +Eventually a raid file daemon will be written to "raidify" files in the background. This will allow fast streaming of data to a single disc, followed by splitting into separate raid files later. + +There is an option to disable raidification for use on systems with only one hard disc, or a hardware RAID array. + + +SUBTITLE Controller + +The raid disc sets are managed by RaidFileController. This reads the configuration file, /etc/box/raidfile.conf, and then stores the sets of discs for use by the other classes. + +In the code, files are referenced using an integer set number, and a filename within that set. For example, (0, "dir/subdir/filename.ext"). The RAID file controller turns this into actual physcial filenames transparently. + +The files are devided into blocks, which should match the fragment/block size of the underlying filesystem for maximum space efficiency. + +If the directories specified in the configuration files are all equal, then userland RAID is disabled. The files will not be processed into the redundant parts. + + +SUBTITLE Writing + +Files are written (and modified) using the RaidFileWrite class. This behaves as a seekable IOStream for writing. + +When all data has been written, the file is committed. This makes it available to be read. Processing to split it into the three files can be delayed. + +If the data is not commited, the temporary write file will be deleted. + +RaidFileWrite will optionally refuse to overwrite another file. If it is being overwritten, a read to that file will get the old file until it has been committed. + + +SUBTITLE Reading + +File are opened and read with RaidFileRead::Open. This returns an IOStream (with some extras) which reads a file. + +If the file has been turned into the RAID representation, then if an EIO error occurs on any operation, it will switch transparently into recovery mode where the file will be regenerated from the remaining two files using the parity file. (and log an error to syslog) + + +SUBTITLE Layout on disc + +The directory structure is replicated onto each of the three directories within the disc set. + +Given a filename x, within the disc set, one of the three discs is chosen as the 0th disc for this file. This is done by hashing the filename. This ensures that an equal load is placed on each disc -- otherwise one disc would only be accessed when a RAID file was being written. + +When the file is being written, the actual physical file is "x.rfwX". This just a striaght normal file -- nothing is done to the data. + +When it is commited, it is renamed to "x.rfw". + +Then, when it is turned into the raid representation, it is split across three discs, each file named "x.rf". + +When trying to open a file, the first attempt is for "x.rfw". If this exists, it is used as is. + +If this fails, two of the "x.rf" files are attempted to be opened. If this succeeds, the file is opened in RAID mode. + +This procedure guarenettes that a file will be opened either as a normal file, or as the correct three RAID file elements, even if a new version is being commited at the same time, or another process is working to turn it into a raid file. + + +SUBTITLE Raid file representation + +The file is split into three files, stripe 1, stripe 2 and parity. + +Considering the file as a set of blocks of the size specified in the configuration, odd numbered blocks go in stripe 1, even blocks in stripe 2, and a XOR of the corresponding blocks in parity. + +Thus only two files are needed to recreate the original. + +The complexity comes because the size of the file is not stored inside the files at any point. This means if stripe 2 is missing, it is ambiguous as to how big the file is. + +Size is stored as a 64 bit integer. This is appended to the parity file, unless it can be stored by XORing it into the last 8 bytes of the parity file. + +This scheme is complex to implement (see the RaidFileRead.cpp!) but minimises the disc spaced wasted. + diff --git a/docs/api-notes/win32_build_on_cygwin_using_mingw.txt b/docs/api-notes/win32_build_on_cygwin_using_mingw.txt new file mode 100644 index 00000000..dccb6be0 --- /dev/null +++ b/docs/api-notes/win32_build_on_cygwin_using_mingw.txt @@ -0,0 +1,107 @@ +How to build Box Backup on Win32 using Cygwin and MinGW +By Chris Wilson, 2009-03-31 + +(To read this document online with better formatting, browse to: +[http://www.boxbackup.org/trac/wiki/CompileWithMinGW]) + +== MinGW C++ == + +Start by installing Cygwin on your Windows machine from [http://www.cygwin.org/cygwin/]. + +Make sure to select the following packages during installation: + + * Devel/automake + * Devel/autoconf + * Devel/gcc-mingw + * Devel/gcc-mingw-core + * Devel/gcc-mingw-g++ + * Devel/make + * Devel/mingw-runtime + * Lib/libxml2 + * Lib/libxslt + * Mingw/mingw-zlib + * Perl/Perl + +If you already have Cygwin installed, please re-run the installer and +ensure that those packages are installed. + +== Base Directory == + +Choose a directory where you will unpack and compile OpenSSL, Zlib and Box Backup. We will call this the ''base directory''. An example might be: + + C:\Cygwin\Home\Your Username + +Make sure you know the full path to this directory. + +== OpenSSL == + +Download OpenSSL from [http://www.openssl.org/source/openssl-0.9.7i.tar.gz] + +Open a Cygwin shell, go to the base directory, and unpack OpenSSL: + + tar xzvf openssl-0.9.7i.tar.gz + +Configure OpenSSL for MinGW compilation, and build and install it: + + cd openssl-0.9.7i + ./Configure --prefix=/usr/i686-pc-mingw32/ mingw + make + make install + +== PCRE == + +This step is only required to support regular expressions in including/excluding files from backups. However, this is a very useful feature. + +Download PCRE from +[http://prdownloads.sourceforge.net/pcre/pcre-6.3.tar.bz2?download] + +Open a Cygwin shell, go to the base directory, and unpack PCRE: + + tar xjvf pcre-6.3.tar.bz2 + +Configure PCRE for MinGW compilation, and build and install it: + + cd pcre-6.3 + export CFLAGS="-mno-cygwin" + ./configure + make winshared + cp .libs/libpcre.a .libs/libpcreposix.a /lib/mingw + cp pcreposix.h /usr/include/mingw + +== Download Box Backup == + +Go back to the base directory, and download the latest Box Backup sources: + + svn co https://www.boxbackup.org/svn/box/trunk/ trunk + +Note: If you have problems during the configure or make stage, please try to eliminate one potential source of problems by running the following command to convert all line endings to Unix format: + + find -type f -not \( -wholename .*svn*. \) -exec dos2unix {} \; + +== Compile Box Backup == + +Enter the source directory and configure like this: + + cd trunk + ./infrastructure/mingw/configure.sh + make + +== Installation == + +Create the destination directory, ''C:\Program Files\Box Backup\bbackupd''. + +Write a configuration file, keys and certificate on a Unix machine, and copy them into the ''Box Backup'' directory, together with the following files from the ''base directory'': + + * boxbackup\release\bin\bbackupd\bbackupd.exe + * boxbackup\release\bin\bbackupquery\bbackupquery.exe + * boxbackup\release\bin\bbackupctl\bbackupctl.exe + * openssl\out32dll\libeay32.dll + * openssl\out32dll\ssleay32.dll + * zlib\zlib1.dll + +Ensure that the user running Box Backup can read from the ''Box Backup'' directory, and write to the ''bbackupd'' directory inside it. + +Run Box Backup by double-clicking on it, and check that it connects to the server. If the window opens and closes immediately, it's probably due to a problem with the configuration file - check the Windows Event Viewer for details. + +See also the service installation and upgrade instructions at +[https://www.boxbackup.org/trac/wiki/CompilationOnWindows]. diff --git a/docs/api-notes/win32_build_on_linux_using_mingw.txt b/docs/api-notes/win32_build_on_linux_using_mingw.txt new file mode 100644 index 00000000..2c3e4fe9 --- /dev/null +++ b/docs/api-notes/win32_build_on_linux_using_mingw.txt @@ -0,0 +1,108 @@ +How to build Box Backup for Windows (Native) on Linux using MinGW +By Chris Wilson, 2005-12-07 + +Install the MinGW cross-compiler for Windows: + +- Debian and Ubuntu users can "apt-get install mingw32" +- Fedora and SuSE users can download RPM packages from + [http://mirzam.it.vu.nl/mingw/] + +You will need to know the prefix used by the cross-compiler executables. +It will usually be something like "ix86-mingw32*-". All the binaries in the +cross-compiler package will start with this prefix. The documentation below +assumes that it is "i386-mingw32-". Adjust to taste. + +You will also need to install Wine and the Linux kernel "binary formats" +(binfmt) support, so that you can run Windows executables on Linux, +otherwise the configure scripts will not work properly with a cross-compiler. +On Ubuntu, run: + + apt-get install wine binfmt-support + /etc/init.d/binfmt-support start + +Start by downloading Zlib from [http://www.zlib.net/], unpack and enter +source directory: + + export CC=i386-mingw32-gcc + export AR="i386-mingw32-ar rc" + export RANLIB="i386-mingw32-ranlib" + ./configure + make + make install prefix=/usr/local/i386-mingw32 + +Download OpenSSL 0.9.8b from +[http://www.openssl.org/source/openssl-0.9.8b.tar.gz] + +Unpack and configure: + + tar xzvf openssl-0.9.8b.tar.gz + cd openssl-0.9.8b + ./Configure --prefix=/usr/local/i386-mingw32 mingw + make makefile.one + wget http://www.boxbackup.org/svn/box/chris/win32/support/openssl-0.9.8b-mingw-cross.patch + patch -p1 < openssl-0.9.8b-mingw-cross.patch + make -f makefile.one + make -f makefile.one install + +Download PCRE from +[http://prdownloads.sourceforge.net/pcre/pcre-6.3.tar.bz2?download] + +Unpack: + + tar xjvf pcre-6.3.tar.bz2 + cd pcre-6.3 + +Configure and make: + + export AR=i386-mingw32-ar + ./configure --host=i386-mingw32 --prefix=/usr/local/i386-mingw32/ + make winshared + +If you get this error: + + ./dftables.exe pcre_chartables.c + /bin/bash: ./dftables.exe: cannot execute binary file + make: *** [pcre_chartables.c] Error 126 + +then run: + + wine ./dftables.exe pcre_chartables.c + make winshared + +to complete the build. Finally: + + cp .libs/libpcre.a /usr/local/i386-pc-mingw32/lib + cp .libs/libpcreposix.a /usr/local/i386-pc-mingw32/lib + cp pcreposix.h /usr/local/i386-pc-mingw32/include + +You will need to find a copy of mingwm10.dll that matches your cross-compiler. +Most MinGW distributions should come with it. On Debian and Ubuntu, for some +bizarre reason, you'll find it compressed as +/usr/share/doc/mingw32-runtime/mingwm10.dll.gz, in which case you'll +have to un-gzip it with "gzip -d". Copy it to a known location, e.g. +/usr/local/i386-mingw32/bin. + +Download and extract Box Backup, and change into the base directory, +e.g. boxbackup-0.11rc2. Change the path to mingwm10.dll in parcels.txt to +match where you found or installed it. + +Now configure Box with: + + ./configure --host=i386-mingw32 \ + CXXFLAGS="-mthreads -I/usr/local/i386-mingw32/include" \ + LDFLAGS=" -mthreads -L/usr/local/i386-mingw32/lib" \ + LIBS="-lcrypto -lws2_32 -lgdi32" + make + +or, if that fails, try this: + + export CXX="i386-mingw32-g++" + export AR=i386-mingw32-ar + export RANLIB=i386-mingw32-ranlib + export CFLAGS="-mthreads" + export CXXFLAGS="-mthreads" + export LDFLAGS="-mthreads" + export LIBS="-lcrypto -lws2_32 -lgdi32" + (if you don't have a "configure" file, run "./bootstrap") + ./configure --target=i386-mingw32 + make CXX="$CXX" AR="$AR" RANLIB="$RANLIB" WINDRES="i386-mingw32-windres" diff --git a/docs/api-notes/windows_porting.txt b/docs/api-notes/windows_porting.txt new file mode 100644 index 00000000..ada3b857 --- /dev/null +++ b/docs/api-notes/windows_porting.txt @@ -0,0 +1,100 @@ +TITLE Notes on porting the backup client to Windows + +It should be relatively easy to port the backup client (bbackupd) to Windows. However, the server relies on unlink() behaviour of the UNIX filesystem which is different on the Windows platform, so it will not be so easy to port. + +An installation of perl is required to build the system. The ActiveState port is the easiest to install. + + +SUBTITLE Build environment + +The build environment generation script, makebuildenv.pl, uses perl to scan all the files and generate makefiles. It's rather orientated towards UNIX systems with gcc. + +Probably the easiest way to handle this is to detect the Windows platform, set up a few variables appropriately (in BoxPlatform.pm) and then post-process the generated Makefiles to mould them into something more handy for the MS Windows compiler and toolchain. + +The script itself is a bit messy. It was fine at first, but then the multi-platform thing got in the way. I do intend to rewrite it at some point in the future. + +Make sure your new version defines PLATFORM_WIN32 on the compile line. + +All files #include "Box.h" as the first include file. Use this for pre-compiled headers. Edit BoxPlatform.h to include the Windows headers required, and include a PLATFORM_WIN32 section. The easiest start will be to leave this blank, apart from typedefs for the basic types and any "not supported" #defines you can find. + +Boring bits of the code, such as exceptions and protocol definitions, are autogenerated using perl scripts. This code should be portable without modification. + +I have tried to avoid the things I know won't work with the MS compiler, so hopefully the code will be fairly clean. However, it might be a little easier to use the MinGW compiler [ http://www.mingw.org/ ] just to be consistent with the UNIX version. But I hope this won't be necessary. + +You'll need the latest version of OpenSSL. This was slightly difficult to get to compile last time I tried -- especially if you're determined to use the optimised assembler version. The main difficulty was getting a version which would link properly with the options used in my project, the default libraries selected got in the way. + + +SUBTITLE Porting as UNIX emulation + +Since the daemon uses so few UNIX system calls and with a limited set of options, it seems to make sense to port it by writing emulations of these functions. It's probably nicest to create a lib/win32 directory, and populate this with header files corresponding to the UNIX header files used. These just contain inline functions which map the UNIX calls to Win32 calls. + +File/socket handles may have to be translated. -1 is used as a failure return value and by the code internally to mark an invalid socket handle. (0 is a valid socket handle) + +Of course, some bits of code aren't relevant, so will just be #ifdefed out, or replaced. But this should be minimal. (Only perhaps the small bit relating to filesystem structure -- there aren't really mount points as such.) + + +SUBTITLE File tracking + +The daemon uses the inode number of a file to keep track of files and directories, so when they're renamed they can be moved efficiently on the store. Some unique (per filesystem) number will have to be found and used instead. + +It uses the Berkeley DB to store these on disc. It's likely another storage system will need to be used. (It just has to map the file's unique number into to a 8 byte struct.) + +There is a in-memory implementation for platforms which don't support Berkeley DB, but this isn't so good when the daemon has to be restarted as all the tracking is lost. But it's an easy start. + + +SUBTITLE Expected filesystem behaviour + +File and directories have (at least) two modification times, for contents and attributes. + +For files, the contents modification time must change when the contents change, and the attributes time when the attributes change (and may change when the contents change too.) + +For directories, the contents modification time must change when files or directories are deleted or added. If it changes any more frequently than this, then the client will be slightly less efficient -- it will download the store's directory listing whenever this time changes. The attributes modification time is less important, as the actual attributes are compared and only uploaded if different. + + +SUBTITLE Attributes + +Attributes means file modification times, flags, and filesystem permissions. + +The BackupClientFileAttribute class will need to be extended. Allocate another "attribute type" for the Win32 attributes, and then serialise it in a compatible way -- put your new attribute type in the header, and then a serialised network byte order structure in the rest. The different size of block is handled for you, and the server never looks inside. + +Add code so that under UNIX, Win32 attributes are ignored, and UNIX attributes under Win32. + +It's probably not necessary to worry too much about these for the first version. Not many people seem to use these attributes anyway. + + +SUBTITLE Times + +The system uses it's own 64 bit time type -- see BoxTime.h. Everything is translated to this from the various different system time types, and calculated and stored internally in this form. + + +SUBTITLE Daemon as a Service + +The client is derived from the Daemon class, which implements a daemon. The interface is simple, and it shouldn't be hard to write a compatible class which implements a Windows Service instead. + +Or cheat and run it as a Win32 application. + +Note that the daemon expects to be able to read every file it wants, and will abort a scan and upload run if it gets an error. The daemon must therefore be run with sufficient privileges. It runs as root under UNIX. + + +SUBTITLE Command Socket + +The backup daemon accepts commands from bbackupctl through a UNIX domain socket. When a connection is made, the user ID of the connecting process is checked to see if it's the same user ID as the daemon is running under. + +This may not have any exact analogue under Win32, so another communications scheme may have to be devised. + +This is only actually necessary if the client is to be run in snapshot mode. It can be safely left unimplemented if snapshot mode is not required, or the prompts for it to sync with the server are implemented some other way. + + +SUBTITLE NTFS streams + +If you want to back up NTFS streams, then a generic solution should probably be defined, so that the Mac OS X resource forks can be backed up with the same mechanism. + + +SUBTITLE Source code + +I work on a slightly different version of the source files. A make distribution script adds the license header and removes private sections of code. This means submitted diffs need a slight bit of translation. + + + + + diff --git a/docs/backup/INDEX.txt b/docs/backup/INDEX.txt deleted file mode 100644 index 50406cac..00000000 --- a/docs/backup/INDEX.txt +++ /dev/null @@ -1,61 +0,0 @@ -TITLE Programmers Notes for Box Backup - -This directory contains the programmers notes for the Box Backup system. They will be of interest only if you want to review or modify the code. - -These notes are intended to be run through a script to produce HTML at some later stage, hence the marks such as 'TITLE'. - - -SUBTITLE Organisation - -The project is split up into several modules. The modules within 'lib' are building blocks for creation of the actual executable programs within 'bin'. 'test' contains unit tests for lib and bin modules. - -The file modules.txt lists the modules which are to be built, and their dependencies. It also allows for platform differences. - - -SUBTITLE Documentation Organisation - -In this directory, the files correspond to modules or areas of interest. Sub directories of the same name contain files documenting specific classes within that module. - - -SUBTITLE Suggested reading order - -* lib_common -* lib_server -* bin_bbackupd -* backup_encryption.txt -* bin_bstored -* lib_raidfile - -and refer to other sections as required. - - -SUBTITLE Building - -The makefiles are generated by makebuildenv.pl. (The top level makefile is generated by makeparcels.pl, but this is for the end user to use, not a programmer.) - -To build a module, cd to it and type make. If the -DRELEASE option is specified (RELEASE=1 with GNU make) the release version will be built. The object files and exes are placed in a directory structure under 'release' or 'debug'. - -It is intended that a test will be written for everything, so in general make commands will be issued only within the test/* modules. Once it has been built, cd to debug/test/ and run the test with ./t . - - -SUBTITLE Programming style - -The code is written to be easy to write. Ease of programming is the primary concern, as this should lead to fewer bugs. Efficiency improvements can be made later when the system as a whole works. - -Much use is made of the STL. - -There is no common base class. - -All errors are reported using exceptions. - -Some of the boring code is generated by perl scripts from description files. - -There are a lot of modules and classes which can easily be used to build other projects in the future -- there is a lot of "framework" code. - - -SUBTITLE Lots more documentation - -The files are extensively commented. Consider this notes as an overview, and then read the source files for detailed and definitive information. - -Each function and class has a very brief decsription of it's purpose in a standard header, and extensive efforts have been maed to comment the code itself. - diff --git a/docs/backup/Win32_Clients.txt b/docs/backup/Win32_Clients.txt deleted file mode 100644 index fad6c4af..00000000 --- a/docs/backup/Win32_Clients.txt +++ /dev/null @@ -1,13 +0,0 @@ -The basic client tools now run on Win32 natively. -The port was done by nick@omniis.com. - -* bbackupd -* bbackupquery -* bbackupctl - -Have been ported. bbackupd runs as a NT style service. - -Known limitations: - -* File attributes and permissions are not backed up. - diff --git a/docs/backup/backup_encryption.txt b/docs/backup/backup_encryption.txt deleted file mode 100644 index 36580581..00000000 --- a/docs/backup/backup_encryption.txt +++ /dev/null @@ -1,109 +0,0 @@ -TITLE Encryption in the backup system - -This document explains how everything is encrypted in the backup system, and points to the various functions which need reviewing to ensure they do actually follow this scheme. - - -SUBTITLE Security objectives - -The crpyto system is designed to keep the following things secret from an attacker who has full access to the server. - -* The names of the files and directories -* The contents of files and directories -* The exact size of files - -Things which are not secret are - -* Directory heirarchy and number of files in each directory -* How the files change over time -* Approximate size of files - - -SUBTITLE Keys - -There are four separate keys used: - -* Filename -* File attributes -* File block index -* File data - -and an additional secret for file attribute hashes. - -The Cipher is Blowfish in CBC mode in most cases, except for the file data. All keys are maximum length 448 bit keys, since the key size only affects the setup time and this is done very infrequently. - -The file data is encrypted with AES in CBC mode, with a 256 bit key (max length). Blowfish is used elsewhere because the larger block size of AES, while more secure, would be terribly space inefficient. Note that Blowfish may also be used when older versions of OpenSSL are in use, and for backwards compatibility with older versions. - -The keys are generated using "openssl rand", and a 1k file of key material is stored in /etc/box/bbackupd. The configuration scripts make this readable only by root. - -Code for review: BackupClientCryptoKeys_Setup() -in lib/backupclient/BackupClientCryptoKeys.cpp - - -SUBTITLE Filenames - -Filenames need to be secret from the attacker, but they need to be compared on the server so it can determine whether or not is it a new version of an old file. - -So, the same Initialisation Vector is used for every single filename, so the same filename encrypted twice will have the same binary representation. - -Filenames use standard PKCS padding implemented by OpenSSL. They are proceeded by two bytes of header which describe the length, and the encoding. - -Code for review: BackupStoreFilenameClear::EncryptClear() -in lib/backupclient/BackupStoreFilenameClear.cpp - - -SUBTITLE File attributes - -These are kept secret as well, since they reveal information. Especially as they contain the target name of symbolic links. - -To encrypt, a random Initialisation Vector is choosen. This is stored first, followed by the attribute data encrypted with PKCS padding. - -Code for review: BackupClientFileAttributes::EncryptAttr() -in lib/backupclient/BackupClientFileAttributes.cpp - - -SUBTITLE File attribute hashes - -To detect and update file attributes efficiently, the file status change time is not used, as this would give suprious results and result in unnecessary updates to the server. Instead, a hash of user id, group id, and mode is used. - -To avoid revealing details about attributes - -1) The filename is added to the hash, so that an attacker cannot determine whether or not two files have identical attributes - -2) A secret is added to the hash, so that an attacker cannot compare attributes between accounts. - -The hash used is the first 64 bits of an MD5 hash. - - -SUBTITLE File block index - -Files are encoded in blocks, so that the rsync algorithm can be used on them. The data is compressed first before encryption. These small blocks don't give the best possible compression, but there is no alternative because the server can't see their contents. - -The file contains a number of blocks, which contain among other things - -* Size of the block when it's not compressed -* MD5 checksum of the block -* RollingChecksum of the block - -We don't want the attacker to know the size, so the first is bad. (Because of compression and padding, there's uncertainty on the size.) - -When the block is only a few bytes long, the latter two reveal it's contents with only a moderate amount of work. So these need to be encrypted. - -In the header of the index, a 64 bit number is chosen. The sensitive parts of the block are then encrypted, without padding, with an Initialisation Vector of this 64 bit number + the block index. - -If a block from an previous file is included in a new version of a file, the same checksum data will be encrypted again, but with a different IV. An eavesdropper will be able to easily find out which data has been re-encrypted, but the plaintext is not revealed. - -Code for review: BackupStoreFileEncodeStream::Read() (IV base choosen about half-way through) -BackupStoreFileEncodeStream::EncodeCurrentBlock() (encrypt index entry) -in lib/backupclient/BackupStoreFileEncodeStream.cpp - - -SUBTITLE File data - -As above, the first is split into chunks and compressed. - -Then, a random initialisation vector is chosen, stored first, followed by the compressed file data encrypted using PKCS padding. - -Code for review: BackupStoreFileEncodeStream::EncodeCurrentBlock() -in lib/backupclient/BackupStoreFileEncodeStream.cpp - - diff --git a/docs/backup/bin_bbackupd.txt b/docs/backup/bin_bbackupd.txt deleted file mode 100644 index 67ad9267..00000000 --- a/docs/backup/bin_bbackupd.txt +++ /dev/null @@ -1,88 +0,0 @@ -TITLE bin/bbackupd - -The backup client daemon. - -This aims to maintain as little information as possible to record which files have been uploaded to the server, while minimising the amount of queries which have to be made to the server. - - -SUBTITLE Scanning - -The daemon is given a length of time, t, which files over this age should be uploaded to the server. This is to stop recently updated files being uploaded immediately to avoid uploading something repeatedly (on the assumption that if a file has been written, it is likely to be modified again shortly). - -It will scan the files at a configured interval, and connect to the server if it needs to upload files or make queries about files and directories. - -The scan interval is actually varied slightly between each run by adding a random number up to a 64th of the configured time. This is to reduce cyclic patterns of load on the backup servers -- otherwise if all the boxes are turned on at about 9am, every morning at 9am there will be a huge spike in load on the server. - -Each scan chooses a time interval, which ends at the current time - t. This will be from 0 to current time - t on the first run, then the next run takes the start time as the end time of the previous run. The scan is only performed if the difference between the start and end times is greater or equal to t. - -For each configured location, the client scans the directories on disc recursively. - -For each directory - -* If the directory has never been scanned before (in this invocation of the daemon) or the modified time on the directory is not that recorded, the listing on the server is downloaded. - -* For each file, if it's modified time is within the time period, it is uploaded. If the directory has been downloaded, it is compared against that, and only uploaded if it's changed. - -* Find all the new files, and upload them if they lie within the time interval. - -* Recurse to sub directories, creating them on the server if necessary. - -Hence, the first time it runs, it will download and compare the entries on the disc to those on the server, but in future runs it will use the file and directory modification times to work out if there is anything which needs uploading. - -If there aren't any changes, it won't even need to connect to the server. - -There are some extra details which allow this to work reliably, but they are documented in the source. - - -SUBTITLE File attributes - -The backup client will update the file attributes on files as soon as it notices they are changed. It records most of the details from stat(), but only a few can be restored. Attributes will only be considered changed if the user id, group id or mode is changed. Detection is by a 64 bit hash, so detection is strictly speaking probablistic. - - -SUBTITLE Encryption - -All the user data is encrypted. There is a separate file, backup_encryption.txt which describes this, and where in the code to look to verify it works as described. - - -SUBTITLE Tracking files and directories - -Renaming files is a difficult problem under this minimal data scanning scheme, because you don't really know whether a file has been renamed, or another file deleted and new one created. - -The solution is to keep (on disc) a map of inode numbers to server object IDs for all directories and files over a certain user configurable threshold. Then, when a new file is discovered, it is first checked to see if it's in this map. If so, a rename is considered, which will take place if the local object corresponding to the name of the tracked object doesn't exist any more. - -Because of the renaming requirement, deletions of objects from the server are recorded and delayed until the end of the scan. - - -SUBTITLE Running out of space - -If the store server indicates on login to the backup client, it will scan, but not upload anything nor adjust it's internal stored details of the local objects. However, deletions and renames happen. - -This is to allow deletions to still work and reduce the amount of storage space used on the server, in the hope that in the future there will be enough space. - -Just not doing anything would mean that one big file created and then deleted at the wrong time would stall the whole backup process. - - -SUBTITLE BackupDaemon - -This is the daemon class for the backup daemon. It handles setting up of all the objects, and implements calulcation of the time intervals for the scanning. - - -SUBTITLE BackupClientContext - -State information for the scans, including maintaining a connection to the store server if required. - - -SUBTITLE BackupClientDirectoryRecord - -A record of state of a directory on the local filesystem. Containing the recursive scanning function, which is long and entertaining, but very necessary. It contains lots of comments which explain the exact details of what's going on. - - -SUBTITLE BackupClientInodeToIDMap - -A implementation of a map of inode number to object ID on the server. If Berkeley DB is available on the platform, it is stored on disc, otherwise there is an in memory version which isn't so good. - - - - - - diff --git a/docs/backup/bin_bbstored.txt b/docs/backup/bin_bbstored.txt deleted file mode 100644 index c9c4e229..00000000 --- a/docs/backup/bin_bbstored.txt +++ /dev/null @@ -1,54 +0,0 @@ -TITLE bin/bbstored - -The backup store daemon. - -Maintains a store of encrypted files, and every so often goes through deleting unnecessary data. - -Uses an implementation of Protocol to communicate with the backup client daemon. See bin/bbstored/backupprotocol.txt for details. - - -SUBTITLE Data storage - -The data is arranged as a set of objects within a RaidFile disc set. Each object has a 64 bit object ID, which is turned into a filename in a mildly complex manner which ensure that directories don't have too many objects in them, but there is a minimal number of nested directories. See StoreStructure::MakeObjectFilename in lib/backupstore/StoreStructure.cpp for more details. - -An object can be a directory or a file. Directories contain files and other directories. - -Files in directories are supersceded by new versions when uploaded, but the old versions are flagged as such. A new version has a different object ID to the old version. - -Every so often, a housekeeping process works out what can be deleted, and deletes unnecessary files to take them below the storage limits set in the store info file. - - -SUBTITLE Note about file storage and downloading - -There's one slight entertainment to file storage, in that the format of the file streamed depends on whether it's being downloaded or uploaded. - -The problem is that it contains an index of all the blocks. For efficiency in managing these blocks, they all need to be in the same place. - -Files are encoded and decoded as they are streamed to and from the server. With encoding, the index is only completely known at the end of the process, so it's sent last, and lives in the filesystem last. - -When it's downloaded, it can't be decoded without knowing the index. So the index is sent first, followed by the data. - - -SUBTITLE BackupContext - -The context of the current connection, and the object which modifies the store. - -Maintains a cache of directories, to avoid reading them continuously, and keeps a track of a BackupStoreInfo object which is written back periodiocally. - - -SUBTITLE BackupStoreDaemon - -A ServerTLS daemon which forks off a separate housekeeping process as it starts up. - -Handling connections is delegated to a Protocol implementation. - - -SUBTITLE BackupCommands.cpp - -Implementation of all the commands. Work which requires writing is handled in the context, read only commands mainly in this file. - - -SUBTITLE HousekeepStoreAccount - -A class which performs housekeeping on a single account. - diff --git a/docs/backup/encrypt_rsync.txt b/docs/backup/encrypt_rsync.txt deleted file mode 100644 index a5db2df2..00000000 --- a/docs/backup/encrypt_rsync.txt +++ /dev/null @@ -1,66 +0,0 @@ -TITLE Encrypted rsync algorithm - -The backup system uses a modified version of the rsync algorithm. A description of the plain algorithm can be found here: - - http://samba.anu.edu.au/rsync/tech_report/ - -The algorithm is modified to allow the server side to be encrypted, yet still benefit from the reduced bandwidth usage. For a single file transfer, the result will be only slightly less efficient than plain rsync. For a backup of a large directory, the overall bandwidth may be less due to the way the backup client daemon detects changes. - -This document assumes you have read the rsync document. - -The code is in lib/backupclient/BackupStoreFile*.*. - - -SUBTITLE Blocks - -Each file is broken up into small blocks. These are individually compressed and encrypted, and have an entry in an index which contains, encrypted, it's weak and strong checksums and decoded plaintext size. This is all done on the client. - -Why not just encrypt the file, and use the standard rsync algorithm? - -1) Compression cannot be used, since encryption turns the file into essentially random data. This is not very compressible. - -2) Any modification to the file will result in all data after that in the file having different ciphertext (in any cipher mode we might want to use). Therefore the rsync algorithm will only be able to detect "same" blocks up until the first modification. This significantly reduces the effectiveness of the process. - -Note that blocks are not all the same size. The last block in the file is unlikely to be a full block, and if data is inserted which is not a integral multiple of the block size, odd sized blocks need to be created. This is because the server cannot reassemble the blocks, because the contents are opaque to the server. - - -SUBTITLE Modifed algorithm - -To produce a list of the changes to send the new version, the client requests the block index of the file. This is the same step as requesting the weak and strong checksums from the remote side with rsync. - -The client then decrypts the index, and builds a list of the 8 most used block sizes above a certain threshold size. - -The new version of the file is then scanned in exactly the same way as rsync for these 8 block sizes. If a block is found, then it is added to a list of found blocks, sorted by position in the file. If a block has already been found at that position, then the old entry is only replaced by the new entry if the new entry is a "better" (bigger) match. - -The block size covering the biggest file area is searched first, so that most of the file can be skipped over after the first pass without expensive checksumming. - -A "recipe" is then built from the found list, by trivially discarding overlapping blocks. Each entry consists of a number of bytes of "new" data, a block start number, and a number of blocks from the old file. The data is stored like this as a memory optimisation, assuming that files mostly stay the same rather than having all their blocks reordered. - -The file is then encoded, with new data being sent as blocks of data, and references to blocks in the old file. The new index is built completely, as the checksums and size need to be rencrypted to match their position in the index. - - -SUBTITLE Combination on server - -The "diff" which is sent from the client is assembled into a full file on the server, simply by adding in blocks from the old file where they are specified in the block index. - - -SUBTITLE Storage on server - -Given that the server will in general store several versions of a file, combining old and new files to form a new file is not terribly efficient on storage space. Particularly for large multi-Gb database files. - -An alternative scheme is outlined below, however, it is significantly more complex to implement, and so is not implemented in this version. - -1) In the block index of the files, store the file ID of the file which each block is source from. This allows a single file to reference blocks from many files. - -2) When the file is downloaded, the server combines the blocks from all the files into a new file as it is streamed to the client. (This is not particuarly complicated to do.) - -This all sounds fine, until housekeeping is considered. Old versions need to be deleted, without losing any blocks necessary for future versions. - -Instead of just deleting a file, the server works out which blocks are still required, and rebuilds the file omitting those blocks which aren't required. - -This complicates working out how much space a file will release when it is "deleted", and indeed, adds a whole new level of complexity to the housekeeping process. (And the tests!) - -The directory structure will need an additional flag, "Partial file", which specifies that the entry cannot be built as previous blocks are no longer available. Entries with this flag should never be sent to the client. - - - diff --git a/docs/backup/lib_backupclient.txt b/docs/backup/lib_backupclient.txt deleted file mode 100644 index 3e4a079b..00000000 --- a/docs/backup/lib_backupclient.txt +++ /dev/null @@ -1,46 +0,0 @@ -TITLE lib/backupclient - -Classes used on the store and on the server. - -See documentation in the files for more details. - - -SUBTITLE BackupStoreDirectory - -The directory listing class, containing a number of entries, representing files. - - -SUBTITLE BackupStoreFile - -Handles compressing and encrypting files, and decoding files downloaded from the server. - - -SUBTITLE BackupStoreFilename - -An encrypted filename. - - -SUBTITLE BackupStoreFilenameClear - -Derived from BackupStoreFilename, but with the ability to encrypt and decrypt filenames. Client side only. - - -SUBTITLE BackupClientFileAttributes - -Only used on the client -- the server treats attributes as blocks of opaque data. - -This reads attributes from files on discs, stores them, encrypts them, and applies them to new files. - -Also has a static function to generate filename attribute hashes given a struct stat and the filename. - - -SUBTITLE BackupClientRestore - -Routines to restore files from the server onto the client filesystem. - - -SUBTITLE BackupClientCryptoKeys - -This reads the key material from disc, and sets up the crypto for storing files, attributes and directories. - - diff --git a/docs/backup/lib_backupstore.txt b/docs/backup/lib_backupstore.txt deleted file mode 100644 index 8f24eb7b..00000000 --- a/docs/backup/lib_backupstore.txt +++ /dev/null @@ -1,30 +0,0 @@ -TITLE lib/backupstore - -Classes which are shared amongst the server side store utilities, bbstored and bbstoreaccounts. Note also depends on lib/backupclient, as a lot of code is shared between the client and server. - - -SUBTITLE BackupStoreAccountDatabase - -A simple implementation of an account database. This will be replaced with a more suitable implementation. - - -SUBTITLE BackupStoreAccounts - -An interface to the account database, and knowledge of how to initialise an account on disc. - - -SUBTITLE BackupStoreConfigVerify - -The same configuration file is used by all the utilities. This is the Configuration verification structure for this file. - - -SUBTITLE BackupStoreInfo - -The "header" information about an account, specifying current disc usage, space limits, etc. - - -SUBTITLE StoreStructure - -Functions specifying how the files are laid out on disc in the store. - - diff --git a/docs/backup/win32_build_on_cygwin_using_mingw.txt b/docs/backup/win32_build_on_cygwin_using_mingw.txt deleted file mode 100644 index e71b5764..00000000 --- a/docs/backup/win32_build_on_cygwin_using_mingw.txt +++ /dev/null @@ -1,53 +0,0 @@ -How to build Box Backup on Win32 using Cygwin and MinGW -By Chris Wilson, 2007-05-26 - -(To read this document online with better formatting, browse to: -http://bbdev.fluffy.co.uk/trac/wiki/CompileWithMinGW) - -Start by installing Cygwin on your Windows machine [http://www.cygwin.org]. -Make sure to select the following packages during installation: - -* Devel/gcc-mingw -* Devel/gcc-mingw-core -* Devel/gcc-mingw-g++ -* Mingw/mingw-zlib - -If you already have Cygwin installed, please re-run the installer and -ensure that those packages are installed. - -Download OpenSSL from -[http://www.openssl.org/source/openssl-0.9.7i.tar.gz] - -Open a Cygwin shell, and unpack OpenSSL: - - tar xzvf openssl-0.9.7i.tar.gz - -Configure OpenSSL for MinGW compilation, and build and install it: - - cd openssl-0.9.7i - ./Configure --prefix=/usr/i686-pc-mingw32/ mingw - make - make install - -Download PCRE from -[http://prdownloads.sourceforge.net/pcre/pcre-6.3.tar.bz2?download] - -Open a Cygwin shell, and unpack PCRE: - - tar xjvf pcre-6.3.tar.bz2 - -Configure PCRE for MinGW compilation, and build and install it: - - cd pcre-6.3 - export CFLAGS="-mno-cygwin" - ./configure - make winshared - cp .libs/libpcre.a .libs/libpcreposix.a /usr/lib/mingw - cp pcreposix.h /usr/include/mingw - -Now unpack the Box Backup sources, enter the source directory, -and configure like this: - - ./infrastructure/mingw/configure.sh - make - diff --git a/docs/backup/win32_build_on_linux_using_mingw.txt b/docs/backup/win32_build_on_linux_using_mingw.txt deleted file mode 100644 index 407096b1..00000000 --- a/docs/backup/win32_build_on_linux_using_mingw.txt +++ /dev/null @@ -1,64 +0,0 @@ -How to build Box Backup for Windows (Native) on Linux using MinGW -By Chris Wilson, 2005-12-07 - -Install the MinGW cross-compiler for Windows: - -- Debian users can "apt-get install mingw32" -- Fedora and SuSE users can download RPM packages from - [http://mirzam.it.vu.nl/mingw/] - -You will need to know the prefix used by the cross-compiler executables. -It will usually be something like "ix86-mingw32*-". All the binaries in the -cross-compiler package will start with this prefix. The documentation below -assumes that it is "i386-mingw32-". Adjust to taste. - -Download Zlib from [http://www.zlib.net/], unpack and enter source directory: - - export CC=i386-mingw32-gcc - export AR="i386-mingw32-ar rc" - export RANLIB="i386-mingw32-ranlib" - ./configure - make - make install prefix=/usr/local/i386-mingw32 - -Download OpenSSL 0.9.8b from -[http://www.openssl.org/source/openssl-0.9.8b.tar.gz] - -Unpack and configure: - - tar xzvf openssl-0.9.8b.tar.gz - cd openssl-0.9.8b - ./Configure --prefix=/usr/local/i386-mingw32 mingw - make makefile.one - wget http://bbdev.fluffy.co.uk/svn/box/chris/win32/support/openssl-0.9.8b-mingw-cross.patch - patch -p1 < openssl-0.9.8b-mingw-cross.patch - make -f makefile.one - make -f makefile.one install - -Download PCRE from -[http://prdownloads.sourceforge.net/pcre/pcre-6.3.tar.bz2?download] - -Unpack: - - tar xjvf pcre-6.3.tar.bz2 - cd pcre-6.3 - -Configure and make: - - ./configure --host=i586-mingw32msvc --prefix=/usr/i386-mingw32/ - make winshared wininstall - cp .libs/libpcreposix.a /usr/i386-pc-mingw32/lib - cp pcreposix.h /usr/i386-pc-mingw32/include/regex.h - -Configure Box with: - - export CXX="i386-mingw32-g++" - export AR=i386-mingw32-ar - export RANLIB=i386-mingw32-ranlib - export CFLAGS="-mthreads" - export CXXFLAGS="-mthreads" - export LDFLAGS="-mthreads" - export LIBS="-lcrypto -lws2_32 -lgdi32" - (if you don't have a "configure" file, run "./bootstrap") - ./configure --target=i386-mingw32 - make CXX="$CXX" AR="$AR" RANLIB="$RANLIB" WINDRES="i386-mingw32-windres" diff --git a/docs/backup/windows_porting.txt b/docs/backup/windows_porting.txt deleted file mode 100644 index ada3b857..00000000 --- a/docs/backup/windows_porting.txt +++ /dev/null @@ -1,100 +0,0 @@ -TITLE Notes on porting the backup client to Windows - -It should be relatively easy to port the backup client (bbackupd) to Windows. However, the server relies on unlink() behaviour of the UNIX filesystem which is different on the Windows platform, so it will not be so easy to port. - -An installation of perl is required to build the system. The ActiveState port is the easiest to install. - - -SUBTITLE Build environment - -The build environment generation script, makebuildenv.pl, uses perl to scan all the files and generate makefiles. It's rather orientated towards UNIX systems with gcc. - -Probably the easiest way to handle this is to detect the Windows platform, set up a few variables appropriately (in BoxPlatform.pm) and then post-process the generated Makefiles to mould them into something more handy for the MS Windows compiler and toolchain. - -The script itself is a bit messy. It was fine at first, but then the multi-platform thing got in the way. I do intend to rewrite it at some point in the future. - -Make sure your new version defines PLATFORM_WIN32 on the compile line. - -All files #include "Box.h" as the first include file. Use this for pre-compiled headers. Edit BoxPlatform.h to include the Windows headers required, and include a PLATFORM_WIN32 section. The easiest start will be to leave this blank, apart from typedefs for the basic types and any "not supported" #defines you can find. - -Boring bits of the code, such as exceptions and protocol definitions, are autogenerated using perl scripts. This code should be portable without modification. - -I have tried to avoid the things I know won't work with the MS compiler, so hopefully the code will be fairly clean. However, it might be a little easier to use the MinGW compiler [ http://www.mingw.org/ ] just to be consistent with the UNIX version. But I hope this won't be necessary. - -You'll need the latest version of OpenSSL. This was slightly difficult to get to compile last time I tried -- especially if you're determined to use the optimised assembler version. The main difficulty was getting a version which would link properly with the options used in my project, the default libraries selected got in the way. - - -SUBTITLE Porting as UNIX emulation - -Since the daemon uses so few UNIX system calls and with a limited set of options, it seems to make sense to port it by writing emulations of these functions. It's probably nicest to create a lib/win32 directory, and populate this with header files corresponding to the UNIX header files used. These just contain inline functions which map the UNIX calls to Win32 calls. - -File/socket handles may have to be translated. -1 is used as a failure return value and by the code internally to mark an invalid socket handle. (0 is a valid socket handle) - -Of course, some bits of code aren't relevant, so will just be #ifdefed out, or replaced. But this should be minimal. (Only perhaps the small bit relating to filesystem structure -- there aren't really mount points as such.) - - -SUBTITLE File tracking - -The daemon uses the inode number of a file to keep track of files and directories, so when they're renamed they can be moved efficiently on the store. Some unique (per filesystem) number will have to be found and used instead. - -It uses the Berkeley DB to store these on disc. It's likely another storage system will need to be used. (It just has to map the file's unique number into to a 8 byte struct.) - -There is a in-memory implementation for platforms which don't support Berkeley DB, but this isn't so good when the daemon has to be restarted as all the tracking is lost. But it's an easy start. - - -SUBTITLE Expected filesystem behaviour - -File and directories have (at least) two modification times, for contents and attributes. - -For files, the contents modification time must change when the contents change, and the attributes time when the attributes change (and may change when the contents change too.) - -For directories, the contents modification time must change when files or directories are deleted or added. If it changes any more frequently than this, then the client will be slightly less efficient -- it will download the store's directory listing whenever this time changes. The attributes modification time is less important, as the actual attributes are compared and only uploaded if different. - - -SUBTITLE Attributes - -Attributes means file modification times, flags, and filesystem permissions. - -The BackupClientFileAttribute class will need to be extended. Allocate another "attribute type" for the Win32 attributes, and then serialise it in a compatible way -- put your new attribute type in the header, and then a serialised network byte order structure in the rest. The different size of block is handled for you, and the server never looks inside. - -Add code so that under UNIX, Win32 attributes are ignored, and UNIX attributes under Win32. - -It's probably not necessary to worry too much about these for the first version. Not many people seem to use these attributes anyway. - - -SUBTITLE Times - -The system uses it's own 64 bit time type -- see BoxTime.h. Everything is translated to this from the various different system time types, and calculated and stored internally in this form. - - -SUBTITLE Daemon as a Service - -The client is derived from the Daemon class, which implements a daemon. The interface is simple, and it shouldn't be hard to write a compatible class which implements a Windows Service instead. - -Or cheat and run it as a Win32 application. - -Note that the daemon expects to be able to read every file it wants, and will abort a scan and upload run if it gets an error. The daemon must therefore be run with sufficient privileges. It runs as root under UNIX. - - -SUBTITLE Command Socket - -The backup daemon accepts commands from bbackupctl through a UNIX domain socket. When a connection is made, the user ID of the connecting process is checked to see if it's the same user ID as the daemon is running under. - -This may not have any exact analogue under Win32, so another communications scheme may have to be devised. - -This is only actually necessary if the client is to be run in snapshot mode. It can be safely left unimplemented if snapshot mode is not required, or the prompts for it to sync with the server are implemented some other way. - - -SUBTITLE NTFS streams - -If you want to back up NTFS streams, then a generic solution should probably be defined, so that the Mac OS X resource forks can be backed up with the same mechanism. - - -SUBTITLE Source code - -I work on a slightly different version of the source files. A make distribution script adds the license header and removes private sections of code. This means submitted diffs need a slight bit of translation. - - - - - diff --git a/docs/common/lib_common.txt b/docs/common/lib_common.txt deleted file mode 100644 index 11d7b02d..00000000 --- a/docs/common/lib_common.txt +++ /dev/null @@ -1,52 +0,0 @@ -TITLE lib/common - -This module is the basic building block of the project. It is included implicitly as a dependency of all other modules. - -It provides basic services such as file and stream functionality, platform information, and various bits of utility code which doesn't fit naturally anywhere else but is useful to many other applications. - - -SUBTITLE Box.h - -The main include file for the project. It should be the first file included in every cpp file, followed by any system headers, then other project headers. This order has been chosen so that eventually it can be used as a target for pre-compiled headers. (This will be important for the Windows port) - - -SUBTITLE BoxPlatform.h - -This contains various bits of information on platform differences. The build scripts include a compile command line definition like PLATFORM_OPENBSD, which is then used to select the required options. - -Some of the stuff is not particularly pleasant, but it aims to produce a consistent compilation environment, and to have a abstracted way of setting other defines to tell the other files what things are available on this platform. - -GNU configure is not used, as it would be just another dependency. Although something does have to be done about compilation on Linux, which is just too different on each distribution to be caught by a common configuration here. - - -SUBTITLE Streams - -Much use is made of streams -- files, connections, buffers, all sorts of things are implemented using streams. - -The abstract base class IOStream defines the interface, see the class file for more details. - -Streams are lib/common's basic contribution to the project. A standard interface for handling data, and a number of utility classes which make performing common stream functions easy. - - -SUBTITLE Serialisation - -Note there is no "serialisable" class. Instead, objects which need to be serialised into Streams simply have two functions, ReadFromStream and WriteToStream. These perform as expected. - -As it turns out, there's no real need for a specific serialisation base class. If you do need to be able to serialise arbitary objects, there are two approaches. - -1) If you're serialising an arbitary object, you're going to know roughly what type it is. So it's likely to be a subclass of a known base class... in which case, you simply use virtual functions. - -2) If it really is an arbitary type, then you just write your function which accepts any type of object to serialise as a template. - - -SUBTITLE BoxException - -The exception base class for the project. All exceptions thrown by this code are dervied from BoxException. In general, each module which has unique errors has it's own exception class. - -CommonException errors are thrown throughout the project. - - -SUBTITLE Other bits - -There are some other extra useful bits which are documented only in the source files. - diff --git a/docs/common/lib_common/BoxTime.txt b/docs/common/lib_common/BoxTime.txt deleted file mode 100644 index 18bef5a5..00000000 --- a/docs/common/lib_common/BoxTime.txt +++ /dev/null @@ -1,7 +0,0 @@ -TITLE BoxTime - -Not strictly a class, but time is presented in the project as 64 bit integers as microseconds since the UNIX Epoch. - -There are a number of utility functions in the BoxTime*.h files, which are useful. - -To get BoxTime values from a struct stat, use the FileModificationTime.h functions. \ No newline at end of file diff --git a/docs/common/lib_common/CollectInBufferStream.txt b/docs/common/lib_common/CollectInBufferStream.txt deleted file mode 100644 index 5f9556a3..00000000 --- a/docs/common/lib_common/CollectInBufferStream.txt +++ /dev/null @@ -1,26 +0,0 @@ -CLASS CollectInBufferStream - -This class is essentially a buffer. Data is written to it, and stored in memory. Then when all data is written which will be written, it can be read out again. - -Useful for building streams in memory. - - -FUNCTION CollectInBufferStream::SetForReading() - -After you've written all the data, call this to set it to read mode. - - -FUNCTION CollectInBufferStream::Reset() - -Reset the stream to the state where is has no data, and is about to have data written. - - -FUNCTION CollectInBufferStream::GetSize() - -Get the size of the buffered data -- the entire data. Use the interface function BytesLeftToRead() in most cases. - - -FUNCTION CollectInBufferStream::GetBuffer() - -Get the buffer, so you can do bad things with it if you're feeling naughty. - diff --git a/docs/common/lib_common/Configuration.txt b/docs/common/lib_common/Configuration.txt deleted file mode 100644 index ef5f38f6..00000000 --- a/docs/common/lib_common/Configuration.txt +++ /dev/null @@ -1,102 +0,0 @@ -CLASS Configuration - -Implements the reading of multi-level configuration files. This is intended to be a generic way of representing configuration implementation. - -Basic validation is performed on files as they are read, specified by a data structure. - -test/common has some examples of usage. - - -SUBTITLE Configuration file format - -The format is simple, a list of Key = Value pairs. For example - -Key1 = Value1 -Key2 = Value2 - -Optionally, multiple values for the same key are allowed. - -Lists of configurations can be nested to any level. These are called Sub-configurations: - -SubConfig -{ - SubKey1 = ValueX - SubKey2 = ValueY -} - - -SUBTITLE Verification - -The verification structure specifies what keys are required, what are optional, and what sub-configurations are expected. Default values can be specified. - -See Configuration.h for the structures. - -RaidFileController::Initialise has a good example of a simple verification layout. - -Wildcards can be used for SubConfigurations, by specifying a name of "*". This allows you to use multiple sections to configure any number of items. - -Each item has a number of flags, which are combined to say whether an item is required, should be an integer or boolean, and rather importantly, whether it's the last item in the list. - -Verification is limited, so you may wish to do more verification the file. There are unimplemented hooks for custom verification functions to be included in the verification definitions. Should be done at some point. - -Boolean keys have possible values "true", "yes", "false", "no" (case insensitive). - - -FUNCTION Configuration::LoadAndVerify() - -Loads the configuration from disc, and verifies it. If there are problems with the verification, it returns some text which can be used to tell the user the problems with the file. These are fairly basic error messages, but do say what should be done to get it to parse properly. - -This returns a Configuration object, which can then be queries for Keys and Values. Sub-configurations are implemented through an interface which returns a reference to another Configuration object. - - - -FUNCTION Configuration::KeyExists() - -Does a specified key exist? - -Use this for optional key values only -- specify them in the verification structure if they are required. - - -FUNCTION Configuration::GetKeyValue() - -Get the value of a key as a string. - -If ConfigTest_MultiValueAllowed is set in the relevant entry in the verify structure, this string may contain multiple values, separated by a single byte with value 0x01 (use Configuration::MultiValueSeparator in code). Use SplitString() defined in Utils.h to split it into components. - -This representation was chosen as multi-values are probably rare, and unlikely to justify writing a nicer (but more memory intensive and complex) solution. - - -FUNCTION Configuration::GetKeyValueInt() - -Get the value of a key as an integer. Make sure the AsInt property is requried in the verification structure. - - -FUNCTION Configuration::GetKeyValueBool() - -Get the value of a key as a boolean value. Make sure the AsBool property is requried in the verification structure. - -Default to "false", should this verification not be performed and an unknown value is specified. - - -FUNCTION Configuration::GetKeyNames() - -Return a list of all the keys in this configuration. - - -FUNCTION Configuration::SubConfigurationExists() - -Does a specified sub-configuration exist? - - -FUNCTION Configuration::GetSubConfiguration() - -Return another Configuration object representing the sub section. - - -FUNCTION Configuration::GetSubConfigurationNames() - -Get a list of all the sub configurations. - -As there isn't a particularly neat way that configurations can be iterated over, mSubConfigurations is public. (BAD!) - - diff --git a/docs/common/lib_common/Conversion.txt b/docs/common/lib_common/Conversion.txt deleted file mode 100644 index 62d1967a..00000000 --- a/docs/common/lib_common/Conversion.txt +++ /dev/null @@ -1,14 +0,0 @@ -TITLE Generic type conversion - -Conversion.h provides generic type conversion. Within the BoxConvert namespace, it defines the templated function - - ToType Convert(FromType From) - -which converts the data type as expected. In general, from and to types have to be specified explicitly. - -Templates rather than overloaded functions are used, firstly to be absolutely explicit about the conversion required, and secondly because overloaded functions can't have differing return types for the same argument type. - -The function is specialised for various types, and the generic version uses C++ type conversion. - -Exceptions may be thrown if the conversion is not possible. These are all of the ConversionException type. - diff --git a/docs/common/lib_common/ExcludeList.txt b/docs/common/lib_common/ExcludeList.txt deleted file mode 100644 index 8a5bf36c..00000000 --- a/docs/common/lib_common/ExcludeList.txt +++ /dev/null @@ -1,21 +0,0 @@ -TITLE ExcludeList - -A class implementing a list of strings which are to be excluded in some way. Entries can be added as strings to be matched exactly, or as regular expressions. - -Multiple entries can be added in a single function call in a manner designed to work with the multi-value entries store in a Configuation object. - - -FUNCTION ExcludeList::AddDefiniteEntries() - -Definite entries are exact strings to match. - - -FUNCTION ExcludeList::AddRegexEntries() - -Regular expressions as defined by POSIX, and supported by the host platform. Will throw an exception if regular expressions are not supported by the platform. - - -FUNCTION ExcludeList::IsExcluded() - -Test a string for being excluded by definite or regex entries. - diff --git a/docs/common/lib_common/FdGetLine.txt b/docs/common/lib_common/FdGetLine.txt deleted file mode 100644 index d92ff94d..00000000 --- a/docs/common/lib_common/FdGetLine.txt +++ /dev/null @@ -1,11 +0,0 @@ -CLASS FdGetLine - -See IOStreamGetLine, difference is that it works on basic UNIX file descriptors rather than full blown streams. Follows basic interface, except... - - -FUNCTION FdGetLine::GetLine - -Returns a string containing the optionally preprocessed line. - -Do not call if IsEOF if true. - diff --git a/docs/common/lib_common/Guards.txt b/docs/common/lib_common/Guards.txt deleted file mode 100644 index 6174fea6..00000000 --- a/docs/common/lib_common/Guards.txt +++ /dev/null @@ -1,5 +0,0 @@ -TITLE Guards.h - -This file contains a couple of classes for using file and memory. When the class goes out of scope, the underlying object is closed or freed, respectively. - - diff --git a/docs/common/lib_common/IOStream.txt b/docs/common/lib_common/IOStream.txt deleted file mode 100644 index 09460656..00000000 --- a/docs/common/lib_common/IOStream.txt +++ /dev/null @@ -1,89 +0,0 @@ -CLASS IOStream - -The base class for streams of data. See IOStream.h for specific details on functions, but the interface is described here. - -Most streams only implement one direction, but some both. - - -SUBTITLE Reading - - -FUNCTION IOStream::Read() - -Reads data from the stream. Returns 0 if there is no data available within the timeout requested. - -Unlike the UNIX API, a return of 0 does not mean that there is no more data to read. Use IOStream::StreamDataLeft() to find this out. - - -FUNCTION IOStream::ReadFullBuffer() - -If you want to read a specific sized block of data from a stream, it is annoying ot use the Read function, because it might return less, so you have to loop. - -This implements that loop. Note that the timeout is the timeout for each individual read -- it might take a lot longer to complete. - -You must check the return value, which is whether or not it managed to get all that data. - - -FUNCTION IOStream::BytesLeftToRead() - -How many bytes are left to read in the stream treated as a whole. May return IOStream::SizeOfStreamUnknown if it is something like a socket which doesn't know how much data is in the stream. - - -FUNCTION IOStream::StreamDataLeft() - -Returns true if more data can be read. Or more specifically, if more data might be able to be read some time in the future. - - -SUBTITLE Writing - - -FUNCTION IOStream::Write() - -Write data to a stream. Writes the entire block, or exceptions. - - -FUNCTION IOStream::WriteAllBuffered() - -Writes any buffered data to the underlying object. - -Call at the end of a series of writes to make sure the data is actually written. (Buffering is not yet implemented anywhere, but it should be soon.) - - -FUNCTION IOStream::StreamClosed() - -Is the stream closed, and writing no longer possible? - - -FUNCTION IOStream::CopyStreamTo() - -A utility function to copy the contents of one stream to another, until the reading stream has no data left. - - -SUBTITLE Other interfaces - -These are slightly nasty interfaces, because they have to match the UNIX API to some extent. - - -FUNCTION IOStream::Close() - -Close the stream -- intended to indicate that nothing more will be written. - -However, also closes reading in most cases. Stream dependent. Probably best just to delete the object and let it sort itself out. - - -FUNCTION IOStream::Seek() - -Seeks within the stream -- mainly for using files within a stream interface, although some of the utility stream classes support it too. - -This is actually a bad interface, because it does not specify whether it applies to reading or writing. This matches the interface provided by files with a single file pointer, but does not map well onto other stream types. - -Should it be changed? Perhaps. But then it means that files would either have to have an inconsitent interface, or the class keep track of two separate file pointers. Which isn't nice. - -So use carefully, and remember whether you're using a file stream, a reading stream, or a writing stream. - - -FUNCTION IOStream::GetPosition() - -Get the current file pointer. Subject to same problems as Seek with regard to semantics. - - diff --git a/docs/common/lib_common/IOStreamGetLine.txt b/docs/common/lib_common/IOStreamGetLine.txt deleted file mode 100644 index 04c56b57..00000000 --- a/docs/common/lib_common/IOStreamGetLine.txt +++ /dev/null @@ -1,29 +0,0 @@ -CLASS IOStreamGetLine - -This class provides a convenient way to read text from a file, line by line. It also can preprocess the line to remove leading and trailing whitespace and comments. Comments are started by the character # and run to the end of the line. - -Create an instance by passing a reference to a stream into the constructor. - -Note the class does internal buffering, so you can only detach it later if the stream supports seeking backwards. - - -FUNCTION IOStreamGetLine::GetLine() - -Returns true if a line could be retreieved without a read timing out. - - -FUNCTION IOStreamGetLine::IsEOF() - -Whether the end of the stream has been reached. Do not call GetLine if this is true. - - -FUNCTION IOStreamGetLine::GetLineNumber() - -Returns the line number. - - -FUNCTION IOStreamGetLine::DetachFile() - -Detaches the stream from the GetLine class. Will seek backwards to "replace" data it's buffered back in the stream. - - diff --git a/docs/common/lib_common/MainHelper.txt b/docs/common/lib_common/MainHelper.txt deleted file mode 100644 index eb5b07f0..00000000 --- a/docs/common/lib_common/MainHelper.txt +++ /dev/null @@ -1,4 +0,0 @@ -TITLE MainHelper.h - -This header contains a couple of macros which add exception handling and reporting to main() functions for executable programs. - diff --git a/docs/common/lib_common/WaitForEvent.txt b/docs/common/lib_common/WaitForEvent.txt deleted file mode 100644 index 0bc55726..00000000 --- a/docs/common/lib_common/WaitForEvent.txt +++ /dev/null @@ -1,16 +0,0 @@ -CLASS WaitForEvent - -This class implements a way to efficiently wait on one or more system objects, for example sockets and file descriptors. Where kqueue() is available, this is used, otherwise poll(). The poll() implementation is less comprehensive and rather more limited. - -To add a compatible object, call Add(). To remove it, call Remove(). To wait for an object to become ready, call Wait(), which returns a pointer to the first ready object, or 0 for a timeout. - -The timeout is set either in the constructor, or using the SetTimout() method. It is specified in milliseconds. - -The kqueue based implementation will automatically remove objects when they are closed (actually, the OS does this), but the poll implementation requires that Remove() be called. - -The flags passed to Add() or Remove() are passed to the object, and are ignored by this class. - -For an object to be able to be added to the list, it should implement FillInKEvent() and FillInPoll() for kqueue and poll implementations respectively. Use the PLATFORM_KQUEUE_NOT_SUPPORTED define to test which is necessary for the platform. - -It does not have to be derived off any particular class, as it uses templates to be compatible with any class. - diff --git a/docs/common/lib_common/xStream.txt b/docs/common/lib_common/xStream.txt deleted file mode 100644 index 91e9c0ea..00000000 --- a/docs/common/lib_common/xStream.txt +++ /dev/null @@ -1,40 +0,0 @@ -TITLE Various stream types - -Some useful streams are implemented in lib/common. - - -SUBTITLE CollectInBufferStream - -Described in it's own page. - - -SUBTITLE FileStream - -Implements a stream from a file, allowing both reading and writing. - - -SUBTITLE MemBlockStream - -Turns a memory block into a readable stream. - -Can also be constructed using StreamableMemBlock, CollectInBufferStream and MemBlockStream as sources of the buffer. - - -SUBTITLE PartialReadStream - -Create a readable stream which will read a set number of bytes from another stream, and then declare itself as closed. - -Useful for extracting a small chunk of another stream to present to a function which expects to consume all of a stream. - - -SUBTITLE ReadGatherStream - -Create a readable stream out of blocks from many streams -- so various sections of any number of streams are composed into a single stream. - -To use, register each stream with AddComponent, then use the returned 'handle' with AddBlock to add blocks to the composed stream. - -Optionally, the object will take ownership of the streams added, and delete them when itself is deleted. - -See the comments in the function blocks in the cpp file for more info. - - diff --git a/docs/common/lib_compress.txt b/docs/common/lib_compress.txt deleted file mode 100644 index f3e26f20..00000000 --- a/docs/common/lib_compress.txt +++ /dev/null @@ -1,8 +0,0 @@ -TITLE lib/compress - -This is a horrid C++ interface to zlib. It is written this way to avoid having to buffer any data, and so be unnecessarily inefficient. But this does end up just being a wrapper to the zlib interface. - -See test/compress for an example of how to use it. But it should be very familiar. - -For an easier interface, use CompressStream. - diff --git a/docs/common/lib_compress/CompressStream.txt b/docs/common/lib_compress/CompressStream.txt deleted file mode 100644 index eca43eb6..00000000 --- a/docs/common/lib_compress/CompressStream.txt +++ /dev/null @@ -1,27 +0,0 @@ -TITLE CompressStream - -This implements a compressing stream class, which compresses and decompresses data sent to an underlying stream object. Data is likely to be buffered. - - -WARNING: Use WriteAllBuffered() (after objects which need to be sent in their entirity) and Close() (after you have finished writing to the stream) otherwise the compression buffering will lose the last bit of data for you. - - -The class works as a filter to an existing stream. Ownership can optionally be taken so that the source stream is deleted when this object is deleted. - -It can operate in one or two directions at any time. Data written to the stream is compressed, data read from the stream is decompressed. If a direction is not being (de)compressed, then it can act as a pass-through without touching the data. - -The constructor specifies the actions to be taken: - -CompressStream(IOStream *pStream, bool TakeOwnership, - bool DecompressRead, bool CompressWrite, - bool PassThroughWhenNotCompressed = false); - -pStream - stream to filter -TakeOwnership - if true, delete the stream when this object is deleted. -DecompressRead - reads pass through a decompress filter -CompressWrite - writes pass through a compression filter -PassThroughWhenNotCompressed - if not filtering a direction, pass through the data unaltered, otherwise exception if an attempt is made on that direction. - - -Note that the buffering and compression will result in different behaviour than the underlying stream: data may not be written in one go, and data being read may come out in different sized chunks. - diff --git a/docs/common/lib_crypto.txt b/docs/common/lib_crypto.txt deleted file mode 100644 index 9ddafe69..00000000 --- a/docs/common/lib_crypto.txt +++ /dev/null @@ -1,28 +0,0 @@ -TITLE lib/crypto - -Provides cryptographic primatives using the OpenSSL implementation. - - -SUBTITLE CipherContexts and CipherDescriptions - -A CipherContext is the interface to encryption and decryption, providing a generic interface. - -It is constructed with a decrypt or encrypt flag, and a CipherDescription. The CipherDescription specifies which cipher is to be used, the key, and all other cipher specific paramteres. - -The CipherContext has support for padding and initialisation vectors. - - -SUBTITLE Random numbers - -An interface to the OpenSSL psuedo random number generater is provided. - - -SUBTITLE Digests - -An interface to the MD5 digest is provided. - - -SUBTITLE RollingChecksum - -The rsync rolling checksum is implemented. The interface is a little tricky to be efficient as the caller must track the position and provide relevant bytes to advance the checksum. - diff --git a/docs/common/lib_crypto/CipherContext.txt b/docs/common/lib_crypto/CipherContext.txt deleted file mode 100644 index 30fb4608..00000000 --- a/docs/common/lib_crypto/CipherContext.txt +++ /dev/null @@ -1,28 +0,0 @@ -CLASS CipherContext - -Encryption and decryption using OpenSSL EVP interface. - -See the cpp file for more documentation in the function headers, and test/crypto for examples. - -General notes below. - - -SUBTITLE Construction - -Construct with the encryption direction, and a CipherDescription of the cipher and key required. - - -SUBTITLE Encrypting or decrypting - -Begin() and Transform() allow piece by piece transformation of a block. - -TransformBlock() transforms an entire block. - - -SUBTITLE Buffering - -All transforms expect to have enough space in the buffers for their entire output. Because of block boundaries and padding, it is necessary that the output buffer should be bigger than the input buffer. The amount of space depends on the cipher. - -InSizeForOutBufferSize() and MaxOutSizeForInBufferSize() perform these space calculations, returning the maximuim in size for a specified out size, and the reverse, respectively. - - diff --git a/docs/common/lib_crypto/RollingChecksum.txt b/docs/common/lib_crypto/RollingChecksum.txt deleted file mode 100644 index d871b3f2..00000000 --- a/docs/common/lib_crypto/RollingChecksum.txt +++ /dev/null @@ -1,36 +0,0 @@ -CLASS RollingChecksum - -Implementing the rsync rolling checksum algorithm. Read it's description first: - -http://samba.anu.edu.au/rsync/tech_report/node3.html - - -SUBTITLE Construction and initial checksum calculation - -The constructor takes a pointer to a block of data and a size, and calculates the checksum of this block. It can now be "rolled forward" to find the checksum of the block of the same size, one byte forward, with minimal calculation. - - -FUNCTION RollingChecksum::GetChecksum() - -Returns the checksum for the current block. - - -FUNCTION RollingChecksum::RollForward() - -This function takes the byte at the start of the current block, and the last byte of the block it's rolling forward to, and moves the checksum on. - -If the block is - - char *pBlock = ; - -with size s, then it should be called with - - RollForward(pBlock[0], pBlock[s]) - -and now GetChecksum will return the checksum of the block (pBlock+1) of size s. - - -FUNCTION RollingChecksum::RollForwardSeveral() - -Similar to RollForward(), but is more efficient for skipping several bytes at once. Takes pointers to the data buffer rather than the actual data values. - diff --git a/docs/common/lib_server.txt b/docs/common/lib_server.txt deleted file mode 100644 index 392f331d..00000000 --- a/docs/common/lib_server.txt +++ /dev/null @@ -1,9 +0,0 @@ -TITLE lib/server - -This provides various classes for implementing client/server applications (and UNIX daemons). - -The stars of the show are ServerTLS and Protocol, which allow client server applications which communicate using TLS (SSL successor) to be built with surprisingly few lines of code. - -All details in the respective class files. - -Look at test/basicserver for examples. diff --git a/docs/common/lib_server/Daemon.txt b/docs/common/lib_server/Daemon.txt deleted file mode 100644 index 6956ec2b..00000000 --- a/docs/common/lib_server/Daemon.txt +++ /dev/null @@ -1,96 +0,0 @@ -CLASS Daemon - -Implement a UNIX daemon which - -* Daemonises (detach from console, etc) -* Sets up signal handlers, and does useful things with them -* Reads configuration files -* Writes PID file -* Opens syslog - -all the usual UNIX basic daemon behaviour. - -The daemon exe takes optional arguments: The first is a configuration file filename which overrides the default. If the second argument is 'SINGLEPROCESS' the daemon will not daemonise. - -The configuration file must have a section like this - -Server -{ - PidFile = /var/run/daemon.pid - User = username -} - -Use DAEMON_VERIFY_SERVER_KEYS (defined in Daemon.h) to include the necessary keys in your configuration file's verify structure. - -The "User" line is optional, and if it is present the Daemon will change user to this username just before it daemonises. Note that unless the directory the PID file is written to has write permission for this user, the PID file will not be deleted on exit. - - -To implement a daemon, derive a class from Daemon, and override Run(). - -Then in your main file, do something like - -int main(int argc, const char *argv[]) -{ - MAINHELPER_START - - BackupDaemon daemon; - return daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); - - MAINHELPER_END -} - -and that's it. Obviously it's worth doing a few more things as well, but that's it. - - -FUNCTION Daemon::Run() - -Override with the main daemon code. It should behave like this - -void SomethingDaemon::Run() -{ - // Read configuration file - - // Do lots of work until StopRun() returns true - while(!StopRun()) - { - ::sleep(10); - } - - // Clean up nicely -} - -which allows the base class to implement the standard start, terminate and -HUP behaviours correctly. - - -FUNCTION Daemon::DaemonName() - -Returns the name of the daemon, for use in syslog. - - -FUNCTION Daemon::DaemonBanner() - -Returns the banner to be displayed on startup, or 0 for no banner. - - -FUNCTION Daemon::GetConfigVerify() - -Returns a configuration verify structure for verifying the config file. Note that this configuration structure should include a sub-configuration called "Server, and have entries defined by DAEMON_VERIFY_SERVER_KEYS. See one of the bin/bbackupd for an example. - - -FUNCTION Daemon::StopRun() - -Returns true when the daemon needs to be terminated or restarted. Use IsReloadConfigWanted() and IsTerminateWanted() to find out which one, if you need to know. - - -FUNCTION Daemon::SetupInInitialProcess() - -Override to perform additional functions in the initial process, before forking and detachment happens. - - -FUNCTION Daemon::EnterChild() - -Called when a child is entered. If you override it, remember to call the base class. - - - - diff --git a/docs/common/lib_server/Protocol.txt b/docs/common/lib_server/Protocol.txt deleted file mode 100644 index 09d3c1f1..00000000 --- a/docs/common/lib_server/Protocol.txt +++ /dev/null @@ -1,120 +0,0 @@ -CLASS Protocol - -Protocol - -* serialises and deserialises data objects -* sends arbitary streams - -through a bi-directional IOStream object, usually a socket. - -These data objects are auto-generated by a perl script, along with the logic to implement a simple protocol where one end is a client, and the other a server. The client sends a command object (and optional streams) and the server returns a reply object (and optional streams). - -The perl script uses a description file which specifies the data inside these objects (which can be extended to include any type which implements the standard serialisation functions), the required responses to the commands, and whether streams are involved. - -It then implements a server object, which given a stream and a user defined context object, waits for commands, processes them, and sends the results back. All you need to do is implement a DoCommand() function for each command object you define. - -On the client side, a Query() function is implemented which takes objects as parameters, and returns the right type of reply object (or throws an exception if it doesn't get it.) Short cut Query() functions are also implemented which don't require objects to be created manually. - -Thus, implementing a server is as simple as deriving a daemon off ServerStream or ServerTLS, and implementing the Connection() function as - - void TestProtocolServer::Connection(SocketStream &rStream) - { - TestProtocolServer server(rStream); - TestContext context; - server.DoServer(context); - } - -and that's it. TestContext is a user defined class which keeps track of all the state of the connection, and is passed to all the DoCommand() functions, which look like this: - - std::auto_ptr - TestProtocolServerSimple::DoCommand(TestProtocolServer &rProtocol, - TestContext &rContext) - { - return std::auto_ptr - (new TestProtocolServerSimpleReply(mValue+1)); - } - -(taken from test/basicserver) - -The client code looks like this - - SocketStream conn; - conn.Open(Socket::TypeUNIX, "testfiles/srv4.sock"); - - TestProtocolClient protocol(conn); - - // Query - { - std::auto_ptr - reply(protocol.QuerySimple(41)); - TEST_THAT(reply->GetValuePlusOne() == 42); - } - - -Finally, debug logging can be generated which allows a list of all commands and their parameters to be logged to syslog or a file. - - -SUBTITLE Protocol Description File - -This file is passed to the lib/server/makeprotocol.pl script, which generates a h and cpp file to implement the protocol. - -It is in two sections, separated by a 'BEGIN_OBJECTS' on a line of it's own. - -In the top half, the following statements must be made. - -Name - The name of the protocol, used in naming classes. - -IdentString - The idenfitifaction string sent over the IOStream to confirm it it - is talking to another Protocol object speaking the same Protocol. - -ServerContextClass - The user defined context class used for the server, and the header - file it is defined in. - -Additionally, the following optional commands can be made. - -ClientType -ServerType (similarly) - Extends the types used in the objects below. Server and client - can use different types for the same object type. - -ImplementLog (Client|Server) (syslog|file) - Implement command logging for client or server into syslog or a file. - -LogTypeToText (Client|Server) - - For extended types, optionally define how to convert them into printf - elements and the code to run to get the argument. Within the evaluate - parameter, VAR is replaced by the name of the variable to display. - If this is not specified for a given type, OPAQUE is output instead. - - -In the object section, an object is defined by a line - - - -followed by lines beginning with whitespace defining the data transmitted in the object. The type may be list, which specifies a list (implemented as a std::vector) of entries of that type. - -The attributes specify exactly how that object is used in the defined protocol. - -Reply - The object is a reply object, sent from the server to the client. - -Command(Reply-Type) - The object is a command, send from the client to the server, and the server - will send back an object of type Reply-Type. - -IsError(Type-Field,SubType-Field) - The command is an error object, and the two files specify the data member - which describes the error type and sub type. - -EndsConversation - When this command is received, the connection is to be terminated. - (ie a logout command) - -StreamWithCommand - When this command is sent as a command, a stream follows it. - - diff --git a/docs/common/lib_server/ServerStream.txt b/docs/common/lib_server/ServerStream.txt deleted file mode 100644 index 6c5932a0..00000000 --- a/docs/common/lib_server/ServerStream.txt +++ /dev/null @@ -1,29 +0,0 @@ -CLASS ServerStream - -ServerStream implementes a Daemon which accepts stream connections over sockets, and forks into a child process to handle them. - -To implement a daemon, derive from - - ServerStream - -The type SocketStream specifies that incoming connections should be treated as normal sockets, and SERVER_LISTEN_PORT is the port to listen to (if it's a inet socket, and if not overridden in the config file). The actual addresses (or names) to bind to are specified in the configuration file by the user. - -Make sure SERVERSTREAM_VERIFY_SERVER_KEYS(0) is included in the configuration verification structure in the "Server" sub configuration. 0 could be replaced with a default address, for example "unix:/var/run/server.sock" to specific a default UNIX socket in the filesystem. - -See test/basicserver for a simple example. - -The ListenAddresses key in the Server subconfiguration is a comma separated list of addresses, specified as family:name. Internet sockets are family 'inet', for example 'inet:localhost' (or 'inet:localhost:1080' to specify a port number as well), and unix domain sockets are 'unix', example above. - - -Override Connection to handle the connection. - -Remember to override Daemon functions like the server name, and start it up, just like a generic Daemon. - - -FUNCTION ServerStream::Connection - -This function takes a connected stream as it's argument. It should then proceed to do whatever it needs to do to talk to the client. - -Using IOStreamGetLine or a Protocol class to communicate may be quick ways of implementing this functionality. - - diff --git a/docs/common/lib_server/ServerTLS.txt b/docs/common/lib_server/ServerTLS.txt deleted file mode 100644 index dbde500f..00000000 --- a/docs/common/lib_server/ServerTLS.txt +++ /dev/null @@ -1,6 +0,0 @@ -CLASS ServerTLS - -Implements a server which uses TLS (SSL) to encrypt and authenticate connections. - -Very similar to ServerStream, except it reads the certificate files for the TLSContext out of the Server sub-configuration to set up a TLSContext ("CertificateFile", "PrivateKeyFile" and "TrustedCAsFile"). Otherwise works exactly the same. - diff --git a/docs/common/lib_server/SocketStream.txt b/docs/common/lib_server/SocketStream.txt deleted file mode 100644 index 82813279..00000000 --- a/docs/common/lib_server/SocketStream.txt +++ /dev/null @@ -1,8 +0,0 @@ -CLASS SocketStream - -A implementation of IOStream which wraps a socket connection. - -It can either be created by attaching to an existing object, or use the Open() function to open a connection to a named host on a specific port (or a local UNIX socket in the filesystem). - -Follows stream interface. - diff --git a/docs/common/lib_server/SocketStreamTLS.txt b/docs/common/lib_server/SocketStreamTLS.txt deleted file mode 100644 index ebb3f233..00000000 --- a/docs/common/lib_server/SocketStreamTLS.txt +++ /dev/null @@ -1,11 +0,0 @@ -CLASS SocketStreamTLS - -An implementation of IOStream which wraps a TLS (SSL) connection over a socket. - -The Open function takes a TLSContext reference which specifies the parameters for the connection. - - -FUNCTION GetPeerCommonName() - -Returns the common name of the certificate presented by the remote end. - diff --git a/docs/common/lib_server/TLSContext.txt b/docs/common/lib_server/TLSContext.txt deleted file mode 100644 index ff50d3e6..00000000 --- a/docs/common/lib_server/TLSContext.txt +++ /dev/null @@ -1,16 +0,0 @@ -CLASS TLSContext - -A wrapper over the OpenSSL context object. - -Note: you need to call SSLLib::Initialise at the beginning of your program to use these functions. - - -SUBTITLE Construction - -The constuctor takes the following parameters - -* Boolean for whether this is acting as a server or a client -* The .pem file containing the certificate which will be used -* The .pem file containing the private key for this certificate -* The .pem file containing the certificates which will certify the other end of the connection. - diff --git a/docs/common/memory_leaks.txt b/docs/common/memory_leaks.txt deleted file mode 100644 index 9a9764ea..00000000 --- a/docs/common/memory_leaks.txt +++ /dev/null @@ -1,44 +0,0 @@ -TITLE Memory leak detection - -The build system contains a primative memory leak detection system in debug builds. - -It works by using #defines to replace calls to malloc, free, realloc, new and delete with debug versions which record their use. When a process ends, it dumps a list of all the blocks or objects which were allocated by not freed, and the file and line of the source where they are originally allocated. - -It's not perfect, but should catch most things and work on most platforms. - -If it doesn't work on your platform, define PLATFORM_DISABLE_MEM_LEAK_TESTING in BoxPlatform.h within the relevant section. - - -SUBTITLE Requirements in source - -It does require some extra lines in the source file. The last included file in each .cpp file must be - - #include "MemLeakFindOn.h" - -and if a .h file uses any of these functions, the contents of the .h file should be bounded with - - #include "MemLeakFindOn.h" - - // ... some code, but absolutely no #includes - - #include "MemLeakFindOff.h" - -The cleanupforcvs.pl script checks for correct usage. - - -SUBTITLE Use in tests - -Tests will automatically dump memory leaks and regard them as a failure for anything executing in the main test process. - -To test for memory leaks in programs run from the test, or daemons, use something like - - TestRemoteProcessMemLeaks("bbackupd.memleaks"); - -If memory leak detection is being used, it will check the specified file for memory leak reports (deleting it afterwards). Any leak is an error. - -The Daemon class automactically arranges for the memory leak reports to be written. Other exes should set this up themselves, preferably using the define in MainHelper.h, as the first thing in their main() function. - - MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, name) - -File is the filename to write the report to, conventionally called ".memleaks", and name is the name of the exe. - diff --git a/docs/docbook/adminguide.xml b/docs/docbook/adminguide.xml new file mode 100644 index 00000000..edb0a58c --- /dev/null +++ b/docs/docbook/adminguide.xml @@ -0,0 +1,1981 @@ + + +]> + + Box Backup administrator's guide + + + License + + Copyright © 2003 - 2007, Ben Summers and contributors. All rights + reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + + + Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + + + All use of this software and associated advertising materials + must display the following acknowledgement: This product includes + software developed by Ben Summers and contributors. + + + + The names of the Authors may not be used to endorse or promote + products derived from this software without specific prior written + permission. + + + + [Where legally impermissible the Authors do not disclaim liability + for direct physical injury or death caused solely by defects in the + software unless it is modified by a third party.] + + THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN + NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + + Configuration + +
+ System configuration + +
+ Server + + After you've downloaded and compiled the programs you need to + install the programs on your server. As root do the following: + + make install-backup-server + + This assumes that you are installing on the same server that you + compiled the software on. If not, copy the + boxbackup-x.xx-backup-server-OSNAME.tgz file to the server you want to + run on, and install there. For example (on Mac OS X): + + tar zxvf boxbackup-0.10-server-darwin8.5.0.tgz +cd boxbackup-0.10-server-darwin8.5.0 +./install-backup-server + + Then create the user for the backup daemon on the server: + + useradd _bbstored + + Box Backup has a built-in software RAID facility (redundant + array of inexpensive disks) for the backup store. This allows you to + spread the store data over three disks, and recover from the loss of + any one disk without losing data. However, this is now deprecated, and + you are recommended to use the software or hardware RAID facilities of + your operating system instead. Use the following command if you want + to create a simple server without Box Backup RAID: + + mkdir /tmp/boxbackupRepository # Create the directory +chown _bbstored /tmp/boxbackupRepository/ # Change the owner to the new boxbackup daemon user + +/usr/local/sbin/raidfile-config /etc/box/ 1024 /tmp/boxbackupRepository + +#substitute 1024 with the desired blocksize +#substitute /tmp/boxbackupRepository with a directory that exists where you want the backup store located +#/usr/local/sbin/raidfile-config --help shows you the options + + Then create the configuration file /etc/box/bbstored.conf The + hostname is tricky as it is used for two things: The name of the + server in the certificate and the address the server is listening on. + Since you might be using NAT, might move the server around or the + domain name might change, choose a name that describes the server. + When the network address of the server changes, you need to update the + ListenAddresses directive in the + /etc/box/bbstored.conf file. + + /usr/local/sbin/bbstored-config /etc/box hostname _bbstored + + This last step outputs 5 instructions that you must execute to + the letter. A lot of questions are raised on the mailing list because + these steps have not been followed properly. + + TODO: Expand on this. Explain the 5 steps in detail. + + If you want to run the server as a non-root user, look here. +
+ +
+ Certificate Management + + There are two steps involved to create an account. You need to + create the account on the server, and sign a certificate to give the + client permission to connect to the server. + + Running a Certification Authority for TLS (SSL) connections is + not trivial. However, a script to does most of the work in a way which + should be good enough for most deployments. + + + The certificate authority directory is intended to be stored + on another server. It should not be kept on the backup server, in + order to limit the impact of a server compromise. The instructions + and the script assume that it will be kept elsewhere, so will ask + you to copy files to and from the CA. + + + + SSL certificates contain validity dates, including a "valid + from" time. If the clock on the machine which signs the certificates + is not syncronised to the clocks of the machines using these + certificates, you will probably get strange errors until the start + time is reached on all machines. If you get strange errors when + attempting to use new certificates, check the clocks on all machines + (client, store and CA). You will probably just need to wait a while + until the certificates become valid, rather than having to + regenerate them. + + +
+ Set up a Certificate Authority + + It is recommended that you keep your Certificate Authority on + a separate machine than either the client or the server, preferably + without direct network access. The contents of this directory + control who can access your backup store server. + + To setup the basic key structure, do the following: + + /usr/local/sbin/bbstored-certs ca init + + (See OpenSSL notes if you + get an OpenSSL error) + + This creates the directory called ca in + the current directory, and initialises it with basic keys. +
+ +
+ Sign a server certificate + + When you use the bbstored-config script to + set up a config file for a server, it will generate a certificate + request (CSR) for you. Transfer it to the machine with your CA, then + do: + + /usr/local/sbin/bbstored-certs ca sign-server hostname-csr.pem + + This signs the certificate for the server. Follow the + instructions in the output on which files to install on the server. + The CSR file is now no longer needed. Make sure you run this command + from the directory above the directory 'ca'. + + TODO: Explain instructions in output. +
+ +
+ Set up an account + + Choose an account number for the user. This must be unique on + the server, and is presented as a 31 bit number in hex greater than + 0, for example, 1 or 75AB23C. Then on the backup store server, + create the account with: + + /usr/local/sbin/bbstoreaccounts create 75AB23C 0 4096M 4505M + + This looks complicated. The numbers are, in order: + + + + The account number allocated (hex) + + + + The RAID disc set (0 if you use raidfile-config and don't + add a new set) + + + + Soft limit (size) + + + + Hard limit (size) + + + + The sizes are are specified in Mb, Gb, or blocks, depending on + the suffix. 1M specifies 1 Mb, 1G specifies 1 Gb, and 1B specifies 1 + block, the size of which depends on how you have configured the + raidfile system with raidfile-config. + + In this example, I have allocated 4Gb (assuming you use 2048 + byte blocks as per my example) as the soft limit, and 4Gb + 10% as + the hard limit. + + NOTE The sizes specified here are pre-RAID. So if you are + using userland RAID, you are actually allocating two-thirds of this + amount. This means that, when you take compression into account, + that if you allocate 2Gb on the server, it'll probably hold about + 2Gb of backed up files (depending on the compressability of those + files). + + The backup client will (voluntarily) try not to upload more + data than is allowed by the soft limit. The store server will refuse + to accept a file if it would take it over the hard limit, and when + doing housekeeping for this account, try and delete old versions and + deleted files to reduce the space taken to below the soft + limit. + + This command will create some files on disc in the raid file + directories (if you run as root, the utility will change to the user + specified in the bbstored.conf file to write them) and update the + accounts file. A server restart is not required. + + NOTE If you get a message saying 'Exception: RaidFile (2/8)', + the directories you specified in the raidfile.conf are not writable + by the _bbstored user -- fix it, and try again. + + Finally, tell the user their account number, and the hostname + of your server. They will use this to set up the backup client, and + send you a CSR. This has the account number embedded in it, and you + should be sure that it has the right account number in it. + + Sign this CSR with this command: + + /usr/local/sbin/bbstored-certs ca sign 75AB23C-csr.pem + + Don't forget to check that the embedded account number is + correct! Then send the two files back to the user, as instructed by + the script. + + Please read the Troubleshooting page if you have + problems. + + TODO: Link to troubleshooting... +
+
+ +
+ Log Files + + You may wish to see what's going on with the server. Edit + /etc/syslog.conf, and add: + + local6.info /var/log/box +local5.info /var/log/raidfile + + Note: Separators must be tabs, + otherwise these entries will be ignored. + + touch /var/log/box +touch /var/log/raidfile + + Set up log rotation for these new log files. For example, if you + have /etc/newsyslog.conf, add the following lines + to it: + + /var/log/box 644 7 2000 * Z +/var/log/raidfile 644 7 2000 * Z + + If you have /etc/logrotate.d, create a new + file in there (for example + /etc/logrotate.d/boxbackup) containing the + following: + + /var/log/box /var/log/raidfile { + weekly + create + compress + rotate 52 +} + + Then restart syslogd, for example: + + /etc/init.d/syslogd restart +
+ +
+ Configuring a client + + Before you can do any configuration, you need to know the + hostname of the server you will be using, and your account number on + that server. + + Later in the process, you will need to send a certificate + request to the administrator of that server for it to be + signed. + + Installation is covered in the compiling and installing section. + You only need the backup-client parcel. + + It is important that you read all the output of the config + scripts. See the end of this page for an example. + + The backup client has to be run as root, because it needs to + read all your files to back them up, although it is possible to back + up a single user's files by running it as that user. (Tip: specify a + directory other than /etc/box, and then give the + alternate config file as the first argument to + bbackupd). However, it will fall over if you don't + give yourself read access to one of your files. + +
+ Basic configuration + + Run the bbackupd-config script to generate + the configuration files and generate a private key and certificate + request. + + /usr/local/sbin/bbackupd-config /etc/box lazy 999 hostname /var/bbackupd /home + + (See OpenSSL notes if you + get an OpenSSL error) + + The items in bold need to be changed. In order, they are the + account number, the hostname of the server you're using, and + finally, the directories you want backed up. You can include as many + you want here. + + However, the directories you specify must not contain other + mounted file systems within them at any depth. Specify them + separately, one per mount point. No checks are currently made to + catch bad configuration of this nature! + + You may also want to consider changing the mode from lazy to + snapshot, depending on what your system is used for: + + + + Lazy Mode + + + This mode regularly scans the files, with only a rough + schedule. It uploads files as and when they are changed, if + the latest version is more than a set age. This is good for + backing up user's documents stored on a server, and spreads + the load out over the day. + + + + + Snapshot Mode + + + This mode emulates the traditional backup behaviour of + taking a snapshot of the filesystem. The backup daemon does + absolutely nothing until it is instructed to make a backup + using the bbackupctl utility (probably as a cron job), at + which point it uploads all files which have been changed since + the last time it uploaded. + + + + + When you run the config script, it will tell you what you need + to do next. Don't forget to read all the output. An example is shown + at the end of this page, but the instructions for your installation + may be different. +
+ +
+ Certificates + + After you have sent your certificate request off to the server + administrator and received your certificate and CA root back, + install them where instructed by the bbackupd-config script during + basic bbackupd configuration. + + You can then run the daemon (as root) by running + /usr/local/sbin/bbackupd, and of course, adding it + to your system's startup scripts. The first time it's run it will + upload everything. Interrupting it and restarting it will only + upload files which were not uploaded before - it's very + tolerant. + + If you run in snapshot mode, you will need to add a cron job + to schedule backups. The config script will tell you the exact + command to use for your system. + + Please read the Troubleshooting page if you have + problems. + + Remember to make a traditional backup of the keys file, as + instructed. You cannot restore files without it. + + It is recommended that you backup up all of /etc/box as it + will make things easier if you need to restore files. But only the + keys are absolutely essential. + + If you want to see what it's doing in more detail (probably a + good idea), follow the instructions in the server setup to create + new log files with syslog. +
+ +
+ Adding and removing backed up locations + + By editing the /etc/box/bbackupd.conf file, you can add and + remove directories to back up - see comments in this file for help. + Send bbackupd a HUP signal after you modify it. + + When you remove a location, it will not be marked as deleted + immediately. Instead, bbackupd waits about two days before doing so, + just in case you change your mind. After this, it will be eventually + removed from the store by the housekeeping process. Run as + root. + + The backup client is designed to be run as root. It is + possible to run without root, but this is not recommended. Clock + synchronisation for file servers. + + If you are using the backup client to backup a filesystem + served from a fileserver, you should ideally ensure that the + fileserver clocks are synchronised with the fileserver. + + bbackupd will cope perfectly well if the clocks are not + synchronised. Errors up to about half an hour cause no problems. + Larger discrepancies cause a loss of efficiency and the potential to + back up a file during a write process. + + There is a configuration parameter MaxFileTimeInFuture, which + specifies how far in the future a file must be for it to be uploaded + as soon as it is seen. You should not need to adjust this (default + is 2 days). Instead, get those clocks synchronised. Excluding files + and directories from the backup. + + Within the bbackupd.conf file, there is a section named + BackupLocations which specifies which locations on disc should be + backed up. It has subsections, each of which is in the + format: + + name + { + Path = /path/of/directory + (optional exclude directives) + } + + name is derived from the Path + by the config script, but should merely be unique. + + The exclude directives are of the form: + + [Exclude|AlwaysInclude][File|Dir][|sRegex] = regex or full pathname + + (The regex suffix is shown as 'sRegex' to make File or Dir + plural) + + For example: + + ExcludeDir = /home/guest-user + ExcludeFilesRegex = *.(mp3|MP3)\$ + AlwaysIncludeFile = /home/username/veryimportant.mp3 + + This excludes the directory /home/guest-user from the backup + along with all mp3 files, except one MP3 file in particular. + + In general, Exclude excludes a file or directory, unless the + directory is explicitly mentioned in a AlwaysInclude + directive. + + If a directive ends in Regex, then it is a regular expression + rather than a explicit full pathname. See + + man 7 re_format + + for the regex syntax on your platform. +
+ +
+ Example configuration output + + This is an example of output from the bbstored-config + script. + + + Follow the instructions output by your script, not the ones + here -- they may be different for your system. + + + /usr/local/sbin/bbackupd-config /etc/box lazy 51 server.example.com /var/bbackupd /home /etc/samba + +Setup bbackupd config utility. + +Configuration: + Writing configuration file: /etc/box/bbackupd.conf + Account: 51 + Server hostname: server.example.com + Directories to back up: + /home + /etc/samba + +Note: If other file systems are mounted inside these directories, then problems may occur +with files on the store server being renamed incorrectly. This will cause efficiency +problems, but not affect the integrity of the backups. + +WARNING: Directories not checked against mountpoints. Check mounted filesystems manually. + +Creating /etc/box... +Creating /etc/box/bbackupd +Generating private key... + [OpenSSL output omitted] + +Generating keys for file backup +Writing notify script /etc/box/bbackupd/NotifyStoreFull.sh +Writing configuration file /etc/box/bbackupd.conf + +=================================================================== + +bbackupd basic configuration complete. + +What you need to do now... + +1) Make a backup of /etc/box/bbackupd/51-FileEncKeys.raw + This should be a secure offsite backup. + Without it, you cannot restore backups. Everything else can + be replaced. But this cannot. + KEEP IT IN A SAFE PLACE, OTHERWISE YOUR BACKUPS ARE USELESS. + +2) Send /etc/box/bbackupd/51-csr.pem + to the administrator of the backup server, and ask for it to + be signed. + +3) The administrator will send you two files. Install them as + /etc/box/bbackupd/51-cert.pem + /etc/box/bbackupd/serverCA.pem + after checking their authenticity. + +4) You may wish to read the configuration file + /etc/box/bbackupd.conf + and adjust as appropraite. + + There are some notes in it on excluding files you do not + wish to be backed up. + +5) Review the script + /etc/box/bbackupd/NotifyStoreFull.sh + and check that it will email the right person when the store + becomes full. This is important -- when the store is full, no + more files will be backed up. You want to know about this. + +6) Start the backup daemon with the command + /usr/local/sbin/bbackupd + in /etc/rc.local, or your local equivalent. + Note that bbackupd must run as root. + +=================================================================== + + Remember to make a secure, offsite backup of your backup keys, + as described in Basic + configuration above. If you do not, and that key is lost, you + have no backups. +
+
+ +
+ Configuration Options + + Box Backup has many options in its configuration file. We will + try to list them all here. + + First of all, here is an example configuration file, for + reference: + + + Example Configuration File + + StoreHostname = localhost +AccountNumber = 0x2 + +KeysFile = /etc/box/2-FileEncKeys.raw +CertificateFile = /etc/box/2-cert.pem +PrivateKeyFile = /etc/box/2-key.pem +TrustedCAsFile = /etc/box/serverCA.pem +DataDirectory = /var/run/boxbackup +NotifyScript = /etc/box/NotifySysadmin.sh +CommandSocket = /var/run/box/bbackupd.sock + +UpdateStoreInterval = 86400 +MinimumFileAge = 3600 +MaxUploadWait = 7200 +FileTrackingSizeThreshold = 65536 +DiffingUploadSizeThreshold = 65536 +MaximumDiffingTime = 20 +ExtendedLogging = no +LogAllFileAccess = yes + +Server +{ + PidFile = /var/run/bbackupd.pid +} +BackupLocations +{ + etc + { + Path = /etc + } + home + { + Path = /home + ExcludeDir = /home/shared + ExcludeDir = /home/chris/.ccache + ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache + } +} + + + As you can see from the example above, the configuration file + has a number of subsections, enclosed in curly braces {}. Some options + appear outside of any subsection, and we will refer to these as root options. The available options in + each section are described below. + + Every option has the form name = value. Names are + not case-sensitive, but values are. Depending on the option, the value + may be: + + + + a path (to a file or directory); + + + + a number (usually in seconds or bytes); + + + + a boolean (the word Yes or No); + + + + a hostname (or IP address). + + + + Paths are specified in native format, i.e. a full Windows path + with drive letter on Windows clients, or a full Unix path on Unix + clients. + + + Example: + + StoreObjectInfoFile = + /var/state/boxbackup/bbackupd.dat + + StoreObjectInfoFile = C:\Program Files\Box + Backup\data\bbackupd.dat + The use of relative paths (which do not start with a + forward slash on Unix, or a drive specification on Windows) is + possible but not recommended, since they are interpreted relative to + the current working directory when bbackupd was started, which is + liable to change unexpectedly over time. + + Numbers which start with "0x" are interpreted as hexadecimal. + Numbers which do not start with "0x" are interpreted as + decimal. + +
+ Root Options + + These options appear outside of any subsection. By convention + they are at the beginning of the configuration file. + + Some options are required, and some are optional. + + + + StoreHostname (required) + + + The Internet host name (DNS name) or IP address of the + server. This is only used to connect to the server. + + + + + AccountNumber (required) + + + The number of the client's account on the server. This + must be provided by the server operator, and must match the + account number in the client's certificate, otherwise the + client will not be able to log into the server. + + The account number may be specified in hexadecimal + (starting with 0x, as in the example above) or in decimal, but + since the server operator works in hexadecimal, that format is + highly recommended and is the default. + + + + + KeysFile (required) + + + The path to the file containing the encryption key used + for data encryption of client file data and filenames. This is + the most important file to keep safe, since without it your + backups cannot be decrypted and are useless. Likewise, if an + attacker gets access to this key and to your encrypted + backups, he can decrypt them and read all your data. + + Do not change the encryption key without deleting all + files from the account on the server first. None of your old + files on the store will be readable if you do so, and if you + change it back, none of the files uploaded with the new key + will be readable. + + + + + CertificateFile (required) + + + The path to the OpenSSL client certificate in PEM + format. This is supplied by the server operator in response to + the certificate request which you send to them. Together with + the PrivateKeyFile, this provides access to the store server + and the encrypted data stored there. + + It is not critical to protect this file or to back it up + safely, since it can be regenerated by creating a new + certificate request, and asking the server operator to sign + it. You may wish to back it up, together with the + PrivateKeyFile, to avoid this inconvenience if you lose all + your data and need quick access to your backups. + + If you do back them up, you should keep them in a + separate location to the KeysFile, since any person holding + the KeysFile and the PrivateKeyFile can gain access to your + encrypted data and decrypt it. + + + + + PrivateKeyFile (required) + + + The path to the OpenSSL private key in PEM format. This + is generated at the same time as the certificate request, but + there is no need to send it to the server operator, and you + should not do so, in case the communication is intercepted by + an attacker. Together with the CertificateFile, this provides + access to the store server and the encrypted data stored + there. + + See the notes under CertificateFile for information + about backing up this file. + + + + + TrustedCAsFile (required) + + + The path to the OpenSSL certificate of the Client + Certificate Authority (CCA), in PEM format. This is supplied + by the server operator along with your account details, or + along with your signed client certificate. This is used to + verify that the server which you are connecting to is + authorised by the person who signed your certificate. It + protects you against DNS and ARP poisoning and IP spoofing + attacks. + + + + + DataDirectory (required) + + + The path to a directory where bbackupd will keep local + state information. This consists of timestamp files which + identify the last backup start and end times, used by + bbackupquery to determine whether files + have changed, and optionally a database of inode numbers, + which are used to check for files being renamed. The database + is only saved if Box Backup is built with Berkeley Database + (BDB) support. + + + + + NotifyScript (optional) + + + The path to the script or command to run when the Box + Backup client detects an error during the backup process. This + is normally used to notify the client system administrator by + e-mail when a backup fails for any reason. + + The script or command is called with one of the + following additional arguments to identify the cause of the + problem: + + + + store-full + + + The backup store is full. No new files are being + uploaded. If some files are marked as deleted, they + should be removed in due course by the server's + housekeeping process. Otherwise, you need to remove some + files from your backup set, or ask the store operator + for more space. + + + + + read-error + + + One or more files which were supposed to be backed + up could not be read. This could be due to: + + running the server as a non-root user; + + + + backing up a mounted filesystem such as + NFS; + + + + access control lists being applied to some + files; + + + + SELinux being enabled; + + + + trying to back up open files under + Windows; + + + + strange directory permissions such as 0000 or + 0400. + + Check the client logs, e.g. + /var/log/bbackupd on Unix, or the Windows Event Viewer + in Control Panel > Administrative Tools, for more + information about which files are not being backed up + and why. + + + + + backup-error + + + There was a communications error with the server, + or an unexpected exception was encountered during a + backup run. Check the client logs, e.g. + /var/log/box on Unix, or the + Windows Event Viewer in Control Panel > + Administrative Tools, for more information about the + problem. + + You may wish to check your Internet access to the + server, check that the server is running, and ask your + server operator to check your account on the + server. + + + + + + + + CommandSocket (optional) + + + The path to the Unix socket which + bbackupd creates when running, and which + bbackupctl uses to communicate with it, for + example to force a sync or a configuration reload. If this + option is omitted, no socket will be created, and + bbackupctl will not function. + + Unix sockets appear within the filesystem on Unix, as a + special type of file, and must be created in a directory which + exists and to which bbackupd has write access, and bbackupctl + has read access. + + On Windows, the path is ignored, and a named + pipe is created instead. This does not currently + have any security attached, so it can be accessed by any user. + Unlike a Unix socket it can also be accessed remotely. Please + use this option with extreme caution on Windows, and only on + fully trusted networks. + + + + + AutomaticBackup (optional) + + + Enable or disable the client from connecting + automatically to the store every + UpdateStoreInterval seconds. When + enabled (set to Yes), the client is in + Lazy Mode. When disabled (set to + No), it is in Snapshot + Mode. This setting is optional, and the default + value is Yes. + + + + + UpdateStoreInterval (required) + + + The approximate time between successive connections to + the server, in seconds, when the client is in Lazy + Mode. The actual time is randomised slightly to + prevent "rush hour" traffic jams on the server, where many + clients try to connect at the same time. + + This value is ignored if the client is in + Snapshot Mode. However, it is still + required. It can be set to zero in this case. + + You will probably need to experiment with the value of + this option. A good value to start with is probably 86400 + seconds, which is one day. + + + + + MinimumFileAge (required) + + + The number of seconds since a file was last modified + before it will be backed up. The reason for this is to avoid + repeatedly backing up files which are repeatedly changing. A + good value is about 3600 seconds (one hour). If set to zero, + files which have changed will always be backed up on the next + backup run. + + The MaxUploadWait option + overrides this option in some circumstances. + + + + + MaxUploadWait (required) + + + The number of seconds since a file was last uploaded + before it will be uploaded again, even if it keeps changing. + The reason for this is to ensure that files which are + continuously modified are eventually uploaded anyway. This + should be no less than the value of + MinimumFileAge. A good value is about + 14400 seconds (4 hours). + + + + + MaxFileTimeInFuture (optional) + + + The maximum time that a file's timestamp can be in the + future, before it will be backed up anyway. Due to clock + synchronisation problems, it is inevitable that you will + occasionally see files timestamped in the future. Normally, + for files which are dated only slightly in the future, you + will want to wait until after the file's date before backing + it up. However, for files whose dates are very wrong (more + than a few hours) you will normally prefer to back them up + immediately. + + A good value is about 7200 seconds (2 hours) to cope + with potential problems when moving in and out of daylight + saving time, if applicable in your timezone. The default + value, if this setting is not provided, is 172800 seconds (2 + days). + + + + + FileTrackingSizeThreshold (required) + + + The minimum size of files which will be tracked by inode + number to detect renames. It is not worth detecting renames of + small files, since they are quick to upload again in full, and + keeping their inode numbers in memory increases the client's + memory usage and slows down searches. Larger files should be + tracked to avoid wasting space on the store and long + uploads. + + A good value is about 65536 bytes (64 kilobytes). + + + + + DiffingUploadSizeThreshold (required) + + + The minimum size of files which will be compared to the + old file on the server, and for which only changes will be + uploaded. It is not worth comparing small files, since they + are quick to upload again in full, and sending the entire file + reduces the risk of data loss if the store is accidentally + corrupted. Larger files should have only their differences + uploaded to avoid wasting space on the store and long + uploads. + + A good value is about 65536 bytes (64 kilobytes). + + + + + MaximumDiffingTime (optional) + + + The maximum time for which the client will attempt to + find differences between the current version and the old + version in the store, before giving up and uploading the + entire file again. Very large files (several gigabytes) may + take a very long time to scan for changes, but would also take + a very long time to upload again and use a lot of space on the + store, so it is normally worth omitting this value. + + Use this option only if, for some bizarre reason, you + prefer to upload really large files in full rather than spend + a long time scanning them for changes. + + + + + KeepAliveTime (optional) + + + The interval (in seconds) between sending Keep-Alive + messages to the server while performing long operations such + as finding differences in large files, or scanning large + directories. + + These messages ensure that the SSL connection is not + closed by the server, or an intervening firewall, due to lack + of activity. + + The server will normally wait up to 15 minutes (900 + seconds) before disconnecting the client, so the value should + be given and should be less than 900. Some firewalls may time + out inactive connections after 10 or 5 minutes. + + A good value is 300 seconds (5 minutes). You may need to + reduce this if you frequently see TLSReadFailed or + TLSWriteFailed errors on the client. + + + + + StoreObjectInfoFile (optional) + + + Enables the use of a state file, which stores the + client's internal state when the client is not running. This + is useful on clients machines which are frequently shut down, + for example desktop and laptop computers, because it removes + the need for the client to recontact the store and rescan all + directories on the first backup run, which may take some time. + This feature is somewhat experimental and not well tested. + + + This is option is disabled by default, in which case the + state is stored in memory only. The value is the path to the + state file. + + + + + ExtendedLogging (optional) + + + Enables the connection debugging mode of the client, + which writes all commands sent to or received from the server + to the system logs. This generates a lot + of output, so it should only be used when instructed, or when + you suspect a connection problem or client-server protocol + error (and you know how to interpret the output). + + This is a boolean value, which may be set to + Yes or No. The default is of + course No. + + + + + ExtendedLogFile (optional, new in 0.11) + + + Enables the same debugging output as + ExtendedLogging, but written to a file + instead of the system logs. This is useful if you need + extended logging, but do not have access to the system logs, + for example if you are not the administrator of the + computer. + + The value is the path to the file where these logs will + be written. If omitted, extended logs will not be written to a + file. This is entirely independent of the + ExtendedLogging option. It does not + make much sense to use both at the same time. + + + + + LogAllFileAccess (optional, new in 0.11) + + + Enables logging of all local file and directory access, + file uploads (full and differential), and excluded files. This + may be useful if the client is failing to upload a particular + file, or crashing while trying to upload it. The logs will be + sent to the system log or Windows Event Viewer. + + This generates a lot + of output, so it should only be used when instructed, or when + you suspect that bbackupd is skipping some files and want to + know why. Because it is verbose, the messages are hidden by + default even if the option is enabled. To see them, you must + run bbackupd with at least one -v option. + + This is a boolean value, which may be set to + Yes or No. The default is of + course No. + + + + + SyncAllowScript (optional) + + + The path to the script or command to run when the client + is about to start an automatic backup run, and wishes to know + whether it is safe to do so. This is useful for clients which + do not always have access to the server, for example laptops + and computers on dial-up Internet connections. + + The script should either output the word + now if the backup should proceed, or else a + number, in seconds, which indicates how long the client should + wait before trying to connect again. Any other output will + result in an error on the client, and the backup will not + run. + + This value is optional, and by default no such script is + used. + + + +
+ +
+ Server Section + + These options appear within the Server subsection, which is at + the root level. + + + + PidFile + + + This option enables the client to write its processs + identifier (PID) to the specified file after starting. The + file will be deleted when the client daemon exits for any + reason. This is disabled by default, but is recommended + whenever you run the client daemon as a daemon (in the + background), which is usually the case. This file can be used + by scripts to determine whether the daemon is still running, + and to send it messages to reload its configuration or to + terminate. + + + Example Server Section + + Server +{ + PidFile = /var/state/boxbackup/bbackupd.pid +} + + + + +
+ +
+ Backup Locations Section + + This section serves only as a container for all defined backup + locations. + + + Example Backup Locations Section + + BackupLocations +{ + etc + { + Path = /etc + } + home + { + Path = /home + ExcludeDir = /home/shared + ExcludeDir = /home/chris/.ccache + ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache + } +} + + + Each subsection is a backup location. The name of the + subsection is the name that will be used on the server. The root + directory of the account on the server contains one subdirectory per + location. The name should be simple, not containing any spaces or + special characters. + + If you do not define any locations, the client will not back + up any files! + + It is currently not recommended to back up the root directory + of the filesystem on Unix. Box Backup is designed to back up + important data and configuration files, not full systems. + Nevertheless, nothing prevents you from doing so if you + desire. + + On Windows, it is currently not possible to back up files + which are open (currently in use), such as open documents in + Microsoft Office, and system files such as the registry and the + paging file. You will get an error for each open file which the + client attempts to back up. Once the file has been closed, it will + be backed up normally. System files will always be open, and should + be excluded from your backups. +
+
+
+
+ + + Administration + + This chapter deals with the dauily running and management of the Box + Backup system. It explains most day-to-day tasks. + +
+ Regular Maintenance + + The steps involved in maintaining and keeping the backup sets + healthy are outlined in this section. + +
+ Controlling a backup client + + The bbackupctl program sends control commands to the bbackupd + daemon. It must be run as the same user as the daemon, and there is no + exception for root. + + The command line syntax is: + + /usr/local/sbin/bbackupctl [-q] [-c config-file] command + + The -q option reduces the amount of output the program emits, + and -c allows an alternative configuration file to be + specified. + + Valid commands are: + + + + terminate + + Stop the bbackupd daemon now (equivalent to kill) + + + + reload + + Reload the configuration file (equivalent to kill + -HUP) + + + + sync + + Connect to the server and synchronise files now + + + + bbackupctl communicates with + the server via a UNIX domain socket, specified in bbackupd.conf with + the CommandSocket directive. This does not need to be specified, and + bbackupd will run without the command + socket, but in this case bbackupctl will not be able to communicate + with the daemon. + + Some platforms cannot check the user id of the connecting + process, so this command socket becomes a denial of service security + risk. bbackupd will warn you when it + starts up if this is the case on your platform, and you should + consider removing the CommandSocket directive on these + platforms. +
+ +
+ Using bbackupctl to perform snapshots + + bbackupctl's main purpose is to + implement snapshot based backups, emulating the behaviour of + traditional backup software. + + Use bbackupd-config to write a configuration file in snapshot + mode, and then run the following command as a cron job. + + /usr/local/sbin/bbackupctl -q sync + + This will cause the backup daemon to upload all changed files + immediately. bbackupctl will exit + almost immediately, and will not output anything unless there is an + error. +
+ +
+ Checking storage space used on the server + +
+ From the client machine + + bbackupquery can tell you how much space is used on the server + for this account. Either use the usage command in interactive mode, + or type: + + /usr/local/sbin/bbackupquery -q usage quit + + to show the space used as a single command. +
+ +
+ On the server + + bbstoreaccounts allows you to query the space used, and change + the limits. To display the space used on the server for an account, + use: + + /usr/local/sbin/bbstoreaccounts info 75AB23C + + To adjust the soft and hard limits on an account, use: + + /usr/local/sbin/bbstoreaccounts setlimit 75AB23C new-soft-limit new-hard-limit + + You do not need to restart the server. +
+
+ +
+ Verify and restore files + + Backups are no use unless you can restore them. The bbackupquery + utility does this and more. + + You don't provide any login information to it, as it just picks + up the data it needs from /etc/box/bbackupd.conf. You should run it as + root so it can find everything it needs. + + Full documentation can be found in the bbackupquery manual page. It follows + the model of a command line sftp client quite closely. + + TODO: Link to bbackupquery man-page here. + + On systems where GNU readline is available (by default) it uses + that for command line history and editing. Otherwise it falls back to + very basic UNIX text entry. + + TODO: Did the readline dependency change to editline? + +
+ Using bbackupquery + + bbackupquery is the tool you use to verify, restore and + investigate your backup files with. When invoked, it simply logs + into the server using the certificates you have listed in + bbackupd.conf. + + After you run bbackupquery, you will see a prompt, allowing + you to execute commands. The list (or ls) command lets you view + files in the store. It works much like unix ls, but with different + options. An example: + + [pthomsen@host bbackupquery]$ bbackupquery +Box Backup Query Tool v0.10, (c) Ben Summers and contributors 2003-2006 +Using configuration file /etc/box/bbackupd.conf +Connecting to store... +Handshake with store... +Login to store... +Login complete. + +Type "help" for a list of commands. + +query > ls +00000002 -d---- mp3 +00000003 -d---- video +00000004 -d---- home-pthomsen +00000005 -d---- root +query > + + The ls commands shows the directories that are backed up. Now + we'll take a closer look at the home-pthomsen directory: + + query > cd home-pthomsen +query > ls +00002809 f----- sample.tiff +0000280a f----- s3.tiff +0000280b f----- s4.tiff +0000280d f----- s2.tiff +0000280e f----- foo.pdf +0000286c f----- core.28720 +0000339a -d---- .emacs.d +0000339d -d---- bbackup-contrib +00003437 f----- calnut.compare.txt +0000345d f----- DSCN1783.jpg +0000345e f----- DSCN1782.jpg +query > + + The ls command takes the following options; + + + + -r -- recursively list + all files + + + + -d -- list deleted + files/directories + + + + -o -- list old versions + of files/directories + + + + -I -- don't display + object ID + + + + -F -- don't display + flags + + + + -t -- show file + modification time (and attr mod time if has the object has + attributes, ~ separated) + + + + -s -- show file size in + blocks used on server (only very approximate indication of size + locally) + + + + The flags displayed from the ls command are as follows: + + + f = file + + d = directory + + X = deleted + + o = old version + + R = remove from server as soon as marked deleted or + old + + a = has attributes stored in directory record which + override attributes in backup file + +
+ +
+ Verify backups + + As with any backup system, you should frequently check that + your backups are working properly by comparing them. Box Backup + makes this very easy and completely automatic. All you have to do is + schedule the bbackupquery compare command to run + regularly, and check its output. You can run the command manually as + follows: + + /usr/local/sbin/bbackupquery "compare -a" quit + + This command will report all the differences found between the + store and the files on disc. It will download everything, so may + take a while. You should expect to see some differences on a typical + compare, because files which have recently changed are unlikely to + have been uploaded yet. It will also tell you how many files have + been modified since the last backup run, since these will normally + have changed, and such failures are expected. + + You are strongly recommended to add this command as a + cron job, at least once a month, and to check the + output for anything suspicious, particularly a large number of + compare failures, failures on files that have not been modified, or + any error (anything except a compare mismatch) that occurs during + the compare operation. + + Consider keeping a record of these messages and comparing them + with a future verification. + + If you would like to do a "quick" check which just downloads + file checksums and compares against that, then run: + + /usr/local/sbin/bbackupquery "compare -aq" quit + + However, this does not check that the file attributes are + correct, and since the checksums are generated on the client they + may not reflect the data on the server if there is a problem -- the + server cannot check the encrypted contents. View this as a quick + indication, rather than a definite check that your backup verifies + correctly. +
+ +
+ Restore backups + + You will need the keys file created when you configured the + server. Without it, you cannot restore the files; this is the + downside of encrypted backups. However, by keeping the small keys + file safe, you indirectly keep your entire backup safe. + + The first step is to recreate the configuration of the backup + client. It's probably best to have stored the /etc/box directory + with your keys. But if you're recreating it, all you really need is + to have got the login infomation correct (ie the certs and + keys). + + Don't run bbackupd yet! It will mark all your files as deleted + if you do, which is not hugely bad in terms of losing data, just a + major inconvenience. (This assumes that you are working from a blank + slate. If you want to restore some files to a different location, + it's fine to restore while bbackupd is running, just do it outside a + backed up directory to make sure it doesn't start uploading the + restored files.) + + Type: + + /usr/local/sbin/bbackupquery + + to run it in interactive mode. + + Type: + + list + + to see a list of the locations stored on the server. + + For each location you want to restore, type: + + restore name-on-server local-dir-name + + The directory specified by local-dir-name must not exist yet. + If the restore is interrupted for any reason, repeat the above + steps, but add the -r flag to the + restore command to tell it to resume. +
+ +
+ Retrieving deleted and old files + + Box Backup makes old versions of files and files you have + deleted available, subject to there being enough disc space on the + server to hold them. + + This is how to retrieve them using bbackupquery. Future + versions will make this far more user-friendly. + + Firstly, run bbackupquery in interactive mode. It behaves in a + similar manner to a command line sftp client. + + /usr/local/sbin/bbackupquery + + Then navigate to the directory containing the file you want, + using list, cd and pwd. + + query > cd home/profiles/USERNAME + + List the directory, using the "o" option to list the files + available without filtering out everything apart from the current + version. (if you want to see deleted files as well, use list + -odt) + + query > list -ot +00000078 f--o- 2004-01-21T20:17:48 NTUSER.DAT +00000079 f--o- 2004-01-21T20:17:48 ntuser.dat.LOG +0000007a f--o- 2004-01-21T17:55:12 ntuser.ini +0000007b f---- 2004-01-12T15:32:00 ntuser.pol +0000007c -d--- 1970-01-01T00:00:00 Templates +00000089 -d--- 1970-01-01T00:00:00 Start Menu +000000a0 -d--- 1970-01-01T00:00:00 SendTo +000000a6 -d--- 1970-01-01T00:00:00 Recent +00000151 -d--- 1970-01-01T00:00:00 PrintHood +00000152 -d--- 1970-01-01T00:00:00 NetHood +00000156 -d--- 1970-01-01T00:00:00 My Documents +0000018d -d--- 1970-01-01T00:00:00 Favorites +00000215 -d--- 1970-01-01T00:00:00 Desktop +00000219 -d--- 1970-01-01T00:00:00 Cookies +0000048b -d--- 1970-01-01T00:00:00 Application Data +000005da -d--- 1970-01-01T00:00:00 UserData +0000437e f--o- 2004-01-24T02:45:43 NTUSER.DAT +0000437f f--o- 2004-01-24T02:45:43 ntuser.dat.LOG +00004380 f--o- 2004-01-23T17:01:29 ntuser.ini +00004446 f--o- 2004-01-24T02:45:43 NTUSER.DAT +00004447 f--o- 2004-01-24T02:45:43 ntuser.dat.LOG +000045f4 f---- 2004-01-26T15:54:16 NTUSER.DAT +000045f5 f---- 2004-01-26T15:54:16 ntuser.dat.LOG +000045f6 f---- 2004-01-26T16:54:31 ntuser.ini + + (this is a listing from a server which is used as a Samba + server for a network of Windows clients.) You now need to fetch the + file using it's ID, rather than it's name. The ID is the hex number + in the first column. Fetch it like this: + + query > get -i 0000437e NTUSER.DAT +Object ID 0000437e fetched successfully. + + The object is now available on your local machine. You can use + lcd to move around, and sh ls to list directories on your local + machine. +
+
+
+ +
+ Fixing corruptions of store data + + This section gives help on what to do if your server has suffered + corruption, for example, after an unclean shutdown or other operating + system or hardware problem. + + In general, as updates to the store are made in an atomic manner, + the most likely result is wasted disc space. However, if really bad + things happen, or you believe that there is a lot of wasted space, then + these instructions will help to restore your data. + + You know you will need to do something if you get strange errors, + and bbackupd attempts to contact the server every 100 seconds or so. Or + if one of the discs in your RAID disc set has failed. + + After following these instructions, the end result will be that + bbackupquery will be able to see all the files which were stored on your + server, and retrieve them. Some of them may be in lost+found directories + in the root of the store (or in their original position if they have + been moved) but they will all be able to be retrieved. + + After you have retrieved the files you want, bbackupd will upload + new versions where necessary, and after about two days, mark any + lost+found directories as deleted. Finally, those directories will be + removed by the housekeeping process on the server. + + These instructions assume you're working on account 1234. Replace + this with the account number that you actually want to check (the one + that is experiencing errors). These steps will need to be repeated for + all affected accounts. + +
+ Stop bbackupd + + First, make sure that bbackupd is not running on the client + machine for the account you are going to recover. Use + bbackupctl terminate to stop it. This step is not + strictly necessary, but is recommended. During any checks on the + account, bbackupd will be unable to log in, and after they are + complete, the account is marked as changed on the server so bbackupd + will perform a complete scan. +
+ +
+ Are you using RAID on the server? + + The raidfile recovery tools have not been written, and probably + will not be, since Box Backup RAID is deprecated. However, when two + out of three files are available, the server will successfully allow + access to your data, even if it complains a lot in the logs. The best + thing to do is to fix the accounts, if necessary, and retrieve any + files you need. Then move the old store directories aside (in case you + need them) and start afresh with new accounts, and let the clients + upload all their data again. +
+ +
+ Check and fix the account + + First, run the check utility, and see what errors it + reports. + + /usr/local/sbin/bbstoreaccounts check 1234 + + This will take some time, and use a fair bit of memory (about 16 + bytes per file and directory). If the output looks plausible and + reports errors which need fixing, run it again but with the fix + flag: + + /usr/local/sbin/bbstoreaccounts check 1234 fix + + This will fix any errors, and remove unrecoverable files. + Directories will be recreated if necessary. + + NOTE: The utility may adjust + the soft and hard limits on the account to make sure that housekeeping + will not remove anything -- check these afterwards. +
+ +
+ Grab any files you need with bbackupquery + + At this point, you will have a working store. Every file which + was on the server, and wasn't corrupt, will be available. + + On the client, use bbackupquery to log in and examine the store. + (type help at the prompt for instructions). Retrieve any files you + need, paying attention to any lost+found directories in the root + directory of the store. + + You can skip this step if you are sure that the client machine + is fine -- in this case, bbackupd will bring the store up to + date. +
+ +
+ Restart bbackupd + + Restart bbackupd on the client machine. The store account will + be brought up to date, and files in the wrong place will be marked for + eventual deletion. +
+
+ +
+ Troubleshooting + + If you are trying to fix a store after your disc has been + corrupted, see Fixing corruptions of + store data. + + Unfortunately, the error messages are not particularly helpful at + the moment. This page lists some of the common errors, and the most + likely causes of them. + + When an error occurs, you will see a message like 'Exception: + RaidFile/OSFileError (2/8)' either on the screen or in your log files. + (it is recommended you set up another log file as recommended in the + server setup instructions.) + + This error may not be particularly helpful, although some do have + extra information about probable causes. To get further information, + check the ExceptionCodes.txt file in the root of the distribution. This + file is generated by the ./configure script, so you will need to have + run that first. + + Some common causes of exceptions are listed below. + + Please email me with any other codes you get, and I will let you + know what they mean, and add notes here. + +
+ RaidFile (2/8) + + This is found either when running bbstoreaccounts or in the + bbstored logs. + + Problem: The directories you + specified in the raidfile.conf are not writable by the _bbstored + user. + + Resolution: Change permissions + appropriately. +
+ +
+ Common (1/2) + + This usually occurs when the configuration files can't be + opened. + + Problem: You created your + configurations in non-standard locations, and the programs cannot find + them. + + Resolution: Explicitly specify + configuration file locations to daemons and programs. For + example + + /usr/local/sbin/bbstored /some/other/dir/bbstored.config /usr/local/sbin/bbackupquery -c /some/other/dir/bbackupd.config + + (daemons specify the name as the first argument, utility + programs with the -c option). + + Problem: bbstored can't find + the raidfile.conf file specified in bbstored.conf. + + Resolution: Edit bbstored.conf + to point to the correct location of this additional configuration + file. +
+ +
+ Server (3/16) + + The server can't listen for connections on the IP address + specified when you configured it. + + Problem: This probably means + you've specified the wrong hostname to bbstored-config -- maybe your + server is behind a NAT firewall? + + Resolution: Edit bbstored.conf + and correct the ListenAddresses line. You should replace the server + address with the IP address of your machine. +
+ +
+ Connection (7/x) + + These errors all relate to connections failing -- you may see + them during operation if there are network failures or other problems + between the client and server. The backup system will recover from + them automatically. + +
+ Connection (7/30) - SSL problems + + Log snippet from client side: + + bbackupd[1904]: Opening connection to server xxxx.xxx... +bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:rsa routines:RSA_padding_check_PKCS1_type_1:block type is not 01 +bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:rsa routines:RSA_EAY_PUBLIC_DECRYPT:padding check failed +bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:asn1 encoding routines:ASN1_verify:EVP lib +bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed +bbackupd[1904]: TRACE: Exception thrown: ConnectionException(Conn_TLSHandshakeFailed) at SocketStreamTLS.cpp(237) +bbackupd[1904]: Exception caught (7/30), reset state and waiting to retry... + + And from the server: + + bbstored[19291]: Incoming connection from xx.xxx.xx.xxx port xxxxx (handling in child xxxxx) +bbstored[21588]: SSL err during Accept: error:xxxxxxxx:SSL routines:SSL3_READ_BYTES:tlsv1 alert decrypt error +bbstored[21588]: in server child, exception Connection TLSHandshakeFailed (7/30) -- terminating child + + Solution: Create a new CA on + the server side and re-generate the client certificate. Re-creating + the client certificate request is not necessary. +
+
+ +
+ Advanced troubleshooting + + If this really doesn't help, then using the DEBUG builds of the + system will give you much more information -- a more descriptive + exception message and the file and line number where the error + occurred. + + For example, if you are having problems with bbstoreaccounts, + build the debug version with: + + cd boxbackup-0.0 +cd bin/bbstoreaccounts +make + + Within the module directories, make defaults to building the + debug version. At the top level, it defaults to release. + + This will build an executable in debug/bin/bbstoreaccounts which + you can then use instead of the release version. It will give far more + useful error messages. + + When you get an error message, use the file and line number to + locate where the error occurs in the code. There will be comments + around that line to explain why the exception happened. + + If you are using a debug version of a daemon, these extended + messages are found in the log files. +
+
+
+ + &__ExceptionCodes__elfjz3fu; + + + Running without root + + It is possible to run both the server and client without root + privileges. + +
+ Server + + The server, by default, runs as a non-root user. However, it + expects to be run as root and changes user to a specified user as soon + as it can, simply for administrative convenience. The server uses a port + greater than 1024, so it doesn't need root to start. + + To run it entirely as a non-root user, edit the bbstored.conf + file, and remove the User directive from the Server section. Then simply + run the server as your desired user. +
+ +
+ Client + + The client requires root for normal operation, since it must be + able to access all files to back them up. However, it is possible to run + the client as a non-root user, with certain limitations. + + Follow the installation instructions, but install the executable + files manually to somewhere in your home directory. Then use + bbackupd-config to configure the daemon, but use a directory other than + /etc/box, probably somewhere in your home directory. + + All directories you specify to be backed up must be readable, and + all files must be owned by the user and readable to that user. + + Important: If any file or directory is not readable by this user, + the backup process will skip that file or directory. Keep an eye on the + logs for reports of this failure. + + Non-root operation of the backup client is recommended only for + testing, and should not be relied on in a production environment. +
+
+
diff --git a/docs/docbook/bb-book.xsl b/docs/docbook/bb-book.xsl new file mode 100644 index 00000000..a4f05fdb --- /dev/null +++ b/docs/docbook/bb-book.xsl @@ -0,0 +1,17 @@ + + + + + + + + + + + + + diff --git a/docs/docbook/bb-man.xsl.tmpl b/docs/docbook/bb-man.xsl.tmpl new file mode 100644 index 00000000..e20eedd5 --- /dev/null +++ b/docs/docbook/bb-man.xsl.tmpl @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/docs/docbook/bb-nochunk-book.xsl b/docs/docbook/bb-nochunk-book.xsl new file mode 100644 index 00000000..86574122 --- /dev/null +++ b/docs/docbook/bb-nochunk-book.xsl @@ -0,0 +1,17 @@ + + + + + + + + + + + + + diff --git a/docs/docbook/bbackupctl.xml b/docs/docbook/bbackupctl.xml new file mode 100644 index 00000000..b4880bfd --- /dev/null +++ b/docs/docbook/bbackupctl.xml @@ -0,0 +1,205 @@ + + + + bbackupctl + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupctl + + Control the Box Backup client daemon + + + + + bbackupctl + + -q + + -c config-file + + command + + + + + Description + + bbackupctl sends commands to a running + bbackupd daemon on a client machine. It can be used to + force an immediate backup, tell the daemon to reload its configuration + files or stop the daemon. If bbackupd is configured in + snapshot mode, it will not back up automatically, and the + bbackupctl must be used to tell it when to start a + backup. + + Communication with the bbackupd daemon takes place over a local + socket (not over the network). Some platforms (notably Windows) can't + determine if the user connecting on this socket has the correct + credentials to execute the commands. On these platforms, ANY local user + can interfere with bbackupd. To avoid this, remove the CommandSocket + option from bbackupd.conf, which will also disable bbackupctl. See the + Client Configuration page for more information. + + bbackupctl needs to read the + bbackupd configuration file to find out the name of the + CommandSocket. If you have to tell bbackupd where to + find the configuration file, you will have to tell + bbackupctl as well. The default on Unix systems is + usually /etc/box/bbackupd.conf. On Windows systems, + it is bbackupd.conf in the same directory where + bbackupd.exe is located. If + bbackupctl cannot find or read the configuration file, + it will log an error message and exit. + + bbackupctl usually writes error messages to the + console and the system logs. If it is not doing what you expect, please + check these outputs first of all. + + + + + + + Run in quiet mode. + + + + + config-file + + + Specify configuration file. + + + + + + Commands + + The following commands are available in bbackupctl: + + + + terminate + + + This command cleanly shuts down bbackupd. + This is better than killing or terminating it any other + way. + + + + + reload + + + Causes the bbackupd daemon to re-read all + its configuration files. Equivalent to kill + -HUP. + + + + + sync + + + Initiates a backup. If no files need to be backed up, no + connection will be made to the server. + + + + + force-sync + + + Initiates a backup, even if the + SyncAllowScript says that no backup should run + now. + + + + + wait-for-sync + + + Passively waits until the next backup starts of its own + accord, and then terminates. + + + + + wait-for-end + + + Passively waits until the next backup starts of its own + accord and finishes, and then terminates. + + + + + sync-and-wait + + + Initiates a backup, waits for it to finish, and then + terminates. + + + + + + + + Files + + /etc/box/bbackupd.conf + + + + See Also + + + bbackupd.conf + + 5 + , + bbackupd-config + + 8 + , + bbackupctl + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbackupd-config.xml b/docs/docbook/bbackupd-config.xml new file mode 100644 index 00000000..41bb6587 --- /dev/null +++ b/docs/docbook/bbackupd-config.xml @@ -0,0 +1,153 @@ + + + + bbackupd-config + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupd-config + + Box Backup client daemon configuration file + generator + + + + + bbackupd-config + + config-dir + + backup-mode + + account-num + + server-hostname + + working-dir + + backup-dir + + backup-dir ... + + + + + Description + + The bbackupd-config script creates configuration files and client + certificates. It takes at least six parameters: + + + + config-dir + + + Configuration directory. Usually + /etc/box. + + + + + backup-mode + + + Either lazy or snapshot. + + + + + account-num + + + The client account number. This is set by the bbstored + administrator. + + + + + server-hostname + + + The hostname or IP address of the bbstored server. + + + + + working-dir + + + A directory to keep temporary state files. This is usually + something like /var/bbackupd. This can be + changed in bbackupd.conf later on if + required. + + + + + backup-dir + + + A space-separated list of directories to be backed up. Note + that this does not traverse mount + points. + + + + + + + Files + + /etc/box/bbackupd.conf + + /etc/box/bbackupd/NotifySysAdmin.sh + + + + See Also + + + bbackupd.conf + + 5 + , + bbackupd + + 8 + , + bbackupctl + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbackupd.conf.xml b/docs/docbook/bbackupd.conf.xml new file mode 100644 index 00000000..43ae2bed --- /dev/null +++ b/docs/docbook/bbackupd.conf.xml @@ -0,0 +1,479 @@ + + + + bbackupd.conf + + 5 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupd.conf + + Box Backup client daemon configuration file + + + + + /etc/box/bbackupd.conf + + + + + Description + + + + AccountNumber + + + The account number of this client. This is set by the admin of + the store server. + + + + + UpdateStoreInterval + + + Specifies the interval between scanning of the local discs. To + avoid cycles of load on the server, this time is randomly adjusted + by a small percentage as the daemon runs. Defaults to 1 hour. + + + + + MinimumFileAge + + + Specifies how long since a file was last modified before it + will be uploaded. Defaults to 6 hours. + + + + + MaxUploadWait + + + If a file is repeatedly modified it won't be uploaded + immediately in case it's modified again. However it should be + uploaded eventually. This is how long we should wait after first + noticing a change. Defaults to 1 day. + + + + + MaxFileTimeInFuture + + + + + + + + AutomaticBackup + + + + + + + + SyncAllowScript + + + Use this to temporarily stop bbackupd from syncronising or + connecting to the store. This specifies a program or script script + which is run just before each sync, and ideally the full path to the + interpreter. It will be run as the same user bbackupd is running as, + usually root. + + The script prints either "now" or a number to STDOUT (and a + terminating newline, no quotes). If the result was "now", then the + sync will happen. If it's a number, then the script will be asked + again in that number of seconds. + + For example, you could use this on a laptop to only backup + when on a specific network. + + + + + MaximumDiffingTime + + + How much time should be spent on diffing files. + + + + + DeleteRedundantLocationsAfter + + + + + + + + FileTrackingSizeThreshold + + + + + + + + DiffingUploadSizeThreshold + + + + + + + + StoreHostname + + + The hostname or IP address of the + bbstored + + 8 + server. + + + + + StorePort + + + The port used by the server. Defaults to 2201. + + + + + ExtendedLogging + + + Logs everything that happens between the client and server. + The + bbackupd + + 8 + client must also be started with + . + + + + + ExtendedLogFile + + + + + + + + LogAllFileAccess + + + + + + + + LogFile + + + + + + + + LogFileLevel + + + + + + + + CommandSocket + + + Where the command socket is created in the filesystem. + + + + + KeepAliveTime + + + + + + + + StoreObjectInfoFile + + + + + + + + NotifyScript + + + The location of the script which runs at certain events. This + script is generated by + bbackupd-config + + 8 + . Defaults to + /etc/box/bbackupd/NotifySysAdmin.sh. + + + + + NotifyAlways + + + + + + + + CertificateFile + + + The path to the client's public certificate. + + + + + PrivateKeyFile + + + The path to the client's private key. This should only be + readable by root. + + + + + TrustedCAsFile + + + The Certificate Authority created by + bbstored-certs + + 8 + . + + + + + KeysFile + + + The data encryption key. This must be kept safe at all costs, your data is + useless without it! + + + + + DataDirectory + + + A directory to keep temporary state files. This is usually + something like /var/bbackupd. + + + + + Server + + + This section relates to the running daemon. + + + + PidFile + + + The location of the process ID file. Defaults to + /var/run/bbackupd.pid. + + + + + + + + BackupLocations + + + This section defines each directory to be backed up. Each + entry must have at least a Path entry and, optionally, include and + exclude directives. + + Multiple include and exclude directives may appear. + + + + Path + + + The path to back up. + + + + + ExcludeFile + + + Exclude a single file. + + + + + ExcludeFilesRegex + + + Exclude multiple files based on a regular expression. + See + re_format + + 7 + . + + + + + ExcludeDir + + + Exclude a single directory. + + + + + ExcludeDirsRegex + + + Exclude multiple directories based on a regular + expression. See + re_format + + 7 + . + + + + + AlwaysIncludeFile + + + Include a single file from a directory which has been + excluded. + + + + + AlwaysIncludeFilesRegex + + + Include multiple files from an excluded directory, + based on a regular expression. + + + + + AlwaysIncludeDir + + + Include a single directory from a directory which has + been excluded. + + + + + AlwaysIncludeDirsRegex + + + Include multiple directories from an excluded + directory, based on a regular expression. + + + + + + + + + + Examples + + The following is an example of a backup location: + + home +{ + Path = /home + ExcludeDir = /home/guest + ExcludeDir = /home/[^/]+/tmp + ExcludeFilesRegex = .*\.(mp3|MP3)$ + AlwaysIncludeFile = /home/someuser/importantspeech.mp3 +} + + + + Files + + /etc/box/bbackupd.conf + + + + See Also + + + bbackupd + + 8 + , + bbackupd-config + + 8 + , + bbackupctl + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbackupd.xml b/docs/docbook/bbackupd.xml new file mode 100644 index 00000000..11de0f3f --- /dev/null +++ b/docs/docbook/bbackupd.xml @@ -0,0 +1,209 @@ + + + + bbackupd + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupd + + Box Backup client daemon + + + + + bbackupd + + -DFkqvVT + + -c config-file + + -t tag + + + + + Description + + bbackupd runs on client computers in the + background, finding new files to back up. When it is time for a backup, + bbackupd will connect to the server + (bbstored) to upload the files. + + A running bbackupd daemon can be controlled with + the bbackupctl command, to make it shut down, reload + its configuration, or start an immediate backup. + + bbackupd needs to be configured to tell it which + files to back up, how often, and to which server (running + bbstored). See the Client Configuration page for more + information. For this, you must write a configuration file. You must + either place it in the default location, or tell + bbackupd where to find it. + + You can check the default location with the + option. The default on Unix systems is usually + /etc/box/bbackupd.conf. On Windows systems, it is + bbackupd.conf in the same directory where + bbackupd.exe is located. If bbackupd cannot find or + read the configuration file, it will log an error message and exit. + + bbackupd usually writes log messages to the + system logs, using the facility local5, which you can + use to filter them to send them to a separate file. It can also write them + to the console, see options below. If bbackupd is not + doing what you expect, please check the logs first of all. + + + Options + + + + config-file + + + Use the specified configuration file. If + is omitted, the last argument is the configuration file. If none + is specified, the default is used (see above). + + + + + + + + Debugging mode. Do not fork into the background (do not run + as a daemon). Not available on Windows. + + + + + + + + No-fork mode. Same as for bbackupd. Not + available on Windows. + + + + + + + + Keep console open after fork, keep writing log messages to + it. Not available on Windows. + + + + + + + + Run more quietly. Reduce verbosity level by one. Available + levels are NOTHING, FATAL, + ERROR, WARNING, + NOTICE, INFO, + TRACE, EVERYTHING. Default + level is NOTICE in non-debugging builds. Use + once to drop to WARNING level, twice for + ERROR level, four times for no logging at + all. + + + + + -v + + + Run more verbosely. Increase verbosity level by one. Use + once to raise to INFO level, twice for + TRACE level, three times for + EVERYTHING (currently the same as + TRACE). + + + + + + + + Run at maximum verbosity (EVERYTHING + level). + + + + + tag + + + Tag each console message with specified marker. Mainly + useful in testing when running multiple daemons on the same + console. + + + + + + + + Timestamp each line of console output. + + + + + + + + Files + + /etc/box/bbackupd.conf + + + + See Also + + + bbackupd.conf + + 5 + , + bbackupd-config + + 8 + , + bbackupctl + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbackupquery.xml b/docs/docbook/bbackupquery.xml new file mode 100644 index 00000000..016d5c7d --- /dev/null +++ b/docs/docbook/bbackupquery.xml @@ -0,0 +1,506 @@ + + + + bbackupquery + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupquery + + Box Backup store query and file retrieval + + + + + bbackupquery + + -q + + -c configfile + + command ... + + + + + Description + + bbackupquery is the main way of interacting with + the backup store from a Box Backup client machine. It supports both + interactive and batch modes of operation. + + It can be used to reviewing the status of a client machine's backup + store, getting status from the store server. The main use is to retrieve + files and directories when needed. + + bbackupquery supports interactive and batch modes + of operation. Interactive mode allows for interaction with the server much + like an interactive FTP client. + + Batch mode is invoked by putting commands into the invocation of + bbackupquery. Example: + + bbackupquery "list home-dirs" quit + + Note that commands that contain spaces are enclosed in double + quotes. If the quit command is omitted, after the + preceding commands are completed, bbackupquery will + enter interactive mode. + + + + Options + + + + + + + Quiet. Suppresses status output while running. + + + + + + + + Use configfile instead of the default bbackupd.conf file. + Can be a relative or full path. + + + + + + + Commands + + The commands that can be used in bbackupquery are listed + below. + + + + help + + + Displays the basic help message, which gives information about + the commands available in bbackupquery. Use the + form help command to get help on a specific + command. + + + + + quit + + + End the session with the store server, and quit + bbackupquery. + + + + + cd options + directory-name + + + Change directory. Options: + + + + + consider deleted directories for traversal + + + + + + + + consider old versions of directories for traversal. + This option should never be useful in a correctly formed + store. + + + + + + + + lcd + local-directory-name + + + Change directory on the client machine. To list the contents + of the local directory, type sh ls (on Unix-like + machines). + + + + + list options + directory-name + + + The list (or its synonym ls) command lists + the content of the current, or specified, directory. The options are + as follows: + + + + + + + recursively list all files + + + + + + + + list deleted files and directories + + + + + + + + list old versions of files and directories + + + + + + + + don't display object IDs + + + + + + + + don't display flags + + + + + + + + show file modification time (and attr mod time, if the + object has attributes). + + + + + + + + show file size in blocks used on server. Note that + this is only a very approximate indication of local file + size. + + + + + + + + ls options + directory-name + + + Synonym for list. + + + + + pwd + + + Print current directory, always relative to the backup store + root. + + + + + sh shell-command + + + Everything after the sh is passed to a shell and run. All + output from the command is displayed in the client. + + Example: to list the contents of the current directory on the + client machine type sh ls. + + + + + compare -a + + + + + + + + compare -l + location-name + + + + + + + + compare store-dir-name + local-dir-name + + + Compare the current data in the store with the data on the + disc. Please note that all the data will be downloaded from the + store, so this can be a very lengthy process depending on the size + of the store, and the size of the part you are comparing. + + Options: + + + + + + + compare all locations. + + + + + + + + compare one backup location as specified in the + configuration file. This compares one of the top level store + directories. + + + + + + + + set return code. The return code is set to the + following values, if quit is the next command. So, if + another command is run after the compare, the return code + will not refer to the compare. This option is very useful + for automating compares. Return code values: + + -- no differences were + found + + + + -- differences were + found + + + + -- an error occured + + + + + + + + + + get object-filename + local-filename + + + + + + + + get -i object-id + local-filename + + + Gets a file from the store. Object is specified as the + filename within the current directory. Local filename is optional. + Ignores old and deleted files when searching the directory for the + file to retrieve. + + To get an old or deleted file, use the + option and select the object as a hex object ID (first column in + listing). The local filename must be specified. + + + + + getobject object-id + local-filename + + + Gets the object specified by the object id (in hex) and stores + the raw contents in the local file specified. Note: This is only + useful for debugging as it does not decode files from the stored + format, which is encrypted and compressed. + + + + + restore -d + directory-name + local-directory-name + + + + + + + + restore -r + + + Restores a directory to the local disc. The local directory + specified must not exist (unless a previous restore is being + restarted). The root cannot be restored -- restore locations + individually. + + Options: + + + + + + + restore a deleted directory + + + + + + + + resume an interrupted restore + + + If a restore operation is interrupted for any + reason, it can be restarted using the switch. + Restore progress information is saved in a file at regular intervals + during the restore operation to allow restarts. + + + + + usage -m + + + Show space used on the server for this account. Display + fields: + + Used: Total amount of space used on + the server + + + + Old files: Space used by old + files + + + + Deleted files: Space used by + deleted files + + + + Directories: Space used by the + directory structure + + + + When Used exceeds the soft limit, the + server will start to remove old and deleted files until the usage + drops below the soft limit. After a while, you should expect to see + the usage stay at just below the soft limit. You only need more + space if the space used by old and deleted files is near + zero. + + The option displays output in + machine-readable form. + + + + + + + Bugs + + If you find a bug in Box Backup and you want to let us know about + it, join the mailing + list and send us a description of the problem there. + + To report a bug, give us at least the following information: + + + + The version of Box Backup you are running + + + + The platform you are running on (hardware and OS), for both + client and server. + + + + If possible attach your config files (bbstored.conf, + bbackupd.conf) to the bug report. + + + + Also attach any log file output that helps shed light on the + problem you are seeing. + + + + And last but certainly not least, a description of what you are + seeing, in as much detail as possible. + + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstoreaccounts.xml b/docs/docbook/bbstoreaccounts.xml new file mode 100644 index 00000000..08092acf --- /dev/null +++ b/docs/docbook/bbstoreaccounts.xml @@ -0,0 +1,386 @@ + + + + bbstoreaccounts + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbstoreaccounts + + Box Backup store accounts manager + + + + + bbstoreaccounts + + -c config-file + + command + + account-id + + command-specific arguments + + + + + Description + + bbstoreaccounts is the tool for managing accounts + on the store server. It can be used to view information related to + accounts, as well as create, change and delete accounts on the store + server. + + bbstoreaccounts always takes at least 2 + parameters: the command name and the account ID. Some commands require + additional parameters, and some commands have optional parameters. + + + Options + + + + + + + The configfile to use for connecting to the store. Default + is /etc/box/bbstored.conf. + + + + + + + Commands + + The commands tells bbstoreaccounts what action to perform. + + + + check account-id + fix + + + The check command verifies the + integrity of the store account given, and optionally fixes any + corruptions. Note: It is + recommended to run the 'simple' check command (without + fix) before using the fix + option. This gives an overview of the extent of any problems, + before attempting to fix them. + + + + + create account-id + disc-set soft-limit + hard-limit + + + Creates a new store account with the parameters given. The + parameters are as follows: + + + + + + + The ID of the new account to be created. A 32-bit + hexadecimal number. Cannot already exist on the + server. + + + + + + + + The disc set from + raidfile.conf + + 5 + where the backups for this client will + be stored. A number. Each RAID-file set has a number in + raidfile.conf. This number is what's used. + + + + + + + + The soft limit is the amount of storage that the + server will guarantee to be available for + storage. + + + + + + + + The amount of storage that the the server will + allow, before rejecting uploads, and starting to + eliminate old and deleted files to get back down to + soft-limit. + + + + + + + + delete account-id + yes + + + Deletes the account from the store server completely. + Removes all backups and deletes all references to the account in + the config files. + + delete will ask for confirmation from + the user, when called. Using the flag, + eliminates that need. This is useful when deleting accounts from + within a script or some other automated means. 0 + + + + + info account-id + + + Display information about the given account. + Example:[root]# bbstoreaccounts info 1 + Account ID: 00000001 + Last object ID: 58757 + Blocks used: 9864063 (38531.50Mb) + Blocks used by old files: 62058 (242.41Mb) +Blocks used by deleted files: 34025 (132.91Mb) + Blocks used by directories: 6679 (26.09Mb) + Block soft limit: 11796480 (46080.00Mb) + Block hard limit: 13107200 (51200.00Mb) + Client store marker: 1139559852000000 + + Explanation: + + + + Account ID + + + The account ID being displayed. + + + + + Last Object ID + + + A counter that keeps track of the objects that + have been backed up. This number refers to the last file + that was written to the store. The ID is displayed as a + decimal number, and the object ID can be converted to a + path name to a file as follows: convert the number to + hex (e.g.: 58757 => 0xE585); The last backed up file + will be (relative from the client's store root): + e5/o85.rfw. Longer numbers infer + more directories in the structure, so as an example + 3952697264 as the last object ID gives 0xEB995FB0, which + translates to a backup pathname of + eb/99/5f/ob0.rfw. + + + + + Blocks used + + + The number of blocks used by the store. The size + in Mb depends on the number of blocks, as well as the + block size for the disc set given in + raidfile.conf + + 5 + . In this case the block size is + 4096. + + + + + Blocks used by old files + + + The number of blocks occupied by files that have + newer versions in the store. This data is at risk for + being removed during housekeeping. + + + + + Blocks used by deleted files + + + The number of blocks used by files that have been + deleted on the client. This data is at risk for being + removed during housekeeping. + + + + + Blocks used by directories + + + The number of blocks used by directories in the + store. + + + + + Block soft limit + + + The soft limit in blocks. The soft limit is the + maximum guaranteed storage space available to the + account. When housekeeping starts, and the old and + deleted files are removed, they are removed in + chronological order (oldest first), until the data used + is less than the soft limit. + + + + + Block hard limit + + + The hard limit in blocks. The hard limit is the + most amount of storage the server will allow in an + account. Any data above this amount will be rejected. + Housekeeping will reduce the storage use, so more data + can be uploaded. + + + + + Client store marker + + + + bbstored + + 8 + uses this number to determine if it + needs to rescan the entire store. If this number is + different from the last time it checked, a rescan will + take place. + + + + + + + + setlimit account-id + soft-limit hard-limit + + + Changes the storage space allocation for the given + account. No server restart is needed. + + Parameters: + + + + + + + The ID of the account to be modified. + + + + + + + + The soft limit is the amount of storage that the + server will guarantee to be available for + storage. + + + + + + + + The amount of storage that the the server will + allow before rejecting uploads and starting to eliminate + old and deleted files to get back down to + . + + + + + + + + + + + Examples + + Create an account with ID 3af on disc set 0, with a 20GB soft-limit + and a 22GB hard-limit:bbstoreaccounts create 3af 0 20G 22GAlter + existing account ID 20 to have a 50GB soft-limit and a 55GB + hard-limit:bbstoreaccounts setlimit 20 50G 55G + + + + Files + + /etc/box/bbstored/accounts.txt + + + + See Also + + + bbstored + + 8 + , + bbstored-config + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstored-certs.xml b/docs/docbook/bbstored-certs.xml new file mode 100644 index 00000000..79308089 --- /dev/null +++ b/docs/docbook/bbstored-certs.xml @@ -0,0 +1,180 @@ + + + + bbstored-certs + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbstored-certs + + Manage certificates for the Box Backup system + + + + + bbstored-certs + + certs-dir + + command + + arguments + + + + + Description + + bbstored-certs creates and signs certificates for + use in Box Backup. It allows the user to create and sign the server keys, + as well as signing client keys. + + All commands must be followed by the certs-dir, + which is the directory in which the certificates are stored. + + + Commands + + There are 3 commands: + + + + init + + + Create the certs-dir, and generate the + server keys for bbstored. certs-dir cannot + exist before running the command. + + + + + sign-server + servercsrfile + + + Sign the server certificate. The + servercsrfile is the file generated by the + init command. + + + + + sign + clientcsrfile + + + Sign a client certificate. The + clientcsrfile is generated during client setup. + See + bbackupd-config + + 8 + . Send the signed certificate back to the client, + and install according to the instructions given by + bbackupd-config. + + + + + + + + Files + + + raidfile-config + + 8 + generates the + raidfile.conf + + 5 + file. + + + + Bugs + + If you find a bug in Box Backup, and you want to let us know about + it, join the mailing + list, and send a description of the problem there. + + To report a bug, give us at least the following information: + + + + The version of Box Backup you are running + + + + The platform you are running on (hardware and OS), for both + client and server. + + + + If possible attach your config files (bbstored.conf, + bbackupd.conf) to the bug report. + + + + Also attach any log file output that helps shed light on the + problem you are seeing. + + + + And last but certainly not least, a description of what you are + seeing, in as much detail as possible. + + + + + + See Also + + + bbstored-config + + 8 + , + bbstored.conf + + 5 + , + bbstoreaccounts + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstored-config.xml b/docs/docbook/bbstored-config.xml new file mode 100644 index 00000000..6d0113c1 --- /dev/null +++ b/docs/docbook/bbstored-config.xml @@ -0,0 +1,148 @@ + + + + bbstored-config + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbstored-config + + Box Backup store daemon configuration file + generator + + + + + bbstored-config + + configdir + + servername + + username + + + + + Description + + The bbstored-config script creates configuration files and server + certificates for a bbstored instance. It takes three parameters: + + + + configdir + + + The directory where config files will reside. A + bbstored subdirectory will be created where + several config files will reside. The + bbstored.conf file will be created in + configdir. + + + + + servername + + + The name of the server that is being configured. Usually the + fully qualified domain name of the machine in question. + + + + + username + + + The name of the user that should be running the + bbstored process. Recommended name: + _bbstored. + + + + + A valid + raidfile.conf + + 5 + must be found in configdir. Several steps are taken + during the run of bbstored-config: + + + + Server certificates are created. This requires interaction from + the operator. + + + + The RAID volumes are checked to ensure that the configuration is + consistent and will work. + + + + Instructions for next steps to take are shown. These steps may + be different for different OS platforms, so pay close attention to + these instructions. + + + + + + Files + + /etc/box/bbstored.conf + + + + See Also + + + bbstored.conf + + 5 + , + bbstored + + 8 + , + bbstored-certs + + 8 + , + raidfile-config + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstored.conf.xml b/docs/docbook/bbstored.conf.xml new file mode 100644 index 00000000..136b1bd0 --- /dev/null +++ b/docs/docbook/bbstored.conf.xml @@ -0,0 +1,211 @@ + + + + bbstored.conf + + 5 + + Box Backup + + Box Backup + + 0.11 + + + + bbstored.conf + + Box Backup store daemon configuration file + + + + + /etc/box/bbstored.conf + + + + + Description + + The following configuration options are valid: + + + + RaidFileConf + + + Specifies the path to the + raidfile.conf + + 5 + . This is normally + /etc/box/raidfile.conf. + + + + + AccountDatabase + + + Specifies the path to the account database created by + + bbstoreaccounts + + 8 + . This is usually + /etc/box/bbstored/accounts.txt. + + + + + ExtendedLogging + + + Specifies whether extended logging should be enabled to show + what commands are being received from clients. + + + + + TimeBetweenHousekeeping + + + How long between scanning for files which need + deleting. + + + + + Server + + + These options relate to the actual daemon. + + PidFile + + + The location of the pidfile, where the daemon's + process ID is kept. + + + + + User + + + The user to run as. + + + + + ListenAddresses + + + The interface addresses to listen on. Hostnames may be + used instead of IP addresses. The format is: + or + . + + + + + CertificateFile + + + The path to the server's public certificate. + + + + + PrivateKeyFile + + + The path to the server's private key. This should only + be readable by root and/or the . + + + + + TrustedCAsFile + + + The Certificate Authority created by + bbstored-certs + + 8 + . + + + + + + + + + + Examples + + The following is an example bbstored.conf: + + RaidFileConf = /etc/box/raidfile.conf +AccountDatabase = /etc/box/bbstored/accounts.txt + +TimeBetweenHousekeeping = 900 + +Server +{ + PidFile = /var/run/bbstored.pid + User = _bbstored + ListenAddresses = inet:server.example.com + CertificateFile = /etc/box/bbstored/server.example.com-cert.pem + PrivateKeyFile = /etc/box/bbstored/server.example.com-key.pem + TrustedCAsFile = /etc/box/bbstored/clientCA.pem +} + + + + Files + + /etc/box/bbstored.conf + + + + See Also + + + bbstored + + 8 + , + bbstored-config + + 8 + , + raidfile-config + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstored.xml b/docs/docbook/bbstored.xml new file mode 100644 index 00000000..81891a8d --- /dev/null +++ b/docs/docbook/bbstored.xml @@ -0,0 +1,90 @@ + + + + bbstored + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbstored + + Box Backup store daemon + + + + + bbstored + + config-file + + + + + Description + + bbstored runs on a central server. Clients + running bbackupd connect to the server and upload + files. + + The only argument is optional and specifies a non-default + configuration file. By default it will look for the configuration file as + /etc/box/bbstored.conf. + + + + Files + + /etc/box/bbstored.conf + + + + See Also + + + bbstored.conf + + 5 + , + bbstored-config + + 8 + , + raidfile-config + + 8 + , + raidfile.conf + + 5 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/html/bbdoc-man.css b/docs/docbook/html/bbdoc-man.css new file mode 100644 index 00000000..0345560e --- /dev/null +++ b/docs/docbook/html/bbdoc-man.css @@ -0,0 +1,104 @@ +body { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + font-size: .75em; + line-height: 180%; + text-align: left; + margin-top: 20px; + margin-right: 100px; + margin-left: 250px; + position: relative; + width: auto; } + +table { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + font-size: 10pt; + line-height: 100%; } + + +code { + font-size: 11pt; } + + + +div.navheader { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + line-height: 100%; } + +#header { + background-color: #e4e6ed; + text-align: left; + padding-top: 10px; + margin-right: -100px; + margin-left: -250px; + top: 20px; + border-top: 1px solid #c4c4d5; + border-bottom: 1px solid white } + +#logo { + position: relative; + margin-left: 200px } + + +#page { + font-size: .75em; + line-height: 180%; + text-align: left; + margin-top: 50px; + margin-right: 100px; + margin-left: 250px; + position: relative; + width: auto } + +#disc { } + +.informaltable td,tr {font-size: 1em; + line-height: 140%; + text-align: left; + background-color: #e4e6ed; + padding: 4px } + +tr,td {font-size: 1em; + line-height: 100%; + background-color: #edeef3; } + +pre, tt { font-size: 1.3em; + color: #088; + letter-spacing: 1px; + word-spacing: 2px} + +h1 { + color: #c00; + font-size: 16pt; + margin-bottom: 2em; + margin-left: -50px } + +h2 { + color: #324e95; + font-size: 12pt; + margin-top: 2em; + margin-left: -50px } + +h3 { + color: #324e95; + font-size: 10pt; + margin-top: 2em; + margin-left: -50px } + +dt { font-weight: bold } + +a:link { + color: #324e95; + text-decoration: none; + background-color: transparent } + +a:visited { + color: #90c; + text-decoration: none } + +a:hover { + color: #c00; + text-decoration: underline; + background-color: transparent } diff --git a/docs/docbook/html/bbdoc.css b/docs/docbook/html/bbdoc.css new file mode 100644 index 00000000..d3b4a1c2 --- /dev/null +++ b/docs/docbook/html/bbdoc.css @@ -0,0 +1,112 @@ +body { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + font-size: .75em; + line-height: 180%; + text-align: left; + margin-top: 20px; + margin-right: 100px; + margin-left: 250px; + position: relative; + width: auto; } + +table { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + font-size: 10pt; + line-height: 100%; } + + + + +div.navheader { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + line-height: 100%; } + +#header { + background-color: #e4e6ed; + text-align: left; + padding-top: 10px; + margin-right: -100px; + margin-left: -250px; + top: 20px; + border-top: 1px solid #c4c4d5; + border-bottom: 1px solid white } + +#logo { + position: relative; + margin-left: 200px } + + +#page { + font-size: .75em; + line-height: 180%; + text-align: left; + margin-top: 50px; + margin-right: 100px; + margin-left: 250px; + position: relative; + width: auto } + +#disc { } + +.informaltable td,tr {font-size: 1em; + line-height: 140%; + text-align: left; + background-color: #e4e6ed; + padding: 4px } + +tr,td {font-size: 1em; + line-height: 100%; + background-color: #edeef3; } + +pre, tt { font-size: 1.3em; + color: #088; + letter-spacing: 1px; + word-spacing: 2px} + +h1 { + color: #c00; + font-size: 16pt; + margin-bottom: 2em; + margin-left: -50px } + +h2 { + color: #324e95; + font-size: 12pt; + margin-top: 2em; + margin-left: -50px } + +h3 { + color: #324e95; + font-size: 10pt; + margin-top: 2em; + margin-left: -50px } + +dt { font-weight: bold } + +ul { + list-style-image: url(images/arrow.png) } + +ul li { + background-color: #e4e6ed; + margin: 1em 6em 1em -2em; + padding: 0.2em 0.5em 0.2em 1em; + border-style: solid; + border-width: 1px; + border-color: #c4c4d5 #fff #fff #c4c4d5 } + +a:link { + color: #324e95; + text-decoration: none; + background-color: transparent } + +a:visited { + color: #90c; + text-decoration: none } + +a:hover { + color: #c00; + text-decoration: underline; + background-color: transparent } diff --git a/docs/docbook/html/favicon.ico b/docs/docbook/html/favicon.ico new file mode 100644 index 00000000..f8cbb3b3 Binary files /dev/null and b/docs/docbook/html/favicon.ico differ diff --git a/docs/docbook/html/images/arrow.png b/docs/docbook/html/images/arrow.png new file mode 100644 index 00000000..4c60805d Binary files /dev/null and b/docs/docbook/html/images/arrow.png differ diff --git a/docs/docbook/html/images/bblogo.png b/docs/docbook/html/images/bblogo.png new file mode 100644 index 00000000..b33230b4 Binary files /dev/null and b/docs/docbook/html/images/bblogo.png differ diff --git a/docs/docbook/html/images/stepahead.png b/docs/docbook/html/images/stepahead.png new file mode 100644 index 00000000..9ac05b8c Binary files /dev/null and b/docs/docbook/html/images/stepahead.png differ diff --git a/docs/docbook/instguide.xml b/docs/docbook/instguide.xml new file mode 100644 index 00000000..addef297 --- /dev/null +++ b/docs/docbook/instguide.xml @@ -0,0 +1,766 @@ + + + + Box Backup Build and Installation Guide + + + License + + Copyright © 2003 - 2007, Ben Summers and contributors. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + + + Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + + + All use of this software and associated advertising materials + must display the following acknowledgement: This product includes + software developed by Ben Summers. + + + + The names of the Authors may not be used to endorse or promote + products derived from this software without specific prior written + permission. + + + + + [Where legally impermissible the Authors do not disclaim liability + for direct physical injury or death caused solely by defects in the + software unless it is modified by a third party.] + + THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS + OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + + + Introduction + + The backup daemon, bbackupd, runs on all machines to be backed up. + The store server daemon, bbstored runs on a central server. Data is sent + to the store server, which stores all data on local filesystems, that is, + only on local hard drives. Tape or other archive media is not used. + + The system is designed to be easy to set up and run, and cheap to + use. Once set up, there should be no need for user or administrative + intervention, apart from usual system maintenance. + +
+ Client daemon + + bbackupd is configured with a list of directories to back up. It + has a lazy approach to backing up data. Every so often, the directories + are scanned, and new data is uploaded to the server. This new data must + be over a set age before it is uploaded. This prevents rapid revisions + of a file resulting in many uploads of the same file in a short period + of time. + + It can also operate in a snapshot mode, which behaves like + traditional backup software. When instructed by an external bbackupctl + program, it will upload all changed files to the server. + + The daemon is always running, although sleeping most of the time. + In lazy mode, it is completely self contained -- scripts running under + cron jobs are not used. The objective is to keep files backed up, not to + make snapshots of the filesystem at particular points in time + available. + + If an old version of the file is present on the server, a modified + version of the rsync algorithm is used to upload only the changed + portions of the file. + + After a new version is uploaded, the old version is still + available (subject to disc space on the server). Similarly, a deleted + file is still available. The only limit to their availability is space + allocated to this account on the server + + Future versions will add the ability to mark the current state of + files on the server, and restore from this mark. This will emulate the + changing of tapes in a tape backup system. + +
+ Restoration + + Restoring files is performed using a query tool, bbackupquery. + This can be used to restore entire directories, or as an 'FTP-like' + tool to list and retrieve individual files. Old versions and deleted + files can be retrieved using this tool for as long as they are kept on + the server. +
+ +
+ Client Resource Usage + + bbackupd uses only a minimal amount of disc space to store + records on uploaded files -- less than 32 bytes per directory and file + over a set size threshold. However, it minimises the amount of queries + it must make to the server by storing, in memory, a data structure + which allows it to determine what data is new. It does not need to + store a record of all files, essentially just the directory names and + last modification times. This is not a huge amount of memory. + + If there are no changes to the directories, then the client will + not even connect to the server. +
+
+ +
+ Security + + Box Backup is designed to be secure in several ways. The data + stored on the backup store server is encrypted using secret-key + cryptography. Additionally, the transport layer is encrypted using TLS, + to ensure that the communications can't be snooped. + +
+ Encryption + + The files, directories, filenames and file attributes are all + encrypted. By examining the stored files on the server, it is only + possible to determine the approximate sizes of a files and the tree + structure of the disc (not names, just number of files and + subdirectories in a directory). By monitoring the actions performed by + a client, it is possible to determine the frequency and approximate + scope of changes to files and directories. + + The connections between the server and client are encrypted + using TLS (latest version of SSL). Traffic analysis is possible to + some degree, but limited in usefulness. + + An attacker will not be able to recover the backed up data + without the encryption keys. Of course, you won't be able to recover + your files without the keys either, so you must make a conventional, + secure, backup of these keys. +
+ +
+ Authentication + + SSL certificates are used to authenticate clients. UNIX user + accounts are not used to minimise the dependence on the configuration + of the operating system hosting the server. + + A script is provided to run the necessary certification + authority with minimal effort. +
+
+ +
+ Server daemon + + The server daemon is designed to be simple to deploy, and run on + the cheapest hardware possible. To avoid the necessity to use expensive + hardware RAID or software RAID with complex setup, it (optionally) + stores files using RAID techniques. + + It does not need to run as a privileged user. + + Each account has a set amount of disc space allocated, with a soft + and a hard limit. If the account exceeds the soft limit, a housekeeping + process will start deleting old versions and deleted files to reduce the + space used to below the soft limit. If the backup client attempts to + upload a file which causes the store to exceed the hard limit, the + upload will be refused. +
+
+ + + Building and installing + +
+ Before you start + + Firstly, check that all the clocks on your clients, servers and + signing machines are accurate and in sync. A disagreement in time + between a client and a server is the biggest cause of installation + difficulties, as the times in the generated certificates will cause + login failures if the start date is in the future. +
+ +
+ Box Backup compile + + In the following instructions, replace 0.00 with the actual + version number of the archive you have downloaded. + + For help building on Windows, see the Windows + Compile Appendix. And if you want to build a Linux RPM, look here. + + You need the latest version of OpenSSL, as some of the extra APIs + it provides are required. You should have this anyway, as earlier + versions have security flaws. (If you have an earlier version installed, + the configuration script will give you instructions on enabling + experimental support for older versions.) + + See OpenSSL notes for more information + on OpenSSL issues. + + There are some notes in the archive about compiling on various + platforms within the boxbackup-0.00 directory -- read them first. For + example, if you are compiling under Linux, look for LINUX.txt as + boxbackup-0.00/LINUX.txt after untaring the archive. + + Download the archive, then in that directory type + + tar xvzf boxbackup-0.00.tgz +cd boxbackup-0.00 +./configure +make + + The server and client will be built and packaged up for + installation on this machine, or ready to be transferred as tar files to + another machine for installation. + + This builds two parcels of binaries and scripts, 'backup-client' + and 'backup-server'. The generated installation scripts assumes you want + everything installed in /usr/local/bin + + Optionally, type make test to run + all the tests. +
+ +
+ Local installation + + Type make install-backup-client + to install the backup client. + + Type make install-backup-server + to install the backup server. +
+ +
+ Remote installation + + In the parcels directory, there are tar files for each parcel. The + name reflects the version and platform you have built it for. + + Transfer this tar file to the remote server, and unpack it, then + run the install script. For example: + + tar xvzf boxbackup-0.00-backup-server-OpenBSD.tgz +cd boxbackup-0.00-backup-server-OpenBSD +./install-backup-server +
+ +
+ Configure options + + You can use arguments to the configure script to adjust the + compile and link lines in the generated Makefiles, should this be + necessary for your platform. The configure script takes the usual GNU + autoconf arguments, a full list of which can be obtained with --help. Additional options for Box Backup + include: + + + + + + --enable-gnu-readline + + Use GNU readline if present. Linking Box Backup against + GNU readline may create licence implications if you then + distribute the binaries. libeditline is also supported as a safe + alternative, and is used by default if available. + + + + --disable-largefile + + Omit support for large files + + + + --with-bdb-lib=DIR + + Specify Berkeley DB library location + + + + --with-bdb-headers=DIR + + Specify Berkeley DB headers location + + + + --with-random=FILE + + Use FILE as random number seed (normally + auto-detected) + + + + --with-tmp-dir=DIR + + Directory for temporary files (normally /tmp) + + + + + + See OpenSSL notes for the OpenSSL + specific options. +
+ +
+ Tests + + There are a number of unit tests provided. To compile and run one + type: + + ./runtest.pl bbackupd release +./runtest.pl common debug +./runtest.pl ALL + + The runtest.pl script will compile and run the test. The first + argument is the test name, and the second the type of build. Use ALL as + a test name to run all the tests. + + The output from the tests is slightly muddled using this script. + If you're developing, porting or trying out new things, it might be + better to use the following scheme: + + cd test/bbackupd +make +cd ../../debug/test/bbackupd +./t + + or in release mode... + + cd test/bbackupd +make -D RELEASE +cd ../../release/test/bbackupd +./t + + (use RELEASE=1 with GNU make) + + I tend to use two windows, one for compilation, and one for + running tests. +
+
+ + + Box Backup and SSL + +
+ General notes + + Ideally, you need to use version 0.9.7 or later of OpenSSL. If + this is installed on your system by default (and it is on most recent + releases of UNIX like OSes) then everything should just work. + + However, if it isn't, you have a few options. + +
+ Upgrade your installation + + The best option is to upgrade your installation to use 0.9.7. + Hopefully your package manager will make this easy for you. This may + require reinstallation of lots of software which depends on OpenSSL, + so may not be ideal. + + (But as there have been a few security flaws in OpenSSL + recently, you probably want to upgrade it anyway.) +
+ +
+ Install another OpenSSL + + The second best option is to install another copy. If you + download and install from source, it will probably install into + /usr/local/ssl. You can then configure Box Backup to use it + using: + + ./configure --with-ssl-headers=/usr/local/ssl/include --with-ssl-lib=/usr/local/ssl/lib + + which will set up the various includes and libraries for + you. + + The configuration scripts may be a problem, depending on your + installation. See below for more information. +
+ +
+ Use the old version of OpenSSL + + If you have an old version installed, the configuration script + will give you instructions on how to enable support for older + versions. Read the warnings, and please, whatever you do, don't + release binary packages or ports which enable this option. + + You may have issues with the configuration scripts, see + below. +
+
+ +
+ If you have problems with the config scripts + + If you get OpenSSL related errors with the configuration scripts, + there are two things to check: + + + + The bin directory within your OpenSSL directory is in the path + (if you have installed another version) + + + + You have an openssl.cnf file which works and can be + found. + + + +
+ OpenSSL config file + + You need to have an openssl.cnf file. The default will generally + work well (see example at end). Make sure the openssl utility can find + it, either set the OPENSSL_CONF environment variable, or install it + into the location that is mentioned in the error messages. + + Example OpenSSL config file: + + # +# OpenSSL example configuration file. +# This is mostly being used for generation of certificate requests. +# + +RANDFILE = /dev/arandom + +#################################################################### +[ req ] +default_bits = 1024 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +attributes = req_attributes + +[ req_distinguished_name ] +countryName = Country Name (2 letter code) +#countryName_default = AU +countryName_min = 2 +countryName_max = 2 + +stateOrProvinceName = State or Province Name (full name) +#stateOrProvinceName_default = Some-State + +localityName = Locality Name (eg, city) + +0.organizationName = Organization Name (eg, company) +#0.organizationName_default = Internet Widgits Pty Ltd + +# we can do this but it is not needed normally :-) +#1.organizationName = Second Organization Name (eg, company) +#1.organizationName_default = CryptSoft Pty Ltd + +organizationalUnitName = Organizational Unit Name (eg, section) +#organizationalUnitName_default = + +commonName = Common Name (eg, fully qualified host name) +commonName_max = 64 + +emailAddress = Email Address +emailAddress_max = 64 + +[ req_attributes ] +challengePassword = A challenge password +challengePassword_min = 4 +challengePassword_max = 20 + +unstructuredName = An optional company name + +[ x509v3_extensions ] + +nsCaRevocationUrl = http://www.cryptsoft.com/ca-crl.pem +nsComment = "This is a comment" + +# under ASN.1, the 0 bit would be encoded as 80 +nsCertType = 0x40 +
+
+
+ + + Compiling bbackupd on Windows using Visual C++ + + This Appendix explains how to build the bbackupd daemon for Windows + using the Visual C++ compiler. + + If you have any problems following these instructions, please sign + up to the mailing + list and report them to us, or send an email to Chris Wilson. Thanks! + + Note: bbstored will not be built + with this process. bbstored is not currently supported on Windows. There + are no plans for bbstored support on Windows. + +
+ Tools + + You will need quite a bit of software to make this work. All of it + is available for free on the Internet, although Visual C++ Express has + license restrictions and a time limit. + +
+ Visual C++ + + Microsoft's Visual C++ compiler and development environment are + part of their commercial product Visual Studio. Visual + Studio 2005 is supported, and 2003 should work as well. + + You can also download + a free copy of Visual C++ 2005 Express. This copy must be registered + (activated) within 30 days, and is free for one year. + + You will need the Platform + SDK to allow you to compile Windows applications. +
+ +
+ Perl + + Download and install ActivePerl for + Windows, which you can probably find here. +
+ +
+ Libraries + + You will need to download and install several libraries. They + must all be built in the same directory, to be able to link + properly. + + Choose a directory where you will unpack and compile OpenSSL, + Zlib and Box Backup. We will call this the base directory. An example + might be: + + C:\Documents and Settings\Your Username\Desktop\Box + + Make sure you know the full path to this directory. + +
+ OpenSSL + + You will need to compile OpenSSL using Visual C++. The latest + release at this time, OpenSSL 0.9.8a, does not compile with Visual + C++ 2005 out of the box, so you need a + patched version. The original + source and patch + are also available. + + To compile OpenSSL: + + + + Use a Windows unzipper such as WinZip (free trial) to + extract the openssl-0.9.8a-vc2005.tar.gz archive, + which you just downloaded, into the base directory. + + + + Rename the folder from openssl-0.9.8a-vc2005 to openssl + + + + Open a command shell (run cmd.exe) and cd to the openssl directory + + + + Run the following commands: + + perl Configure VC-WIN32 +ms\do_ms +"c:\program files\Microsoft Visual Studio 8\Common7\Tools\vsvars32.bat" +nmake -f ms\ntdll.mak + + +
+ +
+ Zlib + + You will need to download the Zlib compiled DLL. + Create a directory called zlib in + the base directory, and unzip the file you just downloaded into that + directory. You don't need to compile anything. +
+
+ +
+ Download Box Backup + + The first version of Box Backup that's known to compile and with + Visual C++ 2005 is available on the Subversion + server. However, this version has not been extensively tested + and may be out of date. + + The changes are expected to be merged into the Subversion trunk + at some point, and this page should then be updated. If in doubt, + please sign up to the mailing + list and ask us. + + To get the source code out of Subversion you will need a Subversion + client for Windows. After installing it, open a new command + prompt, go to the base directory, and type: + + svn co http://www.boxbackup.org/svn/box/chris/win32/vc2005-compile-fixes/ boxbackup + + This should create a directory called boxbackup inside the base directory. +
+ +
+ Configure Box Backup + + Open a command prompt, change to the base directory then + boxbackup, and run win32.bat to configure the sources. Otherwise, + Visual C++ will complain about missing files whose names start with + autogen, and missing config.h. +
+ +
+ Compile Box Backup + + Open Visual C++. Choose "File/Open/Project", navigate to the + base directory, then to boxbackup\infrastructure\msvc\2005 (or + 2003 if using Visual Studio 2003), + and open any project or solution file in that directory. + + Press F7 to compile Box Backup. If the compilation is + successful, boxbackup\Debug\bbackupd.exe will be + created. +
+ +
+ Install Box Backup + + Create the destination directory, C:\Program Files\Box Backup\bbackupd. + + Write a configuration file, keys and certificate on a Unix + machine, and copy them into the Box + Backup directory, together with the following files from + the base directory: + + + + boxbackup\Debug\bbackupd.exe + + + + openssl\out32dll\libeay32.dll + + + + openssl\out32dll\ssleay32.dll + + + + zlib\zlib1.dll + + + + Ensure that the user running Box Backup can read from the + Box Backup directory, and write to + the bbackupd directory inside + it. + + Run Box Backup by double-clicking on it, and check that it + connects to the server. If the window opens and closes immediately, + it's probably due to a problem with the configuration file - check the + Windows Event Viewer for details. +
+ +
+ Windows Service + + Box Backup can also run as a Windows service, in which case it + will be automatically started at boot time in the background. To + install this, open a command prompt, and run: + + cd "C:\Program Files\Box Backup" +bbackupd.exe -i + + This should output Box Backup service installed. +
+
+
+ + + Compilation and installation by building an RPM on + Linux + + It is very easy to build an RPM of Box Backup on Linux platforms. + This should work on all Red Hat distributions (including Fedora), SuSE, + and probably others too. + + Given that you have the correct development packages installed + simply execute this command (replacing the version number): + + rpmbuild -ta boxbackup-0.00.tgz + + rpmbuild will report where the packages have been written to, and + these can be installed in the normal manner. + + If you have never built an RPM before you should set up a convenient + build area as described in the RPM + book. + + If you wish to customise the package you can find the spec file in + the contrib/rpm directory. + +
diff --git a/docs/docbook/raidfile-config.xml b/docs/docbook/raidfile-config.xml new file mode 100644 index 00000000..78a3e94c --- /dev/null +++ b/docs/docbook/raidfile-config.xml @@ -0,0 +1,198 @@ + + + + raidfile-config + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + raidfile-config + + Configure Box Backup's RAID files + + + + + raidfile-config + + config-dir + + blocksize + + dir1 dir2 dir3 + + + + + Description + + raidfile-config creates a raidfile.conf file for Box Backup. This + file holds information about the directories used to store backups in. Box + Backup supports userland RAID, in a restricted RAID5 configuration, where + 3 and only 3 'drives' are supported. You can read more about RAID5 (and + other RAID-levels) here. + + + Parameters + + The parameters are as follows: + + + + config-dir + + + The directory path where configuration files are located. + Usually this is /etc/box. + raidfile.conf will be written in this + directory. + + + + + blocksize + + + The block size used for file storage in the system, in + bytes. Using a multiple of the file system block size is a good + strategy. Depending on the size of the files you will be backing + up, this multiple varies. Of course it also depends on the native + block size of your file system. + + + + + dir1 + + + The first directory in the built-in RAID array. + + + + + dir2 + + + The second directory in the built-in RAID array. If you are + not using the built-in RAID functionality, this field should be + ignored. You should not use the built-in RAID if you have a + hardware RAID solution or if you're using another type of software + RAID (like md on Linux). + + + + + dir3 + + + The third directory in the built-in RAID array. The same + notes that apply to dir2 also apply to + dir3. + + + + + Note that there are currently no way to add multiple disk sets to + the raidfile.conf file using command line tools, etc. See + raidfile.conf + + 5 + for details on adding more disks. + + + + + Bugs + + If you find a bug in Box Backup, and you want to let us know about + it, join the mailing + list, and send a description of the problem there. + + To report a bug, give us at least the following information: + + + + The version of Box Backup you are running + + + + The platform you are running on (hardware and OS), for both + client and server. + + + + If possible attach your config files (bbstored.conf, + bbackupd.conf) to the bug report. + + + + Also attach any log file output that helps shed light on the + problem you are seeing. + + + + And last but certainly not least, a description of what you are + seeing, in as much detail as possible. + + + + + + Files + + raidfile-config generates the + raidfile.conf + + 5 + file. + + + + See Also + + + bbstored-config + + 8 + , + bbstored.conf + + 5 + , + raidfile.conf + + 5 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/raidfile.conf.xml b/docs/docbook/raidfile.conf.xml new file mode 100644 index 00000000..7a8b6410 --- /dev/null +++ b/docs/docbook/raidfile.conf.xml @@ -0,0 +1,143 @@ + + + + raidfile.conf + + 5 + + Box Backup + + Box Backup + + 0.11 + + + + raidfile.conf + + Userland RAID for Box Backup + + + + + /etc/box/raidfile.conf + + + + + Description + + The raidfile.conf is usually generated by + raidfile-config + + 8 + but may be manually edited if the store locations move + or if more than one disc set is required. + + + + discX + + + Specifies a set of discs. + + + + SetNumber + + + The set number of the RAID disc, referenced by each + account. + + + + + BlockSize + + + The block size of the file system (usually 2048). + Under BSD with FFS, set this to your file system's + fragment size (most likely an 8th of the block + size). + + + + + Dir0 + + + The first directory in the RAID array. + + + + + Dir1 + + + The second directory in the RAID array. If you do + not wish to use the built-in RAID functionality, this + field should be set to the same as + Dir0. You should not use the built-in + RAID if you have a hardware RAID solution or if you're + using another type of software RAID (like md on + Linux). + + + + + Dir2 + + + The third directory in the RAID array. The same + notes that apply to Dir2 also apply to + Dir3. + + + + + + + + + + Files + + /etc/box/raidfile.conf + + + + See Also + + + raidfile-config + + 8 + , + bbstored.conf + + 5 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/images/bblogo-alpha.xcf b/docs/images/bblogo-alpha.xcf new file mode 100644 index 00000000..18f494f9 Binary files /dev/null and b/docs/images/bblogo-alpha.xcf differ diff --git a/docs/raidfile/lib_raidfile.txt b/docs/raidfile/lib_raidfile.txt deleted file mode 100644 index 0c4244ce..00000000 --- a/docs/raidfile/lib_raidfile.txt +++ /dev/null @@ -1,73 +0,0 @@ -TITLE lib/raidfile - -Implements RAID type abilities in files in userland, along with simple transaction like semantics for writing and reading files. - -IOStream interfaces are used to read and write files. - -Eventually a raid file daemon will be written to "raidify" files in the background. This will allow fast streaming of data to a single disc, followed by splitting into separate raid files later. - -There is an option to disable raidification for use on systems with only one hard disc, or a hardware RAID array. - - -SUBTITLE Controller - -The raid disc sets are managed by RaidFileController. This reads the configuration file, /etc/box/raidfile.conf, and then stores the sets of discs for use by the other classes. - -In the code, files are referenced using an integer set number, and a filename within that set. For example, (0, "dir/subdir/filename.ext"). The RAID file controller turns this into actual physcial filenames transparently. - -The files are devided into blocks, which should match the fragment/block size of the underlying filesystem for maximum space efficiency. - -If the directories specified in the configuration files are all equal, then userland RAID is disabled. The files will not be processed into the redundant parts. - - -SUBTITLE Writing - -Files are written (and modified) using the RaidFileWrite class. This behaves as a seekable IOStream for writing. - -When all data has been written, the file is committed. This makes it available to be read. Processing to split it into the three files can be delayed. - -If the data is not commited, the temporary write file will be deleted. - -RaidFileWrite will optionally refuse to overwrite another file. If it is being overwritten, a read to that file will get the old file until it has been committed. - - -SUBTITLE Reading - -File are opened and read with RaidFileRead::Open. This returns an IOStream (with some extras) which reads a file. - -If the file has been turned into the RAID representation, then if an EIO error occurs on any operation, it will switch transparently into recovery mode where the file will be regenerated from the remaining two files using the parity file. (and log an error to syslog) - - -SUBTITLE Layout on disc - -The directory structure is replicated onto each of the three directories within the disc set. - -Given a filename x, within the disc set, one of the three discs is chosen as the 0th disc for this file. This is done by hashing the filename. This ensures that an equal load is placed on each disc -- otherwise one disc would only be accessed when a RAID file was being written. - -When the file is being written, the actual physical file is "x.rfwX". This just a striaght normal file -- nothing is done to the data. - -When it is commited, it is renamed to "x.rfw". - -Then, when it is turned into the raid representation, it is split across three discs, each file named "x.rf". - -When trying to open a file, the first attempt is for "x.rfw". If this exists, it is used as is. - -If this fails, two of the "x.rf" files are attempted to be opened. If this succeeds, the file is opened in RAID mode. - -This procedure guarenettes that a file will be opened either as a normal file, or as the correct three RAID file elements, even if a new version is being commited at the same time, or another process is working to turn it into a raid file. - - -SUBTITLE Raid file representation - -The file is split into three files, stripe 1, stripe 2 and parity. - -Considering the file as a set of blocks of the size specified in the configuration, odd numbered blocks go in stripe 1, even blocks in stripe 2, and a XOR of the corresponding blocks in parity. - -Thus only two files are needed to recreate the original. - -The complexity comes because the size of the file is not stored inside the files at any point. This means if stripe 2 is missing, it is ambiguous as to how big the file is. - -Size is stored as a 64 bit integer. This is appended to the parity file, unless it can be stored by XORing it into the last 8 bytes of the parity file. - -This scheme is complex to implement (see the RaidFileRead.cpp!) but minimises the disc spaced wasted. - diff --git a/docs/raidfile/lib_raidfile/RaidFileRead.txt b/docs/raidfile/lib_raidfile/RaidFileRead.txt deleted file mode 100644 index 0a5efbf7..00000000 --- a/docs/raidfile/lib_raidfile/RaidFileRead.txt +++ /dev/null @@ -1,14 +0,0 @@ -CLASS RaidFileRead - -Read a raid file. - -IOStream interface, plus a few extras, including reading directories and checking that files exist. - - -FUNCTION RaidFileRead::Open - -Open a given raid file -- returns a pointer to a new RaidFileRead object. - -Note that one of two types could be returned, depending on the representation of the file. - - diff --git a/docs/raidfile/lib_raidfile/RaidFileWrite.txt b/docs/raidfile/lib_raidfile/RaidFileWrite.txt deleted file mode 100644 index 89500c37..00000000 --- a/docs/raidfile/lib_raidfile/RaidFileWrite.txt +++ /dev/null @@ -1,36 +0,0 @@ -CLASS RaidFileWrite - -Interface to writing raidfiles. - -See IOStream interface. - -Some other useful functions are available, see h and cpp files. - - -FUNCTION RaidFileWrite::RaidFileWrite() - -The constructor takes the disc set number and filename of the file you're interested. - - -FUNCTION RaidFileWrite::Open() - -Open() opens the file for writing, and takes an "allow overwrite" flag. - - -FUNCTION RaidFileWrite::Commit() - -Commmit the file, and make it visible to RaidFileRead. If ConvertToRaidNow == true, it will be converted to raid file representation immediately. - -Setting it to false is not a good idea. Later on, it will tell a daemon to convert it in the background, but for now it simply won't be converted. - - -FUNCTION RaidFileWrite::Discard() - -Abort the creation/update. Equivalent to just deleting the object without calling Commit(). - - -FUNCTION RaidFileWrite::Delete() - -Delete a file -- don't need to Open() it first. - - diff --git a/docs/tools/generate_except_xml.pl b/docs/tools/generate_except_xml.pl new file mode 100644 index 00000000..38ee9074 --- /dev/null +++ b/docs/tools/generate_except_xml.pl @@ -0,0 +1,74 @@ +#!/usr/bin/perl -w +use strict; + +open (EXCEPT, "< $ARGV[0]") or die "Can't open $ARGV[0]: $!\n"; +open (DOCBOOK, "> $ARGV[1]") or die "Can't open $ARGV[1] for writing: $!\n"; + +print DOCBOOK < + + + Exception codes + +EOD +my $sectionName; +my $sectionNum; +my $sectionDesc; +my $exceptionCode; +my $exceptionShortDesc; +my $exceptionLongDesc; +while() +{ + next if(m/^#/); + chomp; + if(m/^EXCEPTION TYPE (\w+) (\d+)/) + { + $sectionName = ucfirst(lc($1)); + $sectionNum = $2; + if($sectionName ne "Common") + { + $sectionDesc = "the " . $sectionName; + } + else + { + $sectionDesc = "any"; + } + print DOCBOOK < + $sectionName Exceptions ($sectionNum) + + These are exceptions that can occur in $sectionDesc module + of the system. + + +EOD + } + + # The END TYPE line + if(m/^END TYPE$/) + { + print DOCBOOK " \n \n"; + } + + # The actual exceptions + if(m/(\(\d+\/\d+\)) - (\w+ \w+)(?: - )?(.*)$/) + { + $exceptionCode = $1; + $exceptionShortDesc = $2; + $exceptionLongDesc = $3; + + print DOCBOOK " \n "; + print DOCBOOK $exceptionCode . ": " . $exceptionShortDesc . ""; + if($exceptionLongDesc ne "") + { + print DOCBOOK " -- " . $exceptionLongDesc; + } + print DOCBOOK "\n \n"; + } +} + +print DOCBOOK "\n"; + +close EXCEPT; +close DOCBOOK; + diff --git a/documentation/Makefile b/documentation/Makefile deleted file mode 100644 index aefa3ee2..00000000 --- a/documentation/Makefile +++ /dev/null @@ -1,66 +0,0 @@ - - -# Process DocBook to HTML - -DBPROC=/usr/bin/xsltproc -BOOKXSL=bb-book.xsl -NOCHUNKBOOKXSL=bb-nochunk-book.xsl -MANXSL=bb-man.xsl -HTMLPREFIX=box-html -VPATH= adminguide -.SUFFIXES: .html .xml - -all: docs - -docs: instguide adminguide manpages - mkdir -p $(HTMLPREFIX)/images - cp html/images/*.png $(HTMLPREFIX)/images/. - cp html/*.css $(HTMLPREFIX)/. - -adminguide: $(HTMLPREFIX)/adminguide/index.html - -$(HTMLPREFIX)/adminguide/index.html: adminguide.xml ExceptionCodes.xml $(BOOKXSL) - # docname=`echo $@ | sed -e 's/\/index.html//'` - $(DBPROC) -o $(HTMLPREFIX)/adminguide/ $(BOOKXSL) adminguide.xml - -instguide: $(HTMLPREFIX)/instguide/index.html - -$(HTMLPREFIX)/instguide/index.html: instguide.xml $(BOOKXSL) - $(DBPROC) -o $(HTMLPREFIX)/instguide/ $(BOOKXSL) instguide.xml - -ExceptionCodes.xml: ../../ExceptionCodes.txt - perl ./generate_except_xml.pl - -manpages: man-dirs man-nroff man-html - -man-dirs: man-pages/.there $(HTMLPREFIX)/man-html/.there - -$(HTMLPREFIX)/man-html/.there: - if [ ! -d man-html ]; then mkdir -p $(HTMLPREFIX)/man-html; touch $(HTMLPREFIX)/man-html/.there; fi - -man-pages/.there: - if [ ! -d man-pages ]; then mkdir man-pages; touch man-pages/.there; fi - -man-nroff: bbackupquery.1 bbackupctl.1 bbstoreaccounts.1 bbstored-config.1 raidfile-config.1 bbstored-certs.1 - -man-html: bbackupquery.html bbackupctl.html bbstoreaccounts.html bbstored-config.html raidfile-config.html bbstored-certs.html - -%.html: %.xml - $(DBPROC) -o $@ $(NOCHUNKBOOKXSL) $< - mv $@ $(HTMLPREFIX)/man-html/. - -%.1: %.xml - $(DBPROC) -o $@ $(MANXSL) $< - mv $@ man-pages/. - gzip -f -9 man-pages/$@ - -dockit: clean docs - tar zcf documentation-kit-0.10.tar.gz $(HTMLPREFIX)/ - -clean: - if [ -d ./$(HTMLPREFIX) ]; then rm -rf $(HTMLPREFIX) ; fi - if [ -d ./man-pages ]; then rm -rf ./man-pages/; fi - if [ -f ExceptionCodes.xml ]; then rm ExceptionCodes.xml; fi - if [ -f documentation-kit-0.10.tar.gz ]; then rm documentation-kit-0.10.tar.gz; fi - - diff --git a/documentation/adminguide.xml b/documentation/adminguide.xml deleted file mode 100644 index d3552725..00000000 --- a/documentation/adminguide.xml +++ /dev/null @@ -1,1981 +0,0 @@ - - -]> - - Box Backup administrator's guide - - - License - - Copyright © 2003 - 2007, Ben Summers and contributors. All rights - reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - - - Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - - - All use of this software and associated advertising materials - must display the following acknowledgement: This product includes - software developed by Ben Summers and contributors. - - - - The names of the Authors may not be used to endorse or promote - products derived from this software without specific prior written - permission. - - - - [Where legally impermissible the Authors do not disclaim liability - for direct physical injury or death caused solely by defects in the - software unless it is modified by a third party.] - - THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - - - Configuration - -
- System configuration - -
- Server - - After you've downloaded and compiled the programs you need to - install the programs on your server. As root do the following: - - make install-backup-server - - This assumes that you are installing on the same server that you - compiled the software on. If not, copy the - boxbackup-x.xx-backup-server-OSNAME.tgz file to the server you want to - run on, and install there. For example (on Mac OS X): - - tar zxvf boxbackup-0.10-server-darwin8.5.0.tgz -cd boxbackup-0.10-server-darwin8.5.0 -./install-backup-server - - Then create the user for the backup daemon on the server: - - useradd _bbstored - - Box Backup has a built-in software RAID facility (redundant - array of inexpensive disks) for the backup store. This allows you to - spread the store data over three disks, and recover from the loss of - any one disk without losing data. However, this is now deprecated, and - you are recommended to use the software or hardware RAID facilities of - your operating system instead. Use the following command if you want - to create a simple server without Box Backup RAID: - - mkdir /tmp/boxbackupRepository # Create the directory -chown _bbstored /tmp/boxbackupRepository/ # Change the owner to the new boxbackup daemon user - -/usr/local/bin/raidfile-config /etc/box/ 1024 /tmp/boxbackupRepository - -#substitute 1024 with the desired blocksize -#substitute /tmp/boxbackupRepository with a directory that exists where you want the backup store located -#/usr/local/bin/raidfile-config --help shows you the options - - Then create the configuration file /etc/box/bbstored.conf The - hostname is tricky as it is used for two things: The name of the - server in the certificate and the address the server is listening on. - Since you might be using NAT, might move the server around or the - domain name might change, choose a name that describes the server. - When the network address of the server changes, you need to update the - ListenAddresses directive in the - /etc/box/bbstored.conf file. - - /usr/local/bin/bbstored-config /etc/box hostname _bbstored - - This last step outputs 5 instructions that you must execute to - the letter. A lot of questions are raised on the mailing list because - these steps have not been followed properly. - - TODO: Expand on this. Explain the 5 steps in detail. - - If you want to run the server as a non-root user, look here. -
- -
- Certificate Management - - There are two steps involved to create an account. You need to - create the account on the server, and sign a certificate to give the - client permission to connect to the server. - - Running a Certification Authority for TLS (SSL) connections is - not trivial. However, a script to does most of the work in a way which - should be good enough for most deployments. - - - The certificate authority directory is intended to be stored - on another server. It should not be kept on the backup server, in - order to limit the impact of a server compromise. The instructions - and the script assume that it will be kept elsewhere, so will ask - you to copy files to and from the CA. - - - - SSL certificates contain validity dates, including a "valid - from" time. If the clock on the machine which signs the certificates - is not syncronised to the clocks of the machines using these - certificates, you will probably get strange errors until the start - time is reached on all machines. If you get strange errors when - attempting to use new certificates, check the clocks on all machines - (client, store and CA). You will probably just need to wait a while - until the certificates become valid, rather than having to - regenerate them. - - -
- Set up a Certificate Authority - - It is recommended that you keep your Certificate Authority on - a separate machine than either the client or the server, preferably - without direct network access. The contents of this directory - control who can access your backup store server. - - To setup the basic key structure, do the following: - - /usr/local/bin/bbstored-certs ca init - - (See OpenSSL notes if you - get an OpenSSL error) - - This creates the directory called ca in - the current directory, and initialises it with basic keys. -
- -
- Sign a server certificate - - When you use the bbstored-config script to - set up a config file for a server, it will generate a certificate - request (CSR) for you. Transfer it to the machine with your CA, then - do: - - /usr/local/bin/bbstored-certs ca sign-server hostname-csr.pem - - This signs the certificate for the server. Follow the - instructions in the output on which files to install on the server. - The CSR file is now no longer needed. Make sure you run this command - from the directory above the directory 'ca'. - - TODO: Explain instructions in output. -
- -
- Set up an account - - Choose an account number for the user. This must be unique on - the server, and is presented as a 31 bit number in hex greater than - 0, for example, 1 or 75AB23C. Then on the backup store server, - create the account with: - - /usr/local/bin/bbstoreaccounts create 75AB23C 0 4096M 4505M - - This looks complicated. The numbers are, in order: - - - - The account number allocated (hex) - - - - The RAID disc set (0 if you use raidfile-config and don't - add a new set) - - - - Soft limit (size) - - - - Hard limit (size) - - - - The sizes are are specified in Mb, Gb, or blocks, depending on - the suffix. 1M specifies 1 Mb, 1G specifies 1 Gb, and 1B specifies 1 - block, the size of which depends on how you have configured the - raidfile system with raidfile-config. - - In this example, I have allocated 4Gb (assuming you use 2048 - byte blocks as per my example) as the soft limit, and 4Gb + 10% as - the hard limit. - - NOTE The sizes specified here are pre-RAID. So if you are - using userland RAID, you are actually allocating two-thirds of this - amount. This means that, when you take compression into account, - that if you allocate 2Gb on the server, it'll probably hold about - 2Gb of backed up files (depending on the compressability of those - files). - - The backup client will (voluntarily) try not to upload more - data than is allowed by the soft limit. The store server will refuse - to accept a file if it would take it over the hard limit, and when - doing housekeeping for this account, try and delete old versions and - deleted files to reduce the space taken to below the soft - limit. - - This command will create some files on disc in the raid file - directories (if you run as root, the utility will change to the user - specified in the bbstored.conf file to write them) and update the - accounts file. A server restart is not required. - - NOTE If you get a message saying 'Exception: RaidFile (2/8)', - the directories you specified in the raidfile.conf are not writable - by the _bbstored user -- fix it, and try again. - - Finally, tell the user their account number, and the hostname - of your server. They will use this to set up the backup client, and - send you a CSR. This has the account number embedded in it, and you - should be sure that it has the right account number in it. - - Sign this CSR with this command: - - /usr/local/bin/bbstored-certs ca sign 75AB23C-csr.pem - - Don't forget to check that the embedded account number is - correct! Then send the two files back to the user, as instructed by - the script. - - Please read the Troubleshooting page if you have - problems. - - TODO: Link to troubleshooting... -
-
- -
- Log Files - - You may wish to see what's going on with the server. Edit - /etc/syslog.conf, and add: - - local6.info /var/log/box -local5.info /var/log/raidfile - - Note: Separators must be tabs, - otherwise these entries will be ignored. - - touch /var/log/box -touch /var/log/raidfile - - Set up log rotation for these new log files. For example, if you - have /etc/newsyslog.conf, add the following lines - to it: - - /var/log/box 644 7 2000 * Z -/var/log/raidfile 644 7 2000 * Z - - If you have /etc/logrotate.d, create a new - file in there (for example - /etc/logrotate.d/boxbackup) containing the - following: - - /var/log/box /var/log/raidfile { - weekly - create - compress - rotate 52 -} - - Then restart syslogd, for example: - - /etc/init.d/syslogd restart -
- -
- Configuring a client - - Before you can do any configuration, you need to know the - hostname of the server you will be using, and your account number on - that server. - - Later in the process, you will need to send a certificate - request to the administrator of that server for it to be - signed. - - Installation is covered in the compiling and installing section. - You only need the backup-client parcel. - - It is important that you read all the output of the config - scripts. See the end of this page for an example. - - The backup client has to be run as root, because it needs to - read all your files to back them up, although it is possible to back - up a single user's files by running it as that user. (Tip: specify a - directory other than /etc/box, and then give the - alternate config file as the first argument to - bbackupd). However, it will fall over if you don't - give yourself read access to one of your files. - -
- Basic configuration - - Run the bbackupd-config script to generate - the configuration files and generate a private key and certificate - request. - - /usr/local/bin/bbackupd-config /etc/box lazy 999 hostname /var/bbackupd /home - - (See OpenSSL notes if you - get an OpenSSL error) - - The items in bold need to be changed. In order, they are the - account number, the hostname of the server you're using, and - finally, the directories you want backed up. You can include as many - you want here. - - However, the directories you specify must not contain other - mounted file systems within them at any depth. Specify them - separately, one per mount point. No checks are currently made to - catch bad configuration of this nature! - - You may also want to consider changing the mode from lazy to - snapshot, depending on what your system is used for: - - - - Lazy Mode - - - This mode regularly scans the files, with only a rough - schedule. It uploads files as and when they are changed, if - the latest version is more than a set age. This is good for - backing up user's documents stored on a server, and spreads - the load out over the day. - - - - - Snapshot Mode - - - This mode emulates the traditional backup behaviour of - taking a snapshot of the filesystem. The backup daemon does - absolutely nothing until it is instructed to make a backup - using the bbackupctl utility (probably as a cron job), at - which point it uploads all files which have been changed since - the last time it uploaded. - - - - - When you run the config script, it will tell you what you need - to do next. Don't forget to read all the output. An example is shown - at the end of this page, but the instructions for your installation - may be different. -
- -
- Certificates - - After you have sent your certificate request off to the server - administrator and received your certificate and CA root back, - install them where instructed by the bbackupd-config script during - basic bbackupd configuration. - - You can then run the daemon (as root) by running - /usr/local/bin/bbackupd, and of course, adding it - to your system's startup scripts. The first time it's run it will - upload everything. Interrupting it and restarting it will only - upload files which were not uploaded before - it's very - tolerant. - - If you run in snapshot mode, you will need to add a cron job - to schedule backups. The config script will tell you the exact - command to use for your system. - - Please read the Troubleshooting page if you have - problems. - - Remember to make a traditional backup of the keys file, as - instructed. You cannot restore files without it. - - It is recommended that you backup up all of /etc/box as it - will make things easier if you need to restore files. But only the - keys are absolutely essential. - - If you want to see what it's doing in more detail (probably a - good idea), follow the instructions in the server setup to create - new log files with syslog. -
- -
- Adding and removing backed up locations - - By editing the /etc/box/bbackupd.conf file, you can add and - remove directories to back up - see comments in this file for help. - Send bbackupd a HUP signal after you modify it. - - When you remove a location, it will not be marked as deleted - immediately. Instead, bbackupd waits about two days before doing so, - just in case you change your mind. After this, it will be eventually - removed from the store by the housekeeping process. Run as - root. - - The backup client is designed to be run as root. It is - possible to run without root, but this is not recommended. Clock - synchronisation for file servers. - - If you are using the backup client to backup a filesystem - served from a fileserver, you should ideally ensure that the - fileserver clocks are synchronised with the fileserver. - - bbackupd will cope perfectly well if the clocks are not - synchronised. Errors up to about half an hour cause no problems. - Larger discrepancies cause a loss of efficiency and the potential to - back up a file during a write process. - - There is a configuration parameter MaxFileTimeInFuture, which - specifies how far in the future a file must be for it to be uploaded - as soon as it is seen. You should not need to adjust this (default - is 2 days). Instead, get those clocks synchronised. Excluding files - and directories from the backup. - - Within the bbackupd.conf file, there is a section named - BackupLocations which specifies which locations on disc should be - backed up. It has subsections, each of which is in the - format: - - name - { - Path = /path/of/directory - (optional exclude directives) - } - - name is derived from the Path - by the config script, but should merely be unique. - - The exclude directives are of the form: - - [Exclude|AlwaysInclude][File|Dir][|sRegex] = regex or full pathname - - (The regex suffix is shown as 'sRegex' to make File or Dir - plural) - - For example: - - ExcludeDir = /home/guest-user - ExcludeFilesRegex = *.(mp3|MP3)\$ - AlwaysIncludeFile = /home/username/veryimportant.mp3 - - This excludes the directory /home/guest-user from the backup - along with all mp3 files, except one MP3 file in particular. - - In general, Exclude excludes a file or directory, unless the - directory is explicitly mentioned in a AlwaysInclude - directive. - - If a directive ends in Regex, then it is a regular expression - rather than a explicit full pathname. See - - man 7 re_format - - for the regex syntax on your platform. -
- -
- Example configuration output - - This is an example of output from the bbstored-config - script. - - - Follow the instructions output by your script, not the ones - here -- they may be different for your system. - - - /usr/local/bin/bbackupd-config /etc/box lazy 51 server.example.com /var/bbackupd /home /etc/samba - -Setup bbackupd config utility. - -Configuration: - Writing configuration file: /etc/box/bbackupd.conf - Account: 51 - Server hostname: server.example.com - Directories to back up: - /home - /etc/samba - -Note: If other file systems are mounted inside these directories, then problems may occur -with files on the store server being renamed incorrectly. This will cause efficiency -problems, but not affect the integrity of the backups. - -WARNING: Directories not checked against mountpoints. Check mounted filesystems manually. - -Creating /etc/box... -Creating /etc/box/bbackupd -Generating private key... - [OpenSSL output omitted] - -Generating keys for file backup -Writing notify script /etc/box/bbackupd/NotifyStoreFull.sh -Writing configuration file /etc/box/bbackupd.conf - -=================================================================== - -bbackupd basic configuration complete. - -What you need to do now... - -1) Make a backup of /etc/box/bbackupd/51-FileEncKeys.raw - This should be a secure offsite backup. - Without it, you cannot restore backups. Everything else can - be replaced. But this cannot. - KEEP IT IN A SAFE PLACE, OTHERWISE YOUR BACKUPS ARE USELESS. - -2) Send /etc/box/bbackupd/51-csr.pem - to the administrator of the backup server, and ask for it to - be signed. - -3) The administrator will send you two files. Install them as - /etc/box/bbackupd/51-cert.pem - /etc/box/bbackupd/serverCA.pem - after checking their authenticity. - -4) You may wish to read the configuration file - /etc/box/bbackupd.conf - and adjust as appropraite. - - There are some notes in it on excluding files you do not - wish to be backed up. - -5) Review the script - /etc/box/bbackupd/NotifyStoreFull.sh - and check that it will email the right person when the store - becomes full. This is important -- when the store is full, no - more files will be backed up. You want to know about this. - -6) Start the backup daemon with the command - /usr/local/bin/bbackupd - in /etc/rc.local, or your local equivalent. - Note that bbackupd must run as root. - -=================================================================== - - Remember to make a secure, offsite backup of your backup keys, - as described in Basic - configuration above. If you do not, and that key is lost, you - have no backups. -
-
- -
- Configuration Options - - Box Backup has many options in its configuration file. We will - try to list them all here. - - First of all, here is an example configuration file, for - reference: - - - Example Configuration File - - StoreHostname = localhost -AccountNumber = 0x2 - -KeysFile = /etc/box/2-FileEncKeys.raw -CertificateFile = /etc/box/2-cert.pem -PrivateKeyFile = /etc/box/2-key.pem -TrustedCAsFile = /etc/box/serverCA.pem -DataDirectory = /var/run/boxbackup -NotifyScript = /etc/box/NotifySysadmin.sh -CommandSocket = /var/run/box/bbackupd.sock - -UpdateStoreInterval = 86400 -MinimumFileAge = 3600 -MaxUploadWait = 7200 -FileTrackingSizeThreshold = 65536 -DiffingUploadSizeThreshold = 65536 -MaximumDiffingTime = 20 -ExtendedLogging = no -LogAllFileAccess = yes - -Server -{ - PidFile = /var/run/bbackupd.pid -} -BackupLocations -{ - etc - { - Path = /etc - } - home - { - Path = /home - ExcludeDir = /home/shared - ExcludeDir = /home/chris/.ccache - ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache - } -} - - - As you can see from the example above, the configuration file - has a number of subsections, enclosed in curly braces {}. Some options - appear outside of any subsection, and we will refer to these as root options. The available options in - each section are described below. - - Every option has the form name = value. Names are - not case-sensitive, but values are. Depending on the option, the value - may be: - - - - a path (to a file or directory); - - - - a number (usually in seconds or bytes); - - - - a boolean (the word Yes or No); - - - - a hostname (or IP address). - - - - Paths are specified in native format, i.e. a full Windows path - with drive letter on Windows clients, or a full Unix path on Unix - clients. - - - Example: - - StoreObjectInfoFile = - /var/state/boxbackup/bbackupd.dat - - StoreObjectInfoFile = C:\Program Files\Box - Backup\data\bbackupd.dat - The use of relative paths (which do not start with a - forward slash on Unix, or a drive specification on Windows) is - possible but not recommended, since they are interpreted relative to - the current working directory when bbackupd was started, which is - liable to change unexpectedly over time. - - Numbers which start with "0x" are interpreted as hexadecimal. - Numbers which do not start with "0x" are interpreted as - decimal. - -
- Root Options - - These options appear outside of any subsection. By convention - they are at the beginning of the configuration file. - - Some options are required, and some are optional. - - - - StoreHostname (required) - - - The Internet host name (DNS name) or IP address of the - server. This is only used to connect to the server. - - - - - AccountNumber (required) - - - The number of the client's account on the server. This - must be provided by the server operator, and must match the - account number in the client's certificate, otherwise the - client will not be able to log into the server. - - The account number may be specified in hexadecimal - (starting with 0x, as in the example above) or in decimal, but - since the server operator works in hexadecimal, that format is - highly recommended and is the default. - - - - - KeysFile (required) - - - The path to the file containing the encryption key used - for data encryption of client file data and filenames. This is - the most important file to keep safe, since without it your - backups cannot be decrypted and are useless. Likewise, if an - attacker gets access to this key and to your encrypted - backups, he can decrypt them and read all your data. - - Do not change the encryption key without deleting all - files from the account on the server first. None of your old - files on the store will be readable if you do so, and if you - change it back, none of the files uploaded with the new key - will be readable. - - - - - CertificateFile (required) - - - The path to the OpenSSL client certificate in PEM - format. This is supplied by the server operator in response to - the certificate request which you send to them. Together with - the PrivateKeyFile, this provides access to the store server - and the encrypted data stored there. - - It is not critical to protect this file or to back it up - safely, since it can be regenerated by creating a new - certificate request, and asking the server operator to sign - it. You may wish to back it up, together with the - PrivateKeyFile, to avoid this inconvenience if you lose all - your data and need quick access to your backups. - - If you do back them up, you should keep them in a - separate location to the KeysFile, since any person holding - the KeysFile and the PrivateKeyFile can gain access to your - encrypted data and decrypt it. - - - - - PrivateKeyFile (required) - - - The path to the OpenSSL private key in PEM format. This - is generated at the same time as the certificate request, but - there is no need to send it to the server operator, and you - should not do so, in case the communication is intercepted by - an attacker. Together with the CertificateFile, this provides - access to the store server and the encrypted data stored - there. - - See the notes under CertificateFile for information - about backing up this file. - - - - - TrustedCAsFile (required) - - - The path to the OpenSSL certificate of the Client - Certificate Authority (CCA), in PEM format. This is supplied - by the server operator along with your account details, or - along with your signed client certificate. This is used to - verify that the server which you are connecting to is - authorised by the person who signed your certificate. It - protects you against DNS and ARP poisoning and IP spoofing - attacks. - - - - - DataDirectory (required) - - - The path to a directory where bbackupd will keep local - state information. This consists of timestamp files which - identify the last backup start and end times, used by - bbackupquery to determine whether files - have changed, and optionally a database of inode numbers, - which are used to check for files being renamed. The database - is only saved if Box Backup is built with Berkeley Database - (BDB) support. - - - - - NotifyScript (optional) - - - The path to the script or command to run when the Box - Backup client detects an error during the backup process. This - is normally used to notify the client system administrator by - e-mail when a backup fails for any reason. - - The script or command is called with one of the - following additional arguments to identify the cause of the - problem: - - - - store-full - - - The backup store is full. No new files are being - uploaded. If some files are marked as deleted, they - should be removed in due course by the server's - housekeeping process. Otherwise, you need to remove some - files from your backup set, or ask the store operator - for more space. - - - - - read-error - - - One or more files which were supposed to be backed - up could not be read. This could be due to: - - running the server as a non-root user; - - - - backing up a mounted filesystem such as - NFS; - - - - access control lists being applied to some - files; - - - - SELinux being enabled; - - - - trying to back up open files under - Windows; - - - - strange directory permissions such as 0000 or - 0400. - - Check the client logs, e.g. - /var/log/bbackupd on Unix, or the Windows Event Viewer - in Control Panel > Administrative Tools, for more - information about which files are not being backed up - and why. - - - - - backup-error - - - There was a communications error with the server, - or an unexpected exception was encountered during a - backup run. Check the client logs, e.g. - /var/log/box on Unix, or the - Windows Event Viewer in Control Panel > - Administrative Tools, for more information about the - problem. - - You may wish to check your Internet access to the - server, check that the server is running, and ask your - server operator to check your account on the - server. - - - - - - - - CommandSocket (optional) - - - The path to the Unix socket which - bbackupd creates when running, and which - bbackupctl uses to communicate with it, for - example to force a sync or a configuration reload. If this - option is omitted, no socket will be created, and - bbackupctl will not function. - - Unix sockets appear within the filesystem on Unix, as a - special type of file, and must be created in a directory which - exists and to which bbackupd has write access, and bbackupctl - has read access. - - On Windows, the path is ignored, and a named - pipe is created instead. This does not currently - have any security attached, so it can be accessed by any user. - Unlike a Unix socket it can also be accessed remotely. Please - use this option with extreme caution on Windows, and only on - fully trusted networks. - - - - - AutomaticBackup (optional) - - - Enable or disable the client from connecting - automatically to the store every - UpdateStoreInterval seconds. When - enabled (set to Yes), the client is in - Lazy Mode. When disabled (set to - No), it is in Snapshot - Mode. This setting is optional, and the default - value is Yes. - - - - - UpdateStoreInterval (required) - - - The approximate time between successive connections to - the server, in seconds, when the client is in Lazy - Mode. The actual time is randomised slightly to - prevent "rush hour" traffic jams on the server, where many - clients try to connect at the same time. - - This value is ignored if the client is in - Snapshot Mode. However, it is still - required. It can be set to zero in this case. - - You will probably need to experiment with the value of - this option. A good value to start with is probably 86400 - seconds, which is one day. - - - - - MinimumFileAge (required) - - - The number of seconds since a file was last modified - before it will be backed up. The reason for this is to avoid - repeatedly backing up files which are repeatedly changing. A - good value is about 3600 seconds (one hour). If set to zero, - files which have changed will always be backed up on the next - backup run. - - The MaxUploadWait option - overrides this option in some circumstances. - - - - - MaxUploadWait (required) - - - The number of seconds since a file was last uploaded - before it will be uploaded again, even if it keeps changing. - The reason for this is to ensure that files which are - continuously modified are eventually uploaded anyway. This - should be no less than the value of - MinimumFileAge. A good value is about - 14400 seconds (4 hours). - - - - - MaxFileTimeInFuture (optional) - - - The maximum time that a file's timestamp can be in the - future, before it will be backed up anyway. Due to clock - synchronisation problems, it is inevitable that you will - occasionally see files timestamped in the future. Normally, - for files which are dated only slightly in the future, you - will want to wait until after the file's date before backing - it up. However, for files whose dates are very wrong (more - than a few hours) you will normally prefer to back them up - immediately. - - A good value is about 7200 seconds (2 hours) to cope - with potential problems when moving in and out of daylight - saving time, if applicable in your timezone. The default - value, if this setting is not provided, is 172800 seconds (2 - days). - - - - - FileTrackingSizeThreshold (required) - - - The minimum size of files which will be tracked by inode - number to detect renames. It is not worth detecting renames of - small files, since they are quick to upload again in full, and - keeping their inode numbers in memory increases the client's - memory usage and slows down searches. Larger files should be - tracked to avoid wasting space on the store and long - uploads. - - A good value is about 65536 bytes (64 kilobytes). - - - - - DiffingUploadSizeThreshold (required) - - - The minimum size of files which will be compared to the - old file on the server, and for which only changes will be - uploaded. It is not worth comparing small files, since they - are quick to upload again in full, and sending the entire file - reduces the risk of data loss if the store is accidentally - corrupted. Larger files should have only their differences - uploaded to avoid wasting space on the store and long - uploads. - - A good value is about 65536 bytes (64 kilobytes). - - - - - MaximumDiffingTime (optional) - - - The maximum time for which the client will attempt to - find differences between the current version and the old - version in the store, before giving up and uploading the - entire file again. Very large files (several gigabytes) may - take a very long time to scan for changes, but would also take - a very long time to upload again and use a lot of space on the - store, so it is normally worth omitting this value. - - Use this option only if, for some bizarre reason, you - prefer to upload really large files in full rather than spend - a long time scanning them for changes. - - - - - KeepAliveTime (optional) - - - The interval (in seconds) between sending Keep-Alive - messages to the server while performing long operations such - as finding differences in large files, or scanning large - directories. - - These messages ensure that the SSL connection is not - closed by the server, or an intervening firewall, due to lack - of activity. - - The server will normally wait up to 15 minutes (900 - seconds) before disconnecting the client, so the value should - be given and should be less than 900. Some firewalls may time - out inactive connections after 10 or 5 minutes. - - A good value is 300 seconds (5 minutes). You may need to - reduce this if you frequently see TLSReadFailed or - TLSWriteFailed errors on the client. - - - - - StoreObjectInfoFile (optional) - - - Enables the use of a state file, which stores the - client's internal state when the client is not running. This - is useful on clients machines which are frequently shut down, - for example desktop and laptop computers, because it removes - the need for the client to recontact the store and rescan all - directories on the first backup run, which may take some time. - This feature is somewhat experimental and not well tested. - - - This is option is disabled by default, in which case the - state is stored in memory only. The value is the path to the - state file. - - - - - ExtendedLogging (optional) - - - Enables the connection debugging mode of the client, - which writes all commands sent to or received from the server - to the system logs. This generates a lot - of output, so it should only be used when instructed, or when - you suspect a connection problem or client-server protocol - error (and you know how to interpret the output). - - This is a boolean value, which may be set to - Yes or No. The default is of - course No. - - - - - ExtendedLogFile (optional, new in 0.11) - - - Enables the same debugging output as - ExtendedLogging, but written to a file - instead of the system logs. This is useful if you need - extended logging, but do not have access to the system logs, - for example if you are not the administrator of the - computer. - - The value is the path to the file where these logs will - be written. If omitted, extended logs will not be written to a - file. This is entirely independent of the - ExtendedLogging option. It does not - make much sense to use both at the same time. - - - - - LogAllFileAccess (optional, new in 0.11) - - - Enables logging of all local file and directory access, - file uploads (full and differential), and excluded files. This - may be useful if the client is failing to upload a particular - file, or crashing while trying to upload it. The logs will be - sent to the system log or Windows Event Viewer. - - This generates a lot - of output, so it should only be used when instructed, or when - you suspect that bbackupd is skipping some files and want to - know why. Because it is verbose, the messages are hidden by - default even if the option is enabled. To see them, you must - run bbackupd with at least one -v option. - - This is a boolean value, which may be set to - Yes or No. The default is of - course No. - - - - - SyncAllowScript (optional) - - - The path to the script or command to run when the client - is about to start an automatic backup run, and wishes to know - whether it is safe to do so. This is useful for clients which - do not always have access to the server, for example laptops - and computers on dial-up Internet connections. - - The script should either output the word - now if the backup should proceed, or else a - number, in seconds, which indicates how long the client should - wait before trying to connect again. Any other output will - result in an error on the client, and the backup will not - run. - - This value is optional, and by default no such script is - used. - - - -
- -
- Server Section - - These options appear within the Server subsection, which is at - the root level. - - - - PidFile - - - This option enables the client to write its processs - identifier (PID) to the specified file after starting. The - file will be deleted when the client daemon exits for any - reason. This is disabled by default, but is recommended - whenever you run the client daemon as a daemon (in the - background), which is usually the case. This file can be used - by scripts to determine whether the daemon is still running, - and to send it messages to reload its configuration or to - terminate. - - - Example Server Section - - Server -{ - PidFile = /var/state/boxbackup/bbackupd.pid -} - - - - -
- -
- Backup Locations Section - - This section serves only as a container for all defined backup - locations. - - - Example Backup Locations Section - - BackupLocations -{ - etc - { - Path = /etc - } - home - { - Path = /home - ExcludeDir = /home/shared - ExcludeDir = /home/chris/.ccache - ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache - } -} - - - Each subsection is a backup location. The name of the - subsection is the name that will be used on the server. The root - directory of the account on the server contains one subdirectory per - location. The name should be simple, not containing any spaces or - special characters. - - If you do not define any locations, the client will not back - up any files! - - It is currently not recommended to back up the root directory - of the filesystem on Unix. Box Backup is designed to back up - important data and configuration files, not full systems. - Nevertheless, nothing prevents you from doing so if you - desire. - - On Windows, it is currently not possible to back up files - which are open (currently in use), such as open documents in - Microsoft Office, and system files such as the registry and the - paging file. You will get an error for each open file which the - client attempts to back up. Once the file has been closed, it will - be backed up normally. System files will always be open, and should - be excluded from your backups. -
-
-
-
- - - Administration - - This chapter deals with the dauily running and management of the Box - Backup system. It explains most day-to-day tasks. - -
- Regular Maintenance - - The steps involved in maintaining and keeping the backup sets - healthy are outlined in this section. - -
- Controlling a backup client - - The bbackupctl program sends control commands to the bbackupd - daemon. It must be run as the same user as the daemon, and there is no - exception for root. - - The command line syntax is: - - /usr/local/bin/bbackupctl [-q] [-c config-file] command - - The -q option reduces the amount of output the program emits, - and -c allows an alternative configuration file to be - specified. - - Valid commands are: - - - - terminate - - Stop the bbackupd daemon now (equivalent to kill) - - - - reload - - Reload the configuration file (equivalent to kill - -HUP) - - - - sync - - Connect to the server and synchronise files now - - - - bbackupctl communicates with - the server via a UNIX domain socket, specified in bbackupd.conf with - the CommandSocket directive. This does not need to be specified, and - bbackupd will run without the command - socket, but in this case bbackupctl will not be able to communicate - with the daemon. - - Some platforms cannot check the user id of the connecting - process, so this command socket becomes a denial of service security - risk. bbackupd will warn you when it - starts up if this is the case on your platform, and you should - consider removing the CommandSocket directive on these - platforms. -
- -
- Using bbackupctl to perform snapshots - - bbackupctl's main purpose is to - implement snapshot based backups, emulating the behaviour of - traditional backup software. - - Use bbackupd-config to write a configuration file in snapshot - mode, and then run the following command as a cron job. - - /usr/local/bin/bbackupctl -q sync - - This will cause the backup daemon to upload all changed files - immediately. bbackupctl will exit - almost immediately, and will not output anything unless there is an - error. -
- -
- Checking storage space used on the server - -
- From the client machine - - bbackupquery can tell you how much space is used on the server - for this account. Either use the usage command in interactive mode, - or type: - - /usr/local/bin/bbackupquery -q usage quit - - to show the space used as a single command. -
- -
- On the server - - bbstoreaccounts allows you to query the space used, and change - the limits. To display the space used on the server for an account, - use: - - /usr/local/bin/bbstoreaccounts info 75AB23C - - To adjust the soft and hard limits on an account, use: - - /usr/local/bin/bbstoreaccounts setlimit 75AB23C new-soft-limit new-hard-limit - - You do not need to restart the server. -
-
- -
- Verify and restore files - - Backups are no use unless you can restore them. The bbackupquery - utility does this and more. - - You don't provide any login information to it, as it just picks - up the data it needs from /etc/box/bbackupd.conf. You should run it as - root so it can find everything it needs. - - Full documentation can be found in the bbackupquery manual page. It follows - the model of a command line sftp client quite closely. - - TODO: Link to bbackupquery man-page here. - - On systems where GNU readline is available (by default) it uses - that for command line history and editing. Otherwise it falls back to - very basic UNIX text entry. - - TODO: Did the readline dependency change to editline? - -
- Using bbackupquery - - bbackupquery is the tool you use to verify, restore and - investigate your backup files with. When invoked, it simply logs - into the server using the certificates you have listed in - bbackupd.conf. - - After you run bbackupquery, you will see a prompt, allowing - you to execute commands. The list (or ls) command lets you view - files in the store. It works much like unix ls, but with different - options. An example: - - [pthomsen@host bbackupquery]$ bbackupquery -Box Backup Query Tool v0.10, (c) Ben Summers and contributors 2003-2006 -Using configuration file /etc/box/bbackupd.conf -Connecting to store... -Handshake with store... -Login to store... -Login complete. - -Type "help" for a list of commands. - -query > ls -00000002 -d---- mp3 -00000003 -d---- video -00000004 -d---- home-pthomsen -00000005 -d---- root -query > - - The ls commands shows the directories that are backed up. Now - we'll take a closer look at the home-pthomsen directory: - - query > cd home-pthomsen -query > ls -00002809 f----- sample.tiff -0000280a f----- s3.tiff -0000280b f----- s4.tiff -0000280d f----- s2.tiff -0000280e f----- foo.pdf -0000286c f----- core.28720 -0000339a -d---- .emacs.d -0000339d -d---- bbackup-contrib -00003437 f----- calnut.compare.txt -0000345d f----- DSCN1783.jpg -0000345e f----- DSCN1782.jpg -query > - - The ls command takes the following options; - - - - -r -- recursively list - all files - - - - -d -- list deleted - files/directories - - - - -o -- list old versions - of files/directories - - - - -I -- don't display - object ID - - - - -F -- don't display - flags - - - - -t -- show file - modification time (and attr mod time if has the object has - attributes, ~ separated) - - - - -s -- show file size in - blocks used on server (only very approximate indication of size - locally) - - - - The flags displayed from the ls command are as follows: - - - f = file - - d = directory - - X = deleted - - o = old version - - R = remove from server as soon as marked deleted or - old - - a = has attributes stored in directory record which - override attributes in backup file - -
- -
- Verify backups - - As with any backup system, you should frequently check that - your backups are working properly by comparing them. Box Backup - makes this very easy and completely automatic. All you have to do is - schedule the bbackupquery compare command to run - regularly, and check its output. You can run the command manually as - follows: - - /usr/local/bin/bbackupquery "compare -a" quit - - This command will report all the differences found between the - store and the files on disc. It will download everything, so may - take a while. You should expect to see some differences on a typical - compare, because files which have recently changed are unlikely to - have been uploaded yet. It will also tell you how many files have - been modified since the last backup run, since these will normally - have changed, and such failures are expected. - - You are strongly recommended to add this command as a - cron job, at least once a month, and to check the - output for anything suspicious, particularly a large number of - compare failures, failures on files that have not been modified, or - any error (anything except a compare mismatch) that occurs during - the compare operation. - - Consider keeping a record of these messages and comparing them - with a future verification. - - If you would like to do a "quick" check which just downloads - file checksums and compares against that, then run: - - /usr/local/bin/bbackupquery "compare -aq" quit - - However, this does not check that the file attributes are - correct, and since the checksums are generated on the client they - may not reflect the data on the server if there is a problem -- the - server cannot check the encrypted contents. View this as a quick - indication, rather than a definite check that your backup verifies - correctly. -
- -
- Restore backups - - You will need the keys file created when you configured the - server. Without it, you cannot restore the files; this is the - downside of encrypted backups. However, by keeping the small keys - file safe, you indirectly keep your entire backup safe. - - The first step is to recreate the configuration of the backup - client. It's probably best to have stored the /etc/box directory - with your keys. But if you're recreating it, all you really need is - to have got the login infomation correct (ie the certs and - keys). - - Don't run bbackupd yet! It will mark all your files as deleted - if you do, which is not hugely bad in terms of losing data, just a - major inconvenience. (This assumes that you are working from a blank - slate. If you want to restore some files to a different location, - it's fine to restore while bbackupd is running, just do it outside a - backed up directory to make sure it doesn't start uploading the - restored files.) - - Type: - - /usr/local/bin/bbackupquery - - to run it in interactive mode. - - Type: - - list - - to see a list of the locations stored on the server. - - For each location you want to restore, type: - - restore name-on-server local-dir-name - - The directory specified by local-dir-name must not exist yet. - If the restore is interrupted for any reason, repeat the above - steps, but add the -r flag to the - restore command to tell it to resume. -
- -
- Retrieving deleted and old files - - Box Backup makes old versions of files and files you have - deleted available, subject to there being enough disc space on the - server to hold them. - - This is how to retrieve them using bbackupquery. Future - versions will make this far more user-friendly. - - Firstly, run bbackupquery in interactive mode. It behaves in a - similar manner to a command line sftp client. - - /usr/local/bin/bbackupquery - - Then navigate to the directory containing the file you want, - using list, cd and pwd. - - query > cd home/profiles/USERNAME - - List the directory, using the "o" option to list the files - available without filtering out everything apart from the current - version. (if you want to see deleted files as well, use list - -odt) - - query > list -ot -00000078 f--o- 2004-01-21T20:17:48 NTUSER.DAT -00000079 f--o- 2004-01-21T20:17:48 ntuser.dat.LOG -0000007a f--o- 2004-01-21T17:55:12 ntuser.ini -0000007b f---- 2004-01-12T15:32:00 ntuser.pol -0000007c -d--- 1970-01-01T00:00:00 Templates -00000089 -d--- 1970-01-01T00:00:00 Start Menu -000000a0 -d--- 1970-01-01T00:00:00 SendTo -000000a6 -d--- 1970-01-01T00:00:00 Recent -00000151 -d--- 1970-01-01T00:00:00 PrintHood -00000152 -d--- 1970-01-01T00:00:00 NetHood -00000156 -d--- 1970-01-01T00:00:00 My Documents -0000018d -d--- 1970-01-01T00:00:00 Favorites -00000215 -d--- 1970-01-01T00:00:00 Desktop -00000219 -d--- 1970-01-01T00:00:00 Cookies -0000048b -d--- 1970-01-01T00:00:00 Application Data -000005da -d--- 1970-01-01T00:00:00 UserData -0000437e f--o- 2004-01-24T02:45:43 NTUSER.DAT -0000437f f--o- 2004-01-24T02:45:43 ntuser.dat.LOG -00004380 f--o- 2004-01-23T17:01:29 ntuser.ini -00004446 f--o- 2004-01-24T02:45:43 NTUSER.DAT -00004447 f--o- 2004-01-24T02:45:43 ntuser.dat.LOG -000045f4 f---- 2004-01-26T15:54:16 NTUSER.DAT -000045f5 f---- 2004-01-26T15:54:16 ntuser.dat.LOG -000045f6 f---- 2004-01-26T16:54:31 ntuser.ini - - (this is a listing from a server which is used as a Samba - server for a network of Windows clients.) You now need to fetch the - file using it's ID, rather than it's name. The ID is the hex number - in the first column. Fetch it like this: - - query > get -i 0000437e NTUSER.DAT -Object ID 0000437e fetched successfully. - - The object is now available on your local machine. You can use - lcd to move around, and sh ls to list directories on your local - machine. -
-
-
- -
- Fixing corruptions of store data - - This section gives help on what to do if your server has suffered - corruption, for example, after an unclean shutdown or other operating - system or hardware problem. - - In general, as updates to the store are made in an atomic manner, - the most likely result is wasted disc space. However, if really bad - things happen, or you believe that there is a lot of wasted space, then - these instructions will help to restore your data. - - You know you will need to do something if you get strange errors, - and bbackupd attempts to contact the server every 100 seconds or so. Or - if one of the discs in your RAID disc set has failed. - - After following these instructions, the end result will be that - bbackupquery will be able to see all the files which were stored on your - server, and retrieve them. Some of them may be in lost+found directories - in the root of the store (or in their original position if they have - been moved) but they will all be able to be retrieved. - - After you have retrieved the files you want, bbackupd will upload - new versions where necessary, and after about two days, mark any - lost+found directories as deleted. Finally, those directories will be - removed by the housekeeping process on the server. - - These instructions assume you're working on account 1234. Replace - this with the account number that you actually want to check (the one - that is experiencing errors). These steps will need to be repeated for - all affected accounts. - -
- Stop bbackupd - - First, make sure that bbackupd is not running on the client - machine for the account you are going to recover. Use - bbackupctl terminate to stop it. This step is not - strictly necessary, but is recommended. During any checks on the - account, bbackupd will be unable to log in, and after they are - complete, the account is marked as changed on the server so bbackupd - will perform a complete scan. -
- -
- Are you using RAID on the server? - - The raidfile recovery tools have not been written, and probably - will not be, since Box Backup RAID is deprecated. However, when two - out of three files are available, the server will successfully allow - access to your data, even if it complains a lot in the logs. The best - thing to do is to fix the accounts, if necessary, and retrieve any - files you need. Then move the old store directories aside (in case you - need them) and start afresh with new accounts, and let the clients - upload all their data again. -
- -
- Check and fix the account - - First, run the check utility, and see what errors it - reports. - - /usr/local/bin/bbstoreaccounts check 1234 - - This will take some time, and use a fair bit of memory (about 16 - bytes per file and directory). If the output looks plausible and - reports errors which need fixing, run it again but with the fix - flag: - - /usr/local/bin/bbstoreaccounts check 1234 fix - - This will fix any errors, and remove unrecoverable files. - Directories will be recreated if necessary. - - NOTE: The utility may adjust - the soft and hard limits on the account to make sure that housekeeping - will not remove anything -- check these afterwards. -
- -
- Grab any files you need with bbackupquery - - At this point, you will have a working store. Every file which - was on the server, and wasn't corrupt, will be available. - - On the client, use bbackupquery to log in and examine the store. - (type help at the prompt for instructions). Retrieve any files you - need, paying attention to any lost+found directories in the root - directory of the store. - - You can skip this step if you are sure that the client machine - is fine -- in this case, bbackupd will bring the store up to - date. -
- -
- Restart bbackupd - - Restart bbackupd on the client machine. The store account will - be brought up to date, and files in the wrong place will be marked for - eventual deletion. -
-
- -
- Troubleshooting - - If you are trying to fix a store after your disc has been - corrupted, see Fixing corruptions of - store data. - - Unfortunately, the error messages are not particularly helpful at - the moment. This page lists some of the common errors, and the most - likely causes of them. - - When an error occurs, you will see a message like 'Exception: - RaidFile/OSFileError (2/8)' either on the screen or in your log files. - (it is recommended you set up another log file as recommended in the - server setup instructions.) - - This error may not be particularly helpful, although some do have - extra information about probable causes. To get further information, - check the ExceptionCodes.txt file in the root of the distribution. This - file is generated by the ./configure script, so you will need to have - run that first. - - Some common causes of exceptions are listed below. - - Please email me with any other codes you get, and I will let you - know what they mean, and add notes here. - -
- RaidFile (2/8) - - This is found either when running bbstoreaccounts or in the - bbstored logs. - - Problem: The directories you - specified in the raidfile.conf are not writable by the _bbstored - user. - - Resolution: Change permissions - appropriately. -
- -
- Common (1/2) - - This usually occurs when the configuration files can't be - opened. - - Problem: You created your - configurations in non-standard locations, and the programs cannot find - them. - - Resolution: Explicitly specify - configuration file locations to daemons and programs. For - example - - /usr/local/bin/bbstored /some/other/dir/bbstored.config /usr/local/bin/bbackupquery -c /some/other/dir/bbackupd.config - - (daemons specify the name as the first argument, utility - programs with the -c option). - - Problem: bbstored can't find - the raidfile.conf file specified in bbstored.conf. - - Resolution: Edit bbstored.conf - to point to the correct location of this additional configuration - file. -
- -
- Server (3/16) - - The server can't listen for connections on the IP address - specified when you configured it. - - Problem: This probably means - you've specified the wrong hostname to bbstored-config -- maybe your - server is behind a NAT firewall? - - Resolution: Edit bbstored.conf - and correct the ListenAddresses line. You should replace the server - address with the IP address of your machine. -
- -
- Connection (7/x) - - These errors all relate to connections failing -- you may see - them during operation if there are network failures or other problems - between the client and server. The backup system will recover from - them automatically. - -
- Connection (7/30) - SSL problems - - Log snippet from client side: - - bbackupd[1904]: Opening connection to server xxxx.xxx... -bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:rsa routines:RSA_padding_check_PKCS1_type_1:block type is not 01 -bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:rsa routines:RSA_EAY_PUBLIC_DECRYPT:padding check failed -bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:asn1 encoding routines:ASN1_verify:EVP lib -bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed -bbackupd[1904]: TRACE: Exception thrown: ConnectionException(Conn_TLSHandshakeFailed) at SocketStreamTLS.cpp(237) -bbackupd[1904]: Exception caught (7/30), reset state and waiting to retry... - - And from the server: - - bbstored[19291]: Incoming connection from xx.xxx.xx.xxx port xxxxx (handling in child xxxxx) -bbstored[21588]: SSL err during Accept: error:xxxxxxxx:SSL routines:SSL3_READ_BYTES:tlsv1 alert decrypt error -bbstored[21588]: in server child, exception Connection TLSHandshakeFailed (7/30) -- terminating child - - Solution: Create a new CA on - the server side and re-generate the client certificate. Re-creating - the client certificate request is not necessary. -
-
- -
- Advanced troubleshooting - - If this really doesn't help, then using the DEBUG builds of the - system will give you much more information -- a more descriptive - exception message and the file and line number where the error - occurred. - - For example, if you are having problems with bbstoreaccounts, - build the debug version with: - - cd boxbackup-0.0 -cd bin/bbstoreaccounts -make - - Within the module directories, make defaults to building the - debug version. At the top level, it defaults to release. - - This will build an executable in debug/bin/bbstoreaccounts which - you can then use instead of the release version. It will give far more - useful error messages. - - When you get an error message, use the file and line number to - locate where the error occurs in the code. There will be comments - around that line to explain why the exception happened. - - If you are using a debug version of a daemon, these extended - messages are found in the log files. -
-
-
- - &__ExceptionCodes__elfjz3fu; - - - Running without root - - It is possible to run both the server and client without root - privileges. - -
- Server - - The server, by default, runs as a non-root user. However, it - expects to be run as root and changes user to a specified user as soon - as it can, simply for administrative convenience. The server uses a port - greater than 1024, so it doesn't need root to start. - - To run it entirely as a non-root user, edit the bbstored.conf - file, and remove the User directive from the Server section. Then simply - run the server as your desired user. -
- -
- Client - - The client requires root for normal operation, since it must be - able to access all files to back them up. However, it is possible to run - the client as a non-root user, with certain limitations. - - Follow the installation instructions, but install the executable - files manually to somewhere in your home directory. Then use - bbackupd-config to configure the daemon, but use a directory other than - /etc/box, probably somewhere in your home directory. - - All directories you specify to be backed up must be readable, and - all files must be owned by the user and readable to that user. - - Important: If any file or directory is not readable by this user, - the backup process will skip that file or directory. Keep an eye on the - logs for reports of this failure. - - Non-root operation of the backup client is recommended only for - testing, and should not be relied on in a production environment. -
-
-
diff --git a/documentation/bb-book.xsl b/documentation/bb-book.xsl deleted file mode 100644 index a4f05fdb..00000000 --- a/documentation/bb-book.xsl +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - diff --git a/documentation/bb-man.xsl b/documentation/bb-man.xsl deleted file mode 100644 index 24d99381..00000000 --- a/documentation/bb-man.xsl +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - diff --git a/documentation/bb-nochunk-book.xsl b/documentation/bb-nochunk-book.xsl deleted file mode 100644 index 86574122..00000000 --- a/documentation/bb-nochunk-book.xsl +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - diff --git a/documentation/bbackupctl.xml b/documentation/bbackupctl.xml deleted file mode 100644 index 09085be8..00000000 --- a/documentation/bbackupctl.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - bbackupctl - - 1 - - - - bbackupctl - - Control the bbackupd daemon - - - - - bbackupctl [-q] [-c config-file] command - - - - - Description - - bbackupctl lets the user control the bbackupd - daemon on a client machine. The main use is to force a sync with the store - server. This is especially important if bbackupd(1) is configured to do - snapshot backups. In that case bbackupctl is the only - way to effect a backup. - - Communication with the bbackupd daemon takes place over a local - socket. Some platforms (notably Windows) can't determine if the user - connecting on this socket has the correct credentials to execute the - commands, leaving a rather sizeable security hole open. To avoid this, - unset the CommandSocket parameter in bbackupd.conf(8). - That disables the command socket, so bbackupd is secure. This does, - however, render bbackupctl unusable. - - - Options - - - - -q -- quiet. Do not output status messages. - - - - -c config_file -- Use a different config file from the default - one. Can be a full or a relative path. - - - - - - Commands - - The following commands are available in bbackupctl: - - - - terminate - - This command stops the bbackupd server. This is the equivalent - of killing (kill -KILL) the bbackupd process. - - - - reload - - Causes the bbackupd daemon to re-read all its configuration - files. Equivalent to kill -HUP. - - - - sync - - Initiates a backup to the store of whatever needs to be backed - up. - - - - - - - Author - - Ben Summers and contributors. For help, please go to the Wiki, or subscribe to the Box - Backup mailing - list. - - - - See Also - - bbackupd.conf(8) - - bbackupd(1) - - - - Files - - bbackupctl uses the Box Backup client - configuration file, usually located in - /etc/box/bbackupd.conf. On Windows this file is - usually located in the installation directory, and is named - bbackupd.conf as well. - - - - Bugs - - If you find a bug in Box Backup, and you want to let us know about - it, join the mailing - list, and send a description of the problem there. - - To report a bug, give us at least the following information: - - - - The version of Box Backup you are running - - - - The platform you are running on (Hardware and OS), for both - client and server. - - - - If possible attach your config files (bbstored.conf, - bbackupd.conf) to the bug report. - - - - Also attach any log file output that helps shed light on the - problem you are seeing. - - - - And last but certainly not least, a description of what you are - seeing, in as much detail as possible. - - - - diff --git a/documentation/bbackupquery.xml b/documentation/bbackupquery.xml deleted file mode 100644 index 2902c189..00000000 --- a/documentation/bbackupquery.xml +++ /dev/null @@ -1,380 +0,0 @@ - - - - bbackupquery - - 1 - - - - bbackupquery - - Box Backup store query and retrieval - - - - - bbackupquery [-q] [-c configfile] [commands ...] - - - - - Description - - bbackupquery is the main way of interacting with - the backup store from a Box Backup client machine. It supports both - interactive and batch modes of operation. - - It can be used to reviewing the status of a client machine's backup - store, getting status from the store server. The main use is to retrieve - files and directories when needed. - - bbackupquery supports interactive and batch modes - of operation. Interactive mode allows for interaction with the server much - like an interactive FTP client. - - Batch mode is invoked by putting commands into the invocation of - bbackupquery. Example: - - bbackupquery "list home-dirs" quit - - Note that commands that contain spaces are enclosed in double - quotes. If the quit command is ommitted, after the - preceding commands are completed, bbackupquery will - enter interactive mode. - - Options - - -q: Quiet. Suppresses status output while - running. - - -c configfile: Use config file, instead of the default - bbackupd.conf file. Can be a relative or full path. - - - Commands - - The commands that can be used in bbackupquery - are listed below. - - - - help - - Displays the basic help message, which gives information about - the commands available in bbackupquery. Use the form help - command, to get help on a specific command. - - - - quit - - End the session with the store server, and quit - bbackupquery. - - - - cd [options] <directory-name> - - Change directory. Options: - - - - -d -- consider deleted directories for - traversal - - - - -o -- consider old versions of - directories for traversal. This option should never be useful in - a correctly formed store. - - - - - - lcd <local-directory-name> - - Change directory on the client machine. To list the contents - of the local directory, type sh ls (on unix-like - machines). TODO: Does sh dir work on - Windows? - - - - list [options] [directory-name] - - The list (or its synonym - ls) command lists the content of the current, or - specified, directory. The options are as follows: - - - - -r -- recursively list all files - - - - -d -- list deleted files and - directories - - - - -o -- list old versions of files and - directories - - - - -I -- don't display object IDs - - - - -F -- don't display flags - - - - -t -- show file modification time (and - attr mod time, if the object has attributes. - - - - -s -- show file size in blocks used on - server. Note that this is only a very approximate indication of - local file size. - - - - - - ls [options] [directory-name] - - Synonym for list. - - - - pwd - - Print current directory, always relative to the backup store - root. - - - - sh <shell command> - - Everything after the sh is passed to a shell and run. All - output from the command is displayed in the client. - - Example: to list the contents of the current directory on the - client machine type sh ls. - - - - compare -a - - compare -l <location-name> - - compare <store-dir-name> - <local-dir-name> - - Compare the current data in the store with the data on the - disc. Please note that all the data will be downloaded from the - store, so this can be a very lengthy process depending on the size - of the store, and the size of the part you are comparing. - - Options: - - - - -a -- compare all locations. - - - - -l -- compare one backup location as - specified in the configuration file. This compares one of the - top level store directories. - - - - -c -- set return code. The return code - is set to the following values, if quit is - the next command. So, if another command is run after the - compare, the return code will not refer to - the compare. This option is very useful for - automating compares. Return code values: - - - - 1 -- no differences were found - - - - 2 -- differences were found - - - - 3 -- an error occured - - - - - - - - get <object-filename> - [<local-filename>] - - get -i <object-id> - <local-filename> - - Gets a file from the store. Object is specified as the - filename within the current directory. Local filename is optional. - Ignores old and deleted files when searching the directory for the - file to retrieve. - - To get an old or deleted file, use the -i - option and select the object as a hex object ID (first column in - listing). The local filename must be specified. - - - - getobject <object-id> - <local-filename> - - Gets the object specified by the object id (in hex) and stores - the raw contents in the local file specified. Note: This is only useful for debugging as it - does not decode files from the stored format, which is encrypted and - compressed. - - - - restore [-d] <directory-name> - <local-directory-name> - - restore -r - - Restores a directory to the local disc. The local directory - specified must not exist (unless a previous restore is being - restarted). The root cannot be restored -- restore locations - individually. - - Options: - - - - -d -- restore a deleted - directory - - - - -r -- resume an interrupted - restore - - - - If a restore operation is interrupted for any reason, it can - be restarted using the -r switch. Restore - progress information is saved in a file at regular intervals during - the restore operation to allow restarts. - - - - usage - - Show space used on the server for this account. Display - fields: - - - - Used: Total amount of space used on the - server - - - - Old files: Space used by old - files - - - - Deleted files: Space used by deleted - files - - - - Directories: Space used by the - directory structure - - - - When Used exceeds the soft limit, the server will start to - remove old and deleted files until the usage drops below the soft - limit. After a while, you should expect to see the usage stay at - just below the soft limit. You only need more space if the space - used by old and deleted files is near zero. - - - - - - - Author - - Ben Summers and contributors. For help, please go to the Wiki, or subscribe to the Box - Backup mailing - list. - - - - See Also - - bbackupd.conf(8) - - - - Files - - bbackupquery uses the Box Backup client - configuration file, usually located in - /etc/box/bbackupd.conf. On Windows this file is - usually located in the installation directory, and is named - bbackupd.conf as well. - - - - Bugs - - If you find a bug in Box Backup, and you want to let us know about - it, join the mailing - list, and send a description of the problem there. - - To report a bug, give us at least the following information: - - - - The version of Box Backup you are running - - - - The platform you are running on (Hardware and OS), for both - client and server. - - - - If possible attach your config files (bbstored.conf, - bbackupd.conf) to the bug report. - - - - Also attach any log file output that helps shed light on the - problem you are seeing. - - - - And last but certainly not least, a description of what you are - seeing, in as much detail as possible. - - - - diff --git a/documentation/bbstoreaccounts.xml b/documentation/bbstoreaccounts.xml deleted file mode 100644 index 8ff3be0d..00000000 --- a/documentation/bbstoreaccounts.xml +++ /dev/null @@ -1,290 +0,0 @@ - - - - bbstoreaccounts - - 1 - - - - bbstoreaccounts - - View and change account information on the store - server - - - - - bbstoreaccounts [-c configfile] command account_id - [command-specific arguments] - - - - - Description - - bbstoreaccounts is the tool for managing accounts - on the store server. It can be used to view information related to - accounts, as well as create, change and delete accounts on the store - server. - - bbstoreaccounts alwas takes at least 2 - parameters: the command name and the account ID. Some commands require - additional parameters, and some commands have optional parameters. - - - Options - - -c <configfile> - - The configfile to use for connecting to the store. Default is - /etc/box/bbstored.conf. - - - - Commands - - The commands tells bbstoreaccounts what action to perform. - - - - check <account-id> [fix] - - The check command verifies the integrity of - the store account given, and optionally fixes any corruptions. - Note: It is recommended to run the - 'simple' check command (without fix) before using - the fix option, This gives an overview of the - extent of any problems, before attempting to fix them. - - - - create <account-id> <discset> - <softlimit> <hardlimit> - - Creates a new store account with the parameters given. The - parameters are as follows: - - - - account-id: the ID of the new account - to be created. A 32-bit hexadecimal number. Cannot already exist - on the server. - - - - discset: the disc set from - raidfile.conf(5) where the backups for this client will be - stored.. A number. Each RAID-file set has a number in - raidfile.conf. This number is what's used. - - - - softlimit: The soft limit is the amount - of storage that the server will guarantee to be available for - storage. - - - - hardlimit: The amount of storage that - the the server will allow, before rejecting uploads, and - starting to eliminate old and deleted files to get back down to - softlimit. - - - - - - delete <account-id> [yes] - - Deletes the account from the store server completely. Removes - all backups and deletes all references to the account in the config - files. - - delete will ask for confirmation from the - user, when called. Using the yes flag, eliminates - that need. This is useful when deleting accounts from within a - script or some other automated means. - - - - info <account-id> - - Display information about the given account. Example: - - [root]# bbstoreaccounts info 1 - Account ID: 00000001 - Last object ID: 58757 - Blocks used: 9864063 (38531.50Mb) - Blocks used by old files: 62058 (242.41Mb) -Blocks used by deleted files: 34025 (132.91Mb) - Blocks used by directories: 6679 (26.09Mb) - Block soft limit: 11796480 (46080.00Mb) - Block hard limit: 13107200 (51200.00Mb) - Client store marker: 1139559852000000 - - Explanation: - - - - Account ID: The account ID being displayed. - - - - Last Object ID: A counter that keeps track of the objects - that have been backed up. This number refers to the last file - that was written to the store. The ID is displayed as a decimal - number, and the object ID can be converted to a path name to a - file as follows: convert the number to hex (e.g.: 58757 => - 0xE585); The last backed up file will be (relative from the - client's store root): e5/o85.rfw. Longer - numbers infer more directories in the structure, so as an - example 3952697264 as the last object ID gives 0xEB995FB0, which - translates to a backup pathname of - eb/99/5f/ob0.rfw. - - - - Blocks used: The number of blocks used by the store. The - size in Mb depends on the number of blocks, as well as the block - size for the disc set given in - raidfile.conf(5). In this case the block size - is 4096. - - - - Blocks used by old files: The number of blocks occupied by - files that have newer versions in the store. This data is at - risk for being removed during housekeeping. - - - - Blocks used by deleted files: The number of blocks used by - files that have been deleted on the client. Thi s data is at - risk for being removed during housekeeping. - - - - Blocks used by directories: The number of blocks used by - directories in the store. - - - - Block soft limit: The soft limit in blocks. The soft limit - is the maximum guaranteed storage space available to the - account. When housekeeping starts, and the old and deleted files - are removed, they are removed in chronological order (oldest - first), until the data used is less than the soft limit. - - - - Block hard limit: The hard limit in blocks. The hard limit - is the most amount of storage the server will allow in an - account. Any data above this amount will be rejected. - Housekeeping will reduce the storage use, so more data can be - uploaded. - - - - Client store marker: TODO What exactly is this? - - - - - - setlimit <account-id> <softlimit> - <hardlimit> - - Changes the storage space allocation for the given account. No - server restart is needed. - - Parameters: - - - - account-id: the ID of the new account - to be created. A 32-bit hexadecimal number. Cannot already exist - on the server. - - - - softlimit: The soft limit is the amount - of storage that the server will guarantee to be available for - storage. - - - - hardlimit: The amount of storage that - the the server will allow, before rejecting uploads, and - starting to eliminate old and deleted files to get back down to - softlimit. - - - - - - - - - Author - - Ben Summers and contributors. For help, please go to the Wiki, or subscribe to the Box - Backup mailing - list. - - - - See Also - - bbstored.conf(5) - - raidfile.conf(5) - - - - Files - - bbstoreaccounts uses the Box Backup server - configuration file, usually located in - /etc/box/bbstored.conf. - - - - Bugs - - If you find a bug in Box Backup, and you want to let us know about - it, join the mailing - list, and send a description of the problem there. - - To report a bug, give us at least the following information: - - - - The version of Box Backup you are running - - - - The platform you are running on (Hardware and OS), for both - client and server. - - - - If possible attach your config files (bbstored.conf, - bbackupd.conf) to the bug report. - - - - Also attach any log file output that helps shed light on the - problem you are seeing. - - - - And last but certainly not least, a description of what you are - seeing, in as much detail as possible. - - - - diff --git a/documentation/bbstored-certs.xml b/documentation/bbstored-certs.xml deleted file mode 100644 index 05d3f852..00000000 --- a/documentation/bbstored-certs.xml +++ /dev/null @@ -1,125 +0,0 @@ - - - - bbstored-certs - - 1 - - - - bbstored-certs - - Manage certificates for the Box Backup system - - - - - bbstored-certs <certs-dir> <command> - [<arguments>] - - - - - Description - - bbstored-certs creates and signs certificates for - use in Box Backup. It allows the user to create and sign the server keys, - as well as signing client keys. - - All commands must be followed by the certs-dir, - which is the directory in which the certificates are stored. - - - Commands - - There are 3 commands: - - - - init: Create the - certs-dir, and generate the server keys for - bbstored. certs-dir cannot exist before running - the command. - - - - sign-server <servercsrfile>: Sign the - server certificate. The servercsrfile is the file - generated by the init command. - - - - sign <clientcsrfile>: Sign a client - certificate. The clientcsrfile is generated - during client setup. See bbackupd-config(1). Send - the signed certificate back to the client, and install according to - the instructions given by bbackupd-config. - - - - - - - Author - - Ben Summers and contributors. For help, please go to the Wiki, or subscribe to the Box - Backup mailing - list. - - - - See Also - - bbstored-config(1) - - bbstored.conf(5) - - bbstoreaccounts(1) - - - - Files - - raidfile-config generates the raidfile.conf(5) - file. - - - - Bugs - - If you find a bug in Box Backup, and you want to let us know about - it, join the mailing - list, and send a description of the problem there. - - To report a bug, give us at least the following information: - - - - The version of Box Backup you are running - - - - The platform you are running on (Hardware and OS), for both - client and server. - - - - If possible attach your config files (bbstored.conf, - bbackupd.conf) to the bug report. - - - - Also attach any log file output that helps shed light on the - problem you are seeing. - - - - And last but certainly not least, a description of what you are - seeing, in as much detail as possible. - - - - diff --git a/documentation/bbstored-config.xml b/documentation/bbstored-config.xml deleted file mode 100644 index b7658782..00000000 --- a/documentation/bbstored-config.xml +++ /dev/null @@ -1,140 +0,0 @@ - - - - bbstored-config - - 1 - - - - bbstored-config - - Create config files for bbstored - - - - - bbstored-config <configdir> <servername> - <username> - - - - - Description - - The bbstored-config script creates config files - and server certificates for a bbstored instance. It takes three - parameters: - - - - configdir: The directory where config files - will reside. A subdirectory bbstored will be created, where several - config files will reside. the bbstored.conf file - will be created in configdir. - - - - servername: The name of the server that is - being configured. Usually the fully qualified domain name of the - machine in question. - - - - username: The name of the user that should be - running the bbstored processes. Recommended name: - _bbstored. - - - - A valid raidfile.conf(5) must be found in configdir. Several steps - are taken during the run of bbstored-config: - - - - Configuration files are created. - - - - Server certificates are created. This requires interaction from - the operator. - - - - The raid volumes are checked, to ensure that the configuration - is consistent, and will work. - - - - Instructions for next steps to take are shown. These steps may - be different for different OS platforms, so pay close attention to - these instructions. - - - - - - Author - - Ben Summers and contributors. For help, please go to the Wiki, or subscribe to the Box - Backup mailing - list. - - - - See Also - - raidfile-config(1) - - bbstored.conf(5) - - raidfile.conf(5) - - - - Files - - bbstoreaccounts uses the Box Backup server - configuration file, usually located in - /etc/box/bbstored.conf. - - - - Bugs - - If you find a bug in Box Backup, and you want to let us know about - it, join the mailing - list, and send a description of the problem there. - - To report a bug, give us at least the following information: - - - - The version of Box Backup you are running - - - - The platform you are running on (Hardware and OS), for both - client and server. - - - - If possible attach your config files (bbstored.conf, - bbackupd.conf) to the bug report. - - - - Also attach any log file output that helps shed light on the - problem you are seeing. - - - - And last but certainly not least, a description of what you are - seeing, in as much detail as possible. - - - - diff --git a/documentation/generate_except_xml.pl b/documentation/generate_except_xml.pl deleted file mode 100644 index 9046d5cf..00000000 --- a/documentation/generate_except_xml.pl +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -open (EXCEPT, "<../../ExceptionCodes.txt") or die "Can't open ../../ExceptionCodes.txt: $!\n"; -open (DOCBOOK, ">ExceptionCodes.xml") or die "Can't open Exceptioncodes.xml for writing: $!\n"; - -print DOCBOOK < - - - Exception codes - -EOD -my $sectionName; -my $sectionNum; -my $sectionDesc; -my $exceptionCode; -my $exceptionShortDesc; -my $exceptionLongDesc; -while() -{ - next if(m/^#/); - chomp; - if(m/^EXCEPTION TYPE (\w+) (\d+)/) - { - $sectionName = ucfirst(lc($1)); - $sectionNum = $2; - if($sectionName ne "Common") - { - $sectionDesc = "the " . $sectionName; - } - else - { - $sectionDesc = "any"; - } - print DOCBOOK < - $sectionName Exceptions ($sectionNum) - - These are exceptions that can occur in $sectionDesc module - of the system. - - -EOD - } - - # The END TYPE line - if(m/^END TYPE$/) - { - print DOCBOOK " \n \n"; - } - - # The actual exceptions - if(m/(\(\d+\/\d+\)) - (\w+ \w+)(?: - )?(.*)$/) - { - $exceptionCode = $1; - $exceptionShortDesc = $2; - $exceptionLongDesc = $3; - - print DOCBOOK " \n "; - print DOCBOOK $exceptionCode . ": " . $exceptionShortDesc . ""; - if($exceptionLongDesc ne "") - { - print DOCBOOK " -- " . $exceptionLongDesc; - } - print DOCBOOK "\n \n"; - } -} - -print DOCBOOK "\n"; - -close EXCEPT; -close DOCBOOK; - \ No newline at end of file diff --git a/documentation/html/bbdoc-man.css b/documentation/html/bbdoc-man.css deleted file mode 100644 index 0345560e..00000000 --- a/documentation/html/bbdoc-man.css +++ /dev/null @@ -1,104 +0,0 @@ -body { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - font-size: .75em; - line-height: 180%; - text-align: left; - margin-top: 20px; - margin-right: 100px; - margin-left: 250px; - position: relative; - width: auto; } - -table { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - font-size: 10pt; - line-height: 100%; } - - -code { - font-size: 11pt; } - - - -div.navheader { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - line-height: 100%; } - -#header { - background-color: #e4e6ed; - text-align: left; - padding-top: 10px; - margin-right: -100px; - margin-left: -250px; - top: 20px; - border-top: 1px solid #c4c4d5; - border-bottom: 1px solid white } - -#logo { - position: relative; - margin-left: 200px } - - -#page { - font-size: .75em; - line-height: 180%; - text-align: left; - margin-top: 50px; - margin-right: 100px; - margin-left: 250px; - position: relative; - width: auto } - -#disc { } - -.informaltable td,tr {font-size: 1em; - line-height: 140%; - text-align: left; - background-color: #e4e6ed; - padding: 4px } - -tr,td {font-size: 1em; - line-height: 100%; - background-color: #edeef3; } - -pre, tt { font-size: 1.3em; - color: #088; - letter-spacing: 1px; - word-spacing: 2px} - -h1 { - color: #c00; - font-size: 16pt; - margin-bottom: 2em; - margin-left: -50px } - -h2 { - color: #324e95; - font-size: 12pt; - margin-top: 2em; - margin-left: -50px } - -h3 { - color: #324e95; - font-size: 10pt; - margin-top: 2em; - margin-left: -50px } - -dt { font-weight: bold } - -a:link { - color: #324e95; - text-decoration: none; - background-color: transparent } - -a:visited { - color: #90c; - text-decoration: none } - -a:hover { - color: #c00; - text-decoration: underline; - background-color: transparent } diff --git a/documentation/html/bbdoc.css b/documentation/html/bbdoc.css deleted file mode 100644 index d3b4a1c2..00000000 --- a/documentation/html/bbdoc.css +++ /dev/null @@ -1,112 +0,0 @@ -body { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - font-size: .75em; - line-height: 180%; - text-align: left; - margin-top: 20px; - margin-right: 100px; - margin-left: 250px; - position: relative; - width: auto; } - -table { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - font-size: 10pt; - line-height: 100%; } - - - - -div.navheader { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - line-height: 100%; } - -#header { - background-color: #e4e6ed; - text-align: left; - padding-top: 10px; - margin-right: -100px; - margin-left: -250px; - top: 20px; - border-top: 1px solid #c4c4d5; - border-bottom: 1px solid white } - -#logo { - position: relative; - margin-left: 200px } - - -#page { - font-size: .75em; - line-height: 180%; - text-align: left; - margin-top: 50px; - margin-right: 100px; - margin-left: 250px; - position: relative; - width: auto } - -#disc { } - -.informaltable td,tr {font-size: 1em; - line-height: 140%; - text-align: left; - background-color: #e4e6ed; - padding: 4px } - -tr,td {font-size: 1em; - line-height: 100%; - background-color: #edeef3; } - -pre, tt { font-size: 1.3em; - color: #088; - letter-spacing: 1px; - word-spacing: 2px} - -h1 { - color: #c00; - font-size: 16pt; - margin-bottom: 2em; - margin-left: -50px } - -h2 { - color: #324e95; - font-size: 12pt; - margin-top: 2em; - margin-left: -50px } - -h3 { - color: #324e95; - font-size: 10pt; - margin-top: 2em; - margin-left: -50px } - -dt { font-weight: bold } - -ul { - list-style-image: url(images/arrow.png) } - -ul li { - background-color: #e4e6ed; - margin: 1em 6em 1em -2em; - padding: 0.2em 0.5em 0.2em 1em; - border-style: solid; - border-width: 1px; - border-color: #c4c4d5 #fff #fff #c4c4d5 } - -a:link { - color: #324e95; - text-decoration: none; - background-color: transparent } - -a:visited { - color: #90c; - text-decoration: none } - -a:hover { - color: #c00; - text-decoration: underline; - background-color: transparent } diff --git a/documentation/html/images/arrow.png b/documentation/html/images/arrow.png deleted file mode 100644 index 4c60805d..00000000 Binary files a/documentation/html/images/arrow.png and /dev/null differ diff --git a/documentation/html/images/bblogo.png b/documentation/html/images/bblogo.png deleted file mode 100644 index b33230b4..00000000 Binary files a/documentation/html/images/bblogo.png and /dev/null differ diff --git a/documentation/html/images/stepahead.png b/documentation/html/images/stepahead.png deleted file mode 100644 index 9ac05b8c..00000000 Binary files a/documentation/html/images/stepahead.png and /dev/null differ diff --git a/documentation/instguide.xml b/documentation/instguide.xml deleted file mode 100644 index c857da0d..00000000 --- a/documentation/instguide.xml +++ /dev/null @@ -1,766 +0,0 @@ - - - - Box Backup Build and Installation Guide - - - License - - Copyright © 2003 - 2007, Ben Summers and contributors. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - - - Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - - - All use of this software and associated advertising materials - must display the following acknowledgement: This product includes - software developed by Ben Summers. - - - - The names of the Authors may not be used to endorse or promote - products derived from this software without specific prior written - permission. - - - - - [Where legally impermissible the Authors do not disclaim liability - for direct physical injury or death caused solely by defects in the - software unless it is modified by a third party.] - - THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS - OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR - ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - - - Introduction - - The backup daemon, bbackupd, runs on all machines to be backed up. - The store server daemon, bbstored runs on a central server. Data is sent - to the store server, which stores all data on local filesystems, that is, - only on local hard drives. Tape or other archive media is not used. - - The system is designed to be easy to set up and run, and cheap to - use. Once set up, there should be no need for user or administrative - intervention, apart from usual system maintenance. - -
- Client daemon - - bbackupd is configured with a list of directories to back up. It - has a lazy approach to backing up data. Every so often, the directories - are scanned, and new data is uploaded to the server. This new data must - be over a set age before it is uploaded. This prevents rapid revisions - of a file resulting in many uploads of the same file in a short period - of time. - - It can also operate in a snapshot mode, which behaves like - traditional backup software. When instructed by an external bbackupctl - program, it will upload all changed files to the server. - - The daemon is always running, although sleeping most of the time. - In lazy mode, it is completely self contained -- scripts running under - cron jobs are not used. The objective is to keep files backed up, not to - make snapshots of the filesystem at particular points in time - available. - - If an old version of the file is present on the server, a modified - version of the rsync algorithm is used to upload only the changed - portions of the file. - - After a new version is uploaded, the old version is still - available (subject to disc space on the server). Similarly, a deleted - file is still available. The only limit to their availability is space - allocated to this account on the server - - Future versions will add the ability to mark the current state of - files on the server, and restore from this mark. This will emulate the - changing of tapes in a tape backup system. - -
- Restoration - - Restoring files is performed using a query tool, bbackupquery. - This can be used to restore entire directories, or as an 'FTP-like' - tool to list and retrieve individual files. Old versions and deleted - files can be retrieved using this tool for as long as they are kept on - the server. -
- -
- Client Resource Usage - - bbackupd uses only a minimal amount of disc space to store - records on uploaded files -- less than 32 bytes per directory and file - over a set size threshold. However, it minimises the amount of queries - it must make to the server by storing, in memory, a data structure - which allows it to determine what data is new. It does not need to - store a record of all files, essentially just the directory names and - last modification times. This is not a huge amount of memory. - - If there are no changes to the directories, then the client will - not even connect to the server. -
-
- -
- Security - - Box Backup is designed to be secure in several ways. The data - stored on the backup store server is encrypted using secret-key - cryptography. Additionally, the transport layer is encrypted using TLS, - to ensure that the communications can't be snooped. - -
- Encryption - - The files, directories, filenames and file attributes are all - encrypted. By examining the stored files on the server, it is only - possible to determine the approximate sizes of a files and the tree - structure of the disc (not names, just number of files and - subdirectories in a directory). By monitoring the actions performed by - a client, it is possible to determine the frequency and approximate - scope of changes to files and directories. - - The connections between the server and client are encrypted - using TLS (latest version of SSL). Traffic analysis is possible to - some degree, but limited in usefulness. - - An attacker will not be able to recover the backed up data - without the encryption keys. Of course, you won't be able to recover - your files without the keys either, so you must make a conventional, - secure, backup of these keys. -
- -
- Authentication - - SSL certificates are used to authenticate clients. UNIX user - accounts are not used to minimise the dependence on the configuration - of the operating system hosting the server. - - A script is provided to run the necessary certification - authority with minimal effort. -
-
- -
- Server daemon - - The server daemon is designed to be simple to deploy, and run on - the cheapest hardware possible. To avoid the necessity to use expensive - hardware RAID or software RAID with complex setup, it (optionally) - stores files using RAID techniques. - - It does not need to run as a privileged user. - - Each account has a set amount of disc space allocated, with a soft - and a hard limit. If the account exceeds the soft limit, a housekeeping - process will start deleting old versions and deleted files to reduce the - space used to below the soft limit. If the backup client attempts to - upload a file which causes the store to exceed the hard limit, the - upload will be refused. -
-
- - - Building and installing - -
- Before you start - - Firstly, check that all the clocks on your clients, servers and - signing machines are accurate and in sync. A disagreement in time - between a client and a server is the biggest cause of installation - difficulties, as the times in the generated certificates will cause - login failures if the start date is in the future. -
- -
- Box Backup compile - - In the following instructions, replace 0.00 with the actual - version number of the archive you have downloaded. - - For help building on Windows, see the Windows - Compile Appendix. And if you want to build a Linux RPM, look here. - - You need the latest version of OpenSSL, as some of the extra APIs - it provides are required. You should have this anyway, as earlier - versions have security flaws. (If you have an earlier version installed, - the configuration script will give you instructions on enabling - experimental support for older versions.) - - See OpenSSL notes for more information - on OpenSSL issues. - - There are some notes in the archive about compiling on various - platforms within the boxbackup-0.00 directory -- read them first. For - example, if you are compiling under Linux, look for LINUX.txt as - boxbackup-0.00/LINUX.txt after untaring the archive. - - Download the archive, then in that directory type - - tar xvzf boxbackup-0.00.tgz -cd boxbackup-0.00 -./configure -make - - The server and client will be built and packaged up for - installation on this machine, or ready to be transferred as tar files to - another machine for installation. - - This builds two parcels of binaries and scripts, 'backup-client' - and 'backup-server'. The generated installation scripts assumes you want - everything installed in /usr/local/bin - - Optionally, type make test to run - all the tests. -
- -
- Local installation - - Type make install-backup-client - to install the backup client. - - Type make install-backup-server - to install the backup server. -
- -
- Remote installation - - In the parcels directory, there are tar files for each parcel. The - name reflects the version and platform you have built it for. - - Transfer this tar file to the remote server, and unpack it, then - run the install script. For example: - - tar xvzf boxbackup-0.00-backup-server-OpenBSD.tgz -cd boxbackup-0.00-backup-server-OpenBSD -./install-backup-server -
- -
- Configure options - - You can use arguments to the configure script to adjust the - compile and link lines in the generated Makefiles, should this be - necessary for your platform. The configure script takes the usual GNU - autoconf arguments, a full list of which can be obtained with --help. Additional options for Box Backup - include: - - - - - - --enable-gnu-readline - - Use GNU readline if present. Linking Box Backup against - GNU readline may create licence implications if you then - distribute the binaries. libeditline is also supported as a safe - alternative, and is used by default if available. - - - - --disable-largefile - - Omit support for large files - - - - --with-bdb-lib=DIR - - Specify Berkeley DB library location - - - - --with-bdb-headers=DIR - - Specify Berkeley DB headers location - - - - --with-random=FILE - - Use FILE as random number seed (normally - auto-detected) - - - - --with-tmp-dir=DIR - - Directory for temporary files (normally /tmp) - - - - - - See OpenSSL notes for the OpenSSL - specific options. -
- -
- Tests - - There are a number of unit tests provided. To compile and run one - type: - - ./runtest.pl bbackupd release -./runtest.pl common debug -./runtest.pl ALL - - The runtest.pl script will compile and run the test. The first - argument is the test name, and the second the type of build. Use ALL as - a test name to run all the tests. - - The output from the tests is slightly muddled using this script. - If you're developing, porting or trying out new things, it might be - better to use the following scheme: - - cd test/bbackupd -make -cd ../../debug/test/bbackupd -./t - - or in release mode... - - cd test/bbackupd -make -D RELEASE -cd ../../release/test/bbackupd -./t - - (use RELEASE=1 with GNU make) - - I tend to use two windows, one for compilation, and one for - running tests. -
-
- - - Box Backup and SSL - -
- General notes - - Ideally, you need to use version 0.9.7 or later of OpenSSL. If - this is installed on your system by default (and it is on most recent - releases of UNIX like OSes) then everything should just work. - - However, if it isn't, you have a few options. - -
- Upgrade your installation - - The best option is to upgrade your installation to use 0.9.7. - Hopefully your package manager will make this easy for you. This may - require reinstallation of lots of software which depends on OpenSSL, - so may not be ideal. - - (But as there have been a few security flaws in OpenSSL - recently, you probably want to upgrade it anyway.) -
- -
- Install another OpenSSL - - The second best option is to install another copy. If you - download and install from source, it will probably install into - /usr/local/ssl. You can then configure Box Backup to use it - using: - - ./configure --with-ssl-headers=/usr/local/ssl/include --with-ssl-lib=/usr/local/ssl/lib - - which will set up the various includes and libraries for - you. - - The configuration scripts may be a problem, depending on your - installation. See below for more information. -
- -
- Use the old version of OpenSSL - - If you have an old version installed, the configuration script - will give you instructions on how to enable support for older - versions. Read the warnings, and please, whatever you do, don't - release binary packages or ports which enable this option. - - You may have issues with the configuration scripts, see - below. -
-
- -
- If you have problems with the config scripts - - If you get OpenSSL related errors with the configuration scripts, - there are two things to check: - - - - The bin directory within your OpenSSL directory is in the path - (if you have installed another version) - - - - You have an openssl.cnf file which works and can be - found. - - - -
- OpenSSL config file - - You need to have an openssl.cnf file. The default will generally - work well (see example at end). Make sure the openssl utility can find - it, either set the OPENSSL_CONF environment variable, or install it - into the location that is mentioned in the error messages. - - Example OpenSSL config file: - - # -# OpenSSL example configuration file. -# This is mostly being used for generation of certificate requests. -# - -RANDFILE = /dev/arandom - -#################################################################### -[ req ] -default_bits = 1024 -default_keyfile = privkey.pem -distinguished_name = req_distinguished_name -attributes = req_attributes - -[ req_distinguished_name ] -countryName = Country Name (2 letter code) -#countryName_default = AU -countryName_min = 2 -countryName_max = 2 - -stateOrProvinceName = State or Province Name (full name) -#stateOrProvinceName_default = Some-State - -localityName = Locality Name (eg, city) - -0.organizationName = Organization Name (eg, company) -#0.organizationName_default = Internet Widgits Pty Ltd - -# we can do this but it is not needed normally :-) -#1.organizationName = Second Organization Name (eg, company) -#1.organizationName_default = CryptSoft Pty Ltd - -organizationalUnitName = Organizational Unit Name (eg, section) -#organizationalUnitName_default = - -commonName = Common Name (eg, fully qualified host name) -commonName_max = 64 - -emailAddress = Email Address -emailAddress_max = 64 - -[ req_attributes ] -challengePassword = A challenge password -challengePassword_min = 4 -challengePassword_max = 20 - -unstructuredName = An optional company name - -[ x509v3_extensions ] - -nsCaRevocationUrl = http://www.cryptsoft.com/ca-crl.pem -nsComment = "This is a comment" - -# under ASN.1, the 0 bit would be encoded as 80 -nsCertType = 0x40 -
-
-
- - - Compiling bbackupd on Windows using Visual C++ - - This Appendix explains how to build the bbackupd daemon for Windows - using the Visual C++ compiler. - - If you have any problems following these instructions, please sign - up to the mailing - list and report them to us, or send an email to Chris Wilson. Thanks! - - Note: bbstored will not be built - with this process. bbstored is not currently supported on Windows. There - are no plans for bbstored support on Windows. - -
- Tools - - You will need quite a bit of software to make this work. All of it - is available for free on the Internet, although Visual C++ Express has - license restrictions and a time limit. - -
- Visual C++ - - Microsoft's Visual C++ compiler and development environment are - part of their commercial product Visual Studio. Visual - Studio 2005 is supported, and 2003 should work as well. - - You can also download - a free copy of Visual C++ 2005 Express. This copy must be registered - (activated) within 30 days, and is free for one year. - - You will need the Platform - SDK to allow you to compile Windows applications. -
- -
- Perl - - Download and install ActivePerl for - Windows, which you can probably find here. -
- -
- Libraries - - You will need to download and install several libraries. They - must all be built in the same directory, to be able to link - properly. - - Choose a directory where you will unpack and compile OpenSSL, - Zlib and Box Backup. We will call this the base directory. An example - might be: - - C:\Documents and Settings\Your Username\Desktop\Box - - Make sure you know the full path to this directory. - -
- OpenSSL - - You will need to compile OpenSSL using Visual C++. The latest - release at this time, OpenSSL 0.9.8a, does not compile with Visual - C++ 2005 out of the box, so you need a - patched version. The original - source and patch - are also available. - - To compile OpenSSL: - - - - Use a Windows unzipper such as WinZip (free trial) to - extract the openssl-0.9.8a-vc2005.tar.gz archive, - which you just downloaded, into the base directory. - - - - Rename the folder from openssl-0.9.8a-vc2005 to openssl - - - - Open a command shell (run cmd.exe) and cd to the openssl directory - - - - Run the following commands: - - perl Configure VC-WIN32 -ms\do_ms -"c:\program files\Microsoft Visual Studio 8\Common7\Tools\vsvars32.bat" -nmake -f ms\ntdll.mak - - -
- -
- Zlib - - You will need to download the Zlib compiled DLL. - Create a directory called zlib in - the base directory, and unzip the file you just downloaded into that - directory. You don't need to compile anything. -
-
- -
- Download Box Backup - - The first version of Box Backup that's known to compile and with - Visual C++ 2005 is available on the Subversion - server. However, this version has not been extensively tested - and may be out of date. - - The changes are expected to be merged into the Subversion trunk - at some point, and this page should then be updated. If in doubt, - please sign up to the mailing - list and ask us. - - To get the source code out of Subversion you will need a Subversion - client for Windows. After installing it, open a new command - prompt, go to the base directory, and type: - - svn co http://bbdev.fluffy.co.uk/svn/box/chris/win32/vc2005-compile-fixes/ boxbackup - - This should create a directory called boxbackup inside the base directory. -
- -
- Configure Box Backup - - Open a command prompt, change to the base directory then - boxbackup, and run win32.bat to configure the sources. Otherwise, - Visual C++ will complain about missing files whose names start with - autogen, and missing config.h. -
- -
- Compile Box Backup - - Open Visual C++. Choose "File/Open/Project", navigate to the - base directory, then to boxbackup\infrastructure\msvc\2005 (or - 2003 if using Visual Studio 2003), - and open any project or solution file in that directory. - - Press F7 to compile Box Backup. If the compilation is - successful, boxbackup\Debug\bbackupd.exe will be - created. -
- -
- Install Box Backup - - Create the destination directory, C:\Program Files\Box Backup\bbackupd. - - Write a configuration file, keys and certificate on a Unix - machine, and copy them into the Box - Backup directory, together with the following files from - the base directory: - - - - boxbackup\Debug\bbackupd.exe - - - - openssl\out32dll\libeay32.dll - - - - openssl\out32dll\ssleay32.dll - - - - zlib\zlib1.dll - - - - Ensure that the user running Box Backup can read from the - Box Backup directory, and write to - the bbackupd directory inside - it. - - Run Box Backup by double-clicking on it, and check that it - connects to the server. If the window opens and closes immediately, - it's probably due to a problem with the configuration file - check the - Windows Event Viewer for details. -
- -
- Windows Service - - Box Backup can also run as a Windows service, in which case it - will be automatically started at boot time in the background. To - install this, open a command prompt, and run: - - cd "C:\Program Files\Box Backup" -bbackupd.exe -i - - This should output Box Backup service installed. -
-
-
- - - Compilation and installation by building an RPM on - Linux - - It is very easy to build an RPM of Box Backup on Linux platforms. - This should work on all Red Hat distributions (including Fedora), SuSE, - and probably others too. - - Given that you have the correct development packages installed - simply execute this command (replacing the version number): - - rpmbuild -ta boxbackup-0.00.tgz - - rpmbuild will report where the packages have been written to, and - these can be installed in the normal manner. - - If you have never built an RPM before you should set up a convenient - build area as described in the RPM - book. - - If you wish to customise the package you can find the spec file in - the contrib/rpm directory. - -
diff --git a/documentation/raidfile-config.xml b/documentation/raidfile-config.xml deleted file mode 100644 index c610ceab..00000000 --- a/documentation/raidfile-config.xml +++ /dev/null @@ -1,143 +0,0 @@ - - - - raidfile-config - - 1 - - - - raidfile-config - - Configure Box Backup's RAID files - - - - - raidfile-config <configdir> <blocksize> - <dir1> [<dir2> <dir3>] - - - - - Description - - raidfile-config creates a raidfile.conf file for Box Backup. This - file holds information about the directories used to store backups in. Box - Backup supports userland RAID, in a restricted RAID5 configuration, where - 3 and only 3 'drives' are supported. You can read more about RAID5 (and - other RAID-levels) here. - - - - Parameters - - The parameters are as follows: - - - - configdir: The directory path where - configuration files are located. Usually this is - /etc/box. raidfile.conf will - be written in this directory. - - - - blocksize: The block size used for file - storage in the system, in bytes. Using a multple of the file system - block size is a good strategy. Depending on the size of the files - you will be backing up, this multiple varies. Of course it also - depends on the native block size of your file system. - - - - dir1: The first directory in the built-in - RAID array. - - - - dir2: The second directory in the built-in - RAID array. If you are not using the built-in RAID functionality, - this field should be ignored. You should not use the built-in RAID, - when you have a hardware RAID solution, or if you're using another - type of software RAID (like md on Linux). - - - - dir3: The third directory in the built-in - RAID array. The same notes that apply to dir2 - also apply to dir3. - - - - Note that there are currently no way to add multiple disk sets to - the raidfile.conf file using command line tools, etc. See - raidfile.conf(5) for details on adding more disks - - - - - Author - - Ben Summers and contributors. For help, please go to the Wiki, or subscribe to the Box - Backup mailing - list. - - - - See Also - - bbstored-config(1) - - bbstored.conf(5) - - raidfile.conf(5) - - - - Files - - raidfile-config generates the raidfile.conf(5) - file. - - - - Bugs - - If you find a bug in Box Backup, and you want to let us know about - it, join the mailing - list, and send a description of the problem there. - - To report a bug, give us at least the following information: - - - - The version of Box Backup you are running - - - - The platform you are running on (Hardware and OS), for both - client and server. - - - - If possible attach your config files (bbstored.conf, - bbackupd.conf) to the bug report. - - - - Also attach any log file output that helps shed light on the - problem you are seeing. - - - - And last but certainly not least, a description of what you are - seeing, in as much detail as possible. - - - - diff --git a/infrastructure/BoxPlatform.pm.in b/infrastructure/BoxPlatform.pm.in index d2510627..8657134e 100644 --- a/infrastructure/BoxPlatform.pm.in +++ b/infrastructure/BoxPlatform.pm.in @@ -5,7 +5,6 @@ use Exporter; BEGIN { - # which OS are we building under? $target_os = '@target_os@'; $target_windows = 0; @@ -32,27 +31,23 @@ BEGIN $build_os ne "SunOS" && $build_os ne 'GNU/kFreeBSD'); # blank extra flags by default - $platform_compile_line_extra = '@CPPFLAGS@ @CXXFLAGS@ @CXXFLAGS_STRICT@'; - $platform_compile_line_extra =~ s/ -O2//; - $platform_link_line_extra = '@LDFLAGS@'; + $platform_compile_line_extra = ''; + $platform_link_line_extra = ''; $platform_lib_files = '@LIBS@'; $platform_exe_ext = '@EXEEXT@'; # get version - if (! -r "VERSION.txt" and -r "../../VERSION.txt") - { - open VERSION,"../../VERSION.txt" or die "../../VERSION.txt: $!"; - } - else - { - open VERSION,"VERSION.txt" or die "VERSION.txt: $!"; - } + my $version_file = "VERSION.txt"; + if (not -r $version_file) { $version_file = "../../$version_file" } + die "missing version file: $version_file" unless $version_file; + open VERSION, $version_file or die "$version_file: $!"; $product_version = ; chomp $product_version; $product_name = ; chomp $product_name; close VERSION; + if($product_version =~ /USE_SVN_VERSION/) { # for developers, use SVN version @@ -69,14 +64,23 @@ BEGIN } } close INFO; - $svnurl =~ m!box/(.+)$!; - my $svndir = $1; + + my $svndir; + if ($svnurl =~ m!/box/(.+)$!) + { + $svndir = $1; + } + elsif ($svnurl =~ m'/(boxi/.+)/boxi/boxbackup') + { + $svndir = $1; + } + $svndir =~ tr/0-9A-Za-z/_/c; $product_version =~ s/USE_SVN_VERSION/$svndir.'_'.$svnversion/e; } # where to put the files - $install_into_dir = '@bindir_expanded@'; + $install_into_dir = '@sbindir_expanded@'; # if it's Darwin, if($build_os eq 'Darwin') @@ -108,5 +112,21 @@ sub make_flag return $_[0].'=1'; } +sub parcel_root +{ + my $tos = $_[1] || $target_os; + return $product_name.'-'.$product_version.'-'.$_[0].'-'.$tos; +} + +sub parcel_dir +{ + 'parcels/'.parcel_root($_[0], $_[1]) +} + +sub parcel_target +{ + parcel_dir($_[0]).'.tgz' +} + 1; diff --git a/infrastructure/buildenv-testmain-template.cpp b/infrastructure/buildenv-testmain-template.cpp index 45ad0933..b646a27b 100644 --- a/infrastructure/buildenv-testmain-template.cpp +++ b/infrastructure/buildenv-testmain-template.cpp @@ -18,26 +18,21 @@ #include "Box.h" -#include "stdio.h" -#include -#include -#include -#include -#include -#include #include -#include +#include +#include +#include +#include +#include #ifdef HAVE_GETOPT_H #include #endif -#ifdef WIN32 - #include "emu.h" -#else - #include -#endif +#include +#include +#include #include #include "Logging.h" @@ -48,7 +43,7 @@ int test(int argc, const char *argv[]); -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define MODE_TEXT "release" #else #define MODE_TEXT "debug" @@ -57,7 +52,17 @@ int test(int argc, const char *argv[]); int failures = 0; int first_fail_line; std::string first_fail_file; -std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; + +#ifdef WIN32 + #define QUIET_PROCESS "-Q" +#else + #define QUIET_PROCESS "" +#endif + +std::string bbackupd_args = QUIET_PROCESS, + bbstored_args = QUIET_PROCESS, + bbackupquery_args, + test_args; int filedes_open_at_beginning = -1; @@ -159,7 +164,15 @@ int main(int argc, char * const * argv) // Start memory leak testing MEMLEAKFINDER_START + Logging::SetProgramName(BOX_MODULE); + #ifdef HAVE_GETOPT_H + #ifdef BOX_RELEASE_BUILD + int logLevel = Log::NOTICE; // need an int to do math with + #else + int logLevel = Log::INFO; // need an int to do math with + #endif + struct option longopts[] = { { "bbackupd-args", required_argument, NULL, 'c' }, @@ -170,7 +183,7 @@ int main(int argc, char * const * argv) int ch; - while ((ch = getopt_long(argc, argv, "c:d:s:t:TUV", longopts, NULL)) + while ((ch = getopt_long(argc, argv, "c:d:qs:t:vPTUV", longopts, NULL)) != -1) { switch(ch) @@ -202,9 +215,50 @@ int main(int argc, char * const * argv) } break; + #ifndef WIN32 + case 'P': + { + Console::SetShowPID(true); + } + break; + #endif + + case 'q': + { + if(logLevel == Log::NOTHING) + { + BOX_FATAL("Too many '-q': " + "Cannot reduce logging " + "level any more"); + return 2; + } + logLevel--; + } + break; + + case 'v': + { + if(logLevel == Log::EVERYTHING) + { + BOX_FATAL("Too many '-v': " + "Cannot increase logging " + "level any more"); + return 2; + } + logLevel++; + } + break; + + case 'V': + { + logLevel = Log::EVERYTHING; + } + break; + case 't': { - Console::SetTag(optarg); + Logging::SetProgramName(optarg); + Console::SetShowTag(true); } break; @@ -221,12 +275,6 @@ int main(int argc, char * const * argv) } break; - case 'V': - { - Logging::SetGlobalLevel(Log::EVERYTHING); - } - break; - case '?': { fprintf(stderr, "Unknown option: '%c'\n", @@ -243,6 +291,8 @@ int main(int argc, char * const * argv) } } + Logging::SetGlobalLevel((Log::Level)logLevel); + argc -= optind - 1; argv += optind - 1; #endif // HAVE_GETOPT_H @@ -276,6 +326,9 @@ int main(int argc, char * const * argv) Timers::Init(); int returncode = test(argc, (const char **)argv); Timers::Cleanup(); + + fflush(stdout); + fflush(stderr); // check for memory leaks, if enabled #ifdef BOX_MEMORY_LEAK_TESTING @@ -292,6 +345,10 @@ int main(int argc, char * const * argv) if(fulltestmode) { bool filesleftopen = checkfilesleftopen(); + + fflush(stdout); + fflush(stderr); + if(filesleftopen) { failures++; diff --git a/infrastructure/m4/ax_bswap64.m4 b/infrastructure/m4/ax_bswap64.m4 index 9bfc84fc..8110743c 100644 --- a/infrastructure/m4/ax_bswap64.m4 +++ b/infrastructure/m4/ax_bswap64.m4 @@ -13,7 +13,7 @@ AC_DEFUN([AX_BSWAP64], [ bswap64_function="" AC_CHECK_HEADERS([sys/endian.h asm/byteorder.h]) if test "x$ac_cv_header_sys_endian_h" = "xyes"; then - AC_CACHE_CHECK([for htobe64], [have_htobe64], + AC_CACHE_CHECK([for htobe64], [box_cv_have_htobe64], [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ $ac_includes_default #include @@ -21,15 +21,15 @@ AC_DEFUN([AX_BSWAP64], [ htobe64(0); return 1; ]])], - [have_htobe64=yes], [have_htobe64=no] + [box_cv_have_htobe64=yes], [box_cv_have_htobe64=no] )]) - if test "x$have_htobe64" = "xyes"; then + if test "x$box_cv_have_htobe64" = "xyes"; then bswap64_function=htobe64 fi fi if test "x$bswap64_function" = "x" && \ test "x$ac_cv_header_asm_byteorder_h" = "xyes"; then - AC_CACHE_CHECK([for __cpu_to_be64], [have___cpu_to_be64], + AC_CACHE_CHECK([for __cpu_to_be64], [box_cv_have___cpu_to_be64], [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ $ac_includes_default #include @@ -37,9 +37,9 @@ AC_DEFUN([AX_BSWAP64], [ __cpu_to_be64(0); return 1; ]])], - [have___cpu_to_be64=yes], [have___cpu_to_be64=no] + [box_cv_have___cpu_to_be64=yes], [box_cv_have___cpu_to_be64=no] )]) - if test "x$have___cpu_to_be64" = "xyes"; then + if test "x$box_cv_have___cpu_to_be64" = "xyes"; then bswap64_function=__cpu_to_be64 fi fi diff --git a/infrastructure/m4/ax_check_define_pragma.m4 b/infrastructure/m4/ax_check_define_pragma.m4 index 176faf3d..e3f7fe89 100644 --- a/infrastructure/m4/ax_check_define_pragma.m4 +++ b/infrastructure/m4/ax_check_define_pragma.m4 @@ -10,14 +10,14 @@ dnl @version 2005/07/03 dnl @license AllPermissive AC_DEFUN([AX_CHECK_DEFINE_PRAGMA], [ - AC_CACHE_CHECK([for pre-processor pragma defines], [have_define_pragma], + AC_CACHE_CHECK([for pre-processor pragma defines], [box_cv_have_define_pragma], [AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ #define TEST_DEFINE #pragma pack(1) TEST_DEFINE ]])], - [have_define_pragma=yes], [have_define_pragma=no] + [box_cv_have_define_pragma=yes], [box_cv_have_define_pragma=no] )]) - if test "x$have_define_pragma" = "xyes"; then + if test "x$box_cv_have_define_pragma" = "xyes"; then AC_DEFINE([HAVE_DEFINE_PRAGMA], 1, [Define to 1 if #define of pragmas works]) m4_ifvaln([$1],[$1],[:])dnl m4_ifvaln([$2],[else $2])dnl diff --git a/infrastructure/m4/ax_check_dirent_d_type.m4 b/infrastructure/m4/ax_check_dirent_d_type.m4 index 1c0e2ec2..9c08d391 100644 --- a/infrastructure/m4/ax_check_dirent_d_type.m4 +++ b/infrastructure/m4/ax_check_dirent_d_type.m4 @@ -15,8 +15,8 @@ dnl @license AllPermissive AC_DEFUN([AX_CHECK_DIRENT_D_TYPE], [ AC_CHECK_MEMBERS([struct dirent.d_type],,, [[#include ]]) if test "x$ac_cv_member_struct_dirent_d_type" = "xyes"; then - AC_CACHE_CHECK([[whether struct dirent.d_type is valid]], [have_valid_dirent_d_type], - [AC_RUN_IFELSE( + AC_CACHE_CHECK([[whether struct dirent.d_type is valid]], [box_cv_have_valid_dirent_d_type], + [AC_TRY_RUN( [AC_LANG_PROGRAM([[ $ac_includes_default #include @@ -26,14 +26,16 @@ AC_DEFUN([AX_CHECK_DIRENT_D_TYPE], [ if(dir) res = readdir(dir); return res ? (res->d_type != DT_FILE && res->d_type != DT_DIR) : 1; ]])], - [have_valid_dirent_d_type=yes], [have_valid_dirent_d_type=no] + [box_cv_have_valid_dirent_d_type=yes], + [box_cv_have_valid_dirent_d_type=no], + [box_cv_have_valid_dirent_d_type=cross] )]) - if test "x$have_valid_dirent_d_type" = "xyes"; then + if test "x$box_cv_have_valid_dirent_d_type" = "xyes"; then AC_DEFINE([HAVE_VALID_DIRENT_D_TYPE], 1, [Define to 1 if struct dirent.d_type is valid]) fi fi if test "x$ac_cv_member_struct_dirent_d_type" = "xyes" || \ - test "x$have_valid_dirent_d_type" = "xyes" + test "x$box_cv_have_valid_dirent_d_type" = "xyes" then m4_ifvaln([$1],[$1],[:])dnl m4_ifvaln([$2],[else $2])dnl diff --git a/infrastructure/m4/ax_check_malloc_workaround.m4 b/infrastructure/m4/ax_check_malloc_workaround.m4 index 9b1c9848..7655c0f7 100644 --- a/infrastructure/m4/ax_check_malloc_workaround.m4 +++ b/infrastructure/m4/ax_check_malloc_workaround.m4 @@ -10,16 +10,16 @@ dnl @license AllPermissive AC_DEFUN([AX_CHECK_MALLOC_WORKAROUND], [ if test "x$GXX" = "xyes"; then - AC_CACHE_CHECK([for gcc version 3 or later], [gcc_3_plus], + AC_CACHE_CHECK([for gcc version 3 or later], [box_cv_gcc_3_plus], [AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ #if __GNUC__ < 3 #error "Old GNU C" #endif ]])], - [gcc_3_plus=yes], [gcc_3_plus=no] + [box_cv_gcc_3_plus=yes], [box_cv_gcc_3_plus=no] )]) - if test "x$gcc_3_plus" = "xno"; then - AC_CACHE_CHECK([for malloc workaround], [malloc_workaround], + if test "x$box_cv_gcc_3_plus" = "xno"; then + AC_CACHE_CHECK([for malloc workaround], [box_cv_malloc_workaround], [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #define __USE_MALLOC #include @@ -27,9 +27,9 @@ AC_DEFUN([AX_CHECK_MALLOC_WORKAROUND], [ std::string s; s = "test"; ]])], - [malloc_workaround=yes], [malloc_workaround=no] + [box_cv_malloc_workaround=yes], [box_cv_malloc_workaround=no] )]) - if test "x$malloc_workaround" = "xyes"; then + if test "x$box_cv_malloc_workaround" = "xyes"; then AC_DEFINE([__USE_MALLOC], 1, [Define to 1 if __USE_MALLOC is required work around STL memory leaks]) fi diff --git a/infrastructure/m4/ax_check_nonaligned_access.m4 b/infrastructure/m4/ax_check_nonaligned_access.m4 index 8a6cd0c6..ab2d0b7e 100644 --- a/infrastructure/m4/ax_check_nonaligned_access.m4 +++ b/infrastructure/m4/ax_check_nonaligned_access.m4 @@ -12,7 +12,7 @@ dnl @version 2005/07/12 dnl @license AllPermissive AC_DEFUN([AX_CHECK_NONALIGNED_ACCESS], [ - AC_CACHE_CHECK([if non-aligned 16 bit word accesses fail], [have_aligned_only_int16], + AC_CACHE_CHECK([if non-aligned 16 bit word accesses fail], [box_cv_have_aligned_only_int16], [AC_RUN_IFELSE([AC_LANG_PROGRAM([[$ac_includes_default]], [[ #ifndef HAVE_UINT16_T #define uint16_t u_int16_t; @@ -21,12 +21,12 @@ AC_DEFUN([AX_CHECK_NONALIGNED_ACCESS], [ memset(scratch, 0, sizeof(scratch)); return *(uint16_t*)((char*)scratch+1); ]])], - [have_aligned_only_int16=no], [have_aligned_only_int16=yes] + [box_cv_have_aligned_only_int16=no], [box_cv_have_aligned_only_int16=yes] )]) - if test "x$have_aligned_only_int16" = "xyes"; then + if test "x$box_cv_have_aligned_only_int16" = "xyes"; then AC_DEFINE([HAVE_ALIGNED_ONLY_INT16], 1, [Define to 1 if non-aligned int16 access will fail]) fi - AC_CACHE_CHECK([if non-aligned 32 bit word accesses fail], [have_aligned_only_int32], + AC_CACHE_CHECK([if non-aligned 32 bit word accesses fail], [box_cv_have_aligned_only_int32], [AC_RUN_IFELSE([AC_LANG_PROGRAM([[$ac_includes_default]], [[ #ifndef HAVE_UINT32_T #define uint32_t u_int32_t; @@ -35,12 +35,12 @@ AC_DEFUN([AX_CHECK_NONALIGNED_ACCESS], [ memset(scratch, 0, sizeof(scratch)); return *(uint32_t*)((char*)scratch+1); ]])], - [have_aligned_only_int32=no], [have_aligned_only_int32=yes] + [box_cv_have_aligned_only_int32=no], [box_cv_have_aligned_only_int32=yes] )]) - if test "x$have_aligned_only_int32" = "xyes"; then + if test "x$box_cv_have_aligned_only_int32" = "xyes"; then AC_DEFINE([HAVE_ALIGNED_ONLY_INT32], 1, [Define to 1 if non-aligned int32 access will fail]) fi - AC_CACHE_CHECK([if non-aligned 64 bit word accesses fail], [have_aligned_only_int64], + AC_CACHE_CHECK([if non-aligned 64 bit word accesses fail], [box_cv_have_aligned_only_int64], [AC_RUN_IFELSE([AC_LANG_PROGRAM([[$ac_includes_default]], [[ #ifndef HAVE_UINT64_T #define uint64_t u_int64_t; @@ -49,9 +49,9 @@ AC_DEFUN([AX_CHECK_NONALIGNED_ACCESS], [ memset(scratch, 0, sizeof(scratch)); return *(uint64_t*)((char*)scratch+1); ]])], - [have_aligned_only_int64=no], [have_aligned_only_int64=yes] + [box_cv_have_aligned_only_int64=no], [box_cv_have_aligned_only_int64=yes] )]) - if test "x$have_aligned_only_int64" = "xyes"; then + if test "x$box_cv_have_aligned_only_int64" = "xyes"; then AC_DEFINE([HAVE_ALIGNED_ONLY_INT64], 1, [Define to 1 if non-aligned int64 access will fail]) fi ])dnl diff --git a/infrastructure/m4/ax_check_syscall_lseek.m4 b/infrastructure/m4/ax_check_syscall_lseek.m4 index 491cc1ed..6c26e025 100644 --- a/infrastructure/m4/ax_check_syscall_lseek.m4 +++ b/infrastructure/m4/ax_check_syscall_lseek.m4 @@ -15,8 +15,8 @@ dnl @license AllPermissive AC_DEFUN([AX_CHECK_SYSCALL_LSEEK], [ AC_REQUIRE([AX_FUNC_SYSCALL])dnl if test "x$ac_cv_header_sys_syscall_h" = "xyes"; then - AC_CACHE_CHECK([[whether syscall lseek requires dummy parameter]], [have_lseek_dummy_param], - [AC_RUN_IFELSE( + AC_CACHE_CHECK([[whether syscall lseek requires dummy parameter]], [box_cv_have_lseek_dummy_param], + [AC_TRY_RUN( [AC_LANG_PROGRAM([[ $ac_includes_default #include @@ -50,14 +50,16 @@ AC_DEFUN([AX_CHECK_SYSCALL_LSEEK], [ unlink("lseektest"); return res!=-1; ]])], - [have_lseek_dummy_param=yes], [have_lseek_dummy_param=no] + [box_cv_have_lseek_dummy_param=yes], + [box_cv_have_lseek_dummy_param=no], + [box_cv_have_lseek_dummy_param=no # assume not for cross-compiling] )]) - if test "x$have_lseek_dummy_param" = "xyes"; then + if test "x$box_cv_have_lseek_dummy_param" = "xyes"; then AC_DEFINE([HAVE_LSEEK_DUMMY_PARAM], 1, [Define to 1 if syscall lseek requires a dummy middle parameter]) fi fi - if test "x$have_lseek_dummy_param" = "xno" + if test "x$box_cv_have_lseek_dummy_param" = "xno" then m4_ifvaln([$1],[$1],[:])dnl m4_ifvaln([$2],[else $2])dnl diff --git a/infrastructure/m4/ax_func_syscall.m4 b/infrastructure/m4/ax_func_syscall.m4 index 40496bf0..2429ca93 100644 --- a/infrastructure/m4/ax_func_syscall.m4 +++ b/infrastructure/m4/ax_func_syscall.m4 @@ -12,26 +12,37 @@ dnl @category C dnl @author Martin Ebourne dnl @version 2005/07/01 dnl @license AllPermissive +dnl +dnl Changed by Chris on 081026: +dnl +dnl Reversed the test for __syscall(), remove the test for syscall(), +dnl remove the definition and reverse the sense in ax_func_syscall.m4 +dnl (which checks for __syscall() needing definition). +dnl +dnl Autoconf's AC_CHECK_FUNC defines it when testing for its presence, +dnl so HAVE___SYSCALL will be true even if __syscall has no definition +dnl in the system libraries, and this is precisely the case that we +dnl want to test for, so now we test whether the test program compiles +dnl with no explicit definition (only the system headers) and if that +dnl fails, we set HAVE___SYSCALL_NEED_DEFN to 1. AC_DEFUN([AX_FUNC_SYSCALL], [ AC_CHECK_HEADERS([sys/syscall.h unistd.h]) AC_CHECK_FUNCS([syscall __syscall]) - if test "x$ac_cv_func_syscall" != "xyes" && - test "x$ac_cv_func___syscall" != "xyes"; then - AC_CACHE_CHECK([for __syscall needing definition], [have___syscall_need_defn], + if test "x$ac_cv_func___syscall" = "xyes"; then + AC_CACHE_CHECK([for __syscall needing definition], [box_cv_have___syscall_need_defn], [AC_RUN_IFELSE([AC_LANG_PROGRAM([[ $ac_includes_default #ifdef HAVE_SYS_SYSCALL_H #include #endif - extern "C" off_t __syscall(quad_t number, ...); ]], [[ __syscall(SYS_exit, 0); return 1; ]])], - [have___syscall_need_defn=yes], [have___syscall_need_defn=no] + [box_cv_have___syscall_need_defn=no], [box_cv_have___syscall_need_defn=yes] )]) - if test "x$have___syscall_need_defn" = "xyes"; then + if test "x$box_cv_have___syscall_need_defn" = "xyes"; then AC_DEFINE([HAVE___SYSCALL_NEED_DEFN], 1, [Define to 1 if __syscall is available but needs a definition]) fi diff --git a/infrastructure/makebuildenv.pl.in b/infrastructure/makebuildenv.pl.in index 456bd265..5ed4a0ab 100755 --- a/infrastructure/makebuildenv.pl.in +++ b/infrastructure/makebuildenv.pl.in @@ -39,16 +39,8 @@ unless(-d 'local') # flags about the environment my %env_flags; -my $windows_include_path = ""; -if ($target_windows) -{ - $module_dependency{"lib/common"} = ["lib/win32"]; - push @implicit_deps, "lib/win32"; -} -else -{ - # $env_flags{'IGNORE_lib/win32'} = 1; -} +$module_dependency{"lib/common"} = ["lib/win32"]; +push @implicit_deps, "lib/win32"; # print "Flag: $_\n" for(keys %env_flags); @@ -408,13 +400,41 @@ for my $mod (@implicit_deps, @modules) "test script file for $module for writing\n"; print TESTFILE "#!/bin/sh\necho TEST: $module\n"; - if(-d "$module/testfiles") + if ($target_windows) + { + print TESTFILE <<__E; +kill_process() +{ + if test -r testfiles/\$1.pid; then + /bin/kill -0 -f `cat testfiles/\$1.pid` \\ + && /bin/kill -f `cat testfiles/\$1.pid` + rm testfiles/\$1.pid + fi +} +__E + } + else + { + print TESTFILE <<__E; +kill_process() +{ + test -r testfiles/\$1.pid \\ + && kill -0 `cat testfiles/\$1.pid` \\ + && kill `cat testfiles/\$1.pid` +} +__E + } + + if (-d "$module/testfiles") { print TESTFILE <<__E; echo Killing any running daemons... -test -r testfiles/bbackupd.pid && kill `cat testfiles/bbackupd.pid` -test -r testfiles/bbstored.pid && kill `cat testfiles/bbstored.pid` +kill_process bbackupd +kill_process bbstored +__E + } + print TESTFILE <<__E; echo Removing old test files... chmod -R a+rwx testfiles rm -rf testfiles @@ -423,9 +443,8 @@ echo Copying new test files... cp -p -R ../../../$module/testfiles . __E - } - if(-e "$module/testextra") + if (-e "$module/testextra") { open FL,"$module/testextra" or die "Can't open $module/testextra"; @@ -435,12 +454,11 @@ __E print TESTFILE "$runcmd\n"; - if(-d "$module/testfiles") + if (-d "$module/testfiles") { print TESTFILE <<__E; -# echo Killing any running daemons... -test -r testfiles/bbackupd.pid && kill `cat testfiles/bbackupd.pid` -test -r testfiles/bbstored.pid && kill `cat testfiles/bbstored.pid` +kill_process bbackupd +kill_process bbstored __E } @@ -487,8 +505,7 @@ __E # make include path - my $include_paths = $windows_include_path . - join(' ',map {'-I../../'.$_} @all_deps_for_module); + my $include_paths = join(' ',map {'-I../../'.$_} @all_deps_for_module); # is target a library? my $target_is_library = ($type ne 'bin' && $type ne 'test'); @@ -515,6 +532,9 @@ __E open MAKE,">$mod/Makefile".$mk_name_extra or die "Can't open Makefile for $mod\n"; my $debug_link_extra = ($target_is_library)?'':'../../debug/lib/debug/debug.a'; + my $default_cxxflags = '@CXXFLAGS@'; + $default_cxxflags =~ s/ -O2//g; + my $release_flags = "-O2"; if ($target_windows) { @@ -531,15 +551,21 @@ CXX = @CXX@ AR = @AR@ RANLIB = @RANLIB@ PERL = @PERL@ -WINDRES = windres +WINDRES = @WINDRES@ + +DEFAULT_CXXFLAGS = @CPPFLAGS@ $default_cxxflags @CXXFLAGS_STRICT@ \\ + $include_paths $extra_platform_defines \\ + -DBOX_VERSION="\\"$product_version\\"" +LDFLAGS = @LDFLAGS@ @LDADD_RDYNAMIC@ + .ifdef RELEASE -CXXFLAGS = -DNDEBUG $release_flags @CXXFLAGS_STRICT@ $include_paths $extra_platform_defines -DBOX_VERSION="\\"$product_version\\"" +CXXFLAGS = -DBOX_RELEASE_BUILD $release_flags \$(DEFAULT_CXXFLAGS) OUTBASE = ../../release OUTDIR = ../../release/$mod DEPENDMAKEFLAGS = -D RELEASE VARIENT = RELEASE .else -CXXFLAGS = -g @CXXFLAGS_STRICT@ $include_paths $extra_platform_defines -DBOX_VERSION="\\"$product_version\\"" +CXXFLAGS = -g \$(DEFAULT_CXXFLAGS) OUTBASE = ../../debug OUTDIR = ../../debug/$mod DEPENDMAKEFLAGS = @@ -560,11 +586,11 @@ _AR = \$(AR) _RANLIB = \$(RANLIB) .else HIDE = @ -_CXX = @ echo "[CXX] " \$(*F) && \$(CXX) -_LINK = @ echo "[LINK] " \$(*F) && \$(CXX) -_WINDRES = @ echo "[WINDRES]" \$(*F) && \$(WINDRES) -_AR = @ echo "[AR] " \$(*F) && \$(AR) -_RANLIB = @ echo "[RANLIB] " \$(*F) && \$(RANLIB) +_CXX = @ echo " [CXX] " \$(*F) && \$(CXX) +_LINK = @ echo " [LINK] " \$(*F) && \$(CXX) +_WINDRES = @ echo " [WINDRES]" \$(*F) && \$(WINDRES) +_AR = @ echo " [AR] " \$(*F) && \$(AR) +_RANLIB = @ echo " [RANLIB] " \$(*F) && \$(RANLIB) .endif __E @@ -573,11 +599,11 @@ __E { print MAKE <<__E; HIDE = \$(if \$(V),,@) -_CXX = \$(if \$(V),\$(CXX), @ echo "[CXX] \$<" && \$(CXX)) -_LINK = \$(if \$(V),\$(CXX), @ echo "[LINK] \$@" && \$(CXX)) -_WINDRES = \$(if \$(V),\$(WINDRES), @ echo "[WINDRES] \$<" && \$(WINDRES)) -_AR = \$(if \$(V),\$(AR), @ echo "[AR] \$@" && \$(AR)) -_RANLIB = \$(if \$(V),\$(RANLIB), @ echo "[RANLIB] \$@" && \$(RANLIB)) +_CXX = \$(if \$(V),\$(CXX), @ echo " [CXX] \$<" && \$(CXX)) +_LINK = \$(if \$(V),\$(CXX), @ echo " [LINK] \$@" && \$(CXX)) +_WINDRES = \$(if \$(V),\$(WINDRES), @ echo " [WINDRES] \$<" && \$(WINDRES)) +_AR = \$(if \$(V),\$(AR), @ echo " [AR] \$@" && \$(AR)) +_RANLIB = \$(if \$(V),\$(RANLIB), @ echo " [RANLIB] \$@" && \$(RANLIB)) __E } @@ -683,6 +709,7 @@ __E if ($is_cpp) { $make .= "\t\$(_CXX) \$(CXXFLAGS) $compile_line_extra ". + "-DBOX_MODULE=\"\\\"$mod\\\"\" " . "-c $file -o $out_name\n\n"; } elsif ($is_rc) @@ -726,8 +753,9 @@ __E # run make for things we require for my $dep (@all_deps_for_module) { - $deps_makeinfo .= "\t\t\$(HIDE) (cd ../../$dep; \$(MAKE)$sub_make_options \$(DEPENDMAKEFLAGS) -D NODEPS)\n"; + $deps_makeinfo .= "\t\t\$(HIDE) (cd ../../$dep; \$(MAKE)$sub_make_options -q \$(DEPENDMAKEFLAGS) -D NODEPS || \$(MAKE)$sub_make_options \$(DEPENDMAKEFLAGS) -D NODEPS)\n"; } + $deps_makeinfo .= ".\tendif\n.endif\n\n"; } print MAKE $deps_makeinfo if $bsd_make; @@ -771,7 +799,7 @@ __E { # make a library archive... print MAKE "\t\$(HIDE) (echo -n > $end_target; rm $end_target)\n"; - print MAKE "\t\$(_AR) -q $end_target $o_file_list\n"; + print MAKE "\t\$(_AR) cq $end_target $o_file_list\n"; print MAKE "\t\$(_RANLIB) $end_target\n"; } else @@ -796,7 +824,9 @@ __E } # link line... - print MAKE "\t\$(_LINK) $link_line_extra -o $end_target $o_file_list $lib_files$lo $platform_lib_files\n"; + print MAKE "\t\$(_LINK) \$(LDFLAGS) $link_line_extra " . + "-o $end_target $o_file_list " . + "$lib_files$lo $platform_lib_files\n"; } # tests need to copy the test file over if($type eq 'test') @@ -814,7 +844,11 @@ __E print MAKE $make,"\n"; # and a clean target - print MAKE "clean:\n\t-rm -rf \$(OUTDIR)/*\n.\tifndef SUBCLEAN\n"; + print MAKE </include $1/; - s/-D\s+(\w+)/$1=1/; + s/-D\s+(\w+)/$1=1/g; print MAKE; } @@ -896,11 +930,18 @@ sub additional_objects_from_make_fragment chomp; if(m/link-extra:\s*(.+)\Z/) { - my @o = split /\s+/,$1; - for(@o) + my $extra = $1; + do { - push @$objs_r,$1 if m/\A(.+)\.o\Z/; + my @o = split /\s+/, $extra; + for(@o) + { + push @$objs_r,$1 if m/\A(.+)\.o\Z/; + } + last unless $extra =~ m'\\$'; + $extra = ; } + while(1); } elsif(m/include-makefile:\s*(\S+)/) { diff --git a/infrastructure/makedistribution.pl.in b/infrastructure/makedistribution.pl.in index 2ef8b960..3914780a 100755 --- a/infrastructure/makedistribution.pl.in +++ b/infrastructure/makedistribution.pl.in @@ -7,7 +7,9 @@ use Symbol; my %comment_chars = ('cpp' => '// ', 'h' => '// ', 'pl' => '# ', 'pm' => '# ', '' => '# '); # other extensions which need text copying, just to remove the private stuff -my %text_files = ('txt' => 1, 'spec' => 1); +# .in is included here, as these could be any kind of source, but clearly +# they have text substitutions run on them by autoconf, so we can too :) +my %text_files = ('txt' => 1, 'spec' => 1, 'in' => 1); # files which don't get the license added my %no_license = (); # 'filename' => 1 @@ -113,6 +115,13 @@ sub copy_from_list elsif($src eq 'NO-LICENSE') { $no_license{$dst} = 1; + + # actually copy it, to remove redundancy in manifests + $src = $dst; + $dst = $other; + $dst = $src if $dst eq ''; + + copy_file($src,$dst); } elsif($src eq 'RUN') { @@ -189,9 +198,9 @@ sub copy_file # licensed or not? if(exists $comment_chars{$ext} && !exists $no_license{$fn} && $license_in_dir) { - my $b = $comment_chars{$ext}; - # copy as text, inserting license + # print "source copy $fn to $base_name/$dst_fn\n"; + my $in = gensym; open $in,$fn; open OUT,">$base_name/$dst_fn"; @@ -204,6 +213,7 @@ sub copy_file } # write license + my $b = $comment_chars{$ext}; for(@license) { print OUT $b,$_,"\n" @@ -225,31 +235,32 @@ sub copy_file close OUT; close $in; } - else + elsif(exists $text_files{$ext}) { - if(exists $text_files{$ext}) - { - # copy this as text, to remove private stuff - my $in = gensym; - open $in,$fn; - open OUT,">$base_name/$dst_fn"; + # copy this as text, to remove private stuff + # print "text copy $fn to $base_name/$dst_fn\n"; - while(<$in>) - { - unless(skip_non_applicable_section($_, $in, $fn)) - { - print OUT - } - } - - close OUT; - close $in; - } - else + my $in = gensym; + open $in,$fn; + open OUT,">$base_name/$dst_fn"; + + while(<$in>) { - # copy as binary - system 'cp',$fn,"$base_name/$dst_fn" - } + unless(skip_non_applicable_section($_, $in, $fn)) + { + print OUT + } + } + + close OUT; + close $in; + } + else + { + # copy as binary + # print "binary copy $fn to $base_name/$dst_fn\n"; + my $cmd = "cp -p $fn $base_name/$dst_fn"; + system($cmd) == 0 or die "copy failed: $cmd"; } # copy executable bit from src @@ -332,7 +343,11 @@ sub copy_dir next if m/\AMakefile\Z/; next if m/\Aautogen/; next if m/-smf-method\Z/; # copy only the .in versions - next if m/-manifest.xml\Z/; # copy onlt the .in versions + next if m/-manifest.xml\Z/; # copy only the .in versions + if($dir eq 'docs') { + next if m/.(x[sm]l|tmpl)\Z/; # don't include doc sources + next if m/generate_except_xml.pl/; + } next if !-f "$dir/$_"; copy_file("$dir/$_","$dst_dir/$_"); diff --git a/infrastructure/makeparcels.pl.in b/infrastructure/makeparcels.pl.in index 125ff346..e9e240db 100755 --- a/infrastructure/makeparcels.pl.in +++ b/infrastructure/makeparcels.pl.in @@ -98,71 +98,194 @@ print MAKE "all:\t",join(' ',map {"build-".$_} @parcels),"\n\n"; print MAKE "clean:\n"; for my $parcel (@parcels) { - print MAKE "\trm -rf ",parcel_dir($parcel),"\n"; - print MAKE "\trm -f ",parcel_target($parcel),"\n"; + print MAKE "\trm -rf ", BoxPlatform::parcel_dir($parcel), "\n"; + print MAKE "\trm -f ", BoxPlatform::parcel_target($parcel), "\n"; } -print MAKE "\n"; -print MAKE "test:\trelease/common/test\n\nrelease/common/test:\n\t./runtest.pl ALL release\n\n"; +if ($build_os eq 'CYGWIN') +{ + print MAKE "\tfind release debug -type f | xargs -r rm -f\n"; +} +else +{ + print MAKE "\tfind release debug -type f -exec rm -f {} \\;\n"; +} + +print MAKE <<__END_OF_FRAGMENT; + \$(MAKE) -C docs clean + +test: release/common/test + +release/common/test: + ./runtest.pl ALL release + +.PHONY: docs +docs: + \$(MAKE) -C docs + +__END_OF_FRAGMENT my $release_flag = BoxPlatform::make_flag('RELEASE'); for my $parcel (@parcels) { - my $target = parcel_target($parcel); - print MAKE "build-$parcel:\t$target\n\n"; - print MAKE $target,":\n"; - - my $dir = parcel_dir($parcel); - print MAKE "\ttest -d $dir || mkdir $dir\n"; - + my $target = BoxPlatform::parcel_target($parcel); + my $dir = BoxPlatform::parcel_dir($parcel); + my @parcel_deps; + unless ($target_windows) { open SCRIPT,">parcels/scripts/install-$parcel" or die "Can't open installer script for $parcel for writing"; print SCRIPT "#!/bin/sh\n\n"; } - + for(@{$parcel_contents{$parcel}}) { - my ($type,$name,$dest) = split /\s+/; - my $optional = ''; + my @args = split /\s+/; + + my ($type,$name,$dest) = @args; + my $optional = 0; + my $install = 1; if ($type eq 'optional') { - ($optional,$type,$name,$dest) = split /\s+/; + $optional = 1; + shift @args; + ($type,$name,$dest) = @args; + } + + if ($type eq 'noinstall') + { + $install = 0; + shift @args; + ($type,$name,$dest) = @args; } if($type eq 'bin') { my $exeext = $platform_exe_ext; - print MAKE "\t(cd bin/$name; \$(MAKE) $release_flag)\n"; - print MAKE "\tcp release/bin/$name/$name$exeext $dir\n"; + print MAKE < \n"; +} + +$basedir = $0; +$basedir =~ s|/.*||; +$basedir .= "/.."; +-d $basedir or die "$basedir: $!"; +chdir $basedir or die "$basedir: $!"; +require "infrastructure/BoxPlatform.pm.in"; + +print BoxPlatform::parcel_dir(@ARGV) . "\n"; + +exit 0; diff --git a/infrastructure/printversion.pl b/infrastructure/printversion.pl new file mode 100644 index 00000000..13815284 --- /dev/null +++ b/infrastructure/printversion.pl @@ -0,0 +1,12 @@ +#!perl + +$basedir = $0; +$basedir =~ s|/.*||; +$basedir .= "/.."; +-d $basedir or die "$basedir: $!"; +chdir $basedir or die "$basedir: $!"; +require "infrastructure/BoxPlatform.pm.in"; + +print "$BoxPlatform::product_version\n"; + +exit 0; diff --git a/lib/backupclient/BackupClientCryptoKeys.cpp b/lib/backupclient/BackupClientCryptoKeys.cpp index 46b77f0a..7a8da7ba 100644 --- a/lib/backupclient/BackupClientCryptoKeys.cpp +++ b/lib/backupclient/BackupClientCryptoKeys.cpp @@ -28,40 +28,58 @@ // Created: 1/12/03 // // -------------------------------------------------------------------------- -void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename) +void BackupClientCryptoKeys_Setup(const std::string& rKeyMaterialFilename) { // Read in the key material unsigned char KeyMaterial[BACKUPCRYPTOKEYS_FILE_SIZE]; // Open the file - FileStream file(KeyMaterialFilename); + FileStream file(rKeyMaterialFilename); + // Read in data if(!file.ReadFullBuffer(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE, 0)) { THROW_EXCEPTION(BackupStoreException, CouldntLoadClientKeyMaterial) } - // Tell the filename how to encrypt - BackupStoreFilenameClear::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH, - KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH); - BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish); + // Setup keys and encoding method for filename encryption + BackupStoreFilenameClear::SetBlowfishKey( + KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_KEY_START, + BACKUPCRYPTOKEYS_FILENAME_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILENAME_IV_START, + BACKUPCRYPTOKEYS_FILENAME_IV_LENGTH); + BackupStoreFilenameClear::SetEncodingMethod( + BackupStoreFilename::Encoding_Blowfish); + + // Setup key for attributes encryption + BackupClientFileAttributes::SetBlowfishKey( + KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH); + + // Setup secret for attribute hashing + BackupClientFileAttributes::SetAttributeHashSecret( + KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH); - // Tell the attributes how to encrypt - BackupClientFileAttributes::SetBlowfishKey(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH); - // and the secret for hashing - BackupClientFileAttributes::SetAttributeHashSecret(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_START, BACKUPCRYPTOKEYS_ATTRIBUTE_HASH_SECRET_LENGTH); + // Setup keys for file data encryption + BackupStoreFile::SetBlowfishKeys( + KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH, + KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH); - // Tell the files how to encrypt - BackupStoreFile::SetBlowfishKeys(KeyMaterial + BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_START, BACKUPCRYPTOKEYS_ATTRIBUTES_KEY_LENGTH, - KeyMaterial + BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_START, BACKUPCRYPTOKEYS_FILE_BLOCK_ENTRY_KEY_LENGTH); #ifndef HAVE_OLD_SSL // Use AES where available - BackupStoreFile::SetAESKey(KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH); + BackupStoreFile::SetAESKey( + KeyMaterial + BACKUPCRYPTOKEYS_FILE_AES_KEY_START, + BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH); #endif // Wipe the key material from memory - ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE); + #ifdef _MSC_VER // not defined on MinGW + SecureZeroMemory(KeyMaterial, BACKUPCRYPTOKEYS_FILE_SIZE); + #else + ::memset(KeyMaterial, 0, BACKUPCRYPTOKEYS_FILE_SIZE); + #endif } - - diff --git a/lib/backupclient/BackupClientCryptoKeys.h b/lib/backupclient/BackupClientCryptoKeys.h index 5e3a7df2..f40e2e03 100644 --- a/lib/backupclient/BackupClientCryptoKeys.h +++ b/lib/backupclient/BackupClientCryptoKeys.h @@ -49,7 +49,7 @@ #define BACKUPCRYPTOKEYS_FILE_AES_KEY_LENGTH 32 -void BackupClientCryptoKeys_Setup(const char *KeyMaterialFilename); +void BackupClientCryptoKeys_Setup(const std::string& rKeyMaterialFilename); #endif // BACKUPCLIENTCRYTOKEYS__H diff --git a/lib/backupclient/BackupClientFileAttributes.cpp b/lib/backupclient/BackupClientFileAttributes.cpp index 3ffeb189..bb17d41f 100644 --- a/lib/backupclient/BackupClientFileAttributes.cpp +++ b/lib/backupclient/BackupClientFileAttributes.cpp @@ -15,11 +15,14 @@ #include #include -#include +#include #include + #include +#include #include #include + #ifdef HAVE_SYS_XATTR_H #include #include @@ -299,9 +302,11 @@ void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroM StreamableMemBlock *pnewAttr = 0; try { - struct stat st; - if(::lstat(Filename, &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) != 0) { + BOX_LOG_SYS_ERROR("Failed to stat file: '" << + Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } @@ -390,7 +395,7 @@ void BackupClientFileAttributes::ReadAttributes(const char *Filename, bool ZeroM // Created: 2003/10/07 // // -------------------------------------------------------------------------- -void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st, bool ZeroModificationTimes) +void BackupClientFileAttributes::FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, EMU_STRUCT_STAT &st, bool ZeroModificationTimes) { outputBlock.ResizeBlock(sizeof(attr_StreamFormat)); attr_StreamFormat *pattr = (attr_StreamFormat*)outputBlock.GetBuffer(); @@ -439,6 +444,7 @@ void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBl int linkedToSize = ::readlink(Filename, linkedTo, PATH_MAX); if(linkedToSize == -1) { + BOX_LOG_SYS_ERROR("Failed to readlink '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError); } @@ -463,7 +469,7 @@ void BackupClientFileAttributes::FillAttributesLink(StreamableMemBlock &outputBl void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename) { #ifdef HAVE_SYS_XATTR_H - int listBufferSize = 1000; + int listBufferSize = 10000; char* list = new char[listBufferSize]; try @@ -529,6 +535,9 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc int valueSize = ::lgetxattr(Filename, attrKey.c_str(), 0, 0); if(valueSize<0) { + BOX_LOG_SYS_ERROR("Failed to get " + "extended attributes size " + "for '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError); } @@ -544,6 +553,9 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc valueSize = ::lgetxattr(Filename, attrKey.c_str(), buffer+xattrSize, xattrBufferSize-xattrSize); if(valueSize<0) { + BOX_LOG_SYS_ERROR("Failed to get " + "extended attributes for " + "'" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError); } xattrSize += valueSize; @@ -559,9 +571,25 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc outputBlock.ResizeBlock(xattrSize); } - else if(listSize<0 && errno!=EOPNOTSUPP && errno!=EACCES) + else if(listSize<0) { - THROW_EXCEPTION(CommonException, OSFileError); + if(errno == EOPNOTSUPP || errno == EACCES) + { + // fail silently + } + else if(errno == ERANGE) + { + BOX_ERROR("Failed to list extended " + "attributes of '" << Filename << "': " + "buffer too small, not backed up"); + } + else + { + BOX_LOG_SYS_ERROR("Failed to list extended " + "attributes of '" << Filename << "', " + "not backed up"); + THROW_EXCEPTION(CommonException, OSFileError); + } } } catch(...) @@ -638,6 +666,8 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename, ::unlink(Filename); if(::symlink((char*)(pattr + 1), Filename) != 0) { + BOX_LOG_SYS_ERROR("Failed to symlink '" << Filename << + "' to '" << (char*)(pattr + 1) << "'"); THROW_EXCEPTION(CommonException, OSFileError) } #endif @@ -655,12 +685,18 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename, // Not a link, use normal chown if(::chown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) { + BOX_LOG_SYS_ERROR("Failed to change " + "owner of file " + "'" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } } #else - if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) // use the version which sets things on symlinks + // use the version which sets things on symlinks + if(::lchown(Filename, ntohl(pattr->UID), ntohl(pattr->GID)) != 0) { + BOX_LOG_SYS_ERROR("Failed to change owner of " + "symbolic link '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } #endif @@ -705,6 +741,8 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename, // Try to apply if(::utimes(Filename, times) != 0) { + BOX_LOG_SYS_ERROR("Failed to change times of " + "file '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } } @@ -715,8 +753,12 @@ void BackupClientFileAttributes::WriteAttributes(const char *Filename, } // Apply everything else... (allowable mode flags only) - if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID | S_ISGID | S_ISVTX)) != 0) // mode must be done last (think setuid) + // Mode must be done last (think setuid) + if(::chmod(Filename, mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISUID + | S_ISGID | S_ISVTX)) != 0) { + BOX_LOG_SYS_ERROR("Failed to change permissions of file " + "'" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError) } } @@ -831,6 +873,8 @@ void BackupClientFileAttributes::WriteExtendedAttr(const char *Filename, int xat // FIXME: Warn on EOPNOTSUPP if(::lsetxattr(Filename, key, buffer+xattrOffset, valueSize, 0)!=0 && errno!=EOPNOTSUPP) { + BOX_LOG_SYS_ERROR("Failed to set extended attributes " + "on file '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileError); } @@ -993,7 +1037,7 @@ void BackupClientFileAttributes::SetAttributeHashSecret(const void *pSecret, int // Created: 25/4/04 // // -------------------------------------------------------------------------- -uint64_t BackupClientFileAttributes::GenerateAttributeHash(struct stat &st, const std::string &filename, const std::string &leafname) +uint64_t BackupClientFileAttributes::GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname) { if(sAttributeHashSecretLength == 0) { diff --git a/lib/backupclient/BackupClientFileAttributes.h b/lib/backupclient/BackupClientFileAttributes.h index fa56ff65..b32c14dd 100644 --- a/lib/backupclient/BackupClientFileAttributes.h +++ b/lib/backupclient/BackupClientFileAttributes.h @@ -15,7 +15,7 @@ #include "StreamableMemBlock.h" #include "BoxTime.h" -struct stat; +EMU_STRUCT_STAT; // declaration // -------------------------------------------------------------------------- // @@ -53,11 +53,13 @@ public: static void SetBlowfishKey(const void *pKey, int KeyLength); static void SetAttributeHashSecret(const void *pSecret, int SecretLength); - static uint64_t GenerateAttributeHash(struct stat &st, const std::string &filename, const std::string &leafname); + static uint64_t GenerateAttributeHash(EMU_STRUCT_STAT &st, const std::string &filename, const std::string &leafname); static void FillExtendedAttr(StreamableMemBlock &outputBlock, const char *Filename); private: - static void FillAttributes(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st, bool ZeroModificationTimes); + static void FillAttributes(StreamableMemBlock &outputBlock, + const char *Filename, EMU_STRUCT_STAT &st, + bool ZeroModificationTimes); static void FillAttributesLink(StreamableMemBlock &outputBlock, const char *Filename, struct stat &st); void WriteExtendedAttr(const char *Filename, int xattrOffset) const; diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp index b5a54964..b1c5cd0f 100644 --- a/lib/backupclient/BackupClientRestore.cpp +++ b/lib/backupclient/BackupClientRestore.cpp @@ -193,6 +193,8 @@ typedef struct { bool PrintDots; bool RestoreDeleted; + bool ContinueAfterErrors; + bool ContinuedAfterError; std::string mRestoreResumeInfoFilename; RestoreResumeInfo mResumeInfo; } RestoreParams; @@ -202,20 +204,26 @@ typedef struct // -------------------------------------------------------------------------- // // Function -// Name: BackupClientRestoreDir(BackupProtocolClient &, int64_t, const char *, bool) +// Name: BackupClientRestoreDir(BackupProtocolClient &, +// int64_t, const char *, bool) // Purpose: Restore a directory // Created: 23/11/03 // // -------------------------------------------------------------------------- -static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t DirectoryID, std::string &rLocalDirectoryName, +static int BackupClientRestoreDir(BackupProtocolClient &rConnection, + int64_t DirectoryID, std::string &rLocalDirectoryName, RestoreParams &Params, RestoreResumeInfo &rLevel) { - // If we're resuming... check that we haven't got a next level to look at + // If we're resuming... check that we haven't got a next level to + // look at if(rLevel.mpNextLevel != 0) { // Recurse immediately - std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName); - BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, localDirname, Params, *rLevel.mpNextLevel); + std::string localDirname(rLocalDirectoryName + + DIRECTORY_SEPARATOR_ASCHAR + + rLevel.mNextLevelLocalName); + BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, + localDirname, Params, *rLevel.mpNextLevel); // Add it to the list of done itmes rLevel.mRestoredObjects.insert(rLevel.mNextLevelID); @@ -259,22 +267,23 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir break; case ObjectExists_File: { - // File exists with this name, which is fun. Get rid of it. + // File exists with this name, which is fun. + // Get rid of it. BOX_WARNING("File present with name '" << - rLocalDirectoryName << "', removing " << + rLocalDirectoryName << "', removing " "out of the way of restored directory. " "Use specific restore with ID to " "restore this object."); if(::unlink(rLocalDirectoryName.c_str()) != 0) { - BOX_ERROR("Failed to delete file " << - rLocalDirectoryName << ": " << - strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to delete " + "file '" << + rLocalDirectoryName << "'"); return Restore_UnknownError; } BOX_TRACE("In restore, directory name " - "collision with file " << - rLocalDirectoryName); + "collision with file '" << + rLocalDirectoryName << "'"); } break; case ObjectExists_NoObject: @@ -378,10 +387,17 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir exists == ObjectExists_File) && ::mkdir(rLocalDirectoryName.c_str(), S_IRWXU) != 0) { - BOX_ERROR("Failed to create directory '" << - rLocalDirectoryName << "': " << - strerror(errno)); - return Restore_UnknownError; + BOX_LOG_SYS_ERROR("Failed to create directory '" << + rLocalDirectoryName << "'"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } // Save the restore info, in case it's needed later @@ -394,23 +410,39 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir BOX_ERROR("Failed to save resume info file '" << Params.mRestoreResumeInfoFilename << "': " << e.what()); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to save resume info file '" << Params.mRestoreResumeInfoFilename << "': unknown error"); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } // Fetch the directory listing from the server -- getting a // list of files which is appropriate to the restore type rConnection.QueryListDirectory( - DirectoryID, - Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING), - BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)), - true /* want attributes */); + DirectoryID, + Params.RestoreDeleted?(BackupProtocolClientListDirectory::Flags_Deleted):(BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING), + BackupProtocolClientListDirectory::Flags_OldVersion | (Params.RestoreDeleted?(0):(BackupProtocolClientListDirectory::Flags_Deleted)), + true /* want attributes */); // Retrieve the directory from the stream following BackupStoreDirectory dir; @@ -429,13 +461,29 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir { BOX_ERROR("Failed to restore attributes for '" << rLocalDirectoryName << "': " << e.what()); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to restore attributes for '" << rLocalDirectoryName << "': unknown error"); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } int64_t bytesWrittenSinceLastRestoreInfoSave = 0; @@ -444,35 +492,51 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir { BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; - while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) != 0) + while((en = i.Next(BackupStoreDirectory::Entry::Flags_File)) + != 0) { // Check ID hasn't already been done - if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end()) + if(rLevel.mRestoredObjects.find(en->GetObjectID()) + == rLevel.mRestoredObjects.end()) { // Local name BackupStoreFilenameClear nm(en->GetName()); - std::string localFilename(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename()); + std::string localFilename(rLocalDirectoryName + + DIRECTORY_SEPARATOR_ASCHAR + + nm.GetClearFilename()); // Unlink anything which already exists: // For resuming restores, we can't overwrite // files already there. - if(ObjectExists(localFilename) != ObjectExists_NoObject && + if(ObjectExists(localFilename) + != ObjectExists_NoObject && ::unlink(localFilename.c_str()) != 0) { - BOX_ERROR("Failed to delete file '" << - localFilename << "': " << - strerror(errno)); - return Restore_UnknownError; + BOX_LOG_SYS_ERROR("Failed to delete " + "file '" << localFilename << + "'"); + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } // Request it from the store - rConnection.QueryGetFile(DirectoryID, en->GetObjectID()); + rConnection.QueryGetFile(DirectoryID, + en->GetObjectID()); // Stream containing encoded file - std::auto_ptr objectStream(rConnection.ReceiveStream()); + std::auto_ptr objectStream( + rConnection.ReceiveStream()); - // Decode the file -- need to do different things depending on whether - // the directory entry has additional attributes + // Decode the file -- need to do different + // things depending on whether the directory + // entry has additional attributes try { if(en->HasAttributes()) @@ -493,14 +557,30 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir BOX_ERROR("Failed to restore file '" << localFilename << "': " << e.what()); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to restore file '" << localFilename << "': unknown error"); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } // Progress display? @@ -515,7 +595,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir // Save restore info? int64_t fileSize; - int exists; + bool exists = false; try { @@ -531,7 +611,15 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir "whether file exists: '" << localFilename << "': " << e.what()); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { @@ -539,17 +627,27 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir "whether file exists: '" << localFilename << "': " "unknown error"); - return Restore_UnknownError; + + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } if(exists) { // File exists... - bytesWrittenSinceLastRestoreInfoSave += fileSize; + bytesWrittenSinceLastRestoreInfoSave + += fileSize; if(bytesWrittenSinceLastRestoreInfoSave > MAX_BYTES_WRITTEN_BETWEEN_RESTORE_INFO_SAVES) { - // Save the restore info, in case it's needed later + // Save the restore info, in + // case it's needed later try { Params.mResumeInfo.Save(Params.mRestoreResumeInfoFilename); @@ -590,14 +688,28 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir BOX_ERROR("Failed to save resume info file '" << Params.mRestoreResumeInfoFilename << "': " << e.what()); - return Restore_UnknownError; + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to save resume info file '" << Params.mRestoreResumeInfoFilename << "': unknown error"); - return Restore_UnknownError; + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } bytesWrittenSinceLastRestoreInfoSave = 0; @@ -608,17 +720,23 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir { BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; - while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0) + while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) + != 0) { // Check ID hasn't already been done - if(rLevel.mRestoredObjects.find(en->GetObjectID()) == rLevel.mRestoredObjects.end()) + if(rLevel.mRestoredObjects.find(en->GetObjectID()) + == rLevel.mRestoredObjects.end()) { // Local name BackupStoreFilenameClear nm(en->GetName()); - std::string localDirname(rLocalDirectoryName + DIRECTORY_SEPARATOR_ASCHAR + nm.GetClearFilename()); + std::string localDirname(rLocalDirectoryName + + DIRECTORY_SEPARATOR_ASCHAR + + nm.GetClearFilename()); // Add the level for the next entry - RestoreResumeInfo &rnextLevel(rLevel.AddLevel(en->GetObjectID(), nm.GetClearFilename())); + RestoreResumeInfo &rnextLevel( + rLevel.AddLevel(en->GetObjectID(), + nm.GetClearFilename())); // Recurse int result = BackupClientRestoreDir( @@ -648,13 +766,27 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir { BOX_ERROR("Failed to restore attributes for '" << rLocalDirectoryName << "': " << e.what()); - return Restore_UnknownError; + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } catch(...) { BOX_ERROR("Failed to restore attributes for '" << rLocalDirectoryName << "': unknown error"); - return Restore_UnknownError; + if (Params.ContinueAfterErrors) + { + Params.ContinuedAfterError = true; + } + else + { + return Restore_UnknownError; + } } return Restore_Complete; @@ -664,33 +796,45 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, int64_t Dir // -------------------------------------------------------------------------- // // Function -// Name: BackupClientRestore(BackupProtocolClient &, int64_t, const char *, bool, bool) -// Purpose: Restore a directory on the server to a local directory on the disc. -// -// The local directory must not already exist. +// Name: BackupClientRestore(BackupProtocolClient &, int64_t, +// const char *, bool, bool, bool, bool, bool) +// Purpose: Restore a directory on the server to a local +// directory on the disc. The local directory must not +// already exist. // -// If a restore is aborted for any reason, then it may be resumed if -// Resume == true. If Resume == false and resumption is possible, then -// Restore_ResumePossible is returned. +// If a restore is aborted for any reason, then it may +// be resumed if Resume == true. If Resume == false +// and resumption is possible, then +// Restore_ResumePossible is returned. // -// Set RestoreDeleted to restore a deleted directory. This may not give the -// directory structure when it was deleted, because files may have been deleted -// within it before it was deleted. +// Set RestoreDeleted to restore a deleted directory. +// This may not give the directory structure when it +// was deleted, because files may have been deleted +// within it before it was deleted. // -// Returns Restore_TargetExists if the target directory exists, but -// there is no restore possible. (Won't attempt to overwrite things.) +// Returns Restore_TargetExists if the target +// directory exists, but there is no restore possible. +// (Won't attempt to overwrite things.) // -// Returns Restore_Complete on success. (Exceptions on error.) +// Returns Restore_Complete on success. (Exceptions +// on error, unless ContinueAfterError is true and +// the error is recoverable, in which case it returns +// Restore_CompleteWithErrors) // Created: 23/11/03 // // -------------------------------------------------------------------------- -int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName, - bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume) +int BackupClientRestore(BackupProtocolClient &rConnection, + int64_t DirectoryID, const char *LocalDirectoryName, + bool PrintDots, bool RestoreDeleted, + bool UndeleteAfterRestoreDeleted, bool Resume, + bool ContinueAfterErrors) { // Parameter block RestoreParams params; params.PrintDots = PrintDots; params.RestoreDeleted = RestoreDeleted; + params.ContinueAfterErrors = ContinueAfterErrors; + params.ContinuedAfterError = false; params.mRestoreResumeInfoFilename = LocalDirectoryName; params.mRestoreResumeInfoFilename += ".boxbackupresume"; @@ -699,12 +843,13 @@ int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, // Does any resumption information exist? bool doingResume = false; - if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && targetExistance == ObjectExists_Dir) + if(FileExists(params.mRestoreResumeInfoFilename.c_str()) && + targetExistance == ObjectExists_Dir) { if(!Resume) { - // Caller didn't specify that resume should be done, so refuse to do it - // but say why. + // Caller didn't specify that resume should be done, + // so refuse to do it but say why. return Restore_ResumePossible; } @@ -752,9 +897,7 @@ int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, // Delete the resume information file ::unlink(params.mRestoreResumeInfoFilename.c_str()); - return Restore_Complete; + return params.ContinuedAfterError ? Restore_CompleteWithErrors + : Restore_Complete; } - - - diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h index f7724030..7e492238 100644 --- a/lib/backupclient/BackupClientRestore.h +++ b/lib/backupclient/BackupClientRestore.h @@ -15,14 +15,21 @@ class BackupProtocolClient; enum { Restore_Complete = 0, - Restore_ResumePossible = 1, - Restore_TargetExists = 2, - Restore_TargetPathNotFound = 3, - Restore_UnknownError = 4, + Restore_ResumePossible, + Restore_TargetExists, + Restore_TargetPathNotFound, + Restore_UnknownError, + Restore_CompleteWithErrors, }; -int BackupClientRestore(BackupProtocolClient &rConnection, int64_t DirectoryID, const char *LocalDirectoryName, - bool PrintDots = false, bool RestoreDeleted = false, bool UndeleteAfterRestoreDeleted = false, bool Resume = false); +int BackupClientRestore(BackupProtocolClient &rConnection, + int64_t DirectoryID, + const char *LocalDirectoryName, + bool PrintDots = false, + bool RestoreDeleted = false, + bool UndeleteAfterRestoreDeleted = false, + bool Resume = false, + bool ContinueAfterErrors = false); #endif // BACKUPSCLIENTRESTORE__H diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index 61033b5b..e70ba865 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -17,15 +17,15 @@ static const ConfigurationVerifyKey backuplocationkeys[] = { - {"ExcludeFile", 0, ConfigTest_MultiValueAllowed, 0}, - {"ExcludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0}, - {"ExcludeDir", 0, ConfigTest_MultiValueAllowed, 0}, - {"ExcludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0}, - {"AlwaysIncludeFile", 0, ConfigTest_MultiValueAllowed, 0}, - {"AlwaysIncludeFilesRegex", 0, ConfigTest_MultiValueAllowed, 0}, - {"AlwaysIncludeDir", 0, ConfigTest_MultiValueAllowed, 0}, - {"AlwaysIncludeDirsRegex", 0, ConfigTest_MultiValueAllowed, 0}, - {"Path", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("ExcludeFile", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("ExcludeFilesRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("ExcludeDir", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("ExcludeDirsRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeFile", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeFilesRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeDir", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("AlwaysIncludeDirsRegex", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("Path", ConfigTest_Exists | ConfigTest_LastEntry) }; static const ConfigurationVerify backuplocations[] = @@ -64,39 +64,62 @@ static const ConfigurationVerify verifyserver[] = static const ConfigurationVerifyKey verifyrootkeys[] = { - {"AccountNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - - {"UpdateStoreInterval", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"MinimumFileAge", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"MaxUploadWait", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"MaxFileTimeInFuture", "172800", ConfigTest_IsInt, 0}, // file is uploaded if the file is this much in the future (2 days default) - - {"AutomaticBackup", "yes", ConfigTest_IsBool, 0}, + ConfigurationVerifyKey("AccountNumber", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("UpdateStoreInterval", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("MinimumFileAge", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("MaxUploadWait", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("MaxFileTimeInFuture", ConfigTest_IsInt, 172800), + // file is uploaded if the file is this much in the future + // (2 days default) + ConfigurationVerifyKey("AutomaticBackup", ConfigTest_IsBool, true), - {"SyncAllowScript", 0, 0, 0}, // optional script to run to see if the sync should be started now - // return "now" if it's allowed, or a number of seconds if it's not + ConfigurationVerifyKey("SyncAllowScript", 0), + // script that returns "now" if backup is allowed now, or a number + // of seconds to wait before trying again if not - {"MaximumDiffingTime", 0, ConfigTest_IsInt, 0}, - {"DeleteRedundantLocationsAfter", "172800", ConfigTest_IsInt, 0}, + ConfigurationVerifyKey("MaximumDiffingTime", ConfigTest_IsInt), + ConfigurationVerifyKey("DeleteRedundantLocationsAfter", + ConfigTest_IsInt, 172800), - {"FileTrackingSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"DiffingUploadSizeThreshold", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"StoreHostname", 0, ConfigTest_Exists, 0}, - {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // extended log to syslog - {"ExtendedLogFile", NULL, 0, 0}, // extended log to a file - {"LogAllFileAccess", "no", ConfigTest_IsBool, 0}, + ConfigurationVerifyKey("FileTrackingSizeThreshold", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("DiffingUploadSizeThreshold", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("StoreHostname", ConfigTest_Exists), + ConfigurationVerifyKey("StorePort", ConfigTest_IsInt, + BOX_PORT_BBSTORED), + ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), + // extended log to syslog + ConfigurationVerifyKey("ExtendedLogFile", 0), + // extended log to a file + ConfigurationVerifyKey("LogAllFileAccess", ConfigTest_IsBool, false), + // enable logging reasons why each file is backed up or not + ConfigurationVerifyKey("LogFile", 0), + // enable logging to a file + ConfigurationVerifyKey("LogFileLevel", 0), + // set the level of verbosity of file logging + ConfigurationVerifyKey("CommandSocket", 0), + // not compulsory to have this + ConfigurationVerifyKey("KeepAliveTime", ConfigTest_IsInt), + ConfigurationVerifyKey("StoreObjectInfoFile", 0), + // optional - {"CommandSocket", 0, 0, 0}, // not compulsory to have this - {"KeepAliveTime", 0, ConfigTest_IsInt, 0}, // optional - {"StoreObjectInfoFile", 0, 0, 0}, // optional - - {"NotifyScript", 0, 0, 0}, // optional script to run when backup needs attention, eg store full + ConfigurationVerifyKey("NotifyScript", 0), + // optional script to run when backup needs attention, eg store full - {"CertificateFile", 0, ConfigTest_Exists, 0}, - {"PrivateKeyFile", 0, ConfigTest_Exists, 0}, - {"TrustedCAsFile", 0, ConfigTest_Exists, 0}, - {"KeysFile", 0, ConfigTest_Exists, 0}, - {"DataDirectory", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("NotifyAlways", ConfigTest_IsBool, false), + // option to disable the suppression of duplicate notifications + + ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), + ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), + ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), + ConfigurationVerifyKey("KeysFile", ConfigTest_Exists), + ConfigurationVerifyKey("DataDirectory", + ConfigTest_Exists | ConfigTest_LastEntry), }; const ConfigurationVerify BackupDaemonConfigVerify = diff --git a/lib/backupclient/BackupStoreFile.cpp b/lib/backupclient/BackupStoreFile.cpp index 7e93d59d..27e12bc8 100644 --- a/lib/backupclient/BackupStoreFile.cpp +++ b/lib/backupclient/BackupStoreFile.cpp @@ -65,22 +65,27 @@ BackupStoreFileStats BackupStoreFile::msStats = {0,0,0}; // Function // Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &) // Purpose: Encode a file into something for storing on file server. -// Requires a real filename so full info can be stored. +// Requires a real filename so full info can be stored. // -// Returns a stream. Most of the work is done by the stream -// when data is actually requested -- the file will be held -// open until the stream is deleted or the file finished. +// Returns a stream. Most of the work is done by the stream +// when data is actually requested -- the file will be held +// open until the stream is deleted or the file finished. // Created: 2003/08/28 // // -------------------------------------------------------------------------- -std::auto_ptr BackupStoreFile::EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime) +std::auto_ptr BackupStoreFile::EncodeFile(const char *Filename, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) { // Create the stream std::auto_ptr stream(new BackupStoreFileEncodeStream); // Do the initial setup - ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, 0 /* no recipe, just encode */, - ContainerID, rStoreFilename, pModificationTime); + ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename, + 0 /* no recipe, just encode */, + ContainerID, rStoreFilename, pModificationTime, pLogger, + pRunStatusProvider); // Return the stream for the caller return stream; @@ -267,8 +272,8 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr) { // Does file exist? - struct stat st; - if(::stat(DecodedFilename, &st) == 0) + EMU_STRUCT_STAT st; + if(EMU_STAT(DecodedFilename, &st) == 0) { THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) } @@ -1255,8 +1260,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, // is it a symlink? bool sourceIsSymlink = false; { - struct stat st; - if(::lstat(Filename, &st) == -1) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) == -1) { THROW_EXCEPTION(CommonException, OSFileError) } diff --git a/lib/backupclient/BackupStoreFile.h b/lib/backupclient/BackupStoreFile.h index 3ee5ddb0..f38cd821 100644 --- a/lib/backupclient/BackupStoreFile.h +++ b/lib/backupclient/BackupStoreFile.h @@ -10,11 +10,14 @@ #ifndef BACKUPSTOREFILE__H #define BACKUPSTOREFILE__H -#include "IOStream.h" +#include +#include + #include "BackupClientFileAttributes.h" #include "BackupStoreFilename.h" - -#include +#include "IOStream.h" +#include "ReadLoggingStream.h" +#include "RunStatusProvider.h" typedef struct { @@ -114,7 +117,11 @@ public: // Main interface - static std::auto_ptr EncodeFile(const char *Filename, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0); + static std::auto_ptr EncodeFile(const char *Filename, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime = 0, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); static std::auto_ptr EncodeFileDiff ( const char *Filename, int64_t ContainerID, @@ -207,7 +214,7 @@ public: static BackupStoreFileStats msStats; // For debug -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD static bool TraceDetailsOfDiffProcess; #endif diff --git a/lib/backupclient/BackupStoreFileDiff.cpp b/lib/backupclient/BackupStoreFileDiff.cpp index f7842a0b..e9da1ee7 100644 --- a/lib/backupclient/BackupStoreFileDiff.cpp +++ b/lib/backupclient/BackupStoreFileDiff.cpp @@ -9,6 +9,8 @@ #include "Box.h" +#include + #include #include @@ -38,7 +40,7 @@ using namespace BackupStoreFileCreation; // By default, don't trace out details of the diff as we go along -- would fill up logs significantly. // But it's useful for the test. -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD bool BackupStoreFile::TraceDetailsOfDiffProcess = false; #endif @@ -128,8 +130,8 @@ std::auto_ptr BackupStoreFile::EncodeFileDiff { // Is it a symlink? { - struct stat st; - if(::lstat(Filename, &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_LSTAT(Filename, &st) != 0) { THROW_EXCEPTION(CommonException, OSFileError) } @@ -149,7 +151,7 @@ std::auto_ptr BackupStoreFile::EncodeFileDiff int64_t blocksInIndex = 0; bool canDiffFromThis = false; LoadIndex(rDiffFromBlockIndex, DiffFromObjectID, &pindex, blocksInIndex, Timeout, canDiffFromThis); - //TRACE1("Diff: Blocks in index: %lld\n", blocksInIndex); + // BOX_TRACE("Diff: Blocks in index: " << blocksInIndex); if(!canDiffFromThis) { @@ -434,12 +436,14 @@ static void FindMostUsedSizes(BlocksAvailableEntry *pIndex, int64_t NumBlocks, i } // trace the size table in debug builds -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD if(BackupStoreFile::TraceDetailsOfDiffProcess) { for(int t = 0; t < BACKUP_FILE_DIFF_MAX_BLOCK_SIZES; ++t) { - TRACE3("Diff block size %d: %d (count = %lld)\n", t, Sizes[t], sizeCounts[t]); + BOX_TRACE("Diff block size " << t << ": " << + Sizes[t] << " (count = " << + sizeCounts[t] << ")"); } } #endif @@ -459,11 +463,12 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map BlocksAvailableEntry *pIndex, int64_t NumBlocks, int32_t Sizes[BACKUP_FILE_DIFF_MAX_BLOCK_SIZES], DiffTimer *pDiffTimer) { - Timer maximumDiffingTime(0); + Timer maximumDiffingTime(0, "MaximumDiffingTime"); if(pDiffTimer && pDiffTimer->IsManaged()) { - maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime()); + maximumDiffingTime = Timer(pDiffTimer->GetMaximumDiffingTime(), + "MaximumDiffingTime"); } std::map goodnessOfFit; @@ -626,7 +631,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map // Block matched, roll the checksum forward to the next block without doing // any more comparisons, because these are pointless (as any more matches will be ignored when - // the receipe is generated) and just take up valuable processor time. Edge cases are + // the recipe is generated) and just take up valuable processor time. Edge cases are // especially nasty, using huge amounts of time and memory. int skip = Sizes[s]; if(offset < bytesInEndings && skip > 0) @@ -723,7 +728,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map throw; } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD if(BackupStoreFile::TraceDetailsOfDiffProcess) { // Trace out the found blocks in debug mode @@ -774,7 +779,7 @@ static void SetupHashTable(BlocksAvailableEntry *pIndex, int64_t NumBlocks, int3 // Already present in table? if(pHashTable[hash] != 0) { - //TRACE1("Another hash entry for %d found\n", hash); + //BOX_TRACE("Another hash entry for " << hash << " found"); // Yes -- need to set the pointer in this entry to the current entry to build the linked list pIndex[b].mpNextInHashList = pHashTable[hash]; } @@ -805,7 +810,7 @@ static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChec ASSERT(pFirstInHashList != 0); ASSERT(pIndex != 0); -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD uint16_t DEBUG_Hash = fastSum.GetComponentForHashing(); #endif uint32_t Checksum = fastSum.GetChecksum(); @@ -840,17 +845,19 @@ static bool SecondStageMatch(BlocksAvailableEntry *pFirstInHashList, RollingChec // Then go through the entries in the hash list, comparing with the strong digest calculated scan = pFirstInHashList; - //TRACE0("second stage match\n"); + //BOX_TRACE("second stage match"); while(scan != 0) { - //TRACE3("scan size %d, block size %d, hash %d\n", scan->mSize, BlockSize, Hash); + //BOX_TRACE("scan size " << scan->mSize << + // ", block size " << BlockSize << + // ", hash " << Hash); ASSERT(scan->mSize == BlockSize); ASSERT(RollingChecksum::ExtractHashingComponent(scan->mWeakChecksum) == DEBUG_Hash); // Compare? if(strong.DigestMatches(scan->mStrongChecksum)) { - //TRACE0("Match!\n"); + //BOX_TRACE("Match!\n"); // Found! Add to list of found blocks... int64_t fileOffset = (FileBlockNumber * BlockSize) + Offset; int64_t blockIndex = (scan - pIndex); // pointer arthmitic is frowned upon. But most efficient way of doing it here -- alternative is to use more memory @@ -909,10 +916,11 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA instruction.mSpaceBefore = SizeOfInputFile; rRecipe.push_back(instruction); - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD if(BackupStoreFile::TraceDetailsOfDiffProcess) { - TRACE1("Diff: Default recipe generated, %lld bytes of file\n", SizeOfInputFile); + BOX_TRACE("Diff: Default recipe generated, " << + SizeOfInputFile << " bytes of file"); } #endif @@ -928,7 +936,7 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA ASSERT(i != rFoundBlocks.end()); // check logic // Counting for debug tracing -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD int64_t debug_NewBytesFound = 0; int64_t debug_OldBlocksUsed = 0; #endif @@ -955,7 +963,7 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA instruction.mSpaceBefore = i->first - loc; // Move location forward to match loc += instruction.mSpaceBefore; -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD debug_NewBytesFound += instruction.mSpaceBefore; #endif } @@ -981,7 +989,7 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA instruction.mBlocks += 1; } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD debug_OldBlocksUsed++; #endif @@ -997,18 +1005,22 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA { RESET_INSTRUCTION instruction.mSpaceBefore = SizeOfInputFile - loc; -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD debug_NewBytesFound += instruction.mSpaceBefore; #endif rRecipe.push_back(instruction); } // dump out the recipe -#ifndef NDEBUG - TRACE2("Diff: %lld new bytes found, %lld old blocks used\n", debug_NewBytesFound, debug_OldBlocksUsed); +#ifndef BOX_RELEASE_BUILD + BOX_TRACE("Diff: " << + debug_NewBytesFound << " new bytes found, " << + debug_OldBlocksUsed << " old blocks used"); if(BackupStoreFile::TraceDetailsOfDiffProcess) { - TRACE1("Diff: Recipe generated (size %d)\n======== ========= ========\nSpace b4 FirstBlk NumBlks\n", rRecipe.size()); + BOX_TRACE("Diff: Recipe generated (size " << rRecipe.size()); + BOX_TRACE("======== ========= ========"); + BOX_TRACE("Space b4 FirstBlk NumBlks"); { for(unsigned int e = 0; e < rRecipe.size(); ++e) { @@ -1018,10 +1030,15 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA #else sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); #endif - TRACE3("%8lld %s %8lld\n", rRecipe[e].mSpaceBefore, (rRecipe[e].mpStartBlock == 0)?" -":b, (int64_t)rRecipe[e].mBlocks); + BOX_TRACE(std::setw(8) << + rRecipe[e].mSpaceBefore << + " " << + ((rRecipe[e].mpStartBlock == 0)?" -":b) << + " " << std::setw(8) << + rRecipe[e].mBlocks); } } - TRACE0("======== ========= ========\n"); + BOX_TRACE("======== ========= ========"); } #endif } diff --git a/lib/backupclient/BackupStoreFileEncodeStream.cpp b/lib/backupclient/BackupStoreFileEncodeStream.cpp index 423c11a3..b2d44697 100644 --- a/lib/backupclient/BackupStoreFileEncodeStream.cpp +++ b/lib/backupclient/BackupStoreFileEncodeStream.cpp @@ -9,18 +9,20 @@ #include "Box.h" -#include "BackupStoreFileEncodeStream.h" +#include + +#include "BackupClientFileAttributes.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" #include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFileWire.h" #include "BackupStoreObjectMagic.h" -#include "BackupStoreException.h" -#include "BackupStoreConstants.h" #include "BoxTime.h" -#include "BackupClientFileAttributes.h" #include "FileStream.h" -#include "RollingChecksum.h" #include "Random.h" +#include "RollingChecksum.h" #include "MemLeakFindOn.h" @@ -39,6 +41,7 @@ BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() : mpRecipe(0), mpFile(0), mpLogging(0), + mpRunStatusProvider(NULL), mStatus(Status_Header), mSendData(true), mTotalBlocks(0), @@ -105,8 +108,11 @@ BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() // Created: 8/12/03 // // -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEncodeStream::Recipe *pRecipe, - int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime) +void BackupStoreFileEncodeStream::Setup(const char *Filename, + BackupStoreFileEncodeStream::Recipe *pRecipe, + int64_t ContainerID, const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) { // Pointer to a blank recipe which we might create BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; @@ -126,9 +132,9 @@ void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEnc pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); BackupStoreFileEncodeStream::RecipeInstruction instruction; - instruction.mSpaceBefore = fileSize; // whole file - instruction.mBlocks = 0; // no blocks - instruction.mpStartBlock = 0; // no block + instruction.mSpaceBefore = fileSize; // whole file + instruction.mBlocks = 0; // no blocks + instruction.mpStartBlock = 0; // no block pblankRecipe->push_back(instruction); pRecipe = pblankRecipe; @@ -208,8 +214,18 @@ void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEnc // Open the file mpFile = new FileStream(Filename); - // Create logging stream - mpLogging = new ReadLoggingStream(*mpFile); + if (pLogger) + { + // Create logging stream + mpLogging = new ReadLoggingStream(*mpFile, + *pLogger); + } + else + { + // re-use FileStream instead + mpLogging = mpFile; + mpFile = NULL; + } // Work out the largest possible block required for the encoded data mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); @@ -220,7 +236,7 @@ void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEnc { throw std::bad_alloc(); } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD // In debug builds, make sure that the reallocation code is exercised. mEncodedBuffer.Allocate(mAllocatedBufferSize / 4); #else @@ -257,6 +273,8 @@ void BackupStoreFileEncodeStream::Setup(const char *Filename, BackupStoreFileEnc } throw; } + + mpRunStatusProvider = pRunStatusProvider; } @@ -314,6 +332,11 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { return 0; } + + if(mpRunStatusProvider && mpRunStatusProvider->StopRun()) + { + THROW_EXCEPTION(BackupStoreException, SignalReceived); + } int bytesToRead = NBytes; uint8_t *buffer = (uint8_t*)pBuffer; @@ -529,22 +552,25 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() ASSERT(blockRawSize < mAllocatedBufferSize); // Check file open - if(mpFile == 0 || mpLogging == 0) + if(mpLogging == 0) { // File should be open, but isn't. So logical error. THROW_EXCEPTION(BackupStoreException, Internal) } // Read the data in - if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */)) + if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, + 0 /* not interested in size if failure */)) { - // TODO: Do something more intelligent, and abort this upload because the file - // has changed - THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer) + // TODO: Do something more intelligent, and abort + // this upload because the file has changed. + THROW_EXCEPTION(BackupStoreException, + Temp_FileEncodeStreamDidntReadBuffer) } // Encode it - mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer); + mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, + blockRawSize, mEncodedBuffer); //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); @@ -555,7 +581,8 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() strongChecksum.Finish(); // Add entry to the index - StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); + StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, + weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); // Set vars to reading this block mPositionInCurrentBlock = 0; diff --git a/lib/backupclient/BackupStoreFileEncodeStream.h b/lib/backupclient/BackupStoreFileEncodeStream.h index fb5d0851..c5fa780a 100644 --- a/lib/backupclient/BackupStoreFileEncodeStream.h +++ b/lib/backupclient/BackupStoreFileEncodeStream.h @@ -18,6 +18,7 @@ #include "MD5Digest.h" #include "BackupStoreFile.h" #include "ReadLoggingStream.h" +#include "RunStatusProvider.h" namespace BackupStoreFileCreation { @@ -74,7 +75,11 @@ public: int64_t mOtherFileID; }; - void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime); + void Setup(const char *Filename, Recipe *pRecipe, int64_t ContainerID, + const BackupStoreFilename &rStoreFilename, + int64_t *pModificationTime, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); virtual int Read(void *pBuffer, int NBytes, int Timeout); virtual void Write(const void *pBuffer, int NBytes); @@ -101,7 +106,8 @@ private: Recipe *mpRecipe; IOStream *mpFile; // source file CollectInBufferStream mData; // buffer for header and index entries - ReadLoggingStream *mpLogging; + IOStream *mpLogging; + RunStatusProvider* mpRunStatusProvider; int mStatus; bool mSendData; // true if there's file data to send (ie not a symlink) int64_t mTotalBlocks; // Total number of blocks in the file diff --git a/lib/backupclient/BackupStoreFilename.cpp b/lib/backupclient/BackupStoreFilename.cpp index fbfe3313..72cd1acd 100644 --- a/lib/backupclient/BackupStoreFilename.cpp +++ b/lib/backupclient/BackupStoreFilename.cpp @@ -37,7 +37,7 @@ BackupStoreFilename::BackupStoreFilename() // // -------------------------------------------------------------------------- BackupStoreFilename::BackupStoreFilename(const BackupStoreFilename &rToCopy) - : BackupStoreFilename_base(rToCopy) + : mEncryptedName(rToCopy.mEncryptedName) { } @@ -65,7 +65,7 @@ bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const { bool ok = true; - if(size() < 2) + if(mEncryptedName.size() < 2) { // Isn't long enough to have a header ok = false; @@ -73,14 +73,14 @@ bool BackupStoreFilename::CheckValid(bool ExceptionIfInvalid) const else { // Check size is consistent - unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(*this); - if(dsize != size()) + unsigned int dsize = BACKUPSTOREFILENAME_GET_SIZE(this->mEncryptedName); + if(dsize != mEncryptedName.size()) { ok = false; } // And encoding is an accepted value - unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this); + unsigned int encoding = BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName); if(encoding < Encoding_Min || encoding > Encoding_Max) { ok = false; @@ -119,8 +119,8 @@ void BackupStoreFilename::ReadFromProtocol(Protocol &rProtocol) rProtocol.Read(data, dsize - 2); // assign to this string, storing the header and the extra data - assign(hdr, 2); - append(data.c_str(), data.size()); + mEncryptedName.assign(hdr, 2); + mEncryptedName.append(data.c_str(), data.size()); // Check it CheckValid(); @@ -141,7 +141,7 @@ void BackupStoreFilename::WriteToProtocol(Protocol &rProtocol) const { CheckValid(); - rProtocol.Write(c_str(), size()); + rProtocol.Write(mEncryptedName.c_str(), mEncryptedName.size()); } // -------------------------------------------------------------------------- @@ -177,7 +177,7 @@ void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) buf[0] = hdr[0]; buf[1] = hdr[1]; // assign to this string, storing the header and the extra data - assign(buf, dsize); + mEncryptedName.assign(buf, dsize); } else { @@ -194,7 +194,7 @@ void BackupStoreFilename::ReadFromStream(IOStream &rStream, int Timeout) data[0] = hdr[0]; data[1] = hdr[1]; // assign to this string, storing the header and the extra data - assign(data, dsize); + mEncryptedName.assign(data, dsize); } // Check it @@ -216,7 +216,7 @@ void BackupStoreFilename::WriteToStream(IOStream &rStream) const { CheckValid(); - rStream.Write(c_str(), size()); + rStream.Write(mEncryptedName.c_str(), mEncryptedName.size()); } // -------------------------------------------------------------------------- @@ -242,7 +242,8 @@ void BackupStoreFilename::EncodedFilenameChanged() // -------------------------------------------------------------------------- bool BackupStoreFilename::IsEncrypted() const { - return BACKUPSTOREFILENAME_GET_ENCODING(*this) != Encoding_Clear; + return BACKUPSTOREFILENAME_GET_ENCODING(this->mEncryptedName) != + Encoding_Clear; } @@ -250,8 +251,9 @@ bool BackupStoreFilename::IsEncrypted() const // // Function // Name: BackupStoreFilename::SetAsClearFilename(const char *) -// Purpose: Sets this object to be a valid filename, but with a filename in the clear. -// Used on the server to create filenames when there's no way of encrypting it. +// Purpose: Sets this object to be a valid filename, but with a +// filename in the clear. Used on the server to create +// filenames when there's no way of encrypting it. // Created: 22/4/04 // // -------------------------------------------------------------------------- @@ -268,7 +270,7 @@ void BackupStoreFilename::SetAsClearFilename(const char *Clear) ASSERT(encoded.size() == toEncode.size() + 2); // Store the encoded string - assign(encoded); + mEncryptedName.assign(encoded); // Stuff which must be done EncodedFilenameChanged(); diff --git a/lib/backupclient/BackupStoreFilename.h b/lib/backupclient/BackupStoreFilename.h index a7b6c437..80db9516 100644 --- a/lib/backupclient/BackupStoreFilename.h +++ b/lib/backupclient/BackupStoreFilename.h @@ -40,8 +40,11 @@ class IOStream; // Created: 2003/08/26 // // -------------------------------------------------------------------------- -class BackupStoreFilename : public BackupStoreFilename_base +class BackupStoreFilename /* : public BackupStoreFilename_base */ { +private: + std::string mEncryptedName; + public: BackupStoreFilename(); BackupStoreFilename(const BackupStoreFilename &rToCopy); @@ -71,8 +74,27 @@ public: Encoding_Max = 2 }; + const std::string& GetEncodedFilename() const + { + return mEncryptedName; + } + + bool operator==(const BackupStoreFilename& rOther) const + { + return mEncryptedName == rOther.mEncryptedName; + } + + bool operator!=(const BackupStoreFilename& rOther) const + { + return mEncryptedName != rOther.mEncryptedName; + } + protected: virtual void EncodedFilenameChanged(); + void SetEncodedFilename(const std::string &rEncoded) + { + mEncryptedName = rEncoded; + } }; // On the wire utilities for class and derived class diff --git a/lib/backupclient/BackupStoreFilenameClear.cpp b/lib/backupclient/BackupStoreFilenameClear.cpp index 9114fdd1..e529d8d3 100644 --- a/lib/backupclient/BackupStoreFilenameClear.cpp +++ b/lib/backupclient/BackupStoreFilenameClear.cpp @@ -160,15 +160,17 @@ void BackupStoreFilenameClear::MakeClearAvailable() const CheckValid(); // Decode the header - int size = BACKUPSTOREFILENAME_GET_SIZE(*this); - int encoding = BACKUPSTOREFILENAME_GET_ENCODING(*this); + int size = BACKUPSTOREFILENAME_GET_SIZE(GetEncodedFilename()); + int encoding = BACKUPSTOREFILENAME_GET_ENCODING(GetEncodedFilename()); // Decode based on encoding given in the header switch(encoding) { case Encoding_Clear: - TRACE0("**** BackupStoreFilename encoded with Clear encoding ****\n"); - mClearFilename.assign(c_str() + 2, size - 2); + BOX_TRACE("**** BackupStoreFilename encoded with " + "Clear encoding ****"); + mClearFilename.assign(GetEncodedFilename().c_str() + 2, + size - 2); break; case Encoding_Blowfish: @@ -193,7 +195,8 @@ static void EnsureEncDecBufferSize(int BufSize) if(spEncDecBuffer == 0) { #ifndef WIN32 - TRACE1("Allocating filename encoding/decoding buffer with size %d\n", BufSize); + BOX_TRACE("Allocating filename encoding/decoding buffer " + "with size " << BufSize); #endif spEncDecBuffer = new MemoryBlockGuard(BufSize); MEMLEAKFINDER_NOT_A_LEAK(spEncDecBuffer); @@ -242,7 +245,7 @@ void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, Cipher BACKUPSTOREFILENAME_MAKE_HDR(buffer, encSize, StoreAsEncoding); // Store the encoded string - assign((char*)buffer, encSize); + SetEncodedFilename(std::string((char*)buffer, encSize)); } @@ -256,8 +259,10 @@ void BackupStoreFilenameClear::EncryptClear(const std::string &rToEncode, Cipher // -------------------------------------------------------------------------- void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) const { + const std::string& rEncoded = GetEncodedFilename(); + // Work out max size - int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(size()) + 4; + int maxOutSize = rCipherContext.MaxOutSizeForInBufferSize(rEncoded.size()) + 4; // Make sure encode/decode buffer has enough space EnsureEncDecBufferSize(maxOutSize); @@ -266,8 +271,8 @@ void BackupStoreFilenameClear::DecryptEncoded(CipherContext &rCipherContext) con uint8_t *buffer = *spEncDecBuffer; // Decrypt - const char *str = c_str() + 2; - int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, size() - 2); + const char *str = rEncoded.c_str() + 2; + int sizeOut = rCipherContext.TransformBlock(buffer, sEncDecBufferSize, str, rEncoded.size() - 2); // Assign to this mClearFilename.assign((char*)buffer, sizeOut); diff --git a/lib/backupclient/BackupStoreObjectDump.cpp b/lib/backupclient/BackupStoreObjectDump.cpp index d3d9cc17..654317c1 100644 --- a/lib/backupclient/BackupStoreObjectDump.cpp +++ b/lib/backupclient/BackupStoreObjectDump.cpp @@ -47,7 +47,7 @@ static void OutputLine(FILE *file, bool ToTrace, const char *format, ...) } if(ToTrace) { - TRACE1("%s", text); + BOX_TRACE(text); } } @@ -70,7 +70,7 @@ void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) mAttributesModTime, mAttributes.GetSize()); // So repeated filenames can be illustrated, even though they can't be decoded - std::map nameNum; + std::map nameNum; int nameNumI = 0; // Dump items @@ -78,7 +78,7 @@ void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) for(std::vector::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i) { // Choose file name index number for this file - std::map::iterator nn(nameNum.find((*i)->GetName())); + std::map::iterator nn(nameNum.find((*i)->GetName().GetEncodedFilename())); int ni = nameNumI; if(nn != nameNum.end()) { @@ -86,7 +86,7 @@ void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) } else { - nameNum[(*i)->GetName()] = nameNumI; + nameNum[(*i)->GetName().GetEncodedFilename()] = nameNumI; ++nameNumI; } @@ -124,7 +124,7 @@ void BackupStoreDirectory::Dump(void *clibFileHandle, bool ToTrace) (*i)->GetSizeInBlocks(), (*i)->GetAttributesHash(), (*i)->GetAttributes().GetSize(), - (*i)->GetName().size(), + (*i)->GetName().GetEncodedFilename().size(), ni, ((f & BackupStoreDirectory::Entry::Flags_File)?" file":""), ((f & BackupStoreDirectory::Entry::Flags_Dir)?" dir":""), @@ -173,7 +173,8 @@ void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFi // Read the next two objects BackupStoreFilename fn; fn.ReadFromStream(rFile, IOStream::TimeOutInfinite); - OutputLine(file, ToTrace, "Filename size: %d\n", fn.size()); + OutputLine(file, ToTrace, "Filename size: %d\n", + fn.GetEncodedFilename().size()); BackupClientFileAttributes attr; attr.ReadFromStream(rFile, IOStream::TimeOutInfinite); @@ -211,14 +212,16 @@ void BackupStoreFile::DumpFile(void *clibFileHandle, bool ToTrace, IOStream &rFi if(s > 0) { nnew++; - TRACE2("%8lld this s=%8lld\n", b, s); + BOX_TRACE(std::setw(8) << b << " this s=" << + std::setw(8) << s); } else { nold++; - TRACE2("%8lld other i=%8lld\n", b, 0 - s); + BOX_TRACE(std::setw(8) << b << " other i=" << + std::setw(8) << 0 - s); } } - TRACE0("======== ===== ==========\n"); + BOX_TRACE("======== ===== =========="); } diff --git a/lib/backupclient/RunStatusProvider.h b/lib/backupclient/RunStatusProvider.h new file mode 100644 index 00000000..89f361ca --- /dev/null +++ b/lib/backupclient/RunStatusProvider.h @@ -0,0 +1,29 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: RunStatusProvider.h +// Purpose: Declares the RunStatusProvider interface. +// Created: 2008/08/14 +// +// -------------------------------------------------------------------------- + +#ifndef RUNSTATUSPROVIDER__H +#define RUNSTATUSPROVIDER__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: RunStatusProvider +// Purpose: Provides a StopRun() method which returns true if +// the current backup should be halted. +// Created: 2005/11/15 +// +// -------------------------------------------------------------------------- +class RunStatusProvider +{ + public: + virtual ~RunStatusProvider() { } + virtual bool StopRun() = 0; +}; + +#endif // RUNSTATUSPROVIDER__H diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp index 91a6b758..46cab68f 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.cpp +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -208,8 +208,8 @@ void BackupStoreAccountDatabase::CheckUpToDate() const // -------------------------------------------------------------------------- box_time_t BackupStoreAccountDatabase::GetDBFileModificationTime() const { - struct stat st; - if(::stat(pImpl->mFilename.c_str(), &st) == -1) + EMU_STRUCT_STAT st; + if(EMU_STAT(pImpl->mFilename.c_str(), &st) == -1) { THROW_EXCEPTION(CommonException, OSFileError) } diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index 176ece8f..7598094e 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -268,7 +268,8 @@ void BackupStoreCheck::CheckObjects() } maxDir = CheckObjectsScanDir(0, 1, mStoreRoot); - TRACE1("Max dir starting ID is %llx\n", maxDir); + BOX_TRACE("Max dir starting ID is " << + BOX_FORMAT_OBJECTID(maxDir)); } // Then go through and scan all the objects within those directories diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h index 3f48312a..1d5c1b1e 100644 --- a/lib/backupstore/BackupStoreCheck.h +++ b/lib/backupstore/BackupStoreCheck.h @@ -48,7 +48,7 @@ The following problems can be fixed: // Size of blocks in the list of IDs -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define BACKUPSTORECHECK_BLOCK_SIZE (64*1024) #else #define BACKUPSTORECHECK_BLOCK_SIZE 8 @@ -150,7 +150,7 @@ private: return (pBlock->mFlags[Index / Flags__NumItemsPerEntry] >> ((Index % Flags__NumItemsPerEntry) * Flags__NumFlags)) & Flags__MASK; } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD void DumpObjectInfo(); #define DUMP_OBJECT_INFO DumpObjectInfo(); #else diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 9c6f2452..bcb5c5e9 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -95,6 +95,21 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain mBlocksInDirectories += size; } +class BackupStoreDirectoryFixer +{ + private: + BackupStoreDirectory mDirectory; + std::string mFilename; + std::string mStoreRoot; + int mDiscSetNumber; + + public: + BackupStoreDirectoryFixer(std::string storeRoot, int discSetNumber, + int64_t ID); + void InsertObject(int64_t ObjectID, bool IsDirectory, + int32_t lostDirNameSerial); + ~BackupStoreDirectoryFixer(); +}; // -------------------------------------------------------------------------- // @@ -106,6 +121,10 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain // -------------------------------------------------------------------------- void BackupStoreCheck::CheckUnattachedObjects() { + typedef std::map fixers_t; + typedef std::pair fixer_pair_t; + fixers_t fixers; + // Scan all objects, finding ones which have no container for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i) { @@ -118,7 +137,9 @@ void BackupStoreCheck::CheckUnattachedObjects() if((flags & Flags_IsContained) == 0) { // Unattached object... - BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached."); + BOX_WARNING("Object " << + BOX_FORMAT_OBJECTID(pblock->mID[e]) << + " is unattached."); ++mNumberErrorsFound; // What's to be done? @@ -196,14 +217,50 @@ void BackupStoreCheck::CheckUnattachedObjects() } ASSERT(putIntoDirectoryID != 0); + if (!mFixErrors) + { + continue; + } + + BackupStoreDirectoryFixer* pFixer; + fixers_t::iterator fi = + fixers.find(putIntoDirectoryID); + if (fi == fixers.end()) + { + // no match, create a new one + pFixer = new BackupStoreDirectoryFixer( + mStoreRoot, mDiscSetNumber, + putIntoDirectoryID); + fixers.insert(fixer_pair_t( + putIntoDirectoryID, pFixer)); + } + else + { + pFixer = fi->second; + } + + int32_t lostDirNameSerial = 0; + + if(flags & Flags_IsDir) + { + lostDirNameSerial = mLostDirNameSerial++; + } + // Add it to the directory - InsertObjectIntoDirectory(pblock->mID[e], putIntoDirectoryID, - ((flags & Flags_IsDir) == Flags_IsDir)); + pFixer->InsertObject(pblock->mID[e], + ((flags & Flags_IsDir) == Flags_IsDir), + lostDirNameSerial); } } } -} + // clean up all the fixers. Deleting them commits them automatically. + for (fixers_t::iterator i = fixers.begin(); i != fixers.end(); i++) + { + BackupStoreDirectoryFixer* pFixer = i->second; + delete pFixer; + } +} // -------------------------------------------------------------------------- // @@ -261,6 +318,86 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) return true; } +BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(std::string storeRoot, + int discSetNumber, int64_t ID) +: mStoreRoot(storeRoot), + mDiscSetNumber(discSetNumber) +{ + // Generate filename + StoreStructure::MakeObjectFilename(ID, mStoreRoot, mDiscSetNumber, + mFilename, false /* don't make sure the dir exists */); + + // Read it in + std::auto_ptr file( + RaidFileRead::Open(mDiscSetNumber, mFilename)); + mDirectory.ReadFromStream(*file, IOStream::TimeOutInfinite); +} + +void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, + int32_t lostDirNameSerial) +{ + // Data for the object + BackupStoreFilename objectStoreFilename; + int64_t modTime = 100; // something which isn't zero or a special time + int32_t sizeInBlocks = 0; // suitable for directories + + if(IsDirectory) + { + // Directory -- simply generate a name for it. + char name[32]; + ::sprintf(name, "dir%08x", lostDirNameSerial); + objectStoreFilename.SetAsClearFilename(name); + } + else + { + // Files require a little more work... + // Open file + std::string fileFilename; + StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, + mDiscSetNumber, fileFilename, + false /* don't make sure the dir exists */); + std::auto_ptr file( + RaidFileRead::Open(mDiscSetNumber, fileFilename)); + + // Fill in size information + sizeInBlocks = file->GetDiscUsageInBlocks(); + + // Read in header + file_StreamFormat hdr; + if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || + (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + )) + { + // This should never happen, everything has been + // checked before. + THROW_EXCEPTION(BackupStoreException, Internal) + } + // This tells us nice things + modTime = box_ntoh64(hdr.mModificationTime); + // And the filename comes next + objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite); + } + + // Add a new entry in an appropriate place + mDirectory.AddUnattactedObject(objectStoreFilename, modTime, + ObjectID, sizeInBlocks, + IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); +} + +BackupStoreDirectoryFixer::~BackupStoreDirectoryFixer() +{ + // Fix any flags which have been broken, which there's a good chance of doing + mDirectory.CheckAndFix(); + + // Write it out + RaidFileWrite root(mDiscSetNumber, mFilename); + root.Open(true /* allow overwriting */); + mDirectory.WriteToStream(root); + root.Commit(true /* convert to raid now */); +} // -------------------------------------------------------------------------- // @@ -332,91 +469,6 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() } -// -------------------------------------------------------------------------- -// -// Function -// Name: BackupStoreCheck::InsertObjectIntoDirectory(int64_t, int64_t, bool) -// Purpose: -// Created: 22/4/04 -// -// -------------------------------------------------------------------------- -void BackupStoreCheck::InsertObjectIntoDirectory(int64_t ObjectID, int64_t DirectoryID, bool IsDirectory) -{ - if(!mFixErrors) - { - // Don't do anything if we're not supposed to fix errors - return; - } - - // Data for the object - BackupStoreFilename objectStoreFilename; - int64_t modTime = 100; // something which isn't zero or a special time - int32_t sizeInBlocks = 0; // suitable for directories - - if(IsDirectory) - { - // Directory -- simply generate a name for it. - char name[32]; - ::sprintf(name, "dir%08x", mLostDirNameSerial++); - objectStoreFilename.SetAsClearFilename(name); - } - else - { - // Files require a little more work... - // Open file - std::string fileFilename; - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mDiscSetNumber, fileFilename, false /* don't make sure the dir exists */); - std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, fileFilename)); - // Fill in size information - sizeInBlocks = file->GetDiscUsageInBlocks(); - // Read in header - file_StreamFormat hdr; - if(file->Read(&hdr, sizeof(hdr)) != sizeof(hdr) || (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 -#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE - && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif - )) - { - // This should never happen, everything has been checked before. - THROW_EXCEPTION(BackupStoreException, Internal) - } - // This tells us nice things - modTime = box_ntoh64(hdr.mModificationTime); - // And the filename comes next - objectStoreFilename.ReadFromStream(*file, IOStream::TimeOutInfinite); - } - - // Directory object - BackupStoreDirectory dir; - - // Generate filename - std::string filename; - StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, false /* don't make sure the dir exists */); - - // Read it in - { - std::auto_ptr file(RaidFileRead::Open(mDiscSetNumber, filename)); - dir.ReadFromStream(*file, IOStream::TimeOutInfinite); - } - - // Add a new entry in an appropraite place - dir.AddUnattactedObject(objectStoreFilename, modTime, ObjectID, sizeInBlocks, - IsDirectory?(BackupStoreDirectory::Entry::Flags_Dir):(BackupStoreDirectory::Entry::Flags_File)); - - // Fix any flags which have been broken, which there's a good change of going - dir.CheckAndFix(); - - // Write it out - if(mFixErrors) - { - RaidFileWrite root(mDiscSetNumber, filename); - root.Open(true /* allow overwriting */); - dir.WriteToStream(root); - root.Commit(true /* convert to raid now */); - } -} - - // -------------------------------------------------------------------------- // // Function @@ -594,6 +646,8 @@ void BackupStoreCheck::WriteNewStoreInfo() } } +#define FMT_OID(x) BOX_FORMAT_OBJECTID(x) +#define FMT_i BOX_FORMAT_OBJECTID((*i)->GetObjectID()) // -------------------------------------------------------------------------- // @@ -620,7 +674,11 @@ bool BackupStoreDirectory::CheckAndFix() if(newerEn == 0) { // Depends on something, but it isn't there. - TRACE2("Entry id %llx removed because depends on newer version %llx which doesn't exist\n", (*i)->GetObjectID(), dependsNewer); + BOX_TRACE("Entry id " << FMT_i << + " removed because depends " + "on newer version " << + FMT_OID(dependsNewer) << + " which doesn't exist"); // Remove delete *i; @@ -638,7 +696,12 @@ bool BackupStoreDirectory::CheckAndFix() if(newerEn->GetDependsOlder() != (*i)->GetObjectID()) { // Wrong entry - TRACE3("Entry id %llx, correcting DependsOlder to %llx, was %llx\n", dependsNewer, (*i)->GetObjectID(), newerEn->GetDependsOlder()); + BOX_TRACE("Entry id " << + FMT_OID(dependsNewer) << + ", correcting DependsOlder to " << + FMT_i << + ", was " << + FMT_OID(newerEn->GetDependsOlder())); newerEn->SetDependsOlder((*i)->GetObjectID()); // Mark as changed changed = true; @@ -657,7 +720,11 @@ bool BackupStoreDirectory::CheckAndFix() if(dependsOlder != 0 && FindEntryByID(dependsOlder) == 0) { // Has an older version marked, but this doesn't exist. Remove this mark - TRACE2("Entry id %llx was marked that %llx depended on it, which doesn't exist, dependency info cleared\n", (*i)->GetObjectID(), dependsOlder); + BOX_TRACE("Entry id " << FMT_i << + " was marked as depended on by " << + FMT_OID(dependsOlder) << ", " + "which doesn't exist, dependency " + "info cleared"); (*i)->SetDependsOlder(0); @@ -683,7 +750,7 @@ bool BackupStoreDirectory::CheckAndFix() // Records of things seen std::set idsEncountered; - std::set filenamesEncountered; + std::set filenamesEncountered; do { @@ -693,7 +760,7 @@ bool BackupStoreDirectory::CheckAndFix() bool removeEntry = false; if((*i) == 0) { - TRACE0("Remove because null pointer found\n"); + BOX_TRACE("Remove because null pointer found"); removeEntry = true; } else @@ -704,7 +771,8 @@ bool BackupStoreDirectory::CheckAndFix() if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File)) { // Bad! Unset the file flag - TRACE1("Entry %llx: File flag set when dir flag set\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": File flag and dir flag both set"); (*i)->RemoveFlags(Entry::Flags_File); changed = true; } @@ -713,7 +781,8 @@ bool BackupStoreDirectory::CheckAndFix() if(idsEncountered.find((*i)->GetObjectID()) != idsEncountered.end()) { // ID already seen, or type doesn't match - TRACE1("Entry %llx: Remove because ID already seen\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": Remove because ID already seen"); removeEntry = true; } else @@ -723,14 +792,15 @@ bool BackupStoreDirectory::CheckAndFix() // Check to see if the name has already been encountered -- if not, then it // needs to have the old version flag set - if(filenamesEncountered.find((*i)->GetName()) != filenamesEncountered.end()) + if(filenamesEncountered.find((*i)->GetName().GetEncodedFilename()) != filenamesEncountered.end()) { // Seen before -- check old version flag set if(((*i)->GetFlags() & Entry::Flags_OldVersion) != Entry::Flags_OldVersion && ((*i)->GetFlags() & Entry::Flags_Deleted) == 0) { // Not set, set it - TRACE1("Entry %llx: Set old flag\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": Set old flag"); (*i)->AddFlags(Entry::Flags_OldVersion); changed = true; } @@ -741,13 +811,14 @@ bool BackupStoreDirectory::CheckAndFix() if(((*i)->GetFlags() & Entry::Flags_OldVersion) == Entry::Flags_OldVersion) { // Set, unset it - TRACE1("Entry %llx: Old flag unset\n", (*i)->GetObjectID()); + BOX_TRACE("Entry " << FMT_i << + ": Old flag unset"); (*i)->RemoveFlags(Entry::Flags_OldVersion); changed = true; } // Remember filename - filenamesEncountered.insert((*i)->GetName()); + filenamesEncountered.insert((*i)->GetName().GetEncodedFilename()); } } } diff --git a/lib/backupstore/BackupStoreCheckData.cpp b/lib/backupstore/BackupStoreCheckData.cpp index f22c8339..fed0c3f1 100644 --- a/lib/backupstore/BackupStoreCheckData.cpp +++ b/lib/backupstore/BackupStoreCheckData.cpp @@ -174,7 +174,7 @@ BackupStoreCheck::IDBlock *BackupStoreCheck::LookupID(BackupStoreCheck_ID_t ID, } -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD // -------------------------------------------------------------------------- // // Function @@ -189,15 +189,18 @@ void BackupStoreCheck::DumpObjectInfo() { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; - TRACE2("BLOCK @ 0x%08x, %d entries\n", pblock, bentries); + BOX_TRACE("BLOCK @ " << BOX_FORMAT_HEX32(pblock) << + ", " << bentries << " entries"); for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); - TRACE4("id %llx, c %llx, %s, %s\n", - pblock->mID[e], pblock->mContainer[e], - (flags & Flags_IsDir)?"dir":"file", - (flags & Flags_IsContained)?"contained":"unattached"); + BOX_TRACE(std::hex << + "id " << pblock->mID[e] << + ", c " << pblock->mContainer[e] << + ", " << ((flags & Flags_IsDir)?"dir":"file") << + ", " << ((flags & Flags_IsContained) ? + "contained":"unattached")); } } } diff --git a/lib/backupstore/BackupStoreConfigVerify.cpp b/lib/backupstore/BackupStoreConfigVerify.cpp index 784adfb8..cc6efcf5 100644 --- a/lib/backupstore/BackupStoreConfigVerify.cpp +++ b/lib/backupstore/BackupStoreConfigVerify.cpp @@ -16,7 +16,8 @@ static const ConfigurationVerifyKey verifyserverkeys[] = { - SERVERTLS_VERIFY_SERVER_KEYS(0) // no default listen addresses + SERVERTLS_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) + // no default listen addresses }; static const ConfigurationVerify verifyserver[] = @@ -32,16 +33,18 @@ static const ConfigurationVerify verifyserver[] = static const ConfigurationVerifyKey verifyrootkeys[] = { - {"AccountDatabase", 0, ConfigTest_Exists, 0}, - {"TimeBetweenHousekeeping", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"ExtendedLogging", "no", ConfigTest_IsBool, 0}, // make value "yes" to enable in config file + ConfigurationVerifyKey("AccountDatabase", ConfigTest_Exists), + ConfigurationVerifyKey("TimeBetweenHousekeeping", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), + // make value "yes" to enable in config file #ifdef WIN32 - {"RaidFileConf", "", ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry) #else - {"RaidFileConf", BOX_FILE_RAIDFILE_DEFAULT_CONFIG, ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("RaidFileConf", ConfigTest_LastEntry, + BOX_FILE_RAIDFILE_DEFAULT_CONFIG) #endif - }; const ConfigurationVerify BackupConfigFileVerify = diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index 3588cc00..1d55fdf0 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -55,7 +55,7 @@ typedef struct END_STRUCTURE_PACKING_FOR_WIRE #endif -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define NUM_DELETED_DIRS_BLOCK 256 #else #define NUM_DELETED_DIRS_BLOCK 2 @@ -182,7 +182,7 @@ std::auto_ptr BackupStoreInfo::Load(int32_t AccountID, const st // Insert info from file info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker); info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed); - info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); + info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); info->mBlocksInOldFiles = box_ntoh64(hdr.mBlocksInOldFiles); info->mBlocksInDeletedFiles = box_ntoh64(hdr.mBlocksInDeletedFiles); info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories); diff --git a/lib/backupstore/StoreStructure.h b/lib/backupstore/StoreStructure.h index 094c0deb..ffbe83dd 100644 --- a/lib/backupstore/StoreStructure.h +++ b/lib/backupstore/StoreStructure.h @@ -12,7 +12,7 @@ #include -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define STORE_ID_SEGMENT_LENGTH 8 #define STORE_ID_SEGMENT_MASK 0xff #else diff --git a/lib/common/Box.h b/lib/common/Box.h index d0e7ab1e..1124a062 100644 --- a/lib/common/Box.h +++ b/lib/common/Box.h @@ -17,13 +17,14 @@ #include "BoxPlatform.h" -// uncomment this line to enable full memory leak finding on all malloc-ed blocks (at least, ones used by the STL) +// uncomment this line to enable full memory leak finding on all +// malloc-ed blocks (at least, ones used by the STL) //#define MEMLEAKFINDER_FULL_MALLOC_MONITORING -#ifndef NDEBUG - #ifdef HAVE_EXECINFO_H - #define SHOW_BACKTRACE_ON_EXCEPTION - #endif +// Show backtraces on exceptions in release builds until further notice +// (they are only logged at TRACE level anyway) +#ifdef HAVE_EXECINFO_H + #define SHOW_BACKTRACE_ON_EXCEPTION #endif #ifdef SHOW_BACKTRACE_ON_EXCEPTION @@ -36,14 +37,23 @@ #include "CommonException.h" #include "Logging.h" -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD extern bool AssertFailuresToSyslog; #define ASSERT_FAILS_TO_SYSLOG_ON {AssertFailuresToSyslog = true;} void BoxDebugAssertFailed(const char *cond, const char *file, int line); - #define ASSERT(cond) {if(!(cond)) {BoxDebugAssertFailed(#cond, __FILE__, __LINE__); THROW_EXCEPTION(CommonException, AssertFailed)}} + #define ASSERT(cond) \ + { \ + if(!(cond)) \ + { \ + BoxDebugAssertFailed(#cond, __FILE__, __LINE__); \ + THROW_EXCEPTION_MESSAGE(CommonException, \ + AssertFailed, #cond); \ + } \ + } - // Note that syslog tracing is independent of BoxDebugTraceOn, but stdout tracing is not + // Note that syslog tracing is independent of BoxDebugTraceOn, + // but stdout tracing is not extern bool BoxDebugTraceToSyslog; #define TRACE_TO_SYSLOG(x) {BoxDebugTraceToSyslog = x;} extern bool BoxDebugTraceToStdout; @@ -52,15 +62,6 @@ extern bool BoxDebugTraceOn; int BoxDebug_printf(const char *format, ...); int BoxDebugTrace(const char *format, ...); - #define TRACE0(msg) {BoxDebugTrace("%s", msg);} - #define TRACE1(msg, a0) {BoxDebugTrace(msg, a0);} - #define TRACE2(msg, a0, a1) {BoxDebugTrace(msg, a0, a1);} - #define TRACE3(msg, a0, a1, a2) {BoxDebugTrace(msg, a0, a1, a2);} - #define TRACE4(msg, a0, a1, a2, a3) {BoxDebugTrace(msg, a0, a1, a2, a3);} - #define TRACE5(msg, a0, a1, a2, a3, a4) {BoxDebugTrace(msg, a0, a1, a2, a3, a4);} - #define TRACE6(msg, a0, a1, a2, a3, a4, a5) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5);} - #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6);} - #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7) {BoxDebugTrace(msg, a0, a1, a2, a3, a4, a5, a6, a7);} #ifndef PLATFORM_DISABLE_MEM_LEAK_TESTING #define BOX_MEMORY_LEAK_TESTING @@ -76,16 +77,6 @@ #define TRACE_TO_SYSLOG(x) {} #define TRACE_TO_STDOUT(x) {} - #define TRACE0(msg) - #define TRACE1(msg, a0) - #define TRACE2(msg, a0, a1) - #define TRACE3(msg, a0, a1, a2) - #define TRACE4(msg, a0, a1, a2, a3) - #define TRACE5(msg, a0, a1, a2, a3, a4) - #define TRACE6(msg, a0, a1, a2, a3, a4, a5) - #define TRACE7(msg, a0, a1, a2, a3, a4, a5, a6) - #define TRACE8(msg, a0, a1, a2, a3, a4, a5, a6, a7) - // Box Backup builds release get extra information for exception logging #define EXCEPTION_CODENAMES_EXTENDED #define EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION @@ -113,13 +104,21 @@ #define THROW_EXCEPTION(type, subtype) \ { \ OPTIONAL_DO_BACKTRACE \ - BOX_WARNING("Exception thrown: " #type "(" #subtype ") at " \ - __FILE__ "(" << __LINE__ << ")") \ + BOX_WARNING("Exception thrown: " #type "(" #subtype ") " \ + "at " __FILE__ "(" << __LINE__ << ")") \ throw type(type::subtype); \ } -// extra macros for converting to network byte order +#define THROW_EXCEPTION_MESSAGE(type, subtype, message) \ + { \ + OPTIONAL_DO_BACKTRACE \ + BOX_WARNING("Exception thrown: " #type "(" #subtype ") " \ + " (" message ") at " \ + __FILE__ "(" << __LINE__ << ")") \ + throw type(type::subtype, message); \ + } +// extra macros for converting to network byte order #ifdef HAVE_NETINET_IN_H #include #endif diff --git a/lib/common/BoxException.h b/lib/common/BoxException.h index eb992f57..a8f5d7a6 100644 --- a/lib/common/BoxException.h +++ b/lib/common/BoxException.h @@ -11,6 +11,7 @@ #define BOXEXCEPTION__H #include +#include // -------------------------------------------------------------------------- // diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h index 2f0096aa..c625a7c7 100644 --- a/lib/common/BoxPlatform.h +++ b/lib/common/BoxPlatform.h @@ -29,9 +29,15 @@ #endif #ifdef WIN32 - // need msvcrt version 6.1 or higher for _gmtime64() - // must define this before importing - #define __MSVCRT_VERSION__ 0x0601 + #ifdef __MSVCRT_VERSION__ + #if __MSVCRT_VERSION__ < 0x0601 + #error Must include Box.h before sys/types.h + #endif + #else + // need msvcrt version 6.1 or higher for _gmtime64() + // must define this before importing + #define __MSVCRT_VERSION__ 0x0601 + #endif #endif #ifdef HAVE_SYS_TYPES_H @@ -66,10 +72,14 @@ #endif // Find out if credentials on UNIX sockets can be obtained -#ifndef HAVE_GETPEEREID - #if !HAVE_DECL_SO_PEERCRED - #define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET - #endif +#ifdef HAVE_GETPEEREID + // +#elif HAVE_DECL_SO_PEERCRED + // +#elif defined HAVE_UCRED_H && HAVE_GETPEERUCRED + // +#else + #define PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET #endif #ifdef HAVE_DEFINE_PRAGMA @@ -150,7 +160,7 @@ #endif // for Unix compatibility with Windows :-) -#if !HAVE_DECL_O_BINARY +#ifndef O_BINARY #define O_BINARY 0 #endif @@ -162,14 +172,15 @@ #ifdef WIN32 #define WIN32_LEAN_AND_MEAN - #include "emu.h" #endif -// Solaris has no dirfd(x) macro or function, and we need one. -// We cannot define macros with arguments directly using AC_DEFINE, -// so do it here instead of in configure.ac. +#include "emu.h" + +// Solaris has no dirfd(x) macro or function, and we need one for +// intercept tests. We cannot define macros with arguments directly +// using AC_DEFINE, so do it here instead of in configure.ac. -#if ! HAVE_DECL_DIRFD +#if ! defined PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE && ! HAVE_DECL_DIRFD #ifdef HAVE_DIR_D_FD #define dirfd(x) (x)->d_fd #elif defined HAVE_DIR_DD_FD diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp index 1ddcffd4..d05c0a6c 100644 --- a/lib/common/BoxTime.cpp +++ b/lib/common/BoxTime.cpp @@ -39,8 +39,8 @@ box_time_t GetCurrentBoxTime() struct timeval tv; if (gettimeofday(&tv, NULL) != 0) { - BOX_ERROR("Failed to gettimeofday(), dropping " - "precision: " << strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to gettimeofday(), " + "dropping precision"); } else { @@ -52,3 +52,45 @@ box_time_t GetCurrentBoxTime() return SecondsToBoxTime(time(0)); } + +std::string FormatTime(box_time_t time, bool includeDate, bool showMicros) +{ + std::ostringstream buf; + + time_t seconds = BoxTimeToSeconds(time); + int micros = BoxTimeToMicroSeconds(time) % MICRO_SEC_IN_SEC; + + struct tm tm_now, *tm_ptr = &tm_now; + + #ifdef WIN32 + if ((tm_ptr = localtime(&seconds)) != NULL) + #else + if (localtime_r(&seconds, &tm_now) != NULL) + #endif + { + buf << std::setfill('0'); + + if (includeDate) + { + buf << std::setw(4) << (tm_ptr->tm_year + 1900) << "-" << + std::setw(2) << (tm_ptr->tm_mon + 1) << "-" << + std::setw(2) << (tm_ptr->tm_mday) << " "; + } + + buf << std::setw(2) << tm_ptr->tm_hour << ":" << + std::setw(2) << tm_ptr->tm_min << ":" << + std::setw(2) << tm_ptr->tm_sec; + + if (showMicros) + { + buf << "." << std::setw(6) << micros; + } + } + else + { + buf << strerror(errno); + } + + return buf.str(); +} + diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h index e62a77ab..6681bbbd 100644 --- a/lib/common/BoxTime.h +++ b/lib/common/BoxTime.h @@ -40,4 +40,7 @@ inline uint64_t BoxTimeToMicroSeconds(box_time_t Time) return Time; } +std::string FormatTime(box_time_t time, bool includeDate, + bool showMicros = false); + #endif // BOXTIME__H diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp index 4d76e0e0..7d2e0bac 100644 --- a/lib/common/Configuration.cpp +++ b/lib/common/Configuration.cpp @@ -9,8 +9,11 @@ #include "Box.h" -#include #include +#include +#include + +#include #include "Configuration.h" #include "CommonException.h" @@ -29,7 +32,105 @@ inline bool iw(int c) static const char *sValueBooleanStrings[] = {"yes", "true", "no", "false", 0}; static const bool sValueBooleanValue[] = {true, true, false, false}; +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + void *testFunction +) +: mName(name), + mHasDefaultValue(false), + mFlags(flags), + mTestFunction(testFunction) +{ } + +// to allow passing NULL for default ListenAddresses + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + NoDefaultValue_t t, + void *testFunction +) +: mName(name), + mHasDefaultValue(false), + mFlags(flags), + mTestFunction(testFunction) +{ } + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + std::string defaultValue, + void *testFunction +) +: mName(name), + mDefaultValue(defaultValue), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ } + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + const char *defaultValue, + void *testFunction +) +: mName(name), + mDefaultValue(defaultValue), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ } + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + int defaultValue, + void *testFunction +) +: mName(name), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ + ASSERT(flags & ConfigTest_IsInt); + std::ostringstream val; + val << defaultValue; + mDefaultValue = val.str(); +} +ConfigurationVerifyKey::ConfigurationVerifyKey +( + std::string name, + int flags, + bool defaultValue, + void *testFunction +) +: mName(name), + mHasDefaultValue(true), + mFlags(flags), + mTestFunction(testFunction) +{ + ASSERT(flags & ConfigTest_IsBool); + mDefaultValue = defaultValue ? "yes" : "no"; +} + +ConfigurationVerifyKey::ConfigurationVerifyKey +( + const ConfigurationVerifyKey& rToCopy +) +: mName(rToCopy.mName), + mDefaultValue(rToCopy.mDefaultValue), + mHasDefaultValue(rToCopy.mHasDefaultValue), + mFlags(rToCopy.mFlags), + mTestFunction(rToCopy.mTestFunction) +{ } // -------------------------------------------------------------------------- // @@ -55,8 +156,8 @@ Configuration::Configuration(const std::string &rName) // -------------------------------------------------------------------------- Configuration::Configuration(const Configuration &rToCopy) : mName(rToCopy.mName), - mSubConfigurations(rToCopy.mSubConfigurations), - mKeys(rToCopy.mKeys) + mKeys(rToCopy.mKeys), + mSubConfigurations(rToCopy.mSubConfigurations) { } @@ -98,32 +199,29 @@ std::auto_ptr Configuration::LoadAndVerify( FdGetLine getline(file); // Object to create - Configuration *pconfig = new Configuration(std::string("")); + std::auto_ptr apConfig( + new Configuration(std::string(""))); try { // Load - LoadInto(*pconfig, getline, rErrorMsg, true); + LoadInto(*apConfig, getline, rErrorMsg, true); if(!rErrorMsg.empty()) { // An error occured, return now - //TRACE1("Error message from LoadInto: %s", rErrorMsg.c_str()); - TRACE0("Error at Configuration::LoadInfo\n"); - delete pconfig; - pconfig = 0; + BOX_ERROR("Error in Configuration::LoadInto: " << + rErrorMsg); return std::auto_ptr(0); } // Verify? if(pVerify) { - if(!Verify(*pconfig, *pVerify, std::string(), rErrorMsg)) + if(!apConfig->Verify(*pVerify, std::string(), rErrorMsg)) { - //TRACE1("Error message from Verify: %s", rErrorMsg.c_str()); - TRACE0("Error at Configuration::Verify\n"); - delete pconfig; - pconfig = 0; + BOX_ERROR("Error verifying configuration: " << + rErrorMsg); return std::auto_ptr(0); } } @@ -131,13 +229,11 @@ std::auto_ptr Configuration::LoadAndVerify( catch(...) { // Clean up - delete pconfig; - pconfig = 0; throw; } // Success. Return result. - return std::auto_ptr(pconfig); + return apConfig; } @@ -173,10 +269,10 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s if(startBlockExpected) { // New config object - Configuration config(blockName); + Configuration subConfig(blockName); // Continue processing into this block - if(!LoadInto(config, rGetLine, rErrorMsg, false)) + if(!LoadInto(subConfig, rGetLine, rErrorMsg, false)) { // Abort error return false; @@ -185,11 +281,12 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s startBlockExpected = false; // Store... - rConfig.mSubConfigurations.push_back(std::pair(blockName, config)); + rConfig.AddSubConfig(blockName, subConfig); } else { - rErrorMsg += "Unexpected start block in " + rConfig.mName + "\n"; + rErrorMsg += "Unexpected start block in " + + rConfig.mName + "\n"; } } else @@ -200,7 +297,7 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s if(RootLevel) { // error -- root level doesn't have a close - rErrorMsg += "Root level has close block -- forget to terminate subblock?\n"; + rErrorMsg += "Root level has close block -- forgot to terminate subblock?\n"; // but otherwise ignore } else @@ -246,24 +343,11 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s { std::string key(line.substr(0, keyend)); std::string value(line.substr(valuestart)); - //TRACE2("KEY: |%s|=|%s|\n", key.c_str(), value.c_str()); - - // Check for duplicate values - if(rConfig.mKeys.find(key) != rConfig.mKeys.end()) - { - // Multi-values allowed here, but checked later on - rConfig.mKeys[key] += MultiValueSeparator; - rConfig.mKeys[key] += value; - } - else - { - // Store - rConfig.mKeys[key] = value; - } + rConfig.AddKeyValue(key, value); } else { - rErrorMsg += "Invalid key in block "+rConfig.mName+"\n"; + rErrorMsg += "Invalid configuration key: " + line + "\n"; } } else @@ -286,40 +370,60 @@ bool Configuration::LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::s return true; } +void Configuration::AddKeyValue(const std::string& rKey, + const std::string& rValue) +{ + // Check for duplicate values + if(mKeys.find(rKey) != mKeys.end()) + { + // Multi-values allowed here, but checked later on + mKeys[rKey] += MultiValueSeparator; + mKeys[rKey] += rValue; + } + else + { + // Store + mKeys[rKey] = rValue; + } +} + +void Configuration::AddSubConfig(const std::string& rName, + const Configuration& rSubConfig) +{ + mSubConfigurations.push_back( + std::pair(rName, rSubConfig)); +} + // -------------------------------------------------------------------------- // // Function -// Name: Configuration::KeyExists(const char *) +// Name: Configuration::KeyExists(const std::string&) // Purpose: Checks to see if a key exists // Created: 2003/07/23 // // -------------------------------------------------------------------------- -bool Configuration::KeyExists(const char *pKeyName) const +bool Configuration::KeyExists(const std::string& rKeyName) const { - if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - - return mKeys.find(pKeyName) != mKeys.end(); + return mKeys.find(rKeyName) != mKeys.end(); } // -------------------------------------------------------------------------- // // Function -// Name: Configuration::GetKeyValue(const char *) +// Name: Configuration::GetKeyValue(const std::string&) // Purpose: Returns the value of a configuration variable // Created: 2003/07/23 // // -------------------------------------------------------------------------- -const std::string &Configuration::GetKeyValue(const char *pKeyName) const +const std::string &Configuration::GetKeyValue(const std::string& rKeyName) const { - if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - - std::map::const_iterator i(mKeys.find(pKeyName)); + std::map::const_iterator i(mKeys.find(rKeyName)); if(i == mKeys.end()) { - BOX_ERROR("Missing configuration key: " << pKeyName); + BOX_ERROR("Missing configuration key: " << rKeyName); THROW_EXCEPTION(CommonException, ConfigNoKey) } else @@ -332,16 +436,14 @@ const std::string &Configuration::GetKeyValue(const char *pKeyName) const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::GetKeyValueInt(const char *) +// Name: Configuration::GetKeyValueInt(const std::string& rKeyName) // Purpose: Gets a key value as an integer // Created: 2003/07/23 // // -------------------------------------------------------------------------- -int Configuration::GetKeyValueInt(const char *pKeyName) const +int Configuration::GetKeyValueInt(const std::string& rKeyName) const { - if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - - std::map::const_iterator i(mKeys.find(pKeyName)); + std::map::const_iterator i(mKeys.find(rKeyName)); if(i == mKeys.end()) { @@ -362,16 +464,14 @@ int Configuration::GetKeyValueInt(const char *pKeyName) const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::GetKeyValueBool(const char *) const +// Name: Configuration::GetKeyValueBool(const std::string&) // Purpose: Gets a key value as a boolean // Created: 17/2/04 // // -------------------------------------------------------------------------- -bool Configuration::GetKeyValueBool(const char *pKeyName) const +bool Configuration::GetKeyValueBool(const std::string& rKeyName) const { - if(pKeyName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - - std::map::const_iterator i(mKeys.find(pKeyName)); + std::map::const_iterator i(mKeys.find(rKeyName)); if(i == mKeys.end()) { @@ -428,22 +528,21 @@ std::vector Configuration::GetKeyNames() const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::SubConfigurationExists(const char *) +// Name: Configuration::SubConfigurationExists(const +// std::string&) // Purpose: Checks to see if a sub configuration exists // Created: 2003/07/23 // // -------------------------------------------------------------------------- -bool Configuration::SubConfigurationExists(const char *pSubName) const +bool Configuration::SubConfigurationExists(const std::string& rSubName) const { - if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - // Attempt to find it... std::list >::const_iterator i(mSubConfigurations.begin()); for(; i != mSubConfigurations.end(); ++i) { // This the one? - if(i->first == pSubName) + if(i->first == rSubName) { // Yes. return true; @@ -458,22 +557,52 @@ bool Configuration::SubConfigurationExists(const char *pSubName) const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::GetSubConfiguration(const char *) +// Name: Configuration::GetSubConfiguration(const +// std::string&) // Purpose: Gets a sub configuration // Created: 2003/07/23 // // -------------------------------------------------------------------------- -const Configuration &Configuration::GetSubConfiguration(const char *pSubName) const +const Configuration &Configuration::GetSubConfiguration(const std::string& + rSubName) const { - if(pSubName == 0) {THROW_EXCEPTION(CommonException, BadArguments)} - // Attempt to find it... std::list >::const_iterator i(mSubConfigurations.begin()); for(; i != mSubConfigurations.end(); ++i) { // This the one? - if(i->first == pSubName) + if(i->first == rSubName) + { + // Yes. + return i->second; + } + } + + THROW_EXCEPTION(CommonException, ConfigNoSubConfig) +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: Configuration::GetSubConfiguration(const +// std::string&) +// Purpose: Gets a sub configuration for editing +// Created: 2008/08/12 +// +// -------------------------------------------------------------------------- +Configuration &Configuration::GetSubConfigurationEditable(const std::string& + rSubName) +{ + // Attempt to find it... + + for(SubConfigListType::iterator + i = mSubConfigurations.begin(); + i != mSubConfigurations.end(); ++i) + { + // This the one? + if(i->first == rSubName) { // Yes. return i->second; @@ -510,12 +639,14 @@ std::vector Configuration::GetSubConfigurationNames() const // -------------------------------------------------------------------------- // // Function -// Name: Configuration::Verify(const Configuration &, const ConfigurationVerify &, const std::string &, std::string &) -// Purpose: Return list of sub configuration names +// Name: Configuration::Verify(const ConfigurationVerify &, const std::string &, std::string &) +// Purpose: Checks that the configuration is valid according to the +// supplied verifier // Created: 2003/07/24 // // -------------------------------------------------------------------------- -bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg) +bool Configuration::Verify(const ConfigurationVerify &rVerify, + const std::string &rLevel, std::string &rErrorMsg) { bool ok = true; @@ -528,15 +659,14 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV do { // Can the key be found? - ASSERT(pvkey->mpName); - if(rConfig.KeyExists(pvkey->mpName)) + if(KeyExists(pvkey->Name())) { // Get value - const std::string &rval = rConfig.GetKeyValue(pvkey->mpName); + const std::string &rval = GetKeyValue(pvkey->Name()); const char *val = rval.c_str(); // Check it's a number? - if((pvkey->Tests & ConfigTest_IsInt) == ConfigTest_IsInt) + if((pvkey->Flags() & ConfigTest_IsInt) == ConfigTest_IsInt) { // Test it... char *end; @@ -545,12 +675,12 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV { // not a good value ok = false; - rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid integer.\n"; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is not a valid integer.\n"; } } // Check it's a bool? - if((pvkey->Tests & ConfigTest_IsBool) == ConfigTest_IsBool) + if((pvkey->Flags() & ConfigTest_IsBool) == ConfigTest_IsBool) { // See if it's one of the allowed strings. bool found = false; @@ -568,37 +698,38 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV if(!found) { ok = false; - rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) is not a valid boolean value.\n"; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is not a valid boolean value.\n"; } } // Check for multi valued statments where they're not allowed - if((pvkey->Tests & ConfigTest_MultiValueAllowed) == 0) + if((pvkey->Flags() & ConfigTest_MultiValueAllowed) == 0) { // Check to see if this key is a multi-value -- it shouldn't be if(rval.find(MultiValueSeparator) != rval.npos) { ok = false; - rErrorMsg += rLevel + rConfig.mName +"." + pvkey->mpName + " (key) multi value not allowed (duplicated key?).\n"; + rErrorMsg += rLevel + mName +"." + pvkey->Name() + " (key) multi value not allowed (duplicated key?).\n"; } } } else { // Is it required to exist? - if((pvkey->Tests & ConfigTest_Exists) == ConfigTest_Exists) + if((pvkey->Flags() & ConfigTest_Exists) == ConfigTest_Exists) { // Should exist, but doesn't. ok = false; - rErrorMsg += rLevel + rConfig.mName + "." + pvkey->mpName + " (key) is missing.\n"; + rErrorMsg += rLevel + mName + "." + pvkey->Name() + " (key) is missing.\n"; } - else if(pvkey->mpDefaultValue) + else if(pvkey->HasDefaultValue()) { - rConfig.mKeys[std::string(pvkey->mpName)] = std::string(pvkey->mpDefaultValue); + mKeys[pvkey->Name()] = + pvkey->DefaultValue(); } } - if((pvkey->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + if((pvkey->Flags() & ConfigTest_LastEntry) == ConfigTest_LastEntry) { // No more! todo = false; @@ -610,22 +741,22 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV } while(todo); // Check for additional keys - for(std::map::const_iterator i = rConfig.mKeys.begin(); - i != rConfig.mKeys.end(); ++i) + for(std::map::const_iterator i = mKeys.begin(); + i != mKeys.end(); ++i) { // Is the name in the list? const ConfigurationVerifyKey *scan = rVerify.mpKeys; bool found = false; while(scan) { - if(scan->mpName == i->first) + if(scan->Name() == i->first) { found = true; break; } // Next? - if((scan->Tests & ConfigTest_LastEntry) == ConfigTest_LastEntry) + if((scan->Flags() & ConfigTest_LastEntry) == ConfigTest_LastEntry) { break; } @@ -636,7 +767,7 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV { // Shouldn't exist, but does. ok = false; - rErrorMsg += rLevel + rConfig.mName + "." + i->first + " (key) is not a known key. Check spelling and placement.\n"; + rErrorMsg += rLevel + mName + "." + i->first + " (key) is not a known key. Check spelling and placement.\n"; } } } @@ -650,8 +781,7 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV const ConfigurationVerify *scan = rVerify.mpSubConfigurations; while(scan) { - ASSERT(scan->mpName); - if(scan->mpName[0] == '*') + if(scan->mName.length() > 0 && scan->mName[0] == '*') { wildcardverify = scan; } @@ -659,24 +789,25 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV // Required? if((scan->Tests & ConfigTest_Exists) == ConfigTest_Exists) { - if(scan->mpName[0] == '*') + if(scan->mName.length() > 0 && + scan->mName[0] == '*') { // Check something exists - if(rConfig.mSubConfigurations.size() < 1) + if(mSubConfigurations.size() < 1) { // A sub config should exist, but doesn't. ok = false; - rErrorMsg += rLevel + rConfig.mName + ".* (block) is missing (a block must be present).\n"; + rErrorMsg += rLevel + mName + ".* (block) is missing (a block must be present).\n"; } } else { // Check real thing exists - if(!rConfig.SubConfigurationExists(scan->mpName)) + if(!SubConfigurationExists(scan->mName)) { // Should exist, but doesn't. ok = false; - rErrorMsg += rLevel + rConfig.mName + "." + scan->mpName + " (block) is missing.\n"; + rErrorMsg += rLevel + mName + "." + scan->mName + " (block) is missing.\n"; } } } @@ -690,8 +821,9 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV } // Go through the sub configurations, one by one - for(std::list >::const_iterator i(rConfig.mSubConfigurations.begin()); - i != rConfig.mSubConfigurations.end(); ++i) + for(SubConfigListType::iterator + i = mSubConfigurations.begin(); + i != mSubConfigurations.end(); ++i) { // Can this be found? const ConfigurationVerify *subverify = 0; @@ -701,7 +833,7 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV ASSERT(name); while(scan) { - if(strcmp(scan->mpName, name) == 0) + if(scan->mName == name) { // found it! subverify = scan; @@ -725,7 +857,8 @@ bool Configuration::Verify(Configuration &rConfig, const ConfigurationVerify &rV if(subverify) { // override const-ness here... - if(!Verify((Configuration&)i->second, *subverify, rConfig.mName + '.', rErrorMsg)) + if(!i->second.Verify(*subverify, mName + '.', + rErrorMsg)) { ok = false; } diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h index 64e7568e..2babd753 100644 --- a/lib/common/Configuration.h +++ b/lib/common/Configuration.h @@ -29,20 +29,51 @@ enum class ConfigurationVerifyKey { public: - const char *mpName; // "*" for all other keys (not implemented yet) - const char *mpDefaultValue; // default for when it's not present - int Tests; - void *TestFunction; // set to zero for now, will implement later + typedef enum + { + NoDefaultValue = 1 + } NoDefaultValue_t; + + ConfigurationVerifyKey(std::string name, int flags, + void *testFunction = NULL); + // to allow passing ConfigurationVerifyKey::NoDefaultValue + // for default ListenAddresses + ConfigurationVerifyKey(std::string name, int flags, + NoDefaultValue_t t, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + std::string defaultValue, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + const char* defaultValue, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + int defaultValue, void *testFunction = NULL); + ConfigurationVerifyKey(std::string name, int flags, + bool defaultValue, void *testFunction = NULL); + const std::string& Name() const { return mName; } + const std::string& DefaultValue() const { return mDefaultValue; } + const bool HasDefaultValue() const { return mHasDefaultValue; } + const int Flags() const { return mFlags; } + const void* TestFunction() const { return mTestFunction; } + ConfigurationVerifyKey(const ConfigurationVerifyKey& rToCopy); + +private: + ConfigurationVerifyKey& operator=(const ConfigurationVerifyKey& + noAssign); + + std::string mName; // "*" for all other keys (not implemented yet) + std::string mDefaultValue; // default for when it's not present + bool mHasDefaultValue; + int mFlags; + void *mTestFunction; // set to zero for now, will implement later }; class ConfigurationVerify { public: - const char *mpName; // "*" for all other sub config names + std::string mName; // "*" for all other sub config names const ConfigurationVerify *mpSubConfigurations; const ConfigurationVerifyKey *mpKeys; int Tests; - void *TestFunction; // set to zero for now, will implement later + void *TestFunction; // set to zero for now, will implement later }; class FdGetLine; @@ -57,9 +88,8 @@ class FdGetLine; // -------------------------------------------------------------------------- class Configuration { -private: - Configuration(const std::string &rName); public: + Configuration(const std::string &rName); Configuration(const Configuration &rToCopy); ~Configuration(); @@ -79,26 +109,36 @@ public: std::string &rErrorMsg) { return LoadAndVerify(rFilename, 0, rErrorMsg); } - bool KeyExists(const char *pKeyName) const; - const std::string &GetKeyValue(const char *pKeyName) const; - int GetKeyValueInt(const char *pKeyName) const; - bool GetKeyValueBool(const char *pKeyName) const; + bool KeyExists(const std::string& rKeyName) const; + const std::string &GetKeyValue(const std::string& rKeyName) const; + int GetKeyValueInt(const std::string& rKeyName) const; + bool GetKeyValueBool(const std::string& rKeyName) const; std::vector GetKeyNames() const; - bool SubConfigurationExists(const char *pSubName) const; - const Configuration &GetSubConfiguration(const char *pSubName) const; + bool SubConfigurationExists(const std::string& rSubName) const; + const Configuration &GetSubConfiguration(const std::string& rSubName) const; + Configuration &GetSubConfigurationEditable(const std::string& rSubName); std::vector GetSubConfigurationNames() const; + void AddKeyValue(const std::string& rKey, const std::string& rValue); + void AddSubConfig(const std::string& rName, const Configuration& rSubConfig); + + bool Verify(const ConfigurationVerify &rVerify, std::string &rErrorMsg) + { + return Verify(rVerify, std::string(), rErrorMsg); + } + +private: std::string mName; + // Order of keys not preserved + std::map mKeys; // Order of sub blocks preserved typedef std::list > SubConfigListType; SubConfigListType mSubConfigurations; - // Order of keys, not preserved - std::map mKeys; -private: static bool LoadInto(Configuration &rConfig, FdGetLine &rGetLine, std::string &rErrorMsg, bool RootLevel); - static bool Verify(Configuration &rConfig, const ConfigurationVerify &rVerify, const std::string &rLevel, std::string &rErrorMsg); + bool Verify(const ConfigurationVerify &rVerify, const std::string &rLevel, + std::string &rErrorMsg); }; #endif // CONFIGURATION__H diff --git a/lib/common/DebugAssertFailed.cpp b/lib/common/DebugAssertFailed.cpp index cceab0ef..e498d641 100644 --- a/lib/common/DebugAssertFailed.cpp +++ b/lib/common/DebugAssertFailed.cpp @@ -7,7 +7,7 @@ // // -------------------------------------------------------------------------- -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD #include "Box.h" @@ -33,5 +33,5 @@ void BoxDebugAssertFailed(const char *cond, const char *file, int line) } -#endif // NDEBUG +#endif // BOX_RELEASE_BUILD diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp index 87cdf00d..230d7163 100644 --- a/lib/common/DebugMemLeakFinder.cpp +++ b/lib/common/DebugMemLeakFinder.cpp @@ -8,7 +8,7 @@ // -------------------------------------------------------------------------- -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD #include "Box.h" @@ -75,6 +75,13 @@ namespace void memleakfinder_init() { ASSERT(!memleakfinder_initialised); + + { + // allocates a permanent buffer on Solaris. + // not a leak? + std::ostringstream oss; + } + memleakfinder_initialised = true; } @@ -146,7 +153,9 @@ void *memleakfinder_realloc(void *ptr, size_t size) std::map::iterator i(sMallocBlocks.find(ptr)); if(ptr && i == sMallocBlocks.end()) { - TRACE1("Block %x realloc(), but not in list. Error? Or allocated in startup static objects?\n", ptr); + BOX_WARNING("Block " << ptr << " realloc()ated, but not " + "in list. Error? Or allocated in startup static " + "objects?"); } void *b = ::realloc(ptr, size); @@ -193,7 +202,9 @@ void memleakfinder_free(void *ptr) } else { - TRACE1("Block %p freed, but not known. Error? Or allocated in startup static allocation?\n", ptr); + BOX_WARNING("Block " << ptr << " freed, but not " + "known. Error? Or allocated in startup " + "static allocation?"); } if(sTrackMallocInSection) @@ -293,16 +304,21 @@ void memleakfinder_traceblocksinsection() std::map::const_iterator i(sMallocBlocks.find(*s)); if(i == sMallocBlocks.end()) { - TRACE0("Logical error in section block finding\n"); + BOX_WARNING("Logical error in section block finding"); } else { - TRACE4("Block %p size %d allocated at %s:%d\n", i->first, i->second.size, i->second.file, i->second.line); + BOX_TRACE("Block " << i->first << " size " << + i->second.size << " allocated at " << + i->second.file << ":" << i->second.line); } } for(std::map::const_iterator i(sSectionObjectBlocks.begin()); i != sSectionObjectBlocks.end(); ++i) { - TRACE5("Object%s %p size %d allocated at %s:%d\n", i->second.array?" []":"", i->first, i->second.size, i->second.file, i->second.line); + BOX_TRACE("Object" << (i->second.array?" []":"") << " " << + i->first << " size " << i->second.size << + " allocated at " << i->second.file << + ":" << i->second.line); } } @@ -335,13 +351,27 @@ void memleakfinder_reportleaks_file(FILE *file) ASSERT(!sTrackingDataDestroyed); - for(std::map::const_iterator i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) + for(std::map::const_iterator + i(sMallocBlocks.begin()); i != sMallocBlocks.end(); ++i) { - if(is_leak(i->first)) ::fprintf(file, "Block 0x%p size %d allocated at %s:%d\n", i->first, i->second.size, i->second.file, i->second.line); + if(is_leak(i->first)) + { + ::fprintf(file, "Block %p size %d allocated at " + "%s:%d\n", i->first, i->second.size, + i->second.file, i->second.line); + } } - for(std::map::const_iterator i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) + + for(std::map::const_iterator + i(sObjectBlocks.begin()); i != sObjectBlocks.end(); ++i) { - if(is_leak(i->first)) ::fprintf(file, "Object%s 0x%p size %d allocated at %s:%d\n", i->second.array?" []":"", i->first, i->second.size, i->second.file, i->second.line); + if(is_leak(i->first)) + { + ::fprintf(file, "Object%s %p size %d allocated at " + "%s:%d\n", i->second.array?" []":"", + i->first, i->second.size, i->second.file, + i->second.line); + } } } @@ -390,8 +420,10 @@ extern "C" void memleakfinder_atexit() void memleakfinder_setup_exit_report(const char *filename, const char *markertext) { - ::strcpy(atexit_filename, filename); - ::strcpy(atexit_markertext, markertext); + ::strncpy(atexit_filename, filename, sizeof(atexit_filename)-1); + ::strncpy(atexit_markertext, markertext, sizeof(atexit_markertext)-1); + atexit_filename[sizeof(atexit_filename)-1] = 0; + atexit_markertext[sizeof(atexit_markertext)-1] = 0; if(!atexit_registered) { atexit(memleakfinder_atexit); @@ -516,4 +548,4 @@ void operator delete(void *ptr) throw () internal_delete(ptr); } -#endif // NDEBUG +#endif // BOX_RELEASE_BUILD diff --git a/lib/common/DebugPrintf.cpp b/lib/common/DebugPrintf.cpp index 8d75f458..1335d473 100644 --- a/lib/common/DebugPrintf.cpp +++ b/lib/common/DebugPrintf.cpp @@ -7,7 +7,7 @@ // // -------------------------------------------------------------------------- -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD #include "Box.h" @@ -80,4 +80,4 @@ int BoxDebugTrace(const char *format, ...) } -#endif // NDEBUG +#endif // BOX_RELEASE_BUILD diff --git a/lib/common/EventWatchFilesystemObject.cpp b/lib/common/EventWatchFilesystemObject.cpp index 84781113..43533fc8 100644 --- a/lib/common/EventWatchFilesystemObject.cpp +++ b/lib/common/EventWatchFilesystemObject.cpp @@ -26,9 +26,10 @@ // -------------------------------------------------------------------------- // // Function -// Name: EventWatchFilesystemObject::EventWatchFilesystemObject(const char *) -// Purpose: Constructor -- opens the file object -// Created: 12/3/04 +// Name: EventWatchFilesystemObject::EventWatchFilesystemObject +// (const char *) +// Purpose: Constructor -- opens the file object +// Created: 12/3/04 // // -------------------------------------------------------------------------- EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) @@ -39,9 +40,8 @@ EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) #ifdef HAVE_KQUEUE if(mDescriptor == -1) { - BOX_ERROR("EventWatchFilesystemObject: " - "Failed to open file '" << Filename << "': " << - strerror(errno)); + BOX_LOG_SYS_ERROR("EventWatchFilesystemObject: " + "Failed to open file '" << Filename << "'"); THROW_EXCEPTION(CommonException, OSFileOpenError) } #else @@ -53,9 +53,9 @@ EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) // -------------------------------------------------------------------------- // // Function -// Name: EventWatchFilesystemObject::~EventWatchFilesystemObject() -// Purpose: Destructor -// Created: 12/3/04 +// Name: EventWatchFilesystemObject::~EventWatchFilesystemObject() +// Purpose: Destructor +// Created: 12/3/04 // // -------------------------------------------------------------------------- EventWatchFilesystemObject::~EventWatchFilesystemObject() @@ -70,12 +70,14 @@ EventWatchFilesystemObject::~EventWatchFilesystemObject() // -------------------------------------------------------------------------- // // Function -// Name: EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &) -// Purpose: Copy constructor -// Created: 12/3/04 +// Name: EventWatchFilesystemObject::EventWatchFilesystemObject +// (const EventWatchFilesystemObject &) +// Purpose: Copy constructor +// Created: 12/3/04 // // -------------------------------------------------------------------------- -EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy) +EventWatchFilesystemObject::EventWatchFilesystemObject( + const EventWatchFilesystemObject &rToCopy) : mDescriptor(::dup(rToCopy.mDescriptor)) { if(mDescriptor == -1) @@ -89,17 +91,20 @@ EventWatchFilesystemObject::EventWatchFilesystemObject(const EventWatchFilesyste // -------------------------------------------------------------------------- // // Function -// Name: EventWatchFilesystemObject::FillInKEvent(struct kevent &, int) -// Purpose: For WaitForEvent -// Created: 12/3/04 +// Name: EventWatchFilesystemObject::FillInKEvent(struct kevent &, int) +// Purpose: For WaitForEvent +// Created: 12/3/04 // // -------------------------------------------------------------------------- -void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, int Flags) const +void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, + int Flags) const { - EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, NOTE_DELETE | NOTE_WRITE, 0, (void*)this); + EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, + NOTE_DELETE | NOTE_WRITE, 0, (void*)this); } #else -void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, int Flags) const +void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, + int Flags) const { THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) } diff --git a/lib/common/FdGetLine.h b/lib/common/FdGetLine.h index a18007a3..df43c3c9 100644 --- a/lib/common/FdGetLine.h +++ b/lib/common/FdGetLine.h @@ -12,7 +12,7 @@ #include -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define FDGETLINE_BUFFER_SIZE 1024 #elif defined WIN32 // need enough space for at least one unicode character diff --git a/lib/common/FileModificationTime.h b/lib/common/FileModificationTime.h index a84df579..5f13c015 100644 --- a/lib/common/FileModificationTime.h +++ b/lib/common/FileModificationTime.h @@ -14,7 +14,7 @@ #include "BoxTime.h" -inline box_time_t FileModificationTime(struct stat &st) +inline box_time_t FileModificationTime(EMU_STRUCT_STAT &st) { #ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); @@ -26,19 +26,26 @@ inline box_time_t FileModificationTime(struct stat &st) return datamodified; } -inline box_time_t FileAttrModificationTime(struct stat &st) +inline box_time_t FileAttrModificationTime(EMU_STRUCT_STAT &st) { -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC - box_time_t statusmodified = ((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL); -#else - box_time_t statusmodified = (((int64_t)st.st_ctimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) - + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); + box_time_t statusmodified = +#ifdef HAVE_STRUCT_STAT_ST_MTIMESPEC + (((int64_t)st.st_ctimespec.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC + (((int64_t)st.st_ctim.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctim.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIMENSEC + (((int64_t)st.st_ctimensec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); +#else // no nanoseconds anywhere + (((int64_t)st.st_ctime) * (MICRO_SEC_IN_SEC_LL)); #endif return statusmodified; } -inline box_time_t FileModificationTimeMaxModAndAttr(struct stat &st) +inline box_time_t FileModificationTimeMaxModAndAttr(EMU_STRUCT_STAT &st) { #ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp index e0806e10..d6a3c5da 100644 --- a/lib/common/FileStream.cpp +++ b/lib/common/FileStream.cpp @@ -24,13 +24,40 @@ // Created: 2003/07/31 // // -------------------------------------------------------------------------- -FileStream::FileStream(const char *Filename, int flags, int mode) +FileStream::FileStream(const std::string& rFilename, int flags, int mode) #ifdef WIN32 - : mOSFileHandle(::openfile(Filename, flags, mode)), + : mOSFileHandle(::openfile(rFilename.c_str(), flags, mode)), #else - : mOSFileHandle(::open(Filename, flags, mode)), + : mOSFileHandle(::open(rFilename.c_str(), flags, mode)), #endif - mIsEOF(false) + mIsEOF(false), + mFileName(rFilename) +{ + AfterOpen(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::FileStream(const char *, int, int) +// Purpose: Alternative constructor, takes a const char *, +// avoids const strings being interpreted as handles! +// Created: 2003/07/31 +// +// -------------------------------------------------------------------------- +FileStream::FileStream(const char *pFilename, int flags, int mode) +#ifdef WIN32 + : mOSFileHandle(::openfile(pFilename, flags, mode)), +#else + : mOSFileHandle(::open(pFilename, flags, mode)), +#endif + mIsEOF(false), + mFileName(pFilename) +{ + AfterOpen(); +} + +void FileStream::AfterOpen() { #ifdef WIN32 if(mOSFileHandle == INVALID_HANDLE_VALUE) @@ -46,12 +73,16 @@ FileStream::FileStream(const char *Filename, int flags, int mode) } else { + #ifdef WIN32 + BOX_LOG_WIN_WARNING_NUMBER("Failed to open file: " << + mFileName, winerrno); + #else + BOX_LOG_SYS_WARNING("Failed to open file: " << + mFileName); + #endif THROW_EXCEPTION(CommonException, OSFileOpenError) } } -#ifdef WIN32 - this->fileName = Filename; -#endif } @@ -65,7 +96,8 @@ FileStream::FileStream(const char *Filename, int flags, int mode) // -------------------------------------------------------------------------- FileStream::FileStream(tOSFileHandle FileDescriptor) : mOSFileHandle(FileDescriptor), - mIsEOF(false) + mIsEOF(false), + mFileName("HANDLE") { #ifdef WIN32 if(mOSFileHandle == INVALID_HANDLE_VALUE) @@ -77,9 +109,6 @@ FileStream::FileStream(tOSFileHandle FileDescriptor) BOX_ERROR("FileStream: called with invalid file handle"); THROW_EXCEPTION(CommonException, OSFileOpenError) } -#ifdef WIN32 - this->fileName = "HANDLE"; -#endif } #if 0 @@ -150,27 +179,32 @@ int FileStream::Read(void *pBuffer, int NBytes, int Timeout) NULL ); - if ( valid ) + if(valid) { r = numBytesRead; } - else if (GetLastError() == ERROR_BROKEN_PIPE) + else if(GetLastError() == ERROR_BROKEN_PIPE) { r = 0; } else { - BOX_ERROR("Failed to read from file: " << - GetErrorMessage(GetLastError())); + BOX_LOG_WIN_ERROR("Failed to read from file: " << mFileName); r = -1; } #else int r = ::read(mOSFileHandle, pBuffer, NBytes); + if(r == -1) + { + BOX_LOG_SYS_ERROR("Failed to read from file: " << mFileName); + } #endif + if(r == -1) { THROW_EXCEPTION(CommonException, OSFileReadError) } + if(r == 0) { mIsEOF = true; @@ -190,8 +224,8 @@ int FileStream::Read(void *pBuffer, int NBytes, int Timeout) // -------------------------------------------------------------------------- IOStream::pos_type FileStream::BytesLeftToRead() { - struct stat st; - if(::fstat(mOSFileHandle, &st) != 0) + EMU_STRUCT_STAT st; + if(EMU_FSTAT(mOSFileHandle, &st) != 0) { THROW_EXCEPTION(CommonException, OSFileError) } @@ -233,6 +267,7 @@ void FileStream::Write(const void *pBuffer, int NBytes) #else if(::write(mOSFileHandle, pBuffer, NBytes) != NBytes) { + BOX_LOG_SYS_ERROR("Failed to write to file: " << mFileName); THROW_EXCEPTION(CommonException, OSFileWriteError) } #endif @@ -368,3 +403,44 @@ bool FileStream::StreamClosed() return mIsEOF; } +// -------------------------------------------------------------------------- +// +// Function +// Name: FileStream::CompareWith(IOStream&, int) +// Purpose: Compare bytes in this file with other stream's data +// Created: 2009/01/03 +// +// -------------------------------------------------------------------------- +bool FileStream::CompareWith(IOStream& rOther, int Timeout) +{ + // Size + IOStream::pos_type mySize = BytesLeftToRead(); + IOStream::pos_type otherSize = 0; + + // Test the contents + char buf1[2048]; + char buf2[2048]; + while(StreamDataLeft() && rOther.StreamDataLeft()) + { + int readSize = rOther.Read(buf1, sizeof(buf1), Timeout); + otherSize += readSize; + + if(Read(buf2, readSize) != readSize || + ::memcmp(buf1, buf2, readSize) != 0) + { + return false; + } + } + + // Check read all the data from the server and file -- can't be + // equal if local and remote aren't the same length. Can't use + // StreamDataLeft() test on local file, because if it's the same + // size, it won't know it's EOF yet. + + if(rOther.StreamDataLeft() || otherSize != mySize) + { + return false; + } + + return true; +} diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h index 721bf3dd..7c4118cd 100644 --- a/lib/common/FileStream.h +++ b/lib/common/FileStream.h @@ -31,13 +31,17 @@ class FileStream : public IOStream { public: - FileStream(const char *Filename, -#ifdef WIN32 + FileStream(const std::string& rFilename, int flags = (O_RDONLY | O_BINARY), -#else - int flags = O_RDONLY, -#endif int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + + // Ensure that const char * name doesn't end up as a handle + // on Windows! + + FileStream(const char *pFilename, + int flags = (O_RDONLY | O_BINARY), + int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); + FileStream(tOSFileHandle FileDescriptor); virtual ~FileStream(); @@ -52,15 +56,16 @@ public: virtual bool StreamDataLeft(); virtual bool StreamClosed(); + bool CompareWith(IOStream& rOther, int Timeout = IOStream::TimeOutInfinite); + private: tOSFileHandle mOSFileHandle; bool mIsEOF; FileStream(const FileStream &rToCopy) { /* do not call */ } + void AfterOpen(); -#ifdef WIN32 // for debugging.. - std::string fileName; -#endif + std::string mFileName; }; diff --git a/lib/common/Guards.h b/lib/common/Guards.h index d2fb84e0..cd2e4628 100644 --- a/lib/common/Guards.h +++ b/lib/common/Guards.h @@ -37,8 +37,8 @@ public: { if(mOSFileHandle < 0) { - BOX_ERROR("FileHandleGuard: failed to open file '" << - rFilename << "': " << strerror(errno)); + BOX_LOG_SYS_ERROR("FileHandleGuard: failed to open " + "file '" << rFilename << "'"); THROW_EXCEPTION(CommonException, OSFileOpenError) } } diff --git a/lib/common/IOStream.cpp b/lib/common/IOStream.cpp index 3c7be561..fc9d0bc3 100644 --- a/lib/common/IOStream.cpp +++ b/lib/common/IOStream.cpp @@ -26,19 +26,6 @@ IOStream::IOStream() { } -// -------------------------------------------------------------------------- -// -// Function -// Name: IOStream::IOStream(const IOStream &) -// Purpose: Copy constructor (exceptions) -// Created: 2003/07/31 -// -// -------------------------------------------------------------------------- -IOStream::IOStream(const IOStream &rToCopy) -{ - THROW_EXCEPTION(CommonException, NotSupported) -} - // -------------------------------------------------------------------------- // // Function @@ -238,4 +225,27 @@ bool IOStream::CopyStreamTo(IOStream &rCopyTo, int Timeout, int BufferSize) return true; // completed } +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::Flush(int Timeout) +// Purpose: Read and discard all remaining data in stream. +// Useful for protocol streams which must be flushed +// to avoid breaking the protocol. +// Created: 2008/08/20 +// +// -------------------------------------------------------------------------- +void IOStream::Flush(int Timeout) +{ + char buffer[4096]; + while(StreamDataLeft()) + { + Read(buffer, sizeof(buffer), Timeout); + } +} + +void IOStream::Write(const char *pBuffer) +{ + Write(pBuffer, strlen(pBuffer)); +} diff --git a/lib/common/IOStream.h b/lib/common/IOStream.h index 042ccca4..0b1cedd3 100644 --- a/lib/common/IOStream.h +++ b/lib/common/IOStream.h @@ -22,9 +22,13 @@ class IOStream { public: IOStream(); - IOStream(const IOStream &rToCopy); virtual ~IOStream(); - + +private: + IOStream(const IOStream &rToCopy); /* forbidden */ + IOStream& operator=(const IOStream &rToCopy); /* forbidden */ + +public: enum { TimeOutInfinite = -1, @@ -44,6 +48,7 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite) = 0; virtual pos_type BytesLeftToRead(); // may return IOStream::SizeOfStreamUnknown (and will for most stream types) virtual void Write(const void *pBuffer, int NBytes) = 0; + virtual void Write(const char *pBuffer); virtual void WriteAllBuffered(); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); @@ -57,6 +62,7 @@ public: // Utility functions bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout = IOStream::TimeOutInfinite); bool CopyStreamTo(IOStream &rCopyTo, int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024); + void Flush(int Timeout = IOStream::TimeOutInfinite); static int ConvertSeekTypeToOSWhence(int SeekType); }; diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h index cf152e5a..9a5d1818 100644 --- a/lib/common/IOStreamGetLine.h +++ b/lib/common/IOStreamGetLine.h @@ -14,7 +14,7 @@ #include "IOStream.h" -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define IOSTREAMGETLINE_BUFFER_SIZE 1024 #else #define IOSTREAMGETLINE_BUFFER_SIZE 4 diff --git a/lib/common/Logging.cpp b/lib/common/Logging.cpp index 2b81b52b..1f872d93 100644 --- a/lib/common/Logging.cpp +++ b/lib/common/Logging.cpp @@ -15,12 +15,15 @@ #ifdef HAVE_SYSLOG_H #include #endif +#ifdef HAVE_UNISTD_H + #include +#endif -#include "Logging.h" - +#include #include #include "BoxTime.h" +#include "Logging.h" bool Logging::sLogToSyslog = false; bool Logging::sLogToConsole = false; @@ -32,6 +35,7 @@ Console* Logging::spConsole = NULL; Syslog* Logging::spSyslog = NULL; Log::Level Logging::sGlobalLevel = Log::EVERYTHING; Logging Logging::sGlobalLogging; //automatic initialisation +std::string Logging::sProgramName; Logging::Logging() { @@ -148,12 +152,54 @@ void Logging::Log(Log::Level level, const std::string& rFile, } } +void Logging::LogToSyslog(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage) +{ + if (!sLogToSyslog) + { + return; + } + + if (level > sGlobalLevel) + { + return; + } + + std::string newMessage; + + if (sContextSet) + { + newMessage += "[" + sContext + "] "; + } + + newMessage += rMessage; + + spSyslog->Log(level, rFile, line, newMessage); +} + void Logging::SetContext(std::string context) { sContext = context; sContextSet = true; } +Log::Level Logging::GetNamedLevel(const std::string& rName) +{ + if (rName == "nothing") { return Log::NOTHING; } + else if (rName == "fatal") { return Log::FATAL; } + else if (rName == "error") { return Log::ERROR; } + else if (rName == "warning") { return Log::WARNING; } + else if (rName == "notice") { return Log::NOTICE; } + else if (rName == "info") { return Log::INFO; } + else if (rName == "trace") { return Log::TRACE; } + else if (rName == "everything") { return Log::EVERYTHING; } + else + { + BOX_ERROR("Unknown verbosity level: " << rName); + return Log::INVALID; + } +} + void Logging::ClearContext() { sContextSet = false; @@ -161,6 +207,8 @@ void Logging::ClearContext() void Logging::SetProgramName(const std::string& rProgramName) { + sProgramName = rProgramName; + for (std::vector::iterator i = sLoggers.begin(); i != sLoggers.end(); i++) { @@ -168,12 +216,23 @@ void Logging::SetProgramName(const std::string& rProgramName) } } +void Logging::SetFacility(int facility) +{ + spSyslog->SetFacility(facility); +} + Logger::Logger() : mCurrentLevel(Log::EVERYTHING) { Logging::Add(this); } +Logger::Logger(Log::Level Level) +: mCurrentLevel(Level) +{ + Logging::Add(this); +} + Logger::~Logger() { Logging::Remove(this); @@ -182,12 +241,17 @@ Logger::~Logger() bool Console::sShowTime = false; bool Console::sShowTimeMicros = false; bool Console::sShowTag = false; +bool Console::sShowPID = false; std::string Console::sTag; -void Console::SetTag(const std::string& rTag) +void Console::SetProgramName(const std::string& rProgramName) +{ + sTag = rProgramName; +} + +void Console::SetShowTag(bool enabled) { - sTag = rTag; - sShowTag = true; + sShowTag = enabled; } void Console::SetShowTime(bool enabled) @@ -200,6 +264,11 @@ void Console::SetShowTimeMicros(bool enabled) sShowTimeMicros = enabled; } +void Console::SetShowPID(bool enabled) +{ + sShowPID = enabled; +} + bool Console::Log(Log::Level level, const std::string& rFile, int line, std::string& rMessage) { @@ -215,69 +284,64 @@ bool Console::Log(Log::Level level, const std::string& rFile, target = stderr; } - std::string msg; + std::ostringstream buf; if (sShowTime) { - box_time_t time_now = GetCurrentBoxTime(); - time_t seconds = BoxTimeToSeconds(time_now); - int micros = BoxTimeToMicroSeconds(time_now) % MICRO_SEC_IN_SEC; - - struct tm tm_now, *tm_ptr = &tm_now; + buf << FormatTime(GetCurrentBoxTime(), false, sShowTimeMicros); + buf << " "; + } - #ifdef WIN32 - if ((tm_ptr = localtime(&seconds)) != NULL) - #else - if (localtime_r(&seconds, &tm_now) != NULL) - #endif + if (sShowTag) + { + if (sShowPID) { - std::ostringstream buf; - - buf << std::setfill('0') << - std::setw(2) << tm_ptr->tm_hour << ":" << - std::setw(2) << tm_ptr->tm_min << ":" << - std::setw(2) << tm_ptr->tm_sec; - - if (sShowTimeMicros) - { - buf << "." << std::setw(6) << micros; - } - - buf << " "; - msg += buf.str(); + buf << "[" << sTag << " " << getpid() << "] "; } else { - msg += strerror(errno); - msg += " "; + buf << "[" << sTag << "] "; } } - - if (sShowTag) + else if (sShowPID) { - msg += "[" + sTag + "] "; + buf << "[" << getpid() << "] "; } if (level <= Log::FATAL) { - msg += "FATAL: "; + buf << "FATAL: "; } else if (level <= Log::ERROR) { - msg += "ERROR: "; + buf << "ERROR: "; } else if (level <= Log::WARNING) { - msg += "WARNING: "; + buf << "WARNING: "; } else if (level <= Log::NOTICE) { - msg += "NOTICE: "; + buf << "NOTICE: "; } - - msg += rMessage; + else if (level <= Log::INFO) + { + buf << "INFO: "; + } + else if (level <= Log::TRACE) + { + buf << "TRACE: "; + } + + buf << rMessage; - fprintf(target, "%s\n", msg.c_str()); + #ifdef WIN32 + std::string output = buf.str(); + ConvertUtf8ToConsole(output.c_str(), output); + fprintf(target, "%s\n", output.c_str()); + #else + fprintf(target, "%s\n", buf.str().c_str()); + #endif return true; } @@ -295,6 +359,7 @@ bool Syslog::Log(Log::Level level, const std::string& rFile, switch(level) { case Log::NOTHING: /* fall through */ + case Log::INVALID: /* fall through */ case Log::FATAL: syslogLevel = LOG_CRIT; break; case Log::ERROR: syslogLevel = LOG_ERR; break; case Log::WARNING: syslogLevel = LOG_WARNING; break; @@ -330,9 +395,9 @@ bool Syslog::Log(Log::Level level, const std::string& rFile, return true; } -Syslog::Syslog() +Syslog::Syslog() : mFacility(LOG_LOCAL6) { - ::openlog("Box Backup", LOG_PID, LOG_LOCAL6); + ::openlog("Box Backup", LOG_PID, mFacility); } Syslog::~Syslog() @@ -344,5 +409,83 @@ void Syslog::SetProgramName(const std::string& rProgramName) { mName = rProgramName; ::closelog(); - ::openlog(mName.c_str(), LOG_PID, LOG_LOCAL6); + ::openlog(mName.c_str(), LOG_PID, mFacility); +} + +void Syslog::SetFacility(int facility) +{ + mFacility = facility; + ::closelog(); + ::openlog(mName.c_str(), LOG_PID, mFacility); +} + +int Syslog::GetNamedFacility(const std::string& rFacility) +{ + #define CASE_RETURN(x) if (rFacility == #x) { return LOG_ ## x; } + CASE_RETURN(LOCAL0) + CASE_RETURN(LOCAL1) + CASE_RETURN(LOCAL2) + CASE_RETURN(LOCAL3) + CASE_RETURN(LOCAL4) + CASE_RETURN(LOCAL5) + CASE_RETURN(LOCAL6) + CASE_RETURN(DAEMON) + #undef CASE_RETURN + + BOX_ERROR("Unknown log facility '" << rFacility << "', " + "using default LOCAL6"); + return LOG_LOCAL6; +} + +bool FileLogger::Log(Log::Level Level, const std::string& rFile, + int line, std::string& rMessage) +{ + if (Level > GetLevel()) + { + return true; + } + + /* avoid infinite loop if this throws an exception */ + Logging::Remove(this); + + std::ostringstream buf; + buf << FormatTime(GetCurrentBoxTime(), true, false); + buf << " "; + + if (Level <= Log::FATAL) + { + buf << "[FATAL] "; + } + else if (Level <= Log::ERROR) + { + buf << "[ERROR] "; + } + else if (Level <= Log::WARNING) + { + buf << "[WARNING] "; + } + else if (Level <= Log::NOTICE) + { + buf << "[NOTICE] "; + } + else if (Level <= Log::INFO) + { + buf << "[INFO] "; + } + else if (Level <= Log::TRACE) + { + buf << "[TRACE] "; + } + + buf << rMessage << "\n"; + std::string output = buf.str(); + + #ifdef WIN32 + ConvertUtf8ToConsole(output.c_str(), output); + #endif + + mLogFile.Write(output.c_str(), output.length()); + + Logging::Add(this); + return true; } diff --git a/lib/common/Logging.h b/lib/common/Logging.h index 78db2bea..9bb2cf6c 100644 --- a/lib/common/Logging.h +++ b/lib/common/Logging.h @@ -10,10 +10,14 @@ #ifndef LOGGING__H #define LOGGING__H +#include +#include #include #include #include +#include "FileStream.h" + /* #define BOX_LOG(level, stuff) \ { \ @@ -27,9 +31,16 @@ #define BOX_LOG(level, stuff) \ { \ - std::ostringstream line; \ - line << stuff; \ - Logging::Log(level, __FILE__, __LINE__, line.str()); \ + std::ostringstream _box_log_line; \ + _box_log_line << stuff; \ + Logging::Log(level, __FILE__, __LINE__, _box_log_line.str()); \ +} + +#define BOX_SYSLOG(level, stuff) \ +{ \ + std::ostringstream _box_log_line; \ + _box_log_line << stuff; \ + Logging::LogToSyslog(level, __FILE__, __LINE__, _box_log_line.str()); \ } #define BOX_FATAL(stuff) BOX_LOG(Log::FATAL, stuff) @@ -41,15 +52,56 @@ if (Logging::IsEnabled(Log::TRACE)) \ { BOX_LOG(Log::TRACE, stuff) } -#define BOX_FORMAT_ACCOUNT(accno) \ +#define BOX_LOG_SYS_WARNING(stuff) \ + BOX_WARNING(stuff << ": " << std::strerror(errno) << " (" << errno << ")") +#define BOX_LOG_SYS_ERROR(stuff) \ + BOX_ERROR(stuff << ": " << std::strerror(errno) << " (" << errno << ")") +#define BOX_LOG_SYS_FATAL(stuff) \ + BOX_FATAL(stuff << ": " << std::strerror(errno) << " (" << errno << ")") + +inline std::string GetNativeErrorMessage() +{ +#ifdef WIN32 + return GetErrorMessage(GetLastError()); +#else + std::ostringstream _box_log_line; + _box_log_line << std::strerror(errno) << " (" << errno << ")"; + return _box_log_line.str(); +#endif +} + +#ifdef WIN32 + #define BOX_LOG_WIN_ERROR(stuff) \ + BOX_ERROR(stuff << ": " << GetErrorMessage(GetLastError())) + #define BOX_LOG_WIN_WARNING(stuff) \ + BOX_WARNING(stuff << ": " << GetErrorMessage(GetLastError())) + #define BOX_LOG_WIN_ERROR_NUMBER(stuff, number) \ + BOX_ERROR(stuff << ": " << GetErrorMessage(number)) + #define BOX_LOG_WIN_WARNING_NUMBER(stuff, number) \ + BOX_WARNING(stuff << ": " << GetErrorMessage(number)) + #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_WIN_ERROR(stuff) + #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_WIN_WARNING(stuff) +#else + #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_SYS_ERROR(stuff) + #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_SYS_WARNING(stuff) +#endif + +#define BOX_LOG_SOCKET_ERROR(_type, _name, _port, stuff) \ + BOX_LOG_NATIVE_ERROR(stuff << " (type " << _type << ", name " << \ + _name << ", port " << _port << ")") + +#define BOX_FORMAT_HEX32(number) \ std::hex << \ std::showbase << \ std::internal << \ std::setw(10) << \ std::setfill('0') << \ - (accno) << \ + (number) << \ std::dec +#define BOX_FORMAT_ACCOUNT(accno) \ + BOX_FORMAT_HEX32(accno) + #define BOX_FORMAT_OBJECTID(objectid) \ std::hex << \ std::showbase << \ @@ -69,7 +121,8 @@ namespace Log NOTICE, INFO, TRACE, - EVERYTHING + EVERYTHING, + INVALID = -1 }; } @@ -89,6 +142,7 @@ class Logger public: Logger(); + Logger(Log::Level level); virtual ~Logger(); virtual bool Log(Log::Level level, const std::string& rFile, @@ -117,20 +171,22 @@ class Logger class Console : public Logger { private: + static bool sShowTag; static bool sShowTime; static bool sShowTimeMicros; - static bool sShowTag; + static bool sShowPID; static std::string sTag; public: virtual bool Log(Log::Level level, const std::string& rFile, int line, std::string& rMessage); virtual const char* GetType() { return "Console"; } - virtual void SetProgramName(const std::string& rProgramName) { } + virtual void SetProgramName(const std::string& rProgramName); - static void SetTag(const std::string& rTag); + static void SetShowTag(bool enabled); static void SetShowTime(bool enabled); static void SetShowTimeMicros(bool enabled); + static void SetShowPID(bool enabled); }; // -------------------------------------------------------------------------- @@ -146,6 +202,7 @@ class Syslog : public Logger { private: std::string mName; + int mFacility; public: Syslog(); @@ -155,6 +212,8 @@ class Syslog : public Logger int line, std::string& rMessage); virtual const char* GetType() { return "Syslog"; } virtual void SetProgramName(const std::string& rProgramName); + virtual void SetFacility(int facility); + static int GetNamedFacility(const std::string& rFacility); }; // -------------------------------------------------------------------------- @@ -178,6 +237,7 @@ class Logging static Syslog* spSyslog; static Log::Level sGlobalLevel; static Logging sGlobalLogging; + static std::string sProgramName; public: Logging (); @@ -190,14 +250,74 @@ class Logging static void Remove (Logger* pOldLogger); static void Log(Log::Level level, const std::string& rFile, int line, const std::string& rMessage); + static void LogToSyslog(Log::Level level, const std::string& rFile, + int line, const std::string& rMessage); static void SetContext(std::string context); static void ClearContext(); static void SetGlobalLevel(Log::Level level) { sGlobalLevel = level; } + static Log::Level GetGlobalLevel() { return sGlobalLevel; } + static Log::Level GetNamedLevel(const std::string& rName); static bool IsEnabled(Log::Level level) { return (int)sGlobalLevel >= (int)level; } static void SetProgramName(const std::string& rProgramName); + static std::string GetProgramName() { return sProgramName; } + static void SetFacility(int facility); + + class Guard + { + private: + Log::Level mOldLevel; + + public: + Guard(Log::Level newLevel) + { + mOldLevel = Logging::GetGlobalLevel(); + Logging::SetGlobalLevel(newLevel); + } + ~Guard() + { + Logging::SetGlobalLevel(mOldLevel); + } + }; + + class Tagger + { + private: + std::string mOldTag; + + public: + Tagger(const std::string& rTempTag) + { + mOldTag = Logging::GetProgramName(); + Logging::SetProgramName(mOldTag + " " + rTempTag); + } + ~Tagger() + { + Logging::SetProgramName(mOldTag); + } + }; +}; + +class FileLogger : public Logger +{ + private: + FileStream mLogFile; + FileLogger(const FileLogger& forbidden) + : mLogFile("") { /* do not call */ } + + public: + FileLogger(const std::string& rFileName, Log::Level Level) + : Logger(Level), + mLogFile(rFileName, O_WRONLY | O_CREAT | O_APPEND) + { } + + virtual bool Log(Log::Level Level, const std::string& rFile, + int Line, std::string& rMessage); + + virtual const char* GetType() { return "FileLogger"; } + virtual void SetProgramName(const std::string& rProgramName) { } }; #endif // LOGGING__H diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp index 16c580ba..b3d0009b 100644 --- a/lib/common/NamedLock.cpp +++ b/lib/common/NamedLock.cpp @@ -2,7 +2,8 @@ // // File // Name: NamedLock.cpp -// Purpose: A global named lock, implemented as a lock file in file system +// Purpose: A global named lock, implemented as a lock file in +// file system // Created: 2003/08/28 // // -------------------------------------------------------------------------- @@ -58,8 +59,9 @@ NamedLock::~NamedLock() // // Function // Name: NamedLock::TryAndGetLock(const char *, int) -// Purpose: Trys to get a lock on the name in the file system. -// IMPORTANT NOTE: If a file exists with this name, it will be deleted. +// Purpose: Tries to get a lock on the name in the file system. +// IMPORTANT NOTE: If a file exists with this name, it +// will be deleted. // Created: 2003/08/28 // // -------------------------------------------------------------------------- diff --git a/lib/common/PartialReadStream.cpp b/lib/common/PartialReadStream.cpp index 76096738..f2f79715 100644 --- a/lib/common/PartialReadStream.cpp +++ b/lib/common/PartialReadStream.cpp @@ -44,7 +44,8 @@ PartialReadStream::~PartialReadStream() // Warn in debug mode if(mBytesLeft != 0) { - TRACE1("PartialReadStream::~PartialReadStream when mBytesLeft = %d\n", mBytesLeft); + BOX_TRACE("PartialReadStream destroyed with " << mBytesLeft << + " bytes remaining"); } } diff --git a/lib/common/ReadLoggingStream.cpp b/lib/common/ReadLoggingStream.cpp index 9023f827..54c99c95 100644 --- a/lib/common/ReadLoggingStream.cpp +++ b/lib/common/ReadLoggingStream.cpp @@ -25,12 +25,13 @@ // Created: 2007/01/16 // // -------------------------------------------------------------------------- -ReadLoggingStream::ReadLoggingStream(IOStream& rSource) +ReadLoggingStream::ReadLoggingStream(IOStream& rSource, Logger& rLogger) : mrSource(rSource), mOffset(0), mLength(mrSource.BytesLeftToRead()), mTotalRead(0), - mStartTime(GetCurrentBoxTime()) + mStartTime(GetCurrentBoxTime()), + mrLogger(rLogger) { } @@ -52,26 +53,21 @@ int ReadLoggingStream::Read(void *pBuffer, int NBytes, int Timeout) mOffset += numBytesRead; } - if (mLength >= 0 && mTotalRead > 0) + if (mLength == 0) { - box_time_t timeNow = GetCurrentBoxTime(); - box_time_t elapsed = timeNow - mStartTime; - box_time_t finish = (elapsed * mLength) / mTotalRead; - box_time_t remain = finish - elapsed; - - BOX_TRACE("Read " << numBytesRead << " bytes at " << mOffset << - ", " << (mLength - mOffset) << " remain, eta " << - BoxTimeToSeconds(remain) << "s"); + mrLogger.Log(numBytesRead, mOffset); } - else if (mLength >= 0 && mTotalRead == 0) + else if (mTotalRead == 0) { - BOX_TRACE("Read " << numBytesRead << " bytes at " << mOffset << - ", " << (mLength - mOffset) << " remain"); + mrLogger.Log(numBytesRead, mOffset, mLength); } else { - BOX_TRACE("Read " << numBytesRead << " bytes at " << mOffset << - ", unknown bytes remaining"); + box_time_t timeNow = GetCurrentBoxTime(); + box_time_t elapsed = timeNow - mStartTime; + box_time_t finish = (elapsed * mLength) / mTotalRead; + // box_time_t remain = finish - elapsed; + mrLogger.Log(numBytesRead, mOffset, mLength, elapsed, finish); } return numBytesRead; diff --git a/lib/common/ReadLoggingStream.h b/lib/common/ReadLoggingStream.h index 15c3ef48..b23b542c 100644 --- a/lib/common/ReadLoggingStream.h +++ b/lib/common/ReadLoggingStream.h @@ -15,13 +15,27 @@ class ReadLoggingStream : public IOStream { +public: + class Logger + { + public: + virtual ~Logger() { } + virtual void Log(int64_t readSize, int64_t offset, + int64_t length, box_time_t elapsed, + box_time_t finish) = 0; + virtual void Log(int64_t readSize, int64_t offset, + int64_t length) = 0; + virtual void Log(int64_t readSize, int64_t offset) = 0; + }; + private: IOStream& mrSource; IOStream::pos_type mOffset, mLength, mTotalRead; box_time_t mStartTime; + Logger& mrLogger; public: - ReadLoggingStream(IOStream& rSource); + ReadLoggingStream(IOStream& rSource, Logger& rLogger); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); @@ -35,7 +49,8 @@ public: private: ReadLoggingStream(const ReadLoggingStream &rToCopy) - : mrSource(rToCopy.mrSource) { /* do not call */ } + : mrSource(rToCopy.mrSource), mrLogger(rToCopy.mrLogger) + { /* do not call */ } }; #endif // READLOGGINGSTREAM__H diff --git a/lib/common/SelfFlushingStream.h b/lib/common/SelfFlushingStream.h new file mode 100644 index 00000000..36e9a4d3 --- /dev/null +++ b/lib/common/SelfFlushingStream.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: SelfFlushingStream.h +// Purpose: A stream wrapper that always flushes the underlying +// stream, to ensure protocol safety. +// Created: 2008/08/20 +// +// -------------------------------------------------------------------------- + +#ifndef SELFFLUSHINGSTREAM__H +#define SELFFLUSHINGSTREAM__H + +#include "IOStream.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: SelfFlushingStream +// Purpose: A stream wrapper that always flushes the underlying +// stream, to ensure protocol safety. +// Created: 2008/08/20 +// +// -------------------------------------------------------------------------- +class SelfFlushingStream : public IOStream +{ +public: + SelfFlushingStream(IOStream &rSource) + : mrSource(rSource) { } + + SelfFlushingStream(const SelfFlushingStream &rToCopy) + : mrSource(rToCopy.mrSource) { } + + ~SelfFlushingStream() + { + Flush(); + } + +private: + // no copying from IOStream allowed + SelfFlushingStream(const IOStream& rToCopy); + +public: + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + return mrSource.Read(pBuffer, NBytes, Timeout); + } + virtual pos_type BytesLeftToRead() + { + return mrSource.BytesLeftToRead(); + } + virtual void Write(const void *pBuffer, int NBytes) + { + mrSource.Write(pBuffer, NBytes); + } + virtual bool StreamDataLeft() + { + return mrSource.StreamDataLeft(); + } + virtual bool StreamClosed() + { + return mrSource.StreamClosed(); + } + +private: + IOStream &mrSource; +}; + +#endif // SELFFLUSHINGSTREAM__H + diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp new file mode 100644 index 00000000..04d778c1 --- /dev/null +++ b/lib/common/Test.cpp @@ -0,0 +1,418 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: Test.cpp +// Purpose: Useful stuff for tests +// Created: 2008/04/05 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include +#include +#include + +#include +#include + +#ifdef HAVE_UNISTD_H + #include +#endif + +#include "Test.h" + +bool TestFileExists(const char *Filename) +{ + EMU_STRUCT_STAT st; + return EMU_STAT(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0; +} + +bool TestFileNotEmpty(const char *Filename) +{ + EMU_STRUCT_STAT st; + return EMU_STAT(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0 && + st.st_size > 0; +} + +bool TestDirExists(const char *Filename) +{ + EMU_STRUCT_STAT st; + return EMU_STAT(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == S_IFDIR; +} + +// -1 if doesn't exist +int TestGetFileSize(const char *Filename) +{ + EMU_STRUCT_STAT st; + if(EMU_STAT(Filename, &st) == 0) + { + return st.st_size; + } + return -1; +} + +std::string ConvertPaths(const std::string& rOriginal) +{ +#ifdef WIN32 + // convert UNIX paths to native + + std::string converted; + for (size_t i = 0; i < rOriginal.size(); i++) + { + if (rOriginal[i] == '/') + { + converted += '\\'; + } + else + { + converted += rOriginal[i]; + } + } + return converted; + +#else // !WIN32 + return rOriginal; +#endif +} + +int RunCommand(const std::string& rCommandLine) +{ + return ::system(ConvertPaths(rCommandLine).c_str()); +} + +#ifdef WIN32 +#include +#endif + +bool ServerIsAlive(int pid) +{ + #ifdef WIN32 + + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, + false, pid); + if (hProcess == NULL) + { + if (GetLastError() != ERROR_INVALID_PARAMETER) + { + BOX_ERROR("Failed to open process " << pid << + ": " << + GetErrorMessage(GetLastError())); + } + return false; + } + + DWORD exitCode; + BOOL result = GetExitCodeProcess(hProcess, &exitCode); + CloseHandle(hProcess); + + if (result == 0) + { + BOX_ERROR("Failed to get exit code for process " << + pid << ": " << + GetErrorMessage(GetLastError())) + return false; + } + + if (exitCode == STILL_ACTIVE) + { + return true; + } + + return false; + + #else // !WIN32 + + if(pid == 0) return false; + return ::kill(pid, 0) != -1; + + #endif // WIN32 +} + +int ReadPidFile(const char *pidFile) +{ + if(!TestFileNotEmpty(pidFile)) + { + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file " + "(perhaps one was already running?)"); + return -1; + } + + int pid = -1; + + FILE *f = fopen(pidFile, "r"); + if(f == NULL || fscanf(f, "%d", &pid) != 1) + { + TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); + return -1; + } + fclose(f); + + return pid; +} + +int LaunchServer(const std::string& rCommandLine, const char *pidFile) +{ + ::fprintf(stdout, "Starting server: %s\n", rCommandLine.c_str()); + +#ifdef WIN32 + + PROCESS_INFORMATION procInfo; + + STARTUPINFO startInfo; + startInfo.cb = sizeof(startInfo); + startInfo.lpReserved = NULL; + startInfo.lpDesktop = NULL; + startInfo.lpTitle = NULL; + startInfo.dwFlags = 0; + startInfo.cbReserved2 = 0; + startInfo.lpReserved2 = NULL; + + std::string cmd = ConvertPaths(rCommandLine); + CHAR* tempCmd = strdup(cmd.c_str()); + + DWORD result = CreateProcess + ( + NULL, // lpApplicationName, naughty! + tempCmd, // lpCommandLine + NULL, // lpProcessAttributes + NULL, // lpThreadAttributes + false, // bInheritHandles + 0, // dwCreationFlags + NULL, // lpEnvironment + NULL, // lpCurrentDirectory + &startInfo, // lpStartupInfo + &procInfo // lpProcessInformation + ); + + free(tempCmd); + + if (result == 0) + { + DWORD err = GetLastError(); + printf("Launch failed: %s: error %d\n", rCommandLine.c_str(), + (int)err); + TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + return -1; + } + + CloseHandle(procInfo.hProcess); + CloseHandle(procInfo.hThread); + + return WaitForServerStartup(pidFile, (int)procInfo.dwProcessId); + +#else // !WIN32 + + if(RunCommand(rCommandLine) != 0) + { + TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + return -1; + } + + return WaitForServerStartup(pidFile, 0); + +#endif // WIN32 +} + +int WaitForServerStartup(const char *pidFile, int pidIfKnown) +{ + #ifdef WIN32 + if (pidFile == NULL) + { + return pidIfKnown; + } + #else + // on other platforms there is no other way to get + // the PID, so a NULL pidFile doesn't make sense. + ASSERT(pidFile != NULL); + #endif + + // time for it to start up + ::fprintf(stdout, "Waiting for server to start: "); + + for (int i = 0; i < 15; i++) + { + if (TestFileNotEmpty(pidFile)) + { + break; + } + + if (pidIfKnown && !ServerIsAlive(pidIfKnown)) + { + break; + } + + ::fprintf(stdout, "."); + ::fflush(stdout); + ::sleep(1); + } + + // on Win32 we can check whether the process is alive + // without even checking the PID file + + if (pidIfKnown && !ServerIsAlive(pidIfKnown)) + { + ::fprintf(stdout, " server died!\n"); + TEST_FAIL_WITH_MESSAGE("Server died!"); + return -1; + } + + if (!TestFileNotEmpty(pidFile)) + { + ::fprintf(stdout, " timed out!\n"); + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); + return -1; + } + + ::fprintf(stdout, " done.\n"); + + // wait a second for the pid to be written to the file + ::sleep(1); + + // read pid file + int pid = ReadPidFile(pidFile); + + // On Win32 we can check whether the PID in the pidFile matches + // the one returned by the system, which it always should. + + if (pidIfKnown && pid != pidIfKnown) + { + printf("Server wrote wrong pid to file (%s): expected %d " + "but found %d\n", pidFile, pidIfKnown, pid); + TEST_FAIL_WITH_MESSAGE("Server wrote wrong pid to file"); + return -1; + } + + return pid; +} + +void TestRemoteProcessMemLeaksFunc(const char *filename, + const char* file, int line) +{ +#ifdef BOX_MEMORY_LEAK_TESTING + // Does the file exist? + if(!TestFileExists(filename)) + { + if (failures == 0) + { + first_fail_file = file; + first_fail_line = line; + } + ++failures; + printf("FAILURE: MemLeak report not available (file %s) " + "at %s:%d\n", filename, file, line); + } + else + { + // Is it empty? + if(TestGetFileSize(filename) > 0) + { + if (failures == 0) + { + first_fail_file = file; + first_fail_line = line; + } + ++failures; + printf("FAILURE: Memory leaks found in other process " + "(file %s) at %s:%d\n==========\n", + filename, file, line); + FILE *f = fopen(filename, "r"); + char linebuf[512]; + while(::fgets(linebuf, sizeof(linebuf), f) != 0) + { + printf("%s", linebuf); + } + fclose(f); + printf("==========\n"); + } + + // Delete it + ::unlink(filename); + } +#endif +} + +void force_sync() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "force-sync") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void wait_for_sync_start() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "wait-for-sync") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void wait_for_sync_end() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "wait-for-end") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void sync_and_wait() +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "sync-and-wait") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); +} + +void terminate_bbackupd(int pid) +{ + TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " + "terminate") == 0); + TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + + for (int i = 0; i < 20; i++) + { + if (!ServerIsAlive(pid)) break; + fprintf(stdout, "."); + fflush(stdout); + sleep(1); + } + + TEST_THAT(!ServerIsAlive(pid)); + TestRemoteProcessMemLeaks("bbackupd.memleaks"); +} + + +// Wait a given number of seconds for something to complete +void wait_for_operation(int seconds) +{ + printf("Waiting: "); + fflush(stdout); + for(int l = 0; l < seconds; ++l) + { + sleep(1); + printf("."); + fflush(stdout); + } + printf(" done.\n"); + fflush(stdout); +} + +void safe_sleep(int seconds) +{ +#ifdef WIN32 + Sleep(seconds * 1000); +#else + struct timespec ts; + memset(&ts, 0, sizeof(ts)); + ts.tv_sec = seconds; + ts.tv_nsec = 0; + BOX_TRACE("sleeping for " << seconds << " seconds"); + while (nanosleep(&ts, &ts) == -1 && errno == EINTR) + { + BOX_TRACE("safe_sleep interrupted with " << + ts.tv_sec << "." << ts.tv_nsec << + " secs remaining, sleeping again"); + /* sleep again */ + } +#endif +} + + diff --git a/lib/common/Test.h b/lib/common/Test.h index 49b0ac66..f4766ddc 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -10,18 +10,6 @@ #ifndef TEST__H #define TEST__H -#include -#include -#include -#include - -#include -#include - -#ifdef HAVE_UNISTD_H - #include -#endif - #include #ifdef WIN32 @@ -53,7 +41,8 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; first_fail_line = __LINE__; \ } \ failures++; \ - printf("FAILURE: " msg " at " __FILE__ "(%d)\n", __LINE__); \ + BOX_ERROR("**** TEST FAILURE: " << msg << " at " << __FILE__ << \ + ":" << __LINE__); \ } #define TEST_ABORT_WITH_MESSAGE(msg) {TEST_FAIL_WITH_MESSAGE(msg); return 1;} @@ -88,367 +77,88 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; } \ } -inline bool TestFileExists(const char *Filename) -{ - struct stat st; - return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == 0; -} - -inline bool TestDirExists(const char *Filename) -{ - struct stat st; - return ::stat(Filename, &st) == 0 && (st.st_mode & S_IFDIR) == S_IFDIR; -} - -// -1 if doesn't exist -inline int TestGetFileSize(const char *Filename) -{ - struct stat st; - if(::stat(Filename, &st) == 0) - { - return st.st_size; - } - return -1; -} - -inline std::string ConvertPaths(const std::string& rOriginal) -{ -#ifdef WIN32 - // convert UNIX paths to native - - std::string converted; - for (size_t i = 0; i < rOriginal.size(); i++) - { - if (rOriginal[i] == '/') - { - converted += '\\'; - } - else - { - converted += rOriginal[i]; - } - } - return converted; - -#else // !WIN32 - return rOriginal; -#endif -} - -inline int RunCommand(const std::string& rCommandLine) -{ - return ::system(ConvertPaths(rCommandLine).c_str()); -} - -#ifdef WIN32 -#include -#endif - -inline bool ServerIsAlive(int pid) -{ -#ifdef WIN32 - HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, false, pid); - if (hProcess == NULL) - { - if (GetLastError() != ERROR_INVALID_PARAMETER) - { - printf("Failed to open process %d: error %d\n", - pid, (int)GetLastError()); - } - return false; - } - CloseHandle(hProcess); - return true; -#else // !WIN32 - if(pid == 0) return false; - return ::kill(pid, 0) != -1; -#endif // WIN32 +// utility macro for comparing two strings in a line +#define TEST_EQUAL(_expected, _found) \ +{ \ + std::ostringstream _oss1; \ + _oss1 << _expected; \ + std::string _exp_str = _oss1.str(); \ + \ + std::ostringstream _oss2; \ + _oss2 << _found; \ + std::string _found_str = _oss2.str(); \ + \ + if(_exp_str != _found_str) \ + { \ + printf("Expected <%s> but found <%s>\n", \ + _exp_str.c_str(), _found_str.c_str()); \ + \ + std::ostringstream _oss3; \ + _oss3 << #_found << " != " << #_expected; \ + \ + TEST_FAIL_WITH_MESSAGE(_oss3.str().c_str()); \ + } \ } -inline int ReadPidFile(const char *pidFile) -{ - if(!TestFileExists(pidFile)) - { - TEST_FAIL_WITH_MESSAGE("Server didn't save PID file " - "(perhaps one was already running?)"); - return -1; - } - - int pid = -1; - - FILE *f = fopen(pidFile, "r"); - if(f == NULL || fscanf(f, "%d", &pid) != 1) - { - TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); - return -1; - } - fclose(f); - - return pid; +// utility macro for comparing two strings in a line +#define TEST_EQUAL_LINE(_expected, _found, _line) \ +{ \ + std::ostringstream _oss1; \ + _oss1 << _expected; \ + std::string _exp_str = _oss1.str(); \ + \ + std::ostringstream _oss2; \ + _oss2 << _found; \ + std::string _found_str = _oss2.str(); \ + \ + if(_exp_str != _found_str) \ + { \ + std::string _line_str = _line; \ + printf("Expected <%s> but found <%s> in <%s>\n", \ + _exp_str.c_str(), _found_str.c_str(), _line_str.c_str()); \ + \ + std::ostringstream _oss3; \ + _oss3 << #_found << " != " << #_expected << " in " << _line; \ + \ + TEST_FAIL_WITH_MESSAGE(_oss3.str().c_str()); \ + } \ } -inline int LaunchServer(const std::string& rCommandLine, const char *pidFile) -{ -#ifdef WIN32 - - PROCESS_INFORMATION procInfo; - - STARTUPINFO startInfo; - startInfo.cb = sizeof(startInfo); - startInfo.lpReserved = NULL; - startInfo.lpDesktop = NULL; - startInfo.lpTitle = NULL; - startInfo.dwFlags = 0; - startInfo.cbReserved2 = 0; - startInfo.lpReserved2 = NULL; - - std::string cmd = ConvertPaths(rCommandLine); - CHAR* tempCmd = strdup(cmd.c_str()); - - DWORD result = CreateProcess - ( - NULL, // lpApplicationName, naughty! - tempCmd, // lpCommandLine - NULL, // lpProcessAttributes - NULL, // lpThreadAttributes - false, // bInheritHandles - 0, // dwCreationFlags - NULL, // lpEnvironment - NULL, // lpCurrentDirectory - &startInfo, // lpStartupInfo - &procInfo // lpProcessInformation - ); - - free(tempCmd); - - if (result == 0) - { - DWORD err = GetLastError(); - printf("Launch failed: %s: error %d\n", rCommandLine.c_str(), - (int)err); - return -1; - } - - CloseHandle(procInfo.hProcess); - CloseHandle(procInfo.hThread); - -#else // !WIN32 - - if(RunCommand(rCommandLine) != 0) - { - printf("Server: %s\n", rCommandLine.c_str()); - TEST_FAIL_WITH_MESSAGE("Couldn't start server"); - return -1; - } - -#endif // WIN32 - - #ifdef WIN32 - // on other platforms there is no other way to get - // the PID, so a NULL pidFile doesn't make sense. - - if (pidFile == NULL) - { - return (int)procInfo.dwProcessId; - } - #endif - - // time for it to start up - ::fprintf(stdout, "Starting server: %s\n", rCommandLine.c_str()); - ::fprintf(stdout, "Waiting for server to start: "); - for (int i = 0; i < 15; i++) - { - if (TestFileExists(pidFile)) - { - break; - } - - #ifdef WIN32 - if (!ServerIsAlive((int)procInfo.dwProcessId)) - { - break; - } - #endif - - ::fprintf(stdout, "."); - ::fflush(stdout); - ::sleep(1); - } - - #ifdef WIN32 - // on Win32 we can check whether the process is alive - // without even checking the PID file - - if (!ServerIsAlive((int)procInfo.dwProcessId)) - { - ::fprintf(stdout, "server died!\n"); - TEST_FAIL_WITH_MESSAGE("Server died!"); - return -1; - } - #endif - - if (!TestFileExists(pidFile)) - { - ::fprintf(stdout, "timed out!\n"); - TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); - return -1; +// utility macro for testing a line +#define TEST_LINE(_condition, _line) \ + TEST_THAT(_condition); \ + if (!(_condition)) \ + { \ + printf("Test failed on <%s>\n", _line.c_str()); \ } - ::fprintf(stdout, "done.\n"); - - // wait a second for the pid to be written to the file - ::sleep(1); +bool TestFileExists(const char *Filename); +bool TestDirExists(const char *Filename); - // read pid file - int pid = ReadPidFile(pidFile); - - #ifdef WIN32 - // On Win32 we can check whether the PID in the pidFile matches - // the one returned by the system, which it always should. - - if (pid != (int)procInfo.dwProcessId) - { - printf("Server wrote wrong pid to file (%s): expected %d " - "but found %d\n", pidFile, - (int)procInfo.dwProcessId, pid); - TEST_FAIL_WITH_MESSAGE("Server wrote wrong pid to file"); - return -1; - } - #endif - - return pid; -} +// -1 if doesn't exist +int TestGetFileSize(const char *Filename); +std::string ConvertPaths(const std::string& rOriginal); +int RunCommand(const std::string& rCommandLine); +bool ServerIsAlive(int pid); +int ReadPidFile(const char *pidFile); +int LaunchServer(const std::string& rCommandLine, const char *pidFile); +int WaitForServerStartup(const char *pidFile, int pidIfKnown); #define TestRemoteProcessMemLeaks(filename) \ TestRemoteProcessMemLeaksFunc(filename, __FILE__, __LINE__) -inline void TestRemoteProcessMemLeaksFunc(const char *filename, - const char* file, int line) -{ -#ifdef BOX_MEMORY_LEAK_TESTING - // Does the file exist? - if(!TestFileExists(filename)) - { - if (failures == 0) - { - first_fail_file = file; - first_fail_line = line; - } - ++failures; - printf("FAILURE: MemLeak report not available (file %s) " - "at %s:%d\n", filename, file, line); - } - else - { - // Is it empty? - if(TestGetFileSize(filename) > 0) - { - if (failures == 0) - { - first_fail_file = file; - first_fail_line = line; - } - ++failures; - printf("FAILURE: Memory leaks found in other process " - "(file %s) at %s:%d\n==========\n", - filename, file, line); - FILE *f = fopen(filename, "r"); - char linebuf[512]; - while(::fgets(linebuf, sizeof(linebuf), f) != 0) - { - printf("%s", linebuf); - } - fclose(f); - printf("==========\n"); - } - - // Delete it - ::unlink(filename); - } -#endif -} - -inline void force_sync() -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "force-sync") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); -} - -inline void wait_for_sync_start() -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "wait-for-sync") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); -} - -inline void wait_for_sync_end() -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "wait-for-end") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); -} - -inline void sync_and_wait() -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "sync-and-wait") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); -} - -inline void terminate_bbackupd(int pid) -{ - TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " - "terminate") == 0); - TestRemoteProcessMemLeaks("bbackupctl.memleaks"); - - for (int i = 0; i < 20; i++) - { - if (!ServerIsAlive(pid)) break; - fprintf(stdout, "."); - fflush(stdout); - sleep(1); - } - - TEST_THAT(!ServerIsAlive(pid)); - TestRemoteProcessMemLeaks("bbackupd.memleaks"); -} +void TestRemoteProcessMemLeaksFunc(const char *filename, + const char* file, int line); +void force_sync(); +void wait_for_sync_start(); +void wait_for_sync_end(); +void sync_and_wait(); +void terminate_bbackupd(int pid); // Wait a given number of seconds for something to complete -inline void wait_for_operation(int seconds) -{ - printf("Waiting: "); - fflush(stdout); - for(int l = 0; l < seconds; ++l) - { - sleep(1); - printf("."); - fflush(stdout); - } - printf(" done.\n"); - fflush(stdout); -} - -inline void safe_sleep(int seconds) -{ -#ifdef WIN32 - Sleep(seconds * 1000); -#else - struct timespec ts; - memset(&ts, 0, sizeof(ts)); - ts.tv_sec = seconds; - ts.tv_nsec = 0; - BOX_TRACE("sleeping for " << seconds << " seconds"); - while (nanosleep(&ts, &ts) == -1 && errno == EINTR) - { - BOX_TRACE("safe_sleep interrupted with " << - ts.tv_sec << "." << ts.tv_nsec << - " secs remaining, sleeping again"); - /* sleep again */ - } -#endif -} +void wait_for_operation(int seconds); +void safe_sleep(int seconds); #endif // TEST__H diff --git a/lib/common/Timer.cpp b/lib/common/Timer.cpp index 16133ecc..137ad45f 100644 --- a/lib/common/Timer.cpp +++ b/lib/common/Timer.cpp @@ -8,9 +8,14 @@ // // -------------------------------------------------------------------------- +#ifdef WIN32 + #define _WIN32_WINNT 0x0500 +#endif + #include "Box.h" #include +#include #include "Timer.h" #include "Logging.h" @@ -20,6 +25,9 @@ std::vector* Timers::spTimers = NULL; bool Timers::sRescheduleNeeded = false; +#define TIMER_ID "timer " << mName << " (" << this << ") " +#define TIMER_ID_OF(t) "timer " << (t).GetName() << " (" << &(t) << ") " + typedef void (*sighandler_t)(int); // -------------------------------------------------------------------------- @@ -35,9 +43,7 @@ void Timers::Init() ASSERT(!spTimers); #if defined WIN32 && ! defined PLATFORM_CYGWIN - // no support for signals at all - InitTimer(); - SetTimerHandler(Timers::SignalHandler); + // no init needed #else struct sigaction newact, oldact; newact.sa_handler = Timers::SignalHandler; @@ -72,9 +78,7 @@ void Timers::Cleanup() } #if defined WIN32 && ! defined PLATFORM_CYGWIN - // no support for signals at all - FiniTimer(); - SetTimerHandler(NULL); + // no cleanup needed #else struct itimerval timeout; memset(&timeout, 0, sizeof(timeout)); @@ -149,9 +153,22 @@ void Timers::Remove(Timer& rTimer) Reschedule(); } +void Timers::RequestReschedule() +{ + sRescheduleNeeded = true; +} + +void Timers::RescheduleIfNeeded() +{ + if (sRescheduleNeeded) + { + Reschedule(); + } +} + #define FORMAT_MICROSECONDS(t) \ (int)(t / 1000000) << "." << \ - (int)(t % 1000000) + (int)(t % 1000000) << " seconds" // -------------------------------------------------------------------------- // @@ -195,6 +212,9 @@ void Timers::Reschedule() // we will do it anyway. sRescheduleNeeded = false; +#ifdef WIN32 + // win32 timers need no management +#else box_time_t timeNow = GetCurrentBoxTime(); // scan for, trigger and remove expired timers. Removal requires @@ -212,8 +232,14 @@ void Timers::Reschedule() if (timeToExpiry <= 0) { + /* BOX_TRACE("timer " << *i << " has expired, " "triggering it"); + */ + BOX_TRACE(TIMER_ID_OF(**i) "has expired, " + "triggering " << + FORMAT_MICROSECONDS(-timeToExpiry) << + " late"); rTimer.OnExpire(); spTimers->erase(i); restart = true; @@ -221,10 +247,12 @@ void Timers::Reschedule() } else { + /* BOX_TRACE("timer " << *i << " has not " "expired, triggering in " << FORMAT_MICROSECONDS(timeToExpiry) << " seconds"); + */ } } } @@ -233,6 +261,7 @@ void Timers::Reschedule() // Scan to find the next one to fire (earliest deadline). int64_t timeToNextEvent = 0; + std::string nameOfNextEvent; for (std::vector::iterator i = spTimers->begin(); i != spTimers->end(); i++) @@ -240,6 +269,7 @@ void Timers::Reschedule() Timer& rTimer = **i; int64_t timeToExpiry = rTimer.GetExpiryTime() - timeNow; + ASSERT(timeToExpiry > 0) if (timeToExpiry <= 0) { timeToExpiry = 1; @@ -248,23 +278,36 @@ void Timers::Reschedule() if (timeToNextEvent == 0 || timeToNextEvent > timeToExpiry) { timeToNextEvent = timeToExpiry; + nameOfNextEvent = rTimer.GetName(); } } ASSERT(timeToNextEvent >= 0); - + + if (timeToNextEvent == 0) + { + BOX_TRACE("timer: no more events, going to sleep."); + } + else + { + BOX_TRACE("timer: next event: " << nameOfNextEvent << + " expires in " << FORMAT_MICROSECONDS(timeToNextEvent)); + } + struct itimerval timeout; memset(&timeout, 0, sizeof(timeout)); timeout.it_value.tv_sec = BoxTimeToSeconds(timeToNextEvent); timeout.it_value.tv_usec = (int) - (BoxTimeToMicroSeconds(timeToNextEvent) % MICRO_SEC_IN_SEC); + (BoxTimeToMicroSeconds(timeToNextEvent) + % MICRO_SEC_IN_SEC); if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) { - BOX_ERROR("Failed to initialise timer\n"); + BOX_ERROR("Failed to initialise system timer\n"); THROW_EXCEPTION(CommonException, Internal) } +#endif } // -------------------------------------------------------------------------- @@ -279,27 +322,42 @@ void Timers::Reschedule() // Created: 5/11/2006 // // -------------------------------------------------------------------------- -void Timers::SignalHandler(int iUnused) +void Timers::SignalHandler(int unused) { // ASSERT(spTimers); Timers::RequestReschedule(); } -Timer::Timer(size_t timeoutSecs) +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Timer(size_t timeoutSecs, +// const std::string& rName) +// Purpose: Standard timer constructor, takes a timeout in +// seconds from now, and an optional name for +// logging purposes. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +Timer::Timer(size_t timeoutSecs, const std::string& rName) : mExpires(GetCurrentBoxTime() + SecondsToBoxTime(timeoutSecs)), - mExpired(false) + mExpired(false), + mName(rName) +#ifdef WIN32 +, mTimerHandle(INVALID_HANDLE_VALUE) +#endif { - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD if (timeoutSecs == 0) { - BOX_TRACE("timer " << this << " initialised for " << - timeoutSecs << " secs, will not fire"); + BOX_TRACE(TIMER_ID "initialised for " << timeoutSecs << + " secs, will not fire"); } else { - BOX_TRACE("timer " << this << " initialised for " << - timeoutSecs << " secs, to fire at " << - FORMAT_MICROSECONDS(mExpires)); + BOX_TRACE(TIMER_ID "initialised for " << timeoutSecs << + " secs, to fire at " << FormatTime(mExpires, false, true)); } #endif @@ -310,37 +368,157 @@ Timer::Timer(size_t timeoutSecs) else { Timers::Add(*this); + Start(timeoutSecs * MICRO_SEC_IN_SEC_LL); } } +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Start() +// Purpose: This internal function initialises an OS TimerQueue +// timer on Windows, while on Unixes there is only a +// single global timer, managed by the Timers class, +// so this method does nothing. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::Start() +{ +#ifdef WIN32 + box_time_t timeNow = GetCurrentBoxTime(); + int64_t timeToExpiry = mExpires - timeNow; + + if (timeToExpiry <= 0) + { + BOX_WARNING(TIMER_ID << "fudging expiry from -" << + FORMAT_MICROSECONDS(-timeToExpiry)) + timeToExpiry = 1; + } + + Start(timeToExpiry); +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Start(int64_t delayInMicros) +// Purpose: This internal function initialises an OS TimerQueue +// timer on Windows, with a specified delay already +// calculated to save us doing it again. Like +// Timer::Start(), on Unixes it does nothing. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::Start(int64_t delayInMicros) +{ +#ifdef WIN32 + // only call me once! + ASSERT(mTimerHandle == INVALID_HANDLE_VALUE); + + int64_t delayInMillis = delayInMicros / 1000; + + // Windows XP always seems to fire timers up to 20 ms late, + // at least on my test laptop. Not critical in practice, but our + // tests are precise enough that they will fail if we don't + // correct for it. + delayInMillis -= 20; + + // Set a system timer to call our timer routine + if (CreateTimerQueueTimer(&mTimerHandle, NULL, TimerRoutine, + (PVOID)this, delayInMillis, 0, WT_EXECUTEINTIMERTHREAD) + == FALSE) + { + BOX_ERROR(TIMER_ID "failed to create timer: " << + GetErrorMessage(GetLastError())); + mTimerHandle = INVALID_HANDLE_VALUE; + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Stop() +// Purpose: This internal function deletes the associated OS +// TimerQueue timer on Windows, and on Unixes does +// nothing. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +void Timer::Stop() +{ +#ifdef WIN32 + if (mTimerHandle != INVALID_HANDLE_VALUE) + { + if (DeleteTimerQueueTimer(NULL, mTimerHandle, + INVALID_HANDLE_VALUE) == FALSE) + { + BOX_ERROR(TIMER_ID "failed to delete timer: " << + GetErrorMessage(GetLastError())); + } + mTimerHandle = INVALID_HANDLE_VALUE; + } +#endif +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::~Timer() +// Purpose: Destructor for Timer objects. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + Timer::~Timer() { - #ifndef NDEBUG - BOX_TRACE("timer " << this << " destroyed"); + #ifndef BOX_RELEASE_BUILD + BOX_TRACE(TIMER_ID "destroyed"); #endif Timers::Remove(*this); + Stop(); } +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::Timer(Timer& rToCopy) +// Purpose: Copy constructor for Timer objects. Creates a new +// timer that will trigger at the same time as the +// original. The original will usually be discarded. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + Timer::Timer(const Timer& rToCopy) : mExpires(rToCopy.mExpires), - mExpired(rToCopy.mExpired) + mExpired(rToCopy.mExpired), + mName(rToCopy.mName) +#ifdef WIN32 +, mTimerHandle(INVALID_HANDLE_VALUE) +#endif { - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD if (mExpired) { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << ", already expired, will not fire"); + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "already expired, will not fire"); } else if (mExpires == 0) { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << ", no expiry, will not fire"); + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "no expiry, will not fire"); } else { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << " to fire at " << + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "to fire at " << (int)(mExpires / 1000000) << "." << (int)(mExpires % 1000000)); } @@ -349,46 +527,100 @@ Timer::Timer(const Timer& rToCopy) if (!mExpired && mExpires != 0) { Timers::Add(*this); + Start(); } } +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::operator=(const Timer& rToCopy) +// Purpose: Assignment operator for Timer objects. Works +// exactly the same as the copy constructor, except +// that if the receiving timer is already running, +// it is stopped first. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + Timer& Timer::operator=(const Timer& rToCopy) { - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD if (rToCopy.mExpired) { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << ", already expired, will not fire"); + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "already expired, will not fire"); } else if (rToCopy.mExpires == 0) { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << ", no expiry, will not fire"); + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "no expiry, will not fire"); } else { - BOX_TRACE("timer " << this << " initialised from timer " << - &rToCopy << " to fire at " << + BOX_TRACE(TIMER_ID "initialised from timer " << &rToCopy << ", " + "to fire at " << (int)(rToCopy.mExpires / 1000000) << "." << (int)(rToCopy.mExpires % 1000000)); } #endif Timers::Remove(*this); + Stop(); + mExpires = rToCopy.mExpires; mExpired = rToCopy.mExpired; + mName = rToCopy.mName; + if (!mExpired && mExpires != 0) { Timers::Add(*this); + Start(); } + return *this; } +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::OnExpire() +// Purpose: Method called by Timers::Reschedule (on Unixes) +// on next poll after timer expires, or from +// Timer::TimerRoutine (on Windows) from a separate +// thread managed by the OS. Marks the timer as +// expired for future reference. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + void Timer::OnExpire() { - #ifndef NDEBUG - BOX_TRACE("timer " << this << " fired"); + #ifndef BOX_RELEASE_BUILD + BOX_TRACE(TIMER_ID "fired"); #endif mExpired = true; } + +// -------------------------------------------------------------------------- +// +// Function +// Name: Timer::TimerRoutine(PVOID lpParam, +// BOOLEAN TimerOrWaitFired) +// Purpose: Static method called by the Windows OS when a +// TimerQueue timer expires. +// Created: 27/07/2008 +// +// -------------------------------------------------------------------------- + +#ifdef WIN32 +VOID CALLBACK Timer::TimerRoutine(PVOID lpParam, + BOOLEAN TimerOrWaitFired) +{ + Timer* pTimer = (Timer*)lpParam; + pTimer->OnExpire(); + // is it safe to write to write debug output from a timer? + // e.g. to write to the Event Log? +} +#endif diff --git a/lib/common/Timer.h b/lib/common/Timer.h index ba6d71f4..42b2e00f 100644 --- a/lib/common/Timer.h +++ b/lib/common/Timer.h @@ -40,35 +40,24 @@ class Timers static bool sRescheduleNeeded; static void SignalHandler(int iUnused); - + public: static void Init(); static void Cleanup(); static void Add (Timer& rTimer); static void Remove(Timer& rTimer); - static void RequestReschedule() - { - sRescheduleNeeded = true; - } - - static void RescheduleIfNeeded() - { - if (sRescheduleNeeded) - { - Reschedule(); - } - } + static void RequestReschedule(); + static void RescheduleIfNeeded(); }; class Timer { public: - Timer(size_t timeoutSecs); + Timer(size_t timeoutSecs, const std::string& rName = ""); virtual ~Timer(); Timer(const Timer &); Timer &operator=(const Timer &); -public: box_time_t GetExpiryTime() { return mExpires; } virtual void OnExpire(); bool HasExpired() @@ -76,10 +65,23 @@ public: Timers::RescheduleIfNeeded(); return mExpired; } + + const std::string& GetName() const { return mName; } private: - box_time_t mExpires; - bool mExpired; + box_time_t mExpires; + bool mExpired; + std::string mName; + + void Start(); + void Start(int64_t delayInMicros); + void Stop(); + + #ifdef WIN32 + HANDLE mTimerHandle; + static VOID CALLBACK TimerRoutine(PVOID lpParam, + BOOLEAN TimerOrWaitFired); + #endif }; #include "MemLeakFindOff.h" diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp index 83a12ccf..f45ed26b 100644 --- a/lib/common/Utils.cpp +++ b/lib/common/Utils.cpp @@ -13,11 +13,17 @@ #include #include +#include + #ifdef SHOW_BACKTRACE_ON_EXCEPTION #include #include #endif +#ifdef HAVE_CXXABI_H + #include +#endif + #include "Utils.h" #include "CommonException.h" #include "Logging.h" @@ -52,11 +58,11 @@ void SplitString(const std::string &String, char SplitOn, std::vector 1024) + { + readableValue /= 1024; + units = "kB"; + } + + if (readableValue > 1024) + { + readableValue /= 1024; + units = "MB"; + } + + if (readableValue > 1024) + { + readableValue /= 1024; + units = "GB"; + } + + std::ostringstream result; + result << std::fixed << std::setprecision(2) << readableValue << + " " << units; + return result.str(); +} + +std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, + bool MachineReadable) +{ + std::ostringstream result; + + + if (MachineReadable) + { + result << (Bytes >> 10) << " kB, " << + std::setprecision(0) << ((Bytes*100)/Max) << "%"; + } + else + { + // Bar graph + char bar[17]; + unsigned int b = (int)((Bytes * (sizeof(bar)-1)) / Max); + if(b > sizeof(bar)-1) {b = sizeof(bar)-1;} + for(unsigned int l = 0; l < b; l++) + { + bar[l] = '*'; + } + for(unsigned int l = b; l < sizeof(bar) - 1; l++) + { + bar[l] = ' '; + } + bar[sizeof(bar)-1] = '\0'; + + result << std::fixed << + std::setw(10) << Blocks << " blocks, " << + std::setw(10) << HumanReadableSize(Bytes) << ", " << + std::setw(3) << std::setprecision(0) << + ((Bytes*100)/Max) << "% |" << bar << "|"; + } + + return result.str(); +} +std::string FormatUsageLineStart(const std::string& rName, + bool MachineReadable) +{ + std::ostringstream result; + if (MachineReadable) + { + result << rName << ": "; + } + else + { + result << std::setw(20) << std::right << rName << ": "; + } + return result.str(); +} diff --git a/lib/common/Utils.h b/lib/common/Utils.h index d0842b51..d6792077 100644 --- a/lib/common/Utils.h +++ b/lib/common/Utils.h @@ -30,6 +30,11 @@ enum ObjectExists_Dir = 2 }; int ObjectExists(const std::string& rFilename); +std::string HumanReadableSize(int64_t Bytes); +std::string FormatUsageBar(int64_t Blocks, int64_t Bytes, int64_t Max, + bool MachineReadable); +std::string FormatUsageLineStart(const std::string& rName, + bool MachineReadable); #include "MemLeakFindOff.h" diff --git a/lib/common/WaitForEvent.h b/lib/common/WaitForEvent.h index 52a073e9..045d6d67 100644 --- a/lib/common/WaitForEvent.h +++ b/lib/common/WaitForEvent.h @@ -10,6 +10,8 @@ #ifndef WAITFOREVENT__H #define WAITFOREVENT__H +#include + #ifdef HAVE_KQUEUE #include #include diff --git a/lib/common/makeexception.pl.in b/lib/common/makeexception.pl.in index 1564b75b..76b9b02b 100755 --- a/lib/common/makeexception.pl.in +++ b/lib/common/makeexception.pl.in @@ -71,13 +71,14 @@ print H <<__E; class ${class}Exception : public BoxException { public: - ${class}Exception(unsigned int SubType) - : mSubType(SubType) + ${class}Exception(unsigned int SubType, + const std::string& rMessage = "") + : mSubType(SubType), mMessage(rMessage) { } ${class}Exception(const ${class}Exception &rToCopy) - : mSubType(rToCopy.mSubType) + : mSubType(rToCopy.mSubType), mMessage(rToCopy.mMessage) { } @@ -108,9 +109,14 @@ print H <<__E; virtual unsigned int GetType() const throw(); virtual unsigned int GetSubType() const throw(); virtual const char *what() const throw(); + virtual const std::string& GetMessage() const + { + return mMessage; + } private: unsigned int mSubType; + std::string mMessage; }; #endif // $guardname diff --git a/lib/compress/Compress.h b/lib/compress/Compress.h index 4a98a59e..de38af2c 100644 --- a/lib/compress/Compress.h +++ b/lib/compress/Compress.h @@ -52,10 +52,12 @@ public: if((r = ((Compressing)?(deflateEnd(&mStream)) :(inflateEnd(&mStream)))) != Z_OK) { - TRACE1("zlib error code = %d\n", r); + BOX_WARNING("zlib error code = " << r); if(r == Z_DATA_ERROR) { - TRACE0("WARNING: End of compress/decompress without all input being consumed -- possible corruption?\n"); + BOX_WARNING("End of compress/decompress " + "without all input being consumed, " + "possible corruption?"); } else { @@ -148,7 +150,7 @@ public: // Check errors if(ret < 0) { - TRACE1("zlib error code = %d\n", ret); + BOX_WARNING("zlib error code = " << ret); THROW_EXCEPTION(CompressException, TransformFailed) } diff --git a/lib/compress/CompressStream.cpp b/lib/compress/CompressStream.cpp index 2839b647..9bb73e3d 100644 --- a/lib/compress/CompressStream.cpp +++ b/lib/compress/CompressStream.cpp @@ -19,7 +19,7 @@ #include "MemLeakFindOn.h" // How big a buffer to use -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD // debug! #define BUFFER_SIZE 256 #else @@ -414,7 +414,7 @@ void CompressStream::CheckBuffer() { size *= 2; } - TRACE1("Allocating CompressStream buffer, size %d\n", size); + BOX_TRACE("Allocating CompressStream buffer, size " << size); mpBuffer = ::malloc(size); if(mpBuffer == 0) { diff --git a/lib/crypto/CipherContext.cpp b/lib/crypto/CipherContext.cpp index 8c3b25b2..e5cd9b0e 100644 --- a/lib/crypto/CipherContext.cpp +++ b/lib/crypto/CipherContext.cpp @@ -106,7 +106,8 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes #else // With the old version, a copy needs to be taken first. mpDescription = rDescription.Clone(); - // Mark it as not a leak, otherwise static cipher contexts cause supriously memory leaks to be reported + // Mark it as not a leak, otherwise static cipher contexts + // cause spurious memory leaks to be reported MEMLEAKFINDER_NOT_A_LEAK(mpDescription); mpDescription->SetupParameters(&ctx); #endif @@ -166,7 +167,8 @@ void CipherContext::Begin() // Warn if in a transformation (not an error, because a context might not have been finalised if an exception occured) if(mWithinTransform) { - TRACE0("CipherContext::Begin called when context flagged as within a transform\n"); + BOX_WARNING("CipherContext::Begin called when context " + "flagged as within a transform"); } // Initialise the cipher context again @@ -423,7 +425,8 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p // Warn if in a transformation if(mWithinTransform) { - TRACE0("CipherContext::TransformBlock called when context flagged as within a transform\n"); + BOX_WARNING("CipherContext::TransformBlock called when " + "context flagged as within a transform"); } // Check output buffer size @@ -521,7 +524,8 @@ void CipherContext::SetIV(const void *pIV) // Warn if in a transformation if(mWithinTransform) { - TRACE0("CipherContext::SetIV called when context flagged as within a transform\n"); + BOX_WARNING("CipherContext::SetIV called when context " + "flagged as within a transform"); } // Set IV @@ -559,7 +563,8 @@ const void *CipherContext::SetRandomIV(int &rLengthOut) // Warn if in a transformation if(mWithinTransform) { - TRACE0("CipherContext::SetRandomIV called when context flagged as within a transform\n"); + BOX_WARNING("CipherContext::SetRandomIV called when " + "context flagged as within a transform"); } // Get length of IV diff --git a/lib/httpserver/HTTPException.txt b/lib/httpserver/HTTPException.txt new file mode 100644 index 00000000..52630cda --- /dev/null +++ b/lib/httpserver/HTTPException.txt @@ -0,0 +1,16 @@ +EXCEPTION HTTP 10 + +Internal 0 +RequestReadFailed 1 +RequestAlreadyBeenRead 2 +BadRequest 3 +UnknownResponseCodeUsed 4 +NoContentTypeSet 5 +POSTContentTooLong 6 +CannotSetRedirectIfReponseHasData 7 +CannotSetNotFoundIfReponseHasData 8 +NotImplemented 9 +RequestNotInitialised 10 +BadResponse 11 +ResponseReadFailed 12 +NoStreamConfigured 13 diff --git a/lib/httpserver/HTTPQueryDecoder.cpp b/lib/httpserver/HTTPQueryDecoder.cpp new file mode 100644 index 00000000..c49ac2ce --- /dev/null +++ b/lib/httpserver/HTTPQueryDecoder.cpp @@ -0,0 +1,159 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPQueryDecoder.cpp +// Purpose: Utility class to decode HTTP query strings +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "HTTPQueryDecoder.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::HTTPQueryDecoder( +// HTTPRequest::Query_t &) +// Purpose: Constructor. Pass in the query contents you want +// to decode the query string into. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPQueryDecoder::HTTPQueryDecoder(HTTPRequest::Query_t &rDecodeInto) + : mrDecodeInto(rDecodeInto), + mInKey(true), + mEscapedState(0) +{ + // Insert the terminator for escaped characters + mEscaped[2] = '\0'; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::~HTTPQueryDecoder() +// Purpose: Destructor. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPQueryDecoder::~HTTPQueryDecoder() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::Decode(const char *, int) +// Purpose: Decode a chunk of query string -- call several times with +// the bits as they are received, and then call Finish() +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPQueryDecoder::DecodeChunk(const char *pQueryString, int QueryStringSize) +{ + for(int l = 0; l < QueryStringSize; ++l) + { + char c = pQueryString[l]; + + // BEFORE unescaping, check to see if we need to flip key / value + if(mEscapedState == 0) + { + if(mInKey && c == '=') + { + // Set to store characters in the value + mInKey = false; + continue; + } + else if(!mInKey && c == '&') + { + // Need to store the current key/value pair + mrDecodeInto.insert(HTTPRequest::QueryEn_t(mCurrentKey, mCurrentValue)); + // Blank the strings + mCurrentKey.erase(); + mCurrentValue.erase(); + + // Set to store characters in the key + mInKey = true; + continue; + } + } + + // Decode an escaped value? + if(mEscapedState == 1) + { + // Waiting for char one of the escaped hex value + mEscaped[0] = c; + mEscapedState = 2; + continue; + } + else if(mEscapedState == 2) + { + // Escaped value, decode it + mEscaped[1] = c; // str terminated in constructor + mEscapedState = 0; // stop being in escaped mode + long ch = ::strtol(mEscaped, NULL, 16); + if(ch <= 0 || ch > 255) + { + // Bad character, just ignore + continue; + } + + // Use this instead + c = (char)ch; + } + else if(c == '+') + { + c = ' '; + } + else if(c == '%') + { + mEscapedState = 1; + continue; + } + + // Store decoded value into the appropriate string + if(mInKey) + { + mCurrentKey += c; + } + else + { + mCurrentValue += c; + } + } + + // Don't do anything here with left over values, DecodeChunk might be called + // again. Let Finish() clean up. +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPQueryDecoder::Finish() +// Purpose: Finish the decoding. Necessary to get the last item! +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPQueryDecoder::Finish() +{ + // Insert any remaining value. + if(!mCurrentKey.empty()) + { + mrDecodeInto.insert(HTTPRequest::QueryEn_t(mCurrentKey, mCurrentValue)); + // Blank values, just in case + mCurrentKey.erase(); + mCurrentValue.erase(); + } +} + + diff --git a/lib/httpserver/HTTPQueryDecoder.h b/lib/httpserver/HTTPQueryDecoder.h new file mode 100644 index 00000000..ca5afe7e --- /dev/null +++ b/lib/httpserver/HTTPQueryDecoder.h @@ -0,0 +1,47 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPQueryDecoder.h +// Purpose: Utility class to decode HTTP query strings +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPQUERYDECODER__H +#define HTTPQUERYDECODER__H + +#include "HTTPRequest.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPQueryDecoder +// Purpose: Utility class to decode HTTP query strings +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPQueryDecoder +{ +public: + HTTPQueryDecoder(HTTPRequest::Query_t &rDecodeInto); + ~HTTPQueryDecoder(); +private: + // no copying + HTTPQueryDecoder(const HTTPQueryDecoder &); + HTTPQueryDecoder &operator=(const HTTPQueryDecoder &); +public: + + void DecodeChunk(const char *pQueryString, int QueryStringSize); + void Finish(); + +private: + HTTPRequest::Query_t &mrDecodeInto; + std::string mCurrentKey; + std::string mCurrentValue; + bool mInKey; + char mEscaped[4]; + int mEscapedState; +}; + +#endif // HTTPQUERYDECODER__H + diff --git a/lib/httpserver/HTTPRequest.cpp b/lib/httpserver/HTTPRequest.cpp new file mode 100644 index 00000000..e146a0a3 --- /dev/null +++ b/lib/httpserver/HTTPRequest.cpp @@ -0,0 +1,783 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPRequest.cpp +// Purpose: Request object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include +#include +#include + +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "HTTPQueryDecoder.h" +#include "autogen_HTTPException.h" +#include "IOStream.h" +#include "IOStreamGetLine.h" +#include "Logging.h" + +#include "MemLeakFindOn.h" + +#define MAX_CONTENT_SIZE (128*1024) + +#define ENSURE_COOKIE_JAR_ALLOCATED \ + if(mpCookies == 0) {mpCookies = new CookieJar_t;} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::HTTPRequest() +// Purpose: Constructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPRequest::HTTPRequest() + : mMethod(Method_UNINITIALISED), + mHostPort(80), // default if not specified + mHTTPVersion(0), + mContentLength(-1), + mpCookies(0), + mClientKeepAliveRequested(false), + mExpectContinue(false), + mpStreamToReadFrom(NULL) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::HTTPRequest(enum Method, +// const std::string&) +// Purpose: Alternate constructor for hand-crafted requests +// Created: 03/01/09 +// +// -------------------------------------------------------------------------- +HTTPRequest::HTTPRequest(enum Method method, const std::string& rURI) + : mMethod(method), + mRequestURI(rURI), + mHostPort(80), // default if not specified + mHTTPVersion(HTTPVersion_1_1), + mContentLength(-1), + mpCookies(0), + mClientKeepAliveRequested(false), + mExpectContinue(false), + mpStreamToReadFrom(NULL) +{ +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::~HTTPRequest() +// Purpose: Destructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPRequest::~HTTPRequest() +{ + // Clean up any cookies + if(mpCookies != 0) + { + delete mpCookies; + mpCookies = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::Receive(IOStreamGetLine &, int) +// Purpose: Read the request from an IOStreamGetLine (and +// attached stream). +// Returns false if there was no valid request, +// probably due to a kept-alive connection closing. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) +{ + // Check caller's logic + if(mMethod != Method_UNINITIALISED) + { + THROW_EXCEPTION(HTTPException, RequestAlreadyBeenRead) + } + + // Read the first line, which is of a different format to the rest of the lines + std::string requestLine; + if(!rGetLine.GetLine(requestLine, false /* no preprocessing */, Timeout)) + { + // Didn't get the request line, probably end of connection which had been kept alive + return false; + } + BOX_TRACE("Request line: " << requestLine); + + // Check the method + size_t p = 0; // current position in string + p = requestLine.find(' '); // end of first word + + if (p == std::string::npos) + { + // No terminating space, looks bad + p = requestLine.size(); + } + else + { + mHttpVerb = requestLine.substr(0, p); + if (mHttpVerb == "GET") + { + mMethod = Method_GET; + } + else if (mHttpVerb == "HEAD") + { + mMethod = Method_HEAD; + } + else if (mHttpVerb == "POST") + { + mMethod = Method_POST; + } + else if (mHttpVerb == "PUT") + { + mMethod = Method_PUT; + } + else + { + mMethod = Method_UNKNOWN; + } + } + + // Skip spaces to find URI + const char *requestLinePtr = requestLine.c_str(); + while(requestLinePtr[p] != '\0' && requestLinePtr[p] == ' ') + { + ++p; + } + + // Check there's a URI following... + if(requestLinePtr[p] == '\0') + { + // Didn't get the request line, probably end of connection which had been kept alive + return false; + } + + // Read the URI, unescaping any %XX hex codes + while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0') + { + // End of URI, on to query string? + if(requestLinePtr[p] == '?') + { + // Put the rest into the query string, without escaping anything + ++p; + while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0') + { + mQueryString += requestLinePtr[p]; + ++p; + } + break; + } + // Needs unescaping? + else if(requestLinePtr[p] == '+') + { + mRequestURI += ' '; + } + else if(requestLinePtr[p] == '%') + { + // Be tolerant about this... bad things are silently accepted, + // rather than throwing an error. + char code[4] = {0,0,0,0}; + code[0] = requestLinePtr[++p]; + if(code[0] != '\0') + { + code[1] = requestLinePtr[++p]; + } + + // Convert into a char code + long c = ::strtol(code, NULL, 16); + + // Accept it? + if(c > 0 && c <= 255) + { + mRequestURI += (char)c; + } + } + else + { + // Simple copy of character + mRequestURI += requestLinePtr[p]; + } + + ++p; + } + + // End of URL? + if(requestLinePtr[p] == '\0') + { + // Assume HTTP 0.9 + mHTTPVersion = HTTPVersion_0_9; + } + else + { + // Skip any more spaces + while(requestLinePtr[p] != '\0' && requestLinePtr[p] == ' ') + { + ++p; + } + + // Check to see if there's the right string next... + if(::strncmp(requestLinePtr + p, "HTTP/", 5) == 0) + { + // Find the version numbers + int major, minor; + if(::sscanf(requestLinePtr + p + 5, "%d.%d", &major, &minor) != 2) + { + THROW_EXCEPTION(HTTPException, BadRequest) + } + + // Store version + mHTTPVersion = (major * HTTPVersion__MajorMultiplier) + minor; + } + else + { + // Not good -- wrong string found + THROW_EXCEPTION(HTTPException, BadRequest) + } + } + + BOX_TRACE("HTTPRequest: method=" << mMethod << ", uri=" << + mRequestURI << ", version=" << mHTTPVersion); + + // If HTTP 1.1 or greater, assume keep-alive + if(mHTTPVersion >= HTTPVersion_1_1) + { + mClientKeepAliveRequested = true; + } + + // Decode query string? + if((mMethod == Method_GET || mMethod == Method_HEAD) && !mQueryString.empty()) + { + HTTPQueryDecoder decoder(mQuery); + decoder.DecodeChunk(mQueryString.c_str(), mQueryString.size()); + decoder.Finish(); + } + + // Now parse the headers + ParseHeaders(rGetLine, Timeout); + + std::string expected; + if (GetHeader("Expect", &expected)) + { + if (expected == "100-continue") + { + mExpectContinue = true; + } + } + + // Parse form data? + if(mMethod == Method_POST && mContentLength >= 0) + { + // Too long? Don't allow people to be nasty by sending lots of data + if(mContentLength > MAX_CONTENT_SIZE) + { + THROW_EXCEPTION(HTTPException, POSTContentTooLong) + } + + // Some data in the request to follow, parsing it bit by bit + HTTPQueryDecoder decoder(mQuery); + // Don't forget any data left in the GetLine object + int fromBuffer = rGetLine.GetSizeOfBufferedData(); + if(fromBuffer > mContentLength) fromBuffer = mContentLength; + if(fromBuffer > 0) + { + BOX_TRACE("Decoding " << fromBuffer << " bytes of " + "data from getline buffer"); + decoder.DecodeChunk((const char *)rGetLine.GetBufferedData(), fromBuffer); + // And tell the getline object to ignore the data we just used + rGetLine.IgnoreBufferedData(fromBuffer); + } + // Then read any more data, as required + int bytesToGo = mContentLength - fromBuffer; + while(bytesToGo > 0) + { + char buf[4096]; + int toRead = sizeof(buf); + if(toRead > bytesToGo) toRead = bytesToGo; + IOStream &rstream(rGetLine.GetUnderlyingStream()); + int r = rstream.Read(buf, toRead, Timeout); + if(r == 0) + { + // Timeout, just error + THROW_EXCEPTION(HTTPException, RequestReadFailed) + } + decoder.DecodeChunk(buf, r); + bytesToGo -= r; + } + // Finish off + decoder.Finish(); + } + else if (mContentLength > 0) + { + IOStream::pos_type bytesToCopy = rGetLine.GetSizeOfBufferedData(); + if (bytesToCopy > mContentLength) + { + bytesToCopy = mContentLength; + } + Write(rGetLine.GetBufferedData(), bytesToCopy); + SetForReading(); + mpStreamToReadFrom = &(rGetLine.GetUnderlyingStream()); + } + + return true; +} + +void HTTPRequest::ReadContent(IOStream& rStreamToWriteTo) +{ + Seek(0, SeekType_Absolute); + + CopyStreamTo(rStreamToWriteTo); + IOStream::pos_type bytesCopied = GetSize(); + + while (bytesCopied < mContentLength) + { + char buffer[1024]; + IOStream::pos_type bytesToCopy = sizeof(buffer); + if (bytesToCopy > mContentLength - bytesCopied) + { + bytesToCopy = mContentLength - bytesCopied; + } + bytesToCopy = mpStreamToReadFrom->Read(buffer, bytesToCopy); + rStreamToWriteTo.Write(buffer, bytesToCopy); + bytesCopied += bytesToCopy; + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::Send(IOStream &, int) +// Purpose: Write the request to an IOStream using HTTP. +// Created: 03/01/09 +// +// -------------------------------------------------------------------------- +bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) +{ + switch (mMethod) + { + case Method_UNINITIALISED: + THROW_EXCEPTION(HTTPException, RequestNotInitialised); break; + case Method_UNKNOWN: + THROW_EXCEPTION(HTTPException, BadRequest); break; + case Method_GET: + rStream.Write("GET"); break; + case Method_HEAD: + rStream.Write("HEAD"); break; + case Method_POST: + rStream.Write("POST"); break; + case Method_PUT: + rStream.Write("PUT"); break; + } + + rStream.Write(" "); + rStream.Write(mRequestURI.c_str()); + rStream.Write(" "); + + switch (mHTTPVersion) + { + case HTTPVersion_0_9: rStream.Write("HTTP/0.9"); break; + case HTTPVersion_1_0: rStream.Write("HTTP/1.0"); break; + case HTTPVersion_1_1: rStream.Write("HTTP/1.1"); break; + default: + THROW_EXCEPTION(HTTPException, NotImplemented); + } + + rStream.Write("\n"); + std::ostringstream oss; + + if (mContentLength != -1) + { + oss << "Content-Length: " << mContentLength << "\n"; + } + + if (mContentType != "") + { + oss << "Content-Type: " << mContentType << "\n"; + } + + if (mHostName != "") + { + if (mHostPort != 80) + { + oss << "Host: " << mHostName << ":" << mHostPort << + "\n"; + } + else + { + oss << "Host: " << mHostName << "\n"; + } + } + + if (mpCookies) + { + THROW_EXCEPTION(HTTPException, NotImplemented); + } + + if (mClientKeepAliveRequested) + { + oss << "Connection: keep-alive\n"; + } + else + { + oss << "Connection: close\n"; + } + + for (std::vector
::iterator i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + oss << i->first << ": " << i->second << "\n"; + } + + if (ExpectContinue) + { + oss << "Expect: 100-continue\n"; + } + + rStream.Write(oss.str().c_str()); + rStream.Write("\n"); + + return true; +} + +void HTTPRequest::SendWithStream(IOStream &rStreamToSendTo, int Timeout, + IOStream* pStreamToSend, HTTPResponse& rResponse) +{ + IOStream::pos_type size = pStreamToSend->BytesLeftToRead(); + + if (size != IOStream::SizeOfStreamUnknown) + { + mContentLength = size; + } + + Send(rStreamToSendTo, Timeout, true); + + rResponse.Receive(rStreamToSendTo, Timeout); + if (rResponse.GetResponseCode() != 100) + { + // bad response, abort now + return; + } + + pStreamToSend->CopyStreamTo(rStreamToSendTo, Timeout); + + // receive the final response + rResponse.Receive(rStreamToSendTo, Timeout); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::ParseHeaders(IOStreamGetLine &, int) +// Purpose: Private. Parse the headers of the request +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) +{ + std::string header; + bool haveHeader = false; + while(true) + { + if(rGetLine.IsEOF()) + { + // Header terminates unexpectedly + THROW_EXCEPTION(HTTPException, BadRequest) + } + + std::string currentLine; + if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) + { + // Timeout + THROW_EXCEPTION(HTTPException, RequestReadFailed) + } + + // Is this a continuation of the previous line? + bool processHeader = haveHeader; + if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) + { + // A continuation, don't process anything yet + processHeader = false; + } + //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); + + // Parse the header -- this will actually process the header + // from the previous run around the loop. + if(processHeader) + { + // Find where the : is in the line + const char *h = header.c_str(); + int p = 0; + while(h[p] != '\0' && h[p] != ':') + { + ++p; + } + // Skip white space + int dataStart = p + 1; + while(h[dataStart] == ' ' || h[dataStart] == '\t') + { + ++dataStart; + } + + if(p == sizeof("Content-Length")-1 + && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) + { + // Decode number + long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK + if(len < 0) len = 0; + // Store + mContentLength = len; + } + else if(p == sizeof("Content-Type")-1 + && ::strncasecmp(h, "Content-Type", sizeof("Content-Type")-1) == 0) + { + // Store rest of string as content type + mContentType = h + dataStart; + } + else if(p == sizeof("Host")-1 + && ::strncasecmp(h, "Host", sizeof("Host")-1) == 0) + { + // Store host header + mHostName = h + dataStart; + + // Is there a port number to split off? + std::string::size_type colon = mHostName.find_first_of(':'); + if(colon != std::string::npos) + { + // There's a port in the string... attempt to turn it into an int + mHostPort = ::strtol(mHostName.c_str() + colon + 1, 0, 10); + + // Truncate the string to just the hostname + mHostName = mHostName.substr(0, colon); + + BOX_TRACE("Host: header, hostname = " << + "'" << mHostName << "', host " + "port = " << mHostPort); + } + } + else if(p == sizeof("Cookie")-1 + && ::strncasecmp(h, "Cookie", sizeof("Cookie")-1) == 0) + { + // Parse cookies + ParseCookies(header, dataStart); + } + else if(p == sizeof("Connection")-1 + && ::strncasecmp(h, "Connection", sizeof("Connection")-1) == 0) + { + // Connection header, what is required? + const char *v = h + dataStart; + if(::strcasecmp(v, "close") == 0) + { + mClientKeepAliveRequested = false; + } + else if(::strcasecmp(v, "keep-alive") == 0) + { + mClientKeepAliveRequested = true; + } + // else don't understand, just assume default for protocol version + } + else + { + std::string name = header.substr(0, p); + mExtraHeaders.push_back(Header(name, + h + dataStart)); + } + + // Unset have header flag, as it's now been processed + haveHeader = false; + } + + // Store the chunk of header the for next time round + if(haveHeader) + { + header += currentLine; + } + else + { + header = currentLine; + haveHeader = true; + } + + // End of headers? + if(currentLine.empty()) + { + // All done! + break; + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::ParseCookies(const std::string &, int) +// Purpose: Parse the cookie header +// Created: 20/8/04 +// +// -------------------------------------------------------------------------- +void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) +{ + const char *data = rHeader.c_str() + DataStarts; + const char *pos = data; + const char *itemStart = pos; + std::string name; + + enum + { + s_NAME, s_VALUE, s_VALUE_QUOTED, s_FIND_NEXT_NAME + } state = s_NAME; + + do + { + switch(state) + { + case s_NAME: + { + if(*pos == '=') + { + // Found the name. Store + name.assign(itemStart, pos - itemStart); + // Looking at values now + state = s_VALUE; + if((*(pos + 1)) == '"') + { + // Actually it's a quoted value, skip over that + ++pos; + state = s_VALUE_QUOTED; + } + // Record starting point for this item + itemStart = pos + 1; + } + } + break; + + case s_VALUE: + { + if(*pos == ';' || *pos == ',' || *pos == '\0') + { + // Name ends + ENSURE_COOKIE_JAR_ALLOCATED + std::string value(itemStart, pos - itemStart); + (*mpCookies)[name] = value; + // And move to the waiting stage + state = s_FIND_NEXT_NAME; + } + } + break; + + case s_VALUE_QUOTED: + { + if(*pos == '"') + { + // That'll do nicely, save it + ENSURE_COOKIE_JAR_ALLOCATED + std::string value(itemStart, pos - itemStart); + (*mpCookies)[name] = value; + // And move to the waiting stage + state = s_FIND_NEXT_NAME; + } + } + break; + + case s_FIND_NEXT_NAME: + { + // Skip over terminators and white space to get to the next name + if(*pos != ';' && *pos != ',' && *pos != ' ' && *pos != '\t') + { + // Name starts here + itemStart = pos; + state = s_NAME; + } + } + break; + + default: + // Ooops + THROW_EXCEPTION(HTTPException, Internal) + break; + } + } + while(*(pos++) != 0); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::GetCookie(const char *, std::string &) const +// Purpose: Fetch a cookie's value. If cookie not present, returns false +// and string is unaltered. +// Created: 20/8/04 +// +// -------------------------------------------------------------------------- +bool HTTPRequest::GetCookie(const char *CookieName, std::string &rValueOut) const +{ + // Got any cookies? + if(mpCookies == 0) + { + return false; + } + + // See if it's there + CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); + if(v != mpCookies->end()) + { + // Return the value + rValueOut = v->second; + return true; + } + + return false; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::GetCookie(const char *) +// Purpose: Return a string for the given cookie, or the null string if the +// cookie has not been recieved. +// Created: 22/8/04 +// +// -------------------------------------------------------------------------- +const std::string &HTTPRequest::GetCookie(const char *CookieName) const +{ + static const std::string noCookie; + + // Got any cookies? + if(mpCookies == 0) + { + return noCookie; + } + + // See if it's there + CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); + if(v != mpCookies->end()) + { + // Return the value + return v->second; + } + + return noCookie; +} + + + diff --git a/lib/httpserver/HTTPRequest.h b/lib/httpserver/HTTPRequest.h new file mode 100644 index 00000000..90215751 --- /dev/null +++ b/lib/httpserver/HTTPRequest.h @@ -0,0 +1,174 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPRequest.h +// Purpose: Request object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPREQUEST__H +#define HTTPREQUEST__H + +#include +#include + +#include "CollectInBufferStream.h" + +class HTTPResponse; +class IOStream; +class IOStreamGetLine; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPRequest +// Purpose: Request object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPRequest : public CollectInBufferStream +{ +public: + enum Method + { + Method_UNINITIALISED = -1, + Method_UNKNOWN = 0, + Method_GET = 1, + Method_HEAD = 2, + Method_POST = 3, + Method_PUT = 4 + }; + + HTTPRequest(); + HTTPRequest(enum Method method, const std::string& rURI); + ~HTTPRequest(); +private: + // no copying + HTTPRequest(const HTTPRequest &); + HTTPRequest &operator=(const HTTPRequest &); +public: + typedef std::multimap Query_t; + typedef std::pair QueryEn_t, Header; + + enum + { + HTTPVersion__MajorMultiplier = 1000, + HTTPVersion_0_9 = 9, + HTTPVersion_1_0 = 1000, + HTTPVersion_1_1 = 1001 + }; + + bool Receive(IOStreamGetLine &rGetLine, int Timeout); + bool Send(IOStream &rStream, int Timeout, bool ExpectContinue = false); + void SendWithStream(IOStream &rStreamToSendTo, int Timeout, + IOStream* pStreamToSend, HTTPResponse& rResponse); + void ReadContent(IOStream& rStreamToWriteTo); + + typedef std::map CookieJar_t; + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPResponse::Get*() + // Purpose: Various Get accessors + // Created: 26/3/04 + // + // -------------------------------------------------------------------------- + enum Method GetMethod() const {return mMethod;} + const std::string &GetRequestURI() const {return mRequestURI;} + + // Note: the HTTPRequest generates and parses the Host: header + // Do not attempt to set one yourself with AddHeader(). + const std::string &GetHostName() const {return mHostName;} + void SetHostName(const std::string& rHostName) + { + mHostName = rHostName; + } + + const int GetHostPort() const {return mHostPort;} + const std::string &GetQueryString() const {return mQueryString;} + int GetHTTPVersion() const {return mHTTPVersion;} + const Query_t &GetQuery() const {return mQuery;} + int GetContentLength() const {return mContentLength;} + const std::string &GetContentType() const {return mContentType;} + const CookieJar_t *GetCookies() const {return mpCookies;} // WARNING: May return NULL + bool GetCookie(const char *CookieName, std::string &rValueOut) const; + const std::string &GetCookie(const char *CookieName) const; + bool GetHeader(const std::string& rName, std::string* pValueOut) const + { + for (std::vector
::const_iterator + i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + if (i->first == rName) + { + *pValueOut = i->second; + return true; + } + } + return false; + } + std::vector
GetHeaders() { return mExtraHeaders; } + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPRequest::GetClientKeepAliveRequested() + // Purpose: Returns true if the client requested that the connection + // should be kept open for further requests. + // Created: 22/12/04 + // + // -------------------------------------------------------------------------- + bool GetClientKeepAliveRequested() const {return mClientKeepAliveRequested;} + void SetClientKeepAliveRequested(bool keepAlive) + { + mClientKeepAliveRequested = keepAlive; + } + + void AddHeader(const std::string& rName, const std::string& rValue) + { + mExtraHeaders.push_back(Header(rName, rValue)); + } + bool IsExpectingContinue() const { return mExpectContinue; } + const char* GetVerb() const + { + if (!mHttpVerb.empty()) + { + return mHttpVerb.c_str(); + } + switch (mMethod) + { + case Method_UNINITIALISED: return "Uninitialized"; + case Method_UNKNOWN: return "Unknown"; + case Method_GET: return "GET"; + case Method_HEAD: return "HEAD"; + case Method_POST: return "POST"; + case Method_PUT: return "PUT"; + } + return "Bad"; + } + +private: + void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); + void ParseCookies(const std::string &rHeader, int DataStarts); + + enum Method mMethod; + std::string mRequestURI; + std::string mHostName; + int mHostPort; + std::string mQueryString; + int mHTTPVersion; + Query_t mQuery; + int mContentLength; + std::string mContentType; + CookieJar_t *mpCookies; + bool mClientKeepAliveRequested; + std::vector
mExtraHeaders; + bool mExpectContinue; + IOStream* mpStreamToReadFrom; + std::string mHttpVerb; +}; + +#endif // HTTPREQUEST__H + diff --git a/lib/httpserver/HTTPResponse.cpp b/lib/httpserver/HTTPResponse.cpp new file mode 100644 index 00000000..1a8c8447 --- /dev/null +++ b/lib/httpserver/HTTPResponse.cpp @@ -0,0 +1,648 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPResponse.cpp +// Purpose: Response object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include + +#include "HTTPResponse.h" +#include "IOStreamGetLine.h" +#include "autogen_HTTPException.h" + +#include "MemLeakFindOn.h" + +// Static variables +std::string HTTPResponse::msDefaultURIPrefix; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::HTTPResponse(IOStream*) +// Purpose: Constructor for response to be sent to a stream +// Created: 04/01/09 +// +// -------------------------------------------------------------------------- +HTTPResponse::HTTPResponse(IOStream* pStreamToSendTo) + : mResponseCode(HTTPResponse::Code_NoContent), + mResponseIsDynamicContent(true), + mKeepAlive(false), + mContentLength(-1), + mpStreamToSendTo(pStreamToSendTo) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::HTTPResponse() +// Purpose: Constructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPResponse::HTTPResponse() + : mResponseCode(HTTPResponse::Code_NoContent), + mResponseIsDynamicContent(true), + mKeepAlive(false), + mContentLength(-1), + mpStreamToSendTo(NULL) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::~HTTPResponse() +// Purpose: Destructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPResponse::~HTTPResponse() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::ResponseCodeToString(int) +// Purpose: Return string equivalent of the response code, +// suitable for Status: headers +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +const char *HTTPResponse::ResponseCodeToString(int ResponseCode) +{ + switch(ResponseCode) + { + case Code_OK: return "200 OK"; break; + case Code_NoContent: return "204 No Content"; break; + case Code_MovedPermanently: return "301 Moved Permanently"; break; + case Code_Found: return "302 Found"; break; + case Code_NotModified: return "304 Not Modified"; break; + case Code_TemporaryRedirect: return "307 Temporary Redirect"; break; + case Code_MethodNotAllowed: return "400 Method Not Allowed"; break; + case Code_Unauthorized: return "401 Unauthorized"; break; + case Code_Forbidden: return "403 Forbidden"; break; + case Code_NotFound: return "404 Not Found"; break; + case Code_InternalServerError: return "500 Internal Server Error"; break; + case Code_NotImplemented: return "501 Not Implemented"; break; + default: + { + THROW_EXCEPTION(HTTPException, UnknownResponseCodeUsed) + } + } + return "500 Internal Server Error"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetResponseCode(int) +// Purpose: Set the response code to be returned +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetResponseCode(int Code) +{ + mResponseCode = Code; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetContentType(const char *) +// Purpose: Set content type +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetContentType(const char *ContentType) +{ + mContentType = ContentType; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::Send(IOStream &, bool) +// Purpose: Build the response, and send via the stream. +// Optionally omitting the content. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::Send(bool OmitContent) +{ + if (!mpStreamToSendTo) + { + THROW_EXCEPTION(HTTPException, NoStreamConfigured); + } + + if (GetSize() != 0 && mContentType.empty()) + { + THROW_EXCEPTION(HTTPException, NoContentTypeSet); + } + + // Build and send header + { + std::string header("HTTP/1.1 "); + header += ResponseCodeToString(mResponseCode); + header += "\r\nContent-Type: "; + header += mContentType; + header += "\r\nContent-Length: "; + { + char len[32]; + ::sprintf(len, "%d", OmitContent?(0):(GetSize())); + header += len; + } + // Extra headers... + for(std::vector >::const_iterator i(mExtraHeaders.begin()); i != mExtraHeaders.end(); ++i) + { + header += "\r\n"; + header += i->first + ": " + i->second; + } + // NOTE: a line ending must be included here in all cases + // Control whether the response is cached + if(mResponseIsDynamicContent) + { + // dynamic is private and can't be cached + header += "\r\nCache-Control: no-cache, private"; + } + else + { + // static is allowed to be cached for a day + header += "\r\nCache-Control: max-age=86400"; + } + if(mKeepAlive) + { + header += "\r\nConnection: keep-alive\r\n\r\n"; + } + else + { + header += "\r\nConnection: close\r\n\r\n"; + } + // NOTE: header ends with blank line in all cases + + // Write to stream + mpStreamToSendTo->Write(header.c_str(), header.size()); + } + + // Send content + if(!OmitContent) + { + mpStreamToSendTo->Write(GetBuffer(), GetSize()); + } +} + +void HTTPResponse::SendContinue() +{ + mpStreamToSendTo->Write("HTTP/1.1 100 Continue\r\n"); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::ParseHeaders(IOStreamGetLine &, int) +// Purpose: Private. Parse the headers of the response +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) +{ + std::string header; + bool haveHeader = false; + while(true) + { + if(rGetLine.IsEOF()) + { + // Header terminates unexpectedly + THROW_EXCEPTION(HTTPException, BadRequest) + } + + std::string currentLine; + if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) + { + // Timeout + THROW_EXCEPTION(HTTPException, RequestReadFailed) + } + + // Is this a continuation of the previous line? + bool processHeader = haveHeader; + if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) + { + // A continuation, don't process anything yet + processHeader = false; + } + //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); + + // Parse the header -- this will actually process the header + // from the previous run around the loop. + if(processHeader) + { + // Find where the : is in the line + const char *h = header.c_str(); + int p = 0; + while(h[p] != '\0' && h[p] != ':') + { + ++p; + } + // Skip white space + int dataStart = p + 1; + while(h[dataStart] == ' ' || h[dataStart] == '\t') + { + ++dataStart; + } + + if(p == sizeof("Content-Length")-1 + && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) + { + // Decode number + long len = ::strtol(h + dataStart, NULL, 10); // returns zero in error case, this is OK + if(len < 0) len = 0; + // Store + mContentLength = len; + } + else if(p == sizeof("Content-Type")-1 + && ::strncasecmp(h, "Content-Type", sizeof("Content-Type")-1) == 0) + { + // Store rest of string as content type + mContentType = h + dataStart; + } + else if(p == sizeof("Cookie")-1 + && ::strncasecmp(h, "Cookie", sizeof("Cookie")-1) == 0) + { + THROW_EXCEPTION(HTTPException, NotImplemented); + /* + // Parse cookies + ParseCookies(header, dataStart); + */ + } + else if(p == sizeof("Connection")-1 + && ::strncasecmp(h, "Connection", sizeof("Connection")-1) == 0) + { + // Connection header, what is required? + const char *v = h + dataStart; + if(::strcasecmp(v, "close") == 0) + { + mKeepAlive = false; + } + else if(::strcasecmp(v, "keep-alive") == 0) + { + mKeepAlive = true; + } + // else don't understand, just assume default for protocol version + } + else + { + std::string headerName = header.substr(0, p); + AddHeader(headerName, h + dataStart); + } + + // Unset have header flag, as it's now been processed + haveHeader = false; + } + + // Store the chunk of header the for next time round + if(haveHeader) + { + header += currentLine; + } + else + { + header = currentLine; + haveHeader = true; + } + + // End of headers? + if(currentLine.empty()) + { + // All done! + break; + } + } +} + +void HTTPResponse::Receive(IOStream& rStream, int Timeout) +{ + IOStreamGetLine rGetLine(rStream); + + if(rGetLine.IsEOF()) + { + // Connection terminated unexpectedly + THROW_EXCEPTION(HTTPException, BadResponse) + } + + std::string statusLine; + if(!rGetLine.GetLine(statusLine, false /* no preprocess */, Timeout)) + { + // Timeout + THROW_EXCEPTION(HTTPException, ResponseReadFailed) + } + + if (statusLine.substr(0, 7) != "HTTP/1." || + statusLine[8] != ' ') + { + // Status line terminated unexpectedly + BOX_ERROR("Bad response status line: " << statusLine); + THROW_EXCEPTION(HTTPException, BadResponse) + } + + if (statusLine[5] == '1' && statusLine[7] == '1') + { + // HTTP/1.1 default is to keep alive + mKeepAlive = true; + } + + // Decode the status code + long status = ::strtol(statusLine.substr(9, 3).c_str(), NULL, 10); + // returns zero in error case, this is OK + if (status < 0) status = 0; + // Store + mResponseCode = status; + + // 100 Continue responses have no headers, terminating newline, or body + if (status == 100) + { + return; + } + + ParseHeaders(rGetLine, Timeout); + + // push back whatever bytes we have left + // rGetLine.DetachFile(); + if (mContentLength > 0) + { + if (mContentLength < rGetLine.GetSizeOfBufferedData()) + { + // very small response, not good! + THROW_EXCEPTION(HTTPException, NotImplemented); + } + + mContentLength -= rGetLine.GetSizeOfBufferedData(); + + Write(rGetLine.GetBufferedData(), + rGetLine.GetSizeOfBufferedData()); + } + + while (mContentLength != 0) // could be -1 as well + { + char buffer[4096]; + int readSize = sizeof(buffer); + if (mContentLength > 0 && mContentLength < readSize) + { + readSize = mContentLength; + } + readSize = rStream.Read(buffer, readSize, Timeout); + if (readSize == 0) + { + break; + } + mContentLength -= readSize; + Write(buffer, readSize); + } + + SetForReading(); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const char *) +// Purpose: Add header, given entire line +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +/* +void HTTPResponse::AddHeader(const char *EntireHeaderLine) +{ + mExtraHeaders.push_back(std::string(EntireHeaderLine)); +} +*/ + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const std::string &) +// Purpose: Add header, given entire line +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +/* +void HTTPResponse::AddHeader(const std::string &rEntireHeaderLine) +{ + mExtraHeaders.push_back(rEntireHeaderLine); +} +*/ + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const char *, const char *) +// Purpose: Add header, given header name and it's value +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::AddHeader(const char *pHeader, const char *pValue) +{ + mExtraHeaders.push_back(Header(pHeader, pValue)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const char *, const std::string &) +// Purpose: Add header, given header name and it's value +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::AddHeader(const char *pHeader, const std::string &rValue) +{ + mExtraHeaders.push_back(Header(pHeader, rValue)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::AddHeader(const std::string &, const std::string &) +// Purpose: Add header, given header name and it's value +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::AddHeader(const std::string &rHeader, const std::string &rValue) +{ + mExtraHeaders.push_back(Header(rHeader, rValue)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetCookie(const char *, const char *, const char *, int) +// Purpose: Sets a cookie, using name, value, path and expiry time. +// Created: 20/8/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetCookie(const char *Name, const char *Value, const char *Path, int ExpiresAt) +{ + if(ExpiresAt != 0) + { + THROW_EXCEPTION(HTTPException, NotImplemented) + } + + // Appears you shouldn't use quotes when you generate set-cookie headers. + // Oh well. It was fun finding that out. +/* std::string h("Set-Cookie: "); + h += Name; + h += "=\""; + h += Value; + h += "\"; Version=\"1\"; Path=\""; + h += Path; + h += "\""; +*/ + std::string h; + h += Name; + h += "="; + h += Value; + h += "; Version=1; Path="; + h += Path; + + mExtraHeaders.push_back(Header("Set-Cookie", h)); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetAsRedirect(const char *, bool) +// Purpose: Sets the response objects to be a redirect to another page. +// If IsLocalURL == true, the default prefix will be added. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetAsRedirect(const char *RedirectTo, bool IsLocalURI) +{ + if(mResponseCode != HTTPResponse::Code_NoContent + || !mContentType.empty() + || GetSize() != 0) + { + THROW_EXCEPTION(HTTPException, CannotSetRedirectIfReponseHasData) + } + + // Set response code + mResponseCode = Code_Found; + + // Set location to redirect to + std::string header; + if(IsLocalURI) header += msDefaultURIPrefix; + header += RedirectTo; + mExtraHeaders.push_back(Header("Location", header)); + + // Set up some default content + mContentType = "text/html"; + #define REDIRECT_HTML_1 "Redirection\n

Redirect to content

\n" + Write(REDIRECT_HTML_1, sizeof(REDIRECT_HTML_1) - 1); + if(IsLocalURI) Write(msDefaultURIPrefix.c_str(), msDefaultURIPrefix.size()); + Write(RedirectTo, ::strlen(RedirectTo)); + Write(REDIRECT_HTML_2, sizeof(REDIRECT_HTML_2) - 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::SetAsNotFound(const char *) +// Purpose: Set the response object to be a standard page not found 404 response. +// Created: 7/4/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::SetAsNotFound(const char *URI) +{ + if(mResponseCode != HTTPResponse::Code_NoContent + || mExtraHeaders.size() != 0 + || !mContentType.empty() + || GetSize() != 0) + { + THROW_EXCEPTION(HTTPException, CannotSetNotFoundIfReponseHasData) + } + + // Set response code + mResponseCode = Code_NotFound; + + // Set data + mContentType = "text/html"; + #define NOT_FOUND_HTML_1 "404 Not Found\n

404 Not Found

\n

The URI " + #define NOT_FOUND_HTML_2 " was not found on this server.

\n" + Write(NOT_FOUND_HTML_1, sizeof(NOT_FOUND_HTML_1) - 1); + WriteStringDefang(std::string(URI)); + Write(NOT_FOUND_HTML_2, sizeof(NOT_FOUND_HTML_2) - 1); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPResponse::WriteStringDefang(const char *, unsigned int) +// Purpose: Writes a string 'defanged', ie has HTML special characters escaped +// so that people can't output arbitary HTML by playing with +// URLs and form parameters, and it's safe to write strings into +// HTML element attribute values. +// Created: 9/4/04 +// +// -------------------------------------------------------------------------- +void HTTPResponse::WriteStringDefang(const char *String, unsigned int StringLen) +{ + while(StringLen > 0) + { + unsigned int toWrite = 0; + while(toWrite < StringLen + && String[toWrite] != '<' + && String[toWrite] != '>' + && String[toWrite] != '&' + && String[toWrite] != '"') + { + ++toWrite; + } + if(toWrite > 0) + { + Write(String, toWrite); + StringLen -= toWrite; + String += toWrite; + } + + // Is it a bad character next? + while(StringLen > 0) + { + bool notSpecial = false; + switch(*String) + { + case '<': Write("<", 4); break; + case '>': Write(">", 4); break; + case '&': Write("&", 5); break; + case '"': Write(""", 6); break; + default: + // Stop this loop + notSpecial = true; + break; + } + if(notSpecial) break; + ++String; + --StringLen; + } + } +} + + diff --git a/lib/httpserver/HTTPResponse.h b/lib/httpserver/HTTPResponse.h new file mode 100644 index 00000000..04051958 --- /dev/null +++ b/lib/httpserver/HTTPResponse.h @@ -0,0 +1,175 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPResponse.h +// Purpose: Response object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPRESPONSE__H +#define HTTPRESPONSE__H + +#include +#include + +#include "CollectInBufferStream.h" + +class IOStreamGetLine; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPResponse +// Purpose: Response object for HTTP connections +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPResponse : public CollectInBufferStream +{ +public: + HTTPResponse(IOStream* pStreamToSendTo); + HTTPResponse(); + ~HTTPResponse(); + + // allow copying, but be very careful with the response stream, + // you can only read it once! (this class doesn't police it). + HTTPResponse(const HTTPResponse& rOther) + : mResponseCode(rOther.mResponseCode), + mResponseIsDynamicContent(rOther.mResponseIsDynamicContent), + mKeepAlive(rOther.mKeepAlive), + mContentType(rOther.mContentType), + mExtraHeaders(rOther.mExtraHeaders), + mContentLength(rOther.mContentLength), + mpStreamToSendTo(rOther.mpStreamToSendTo) + { + Write(rOther.GetBuffer(), rOther.GetSize()); + } + + HTTPResponse &operator=(const HTTPResponse &rOther) + { + Reset(); + Write(rOther.GetBuffer(), rOther.GetSize()); + mResponseCode = rOther.mResponseCode; + mResponseIsDynamicContent = rOther.mResponseIsDynamicContent; + mKeepAlive = rOther.mKeepAlive; + mContentType = rOther.mContentType; + mExtraHeaders = rOther.mExtraHeaders; + mContentLength = rOther.mContentLength; + mpStreamToSendTo = rOther.mpStreamToSendTo; + return *this; + } + + typedef std::pair Header; + + void SetResponseCode(int Code); + int GetResponseCode() { return mResponseCode; } + void SetContentType(const char *ContentType); + const std::string& GetContentType() { return mContentType; } + + void SetAsRedirect(const char *RedirectTo, bool IsLocalURI = true); + void SetAsNotFound(const char *URI); + + void Send(bool OmitContent = false); + void SendContinue(); + void Receive(IOStream& rStream, int Timeout = IOStream::TimeOutInfinite); + + // void AddHeader(const char *EntireHeaderLine); + // void AddHeader(const std::string &rEntireHeaderLine); + void AddHeader(const char *Header, const char *Value); + void AddHeader(const char *Header, const std::string &rValue); + void AddHeader(const std::string &rHeader, const std::string &rValue); + bool GetHeader(const std::string& rName, std::string* pValueOut) const + { + for (std::vector
::const_iterator + i = mExtraHeaders.begin(); + i != mExtraHeaders.end(); i++) + { + if (i->first == rName) + { + *pValueOut = i->second; + return true; + } + } + return false; + } + std::string GetHeaderValue(const std::string& rName) + { + std::string value; + if (!GetHeader(rName, &value)) + { + THROW_EXCEPTION(CommonException, ConfigNoKey); + } + return value; + } + + // Set dynamic content flag, default is content is dynamic + void SetResponseIsDynamicContent(bool IsDynamic) {mResponseIsDynamicContent = IsDynamic;} + // Set keep alive control, default is to mark as to be closed + void SetKeepAlive(bool KeepAlive) {mKeepAlive = KeepAlive;} + + void SetCookie(const char *Name, const char *Value, const char *Path = "/", int ExpiresAt = 0); + + enum + { + Code_OK = 200, + Code_NoContent = 204, + Code_MovedPermanently = 301, + Code_Found = 302, // redirection + Code_NotModified = 304, + Code_TemporaryRedirect = 307, + Code_MethodNotAllowed = 400, + Code_Unauthorized = 401, + Code_Forbidden = 403, + Code_NotFound = 404, + Code_InternalServerError = 500, + Code_NotImplemented = 501 + }; + + static const char *ResponseCodeToString(int ResponseCode); + + void WriteStringDefang(const char *String, unsigned int StringLen); + void WriteStringDefang(const std::string &rString) {WriteStringDefang(rString.c_str(), rString.size());} + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPResponse::WriteString(const std::string &) + // Purpose: Write a string to the response (simple sugar function) + // Created: 9/4/04 + // + // -------------------------------------------------------------------------- + void WriteString(const std::string &rString) + { + Write(rString.c_str(), rString.size()); + } + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPResponse::SetDefaultURIPrefix(const std::string &) + // Purpose: Set default prefix used to local redirections + // Created: 26/3/04 + // + // -------------------------------------------------------------------------- + static void SetDefaultURIPrefix(const std::string &rPrefix) + { + msDefaultURIPrefix = rPrefix; + } + +private: + int mResponseCode; + bool mResponseIsDynamicContent; + bool mKeepAlive; + std::string mContentType; + std::vector
mExtraHeaders; + int mContentLength; // only used when reading response from stream + IOStream* mpStreamToSendTo; // nonzero only when constructed with a stream + + static std::string msDefaultURIPrefix; + + void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); +}; + +#endif // HTTPRESPONSE__H + diff --git a/lib/httpserver/HTTPServer.cpp b/lib/httpserver/HTTPServer.cpp new file mode 100644 index 00000000..b8b02249 --- /dev/null +++ b/lib/httpserver/HTTPServer.cpp @@ -0,0 +1,249 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPServer.cpp +// Purpose: HTTP server class +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +#include "HTTPServer.h" +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "IOStreamGetLine.h" + +#include "MemLeakFindOn.h" + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::HTTPServer() +// Purpose: Constructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPServer::HTTPServer() + : mTimeout(20000) // default timeout leaves a little while for clients to get the second request in. +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::~HTTPServer() +// Purpose: Destructor +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +HTTPServer::~HTTPServer() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::DaemonName() +// Purpose: As interface, generic name for daemon +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +const char *HTTPServer::DaemonName() const +{ + return "generic-httpserver"; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::GetConfigVerify() +// Purpose: As interface -- return most basic config so it's only necessary to +// provide this if you want to add extra directives. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +const ConfigurationVerify *HTTPServer::GetConfigVerify() const +{ + static ConfigurationVerifyKey verifyserverkeys[] = + { + HTTPSERVER_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default addresses + }; + + static ConfigurationVerify verifyserver[] = + { + { + "Server", + 0, + verifyserverkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + } + }; + + static ConfigurationVerifyKey verifyrootkeys[] = + { + HTTPSERVER_VERIFY_ROOT_KEYS + }; + + static ConfigurationVerify verify = + { + "root", + verifyserver, + verifyrootkeys, + ConfigTest_Exists | ConfigTest_LastEntry, + 0 + }; + + return &verify; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::Run() +// Purpose: As interface. +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::Run() +{ + // Do some configuration stuff + const Configuration &conf(GetConfiguration()); + HTTPResponse::SetDefaultURIPrefix(conf.GetKeyValue("AddressPrefix")); + + // Let the base class do the work + ServerStream::Run(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::Connection(SocketStream &) +// Purpose: As interface, handle connection +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::Connection(SocketStream &rStream) +{ + // Create a get line object to use + IOStreamGetLine getLine(rStream); + + // Notify dervived claases + HTTPConnectionOpening(); + + bool handleRequests = true; + while(handleRequests) + { + // Parse the request + HTTPRequest request; + if(!request.Receive(getLine, mTimeout)) + { + // Didn't get request, connection probably closed. + break; + } + + // Generate a response + HTTPResponse response(&rStream); + try + { + Handle(request, response); + } + catch(BoxException &e) + { + char exceptionCode[64]; + ::sprintf(exceptionCode, "(%d/%d)", e.GetType(), e.GetSubType()); + SendInternalErrorResponse(exceptionCode, rStream); + return; + } + catch(...) + { + SendInternalErrorResponse("unknown", rStream); + return; + } + + // Keep alive? + if(request.GetClientKeepAliveRequested()) + { + // Mark the response to the client as supporting keepalive + response.SetKeepAlive(true); + } + else + { + // Stop now + handleRequests = false; + } + + // Send the response (omit any content if this is a HEAD method request) + response.Send(request.GetMethod() == HTTPRequest::Method_HEAD); + } + + // Notify derived claases + HTTPConnectionClosing(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::SendInternalErrorResponse(const char *, SocketStream &) +// Purpose: Sends an error response to the remote side +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::SendInternalErrorResponse(const char *Error, SocketStream &rStream) +{ + #define ERROR_HTML_1 "Internal Server Error\n" \ + "

Internal Server Error

\n" \ + "

An error, type " + #define ERROR_HTML_2 " occured when processing the request.

" \ + "

Please try again later.

" \ + "\n\n" + + // Generate the error page + HTTPResponse response(&rStream); + response.SetResponseCode(HTTPResponse::Code_InternalServerError); + response.SetContentType("text/html"); + response.Write(ERROR_HTML_1, sizeof(ERROR_HTML_1) - 1); + response.Write(Error, ::strlen(Error)); + response.Write(ERROR_HTML_2, sizeof(ERROR_HTML_2) - 1); + + // Send the error response + response.Send(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::HTTPConnectionOpening() +// Purpose: Override to get notifications of connections opening +// Created: 22/12/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::HTTPConnectionOpening() +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPServer::HTTPConnectionClosing() +// Purpose: Override to get notifications of connections closing +// Created: 22/12/04 +// +// -------------------------------------------------------------------------- +void HTTPServer::HTTPConnectionClosing() +{ +} + + diff --git a/lib/httpserver/HTTPServer.h b/lib/httpserver/HTTPServer.h new file mode 100644 index 00000000..8009438d --- /dev/null +++ b/lib/httpserver/HTTPServer.h @@ -0,0 +1,79 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: HTTPServer.h +// Purpose: HTTP server class +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#ifndef HTTPSERVER__H +#define HTTPSERVER__H + +#include "ServerStream.h" +#include "SocketStream.h" + +class HTTPRequest; +class HTTPResponse; + +// -------------------------------------------------------------------------- +// +// Class +// Name: HTTPServer +// Purpose: HTTP server +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- +class HTTPServer : public ServerStream +{ +public: + HTTPServer(); + ~HTTPServer(); +private: + // no copying + HTTPServer(const HTTPServer &); + HTTPServer &operator=(const HTTPServer &); +public: + + int GetTimeout() const {return mTimeout;} + + // -------------------------------------------------------------------------- + // + // Function + // Name: HTTPServer::Handle(const HTTPRequest &, HTTPResponse &) + // Purpose: Response to a request, filling in the response object for sending + // at some point in the future. + // Created: 26/3/04 + // + // -------------------------------------------------------------------------- + virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) = 0; + + // For notifications to derived classes + virtual void HTTPConnectionOpening(); + virtual void HTTPConnectionClosing(); + +private: + const char *DaemonName() const; + const ConfigurationVerify *GetConfigVerify() const; + void Run(); + void Connection(SocketStream &rStream); + void SendInternalErrorResponse(const char *Error, SocketStream &rStream); + +private: + int mTimeout; // Timeout for read operations +}; + +// Root level +#define HTTPSERVER_VERIFY_ROOT_KEYS \ + ConfigurationVerifyKey("AddressPrefix", \ + ConfigTest_Exists | ConfigTest_LastEntry) + +// AddressPrefix is, for example, http://localhost:1080 -- ie the beginning of the URI +// This is used for handling redirections. + +// Server level +#define HTTPSERVER_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ + SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) + +#endif // HTTPSERVER__H + diff --git a/lib/httpserver/Makefile.extra b/lib/httpserver/Makefile.extra new file mode 100644 index 00000000..f0ca62be --- /dev/null +++ b/lib/httpserver/Makefile.extra @@ -0,0 +1,7 @@ + +MAKEEXCEPTION = ../../lib/common/makeexception.pl + +# AUTOGEN SEEDING +autogen_HTTPException.h autogen_HTTPException.cpp: $(MAKEEXCEPTION) HTTPException.txt + perl $(MAKEEXCEPTION) HTTPException.txt + diff --git a/lib/httpserver/S3Client.cpp b/lib/httpserver/S3Client.cpp new file mode 100644 index 00000000..cd5988d5 --- /dev/null +++ b/lib/httpserver/S3Client.cpp @@ -0,0 +1,243 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Client.cpp +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include + +// #include +// #include + +#include + +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "HTTPServer.h" +#include "autogen_HTTPException.h" +#include "IOStream.h" +#include "Logging.h" +#include "S3Client.h" +#include "decode.h" +#include "encode.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::GetObject(const std::string& rObjectURI) +// Purpose: Retrieve the object with the specified URI (key) +// from your S3 bucket. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::GetObject(const std::string& rObjectURI) +{ + return FinishAndSendRequest(HTTPRequest::Method_GET, rObjectURI); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::PutObject(const std::string& rObjectURI, +// IOStream& rStreamToSend, const char* pContentType) +// Purpose: Upload the stream to S3, creating or overwriting the +// object with the specified URI (key) in your S3 +// bucket. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::PutObject(const std::string& rObjectURI, + IOStream& rStreamToSend, const char* pContentType) +{ + return FinishAndSendRequest(HTTPRequest::Method_PUT, rObjectURI, + &rStreamToSend, pContentType); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::FinishAndSendRequest( +// HTTPRequest::Method Method, +// const std::string& rRequestURI, +// IOStream* pStreamToSend, +// const char* pStreamContentType) +// Purpose: Internal method which creates an HTTP request to S3, +// populates the date and authorization header fields, +// and sends it to S3 (or the simulator), attaching +// the specified stream if any to the request. Opens a +// connection to the server if necessary, which may +// throw a ConnectionException. Returns the HTTP +// response returned by S3, which may be a 500 error. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, + const std::string& rRequestURI, IOStream* pStreamToSend, + const char* pStreamContentType) +{ + HTTPRequest request(Method, rRequestURI); + request.SetHostName(mHostName); + + std::ostringstream date; + time_t tt = time(NULL); + struct tm *tp = gmtime(&tt); + if (!tp) + { + BOX_ERROR("Failed to get current time"); + THROW_EXCEPTION(HTTPException, Internal); + } + const char *dow[] = {"Sun","Mon","Tue","Wed","Thu","Fri","Sat"}; + date << dow[tp->tm_wday] << ", "; + const char *month[] = {"Jan","Feb","Mar","Apr","May","Jun", + "Jul","Aug","Sep","Oct","Nov","Dec"}; + date << std::internal << std::setfill('0') << + std::setw(2) << tp->tm_mday << " " << + month[tp->tm_mon] << " " << + (tp->tm_year + 1900) << " "; + date << std::setw(2) << tp->tm_hour << ":" << + std::setw(2) << tp->tm_min << ":" << + std::setw(2) << tp->tm_sec << " GMT"; + request.AddHeader("Date", date.str()); + + if (pStreamContentType) + { + request.AddHeader("Content-Type", pStreamContentType); + } + + std::string s3suffix = ".s3.amazonaws.com"; + std::string bucket; + if (mHostName.size() > s3suffix.size()) + { + std::string suffix = mHostName.substr(mHostName.size() - + s3suffix.size(), s3suffix.size()); + if (suffix == s3suffix) + { + bucket = mHostName.substr(0, mHostName.size() - + s3suffix.size()); + } + } + + std::ostringstream data; + data << request.GetVerb() << "\n"; + data << "\n"; /* Content-MD5 */ + data << request.GetContentType() << "\n"; + data << date.str() << "\n"; + + if (! bucket.empty()) + { + data << "/" << bucket; + } + + data << request.GetRequestURI(); + std::string data_string = data.str(); + + unsigned char digest_buffer[EVP_MAX_MD_SIZE]; + unsigned int digest_size = sizeof(digest_buffer); + /* unsigned char* mac = */ HMAC(EVP_sha1(), + mSecretKey.c_str(), mSecretKey.size(), + (const unsigned char*)data_string.c_str(), + data_string.size(), digest_buffer, &digest_size); + std::string digest((const char *)digest_buffer, digest_size); + + base64::encoder encoder; + std::string auth_code = "AWS " + mAccessKey + ":" + + encoder.encode(digest); + + if (auth_code[auth_code.size() - 1] == '\n') + { + auth_code = auth_code.substr(0, auth_code.size() - 1); + } + + request.AddHeader("Authorization", auth_code); + + if (mpSimulator) + { + if (pStreamToSend) + { + pStreamToSend->CopyStreamTo(request); + } + + request.SetForReading(); + CollectInBufferStream response_buffer; + HTTPResponse response(&response_buffer); + + mpSimulator->Handle(request, response); + return response; + } + else + { + try + { + if (!mapClientSocket.get()) + { + mapClientSocket.reset(new SocketStream()); + mapClientSocket->Open(Socket::TypeINET, + mHostName, mPort); + } + return SendRequest(request, pStreamToSend, + pStreamContentType); + } + catch (ConnectionException &ce) + { + if (ce.GetType() == ConnectionException::SocketWriteError) + { + // server may have disconnected us, + // try to reconnect, just once + mapClientSocket->Open(Socket::TypeINET, + mHostName, mPort); + return SendRequest(request, pStreamToSend, + pStreamContentType); + } + else + { + throw; + } + } + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::SendRequest(HTTPRequest& rRequest, +// IOStream* pStreamToSend, +// const char* pStreamContentType) +// Purpose: Internal method which sends a pre-existing HTTP +// request to S3. Attaches the specified stream if any +// to the request. Opens a connection to the server if +// necessary, which may throw a ConnectionException. +// Returns the HTTP response returned by S3, which may +// be a 500 error. +// Created: 09/01/09 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::SendRequest(HTTPRequest& rRequest, + IOStream* pStreamToSend, const char* pStreamContentType) +{ + HTTPResponse response; + + if (pStreamToSend) + { + rRequest.SendWithStream(*mapClientSocket, + 30000 /* milliseconds */, + pStreamToSend, response); + } + else + { + rRequest.Send(*mapClientSocket, 30000 /* milliseconds */); + response.Receive(*mapClientSocket, 30000 /* milliseconds */); + } + + return response; +} diff --git a/lib/httpserver/S3Client.h b/lib/httpserver/S3Client.h new file mode 100644 index 00000000..3c4126ac --- /dev/null +++ b/lib/httpserver/S3Client.h @@ -0,0 +1,72 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: S3Client.h +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- + +#ifndef S3CLIENT__H +#define S3CLIENT__H + +#include +#include + +#include "HTTPRequest.h" +#include "SocketStream.h" + +class HTTPResponse; +class HTTPServer; +class IOStream; + +// -------------------------------------------------------------------------- +// +// Class +// Name: S3Client +// Purpose: Amazon S3 client helper implementation class +// Created: 09/01/2009 +// +// -------------------------------------------------------------------------- +class S3Client +{ + public: + S3Client(HTTPServer* pSimulator, const std::string& rHostName, + const std::string& rAccessKey, const std::string& rSecretKey) + : mpSimulator(pSimulator), + mHostName(rHostName), + mAccessKey(rAccessKey), + mSecretKey(rSecretKey) + { } + + S3Client(std::string HostName, int Port, const std::string& rAccessKey, + const std::string& rSecretKey) + : mpSimulator(NULL), + mHostName(HostName), + mPort(Port), + mAccessKey(rAccessKey), + mSecretKey(rSecretKey) + { } + + HTTPResponse GetObject(const std::string& rObjectURI); + HTTPResponse PutObject(const std::string& rObjectURI, + IOStream& rStreamToSend, const char* pContentType = NULL); + + private: + HTTPServer* mpSimulator; + std::string mHostName; + int mPort; + std::auto_ptr mapClientSocket; + std::string mAccessKey, mSecretKey; + + HTTPResponse FinishAndSendRequest(HTTPRequest::Method Method, + const std::string& rRequestURI, + IOStream* pStreamToSend = NULL, + const char* pStreamContentType = NULL); + HTTPResponse SendRequest(HTTPRequest& rRequest, + IOStream* pStreamToSend = NULL, + const char* pStreamContentType = NULL); +}; + +#endif // S3CLIENT__H + diff --git a/lib/httpserver/cdecode.cpp b/lib/httpserver/cdecode.cpp new file mode 100644 index 00000000..e632f182 --- /dev/null +++ b/lib/httpserver/cdecode.cpp @@ -0,0 +1,92 @@ +/* +cdecoder.c - c source to a base64 decoding algorithm implementation + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +extern "C" +{ + +#include "cdecode.h" + +int base64_decode_value(char value_in) +{ + static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; + static const char decoding_size = sizeof(decoding); + value_in -= 43; + if (value_in < 0 || value_in > decoding_size) return -1; + return decoding[(int)value_in]; +} + +void base64_init_decodestate(base64_decodestate* state_in) +{ + state_in->step = step_a; + state_in->plainchar = 0; +} + +int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in) +{ + const char* codechar = code_in; + char* plainchar = plaintext_out; + char fragment; + + *plainchar = state_in->plainchar; + + switch (state_in->step) + { + while (1) + { + case step_a: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_a; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar = (fragment & 0x03f) << 2; + case step_b: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_b; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x030) >> 4; + *plainchar = (fragment & 0x00f) << 4; + case step_c: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_c; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x03c) >> 2; + *plainchar = (fragment & 0x003) << 6; + case step_d: + do { + if (codechar == code_in+length_in) + { + state_in->step = step_d; + state_in->plainchar = *plainchar; + return plainchar - plaintext_out; + } + fragment = (char)base64_decode_value(*codechar++); + } while (fragment < 0); + *plainchar++ |= (fragment & 0x03f); + } + } + /* control should not reach here */ + return plainchar - plaintext_out; +} + +} diff --git a/lib/httpserver/cdecode.h b/lib/httpserver/cdecode.h new file mode 100644 index 00000000..d0d7f489 --- /dev/null +++ b/lib/httpserver/cdecode.h @@ -0,0 +1,28 @@ +/* +cdecode.h - c header for a base64 decoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_CDECODE_H +#define BASE64_CDECODE_H + +typedef enum +{ + step_a, step_b, step_c, step_d +} base64_decodestep; + +typedef struct +{ + base64_decodestep step; + char plainchar; +} base64_decodestate; + +void base64_init_decodestate(base64_decodestate* state_in); + +int base64_decode_value(char value_in); + +int base64_decode_block(const char* code_in, const int length_in, char* plaintext_out, base64_decodestate* state_in); + +#endif /* BASE64_CDECODE_H */ diff --git a/lib/httpserver/cencode.cpp b/lib/httpserver/cencode.cpp new file mode 100644 index 00000000..b33c0683 --- /dev/null +++ b/lib/httpserver/cencode.cpp @@ -0,0 +1,113 @@ +/* +cencoder.c - c source to a base64 encoding algorithm implementation + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +extern "C" +{ + +#include "cencode.h" + +const int CHARS_PER_LINE = 72; + +void base64_init_encodestate(base64_encodestate* state_in) +{ + state_in->step = step_A; + state_in->result = 0; + state_in->stepcount = 0; +} + +char base64_encode_value(char value_in) +{ + static const char* encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + if (value_in > 63) return '='; + return encoding[(int)value_in]; +} + +int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in) +{ + const char* plainchar = plaintext_in; + const char* const plaintextend = plaintext_in + length_in; + char* codechar = code_out; + char result; + char fragment; + + result = state_in->result; + + switch (state_in->step) + { + while (1) + { + case step_A: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_A; + return codechar - code_out; + } + fragment = *plainchar++; + result = (fragment & 0x0fc) >> 2; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x003) << 4; + case step_B: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_B; + return codechar - code_out; + } + fragment = *plainchar++; + result |= (fragment & 0x0f0) >> 4; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x00f) << 2; + case step_C: + if (plainchar == plaintextend) + { + state_in->result = result; + state_in->step = step_C; + return codechar - code_out; + } + fragment = *plainchar++; + result |= (fragment & 0x0c0) >> 6; + *codechar++ = base64_encode_value(result); + result = (fragment & 0x03f) >> 0; + *codechar++ = base64_encode_value(result); + + ++(state_in->stepcount); + if (state_in->stepcount == CHARS_PER_LINE/4) + { + *codechar++ = '\n'; + state_in->stepcount = 0; + } + } + } + /* control should not reach here */ + return codechar - code_out; +} + +int base64_encode_blockend(char* code_out, base64_encodestate* state_in) +{ + char* codechar = code_out; + + switch (state_in->step) + { + case step_B: + *codechar++ = base64_encode_value(state_in->result); + *codechar++ = '='; + *codechar++ = '='; + break; + case step_C: + *codechar++ = base64_encode_value(state_in->result); + *codechar++ = '='; + break; + case step_A: + break; + } + *codechar++ = '\n'; + + return codechar - code_out; +} + +} diff --git a/lib/httpserver/cencode.h b/lib/httpserver/cencode.h new file mode 100644 index 00000000..cf321312 --- /dev/null +++ b/lib/httpserver/cencode.h @@ -0,0 +1,32 @@ +/* +cencode.h - c header for a base64 encoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_CENCODE_H +#define BASE64_CENCODE_H + +typedef enum +{ + step_A, step_B, step_C +} base64_encodestep; + +typedef struct +{ + base64_encodestep step; + char result; + int stepcount; +} base64_encodestate; + +void base64_init_encodestate(base64_encodestate* state_in); + +char base64_encode_value(char value_in); + +int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in); + +int base64_encode_blockend(char* code_out, base64_encodestate* state_in); + +#endif /* BASE64_CENCODE_H */ + diff --git a/lib/httpserver/decode.h b/lib/httpserver/decode.h new file mode 100644 index 00000000..fe59ef7a --- /dev/null +++ b/lib/httpserver/decode.h @@ -0,0 +1,77 @@ +// :mode=c++: +/* +decode.h - c++ wrapper for a base64 decoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_DECODE_H +#define BASE64_DECODE_H + +#include + +namespace base64 +{ + + extern "C" + { + #include "cdecode.h" + } + + struct decoder + { + base64_decodestate _state; + int _buffersize; + + decoder(int buffersize_in = 4096) + : _buffersize(buffersize_in) + {} + int decode(char value_in) + { + return base64_decode_value(value_in); + } + int decode(const char* code_in, const int length_in, char* plaintext_out) + { + return base64_decode_block(code_in, length_in, plaintext_out, &_state); + } + std::string decode(const std::string& input) + { + base64_init_decodestate(&_state); + char* output = new char[2*input.size()]; + int outlength = decode(input.c_str(), input.size(), + output); + std::string output_string(output, outlength); + base64_init_decodestate(&_state); + delete [] output; + return output_string; + } + void decode(std::istream& istream_in, std::ostream& ostream_in) + { + base64_init_decodestate(&_state); + // + const int N = _buffersize; + char* code = new char[N]; + char* plaintext = new char[N]; + int codelength; + int plainlength; + + do + { + istream_in.read((char*)code, N); + codelength = istream_in.gcount(); + plainlength = decode(code, codelength, plaintext); + ostream_in.write((const char*)plaintext, plainlength); + } + while (istream_in.good() && codelength > 0); + // + base64_init_decodestate(&_state); + + delete [] code; + delete [] plaintext; + } + }; + +} // namespace base64 + +#endif // BASE64_DECODE_H diff --git a/lib/httpserver/encode.h b/lib/httpserver/encode.h new file mode 100644 index 00000000..81957a0f --- /dev/null +++ b/lib/httpserver/encode.h @@ -0,0 +1,87 @@ +// :mode=c++: +/* +encode.h - c++ wrapper for a base64 encoding algorithm + +This is part of the libb64 project, and has been placed in the public domain. +For details, see http://sourceforge.net/projects/libb64 +*/ + +#ifndef BASE64_ENCODE_H +#define BASE64_ENCODE_H + +#include + +namespace base64 +{ + + extern "C" + { + #include "cencode.h" + } + + struct encoder + { + base64_encodestate _state; + int _buffersize; + + encoder(int buffersize_in = 4096) + : _buffersize(buffersize_in) + {} + int encode(char value_in) + { + return base64_encode_value(value_in); + } + int encode(const char* code_in, const int length_in, char* plaintext_out) + { + return base64_encode_block(code_in, length_in, plaintext_out, &_state); + } + int encode_end(char* plaintext_out) + { + return base64_encode_blockend(plaintext_out, &_state); + } + std::string encode(const std::string& input) + { + base64_init_encodestate(&_state); + char* output = new char[2*input.size()]; + int outlength = encode(input.c_str(), input.size(), + output); + outlength += encode_end(output + outlength); + std::string output_string(output, outlength); + base64_init_encodestate(&_state); + delete [] output; + return output_string; + } + void encode(std::istream& istream_in, std::ostream& ostream_in) + { + base64_init_encodestate(&_state); + // + const int N = _buffersize; + char* plaintext = new char[N]; + char* code = new char[2*N]; + int plainlength; + int codelength; + + do + { + istream_in.read(plaintext, N); + plainlength = istream_in.gcount(); + // + codelength = encode(plaintext, plainlength, code); + ostream_in.write(code, codelength); + } + while (istream_in.good() && plainlength > 0); + + codelength = encode_end(code); + ostream_in.write(code, codelength); + // + base64_init_encodestate(&_state); + + delete [] code; + delete [] plaintext; + } + }; + +} // namespace base64 + +#endif // BASE64_ENCODE_H + diff --git a/lib/intercept/intercept.cpp b/lib/intercept/intercept.cpp index 65514ae2..7a33b610 100644 --- a/lib/intercept/intercept.cpp +++ b/lib/intercept/intercept.cpp @@ -22,6 +22,7 @@ #endif #include +#include #ifdef HAVE_DLFCN_H #include @@ -84,6 +85,12 @@ static closedir_t* closedir_real = NULL; static lstat_t* lstat_real = NULL; static lstat_t* lstat_hook = NULL; static const char* lstat_file = NULL; +static lstat_t* stat_real = NULL; +static lstat_t* stat_hook = NULL; +static const char* stat_file = NULL; + +static lstat_post_hook_t* lstat_post_hook = NULL; +static lstat_post_hook_t* stat_post_hook = NULL; #define SIZE_ALWAYS_ERROR -773 @@ -97,7 +104,10 @@ void intercept_clear_setup() intercept_filepos = 0; intercept_delay_ms = 0; readdir_hook = NULL; + stat_hook = NULL; lstat_hook = NULL; + stat_post_hook = NULL; + lstat_post_hook = NULL; } bool intercept_triggered() @@ -222,8 +232,16 @@ int intercept_reterr() } \ } +#if defined _FILE_OFFSET_BITS && _FILE_OFFSET_BITS == 64 + #define DEFINE_ONLY_OPEN64 +#endif + extern "C" int -open(const char *path, int flags, mode_t mode) +#ifdef DEFINE_ONLY_OPEN64 + open64(const char *path, int flags, ...) +#else + open(const char *path, int flags, ...) +#endif // DEFINE_ONLY_OPEN64 { if(intercept_count > 0) { @@ -236,6 +254,15 @@ open(const char *path, int flags, mode_t mode) } } + mode_t mode = 0; + if (flags & O_CREAT) + { + va_list ap; + va_start(ap, flags); + mode = va_arg(ap, int); + va_end(ap); + } + #ifdef PLATFORM_NO_SYSCALL int r = TEST_open(path, flags, mode); #else @@ -257,14 +284,27 @@ open(const char *path, int flags, mode_t mode) return r; } +#ifndef DEFINE_ONLY_OPEN64 extern "C" int -open64(const char *path, int flags, mode_t mode) +// open64(const char *path, int flags, mode_t mode) +// open64(const char *path, int flags, ...) +open64 (__const char *path, int flags, ...) { + mode_t mode = 0; + if (flags & O_CREAT) + { + va_list ap; + va_start(ap, flags); + mode = va_arg(ap, int); + va_end(ap); + } + // With _FILE_OFFSET_BITS set to 64 this should really use (flags | // O_LARGEFILE) here, but not actually necessary for the tests and not // worth the trouble finding O_LARGEFILE return open(path, flags, mode); } +#endif // !DEFINE_ONLY_OPEN64 extern "C" int close(int d) @@ -375,12 +415,12 @@ void intercept_setup_readdir_hook(const char *dirname, readdir_t hookfn) if (hookfn != NULL) { - TRACE2("readdir hooked to %p for %s\n", hookfn, dirname); + BOX_TRACE("readdir hooked to " << hookfn << " for " << dirname); } else if (intercept_filename != NULL) { - TRACE2("readdir unhooked from %p for %s\n", readdir_hook, - intercept_filename); + BOX_TRACE("readdir unhooked from " << readdir_hook << + " for " << intercept_filename); } intercept_filename = dirname; @@ -392,11 +432,11 @@ void intercept_setup_lstat_hook(const char *filename, lstat_t hookfn) /* if (hookfn != NULL) { - TRACE2("lstat hooked to %p for %s\n", hookfn, filename); + BOX_TRACE("lstat hooked to " << hookfn << " for " << filename); } else { - TRACE2("lstat unhooked from %p for %s\n", lstat_hook, + BOX_TRACE("lstat unhooked from " << lstat_hook << " for " << lstat_file); } */ @@ -405,6 +445,40 @@ void intercept_setup_lstat_hook(const char *filename, lstat_t hookfn) lstat_hook = hookfn; } +void intercept_setup_lstat_post_hook(lstat_post_hook_t hookfn) +{ + /* + if (hookfn != NULL) + { + BOX_TRACE("lstat hooked to " << hookfn << " for " << filename); + } + else + { + BOX_TRACE("lstat unhooked from " << lstat_hook << " for " << + lstat_file); + } + */ + + lstat_post_hook = hookfn; +} + +void intercept_setup_stat_post_hook(lstat_post_hook_t hookfn) +{ + /* + if (hookfn != NULL) + { + BOX_TRACE("lstat hooked to " << hookfn << " for " << filename); + } + else + { + BOX_TRACE("lstat unhooked from " << lstat_hook << " for " << + lstat_file); + } + */ + + stat_post_hook = hookfn; +} + static void * find_function(const char *pName) { dlerror(); @@ -534,10 +608,15 @@ lstat(const char *file_name, STAT_STRUCT *buf) if (lstat_hook == NULL || strcmp(file_name, lstat_file) != 0) { #ifdef LINUX_WEIRD_LSTAT - return lstat_real(ver, file_name, buf); + int ret = lstat_real(ver, file_name, buf); #else - return lstat_real(file_name, buf); + int ret = lstat_real(file_name, buf); #endif + if (lstat_post_hook != NULL) + { + ret = lstat_post_hook(ret, file_name, buf); + } + return ret; } #ifdef LINUX_WEIRD_LSTAT @@ -547,4 +626,48 @@ lstat(const char *file_name, STAT_STRUCT *buf) #endif } +extern "C" int +#ifdef LINUX_WEIRD_LSTAT +__xstat(int ver, const char *file_name, STAT_STRUCT *buf) +#else +stat(const char *file_name, STAT_STRUCT *buf) +#endif +{ + if (stat_real == NULL) + { + #ifdef LINUX_WEIRD_LSTAT + stat_real = (lstat_t*)find_function("__xstat"); + #else + stat_real = (lstat_t*)find_function("stat"); + #endif + } + + if (stat_real == NULL) + { + perror("cannot find real stat"); + errno = ENOSYS; + return -1; + } + + if (stat_hook == NULL || strcmp(file_name, stat_file) != 0) + { + #ifdef LINUX_WEIRD_LSTAT + int ret = stat_real(ver, file_name, buf); + #else + int ret = stat_real(file_name, buf); + #endif + if (stat_post_hook != NULL) + { + ret = stat_post_hook(ret, file_name, buf); + } + return ret; + } + + #ifdef LINUX_WEIRD_LSTAT + return stat_hook(ver, file_name, buf); + #else + return stat_hook(file_name, buf); + #endif +} + #endif // n PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE diff --git a/lib/intercept/intercept.h b/lib/intercept/intercept.h index bc6557f3..80a17d3f 100644 --- a/lib/intercept/intercept.h +++ b/lib/intercept/intercept.h @@ -23,17 +23,20 @@ extern "C" typedef struct dirent *(readdir_t) (DIR *dir); typedef int (closedir_t)(DIR *dir); #if defined __GNUC__ && __GNUC__ >= 2 -#define LINUX_WEIRD_LSTAT -#define STAT_STRUCT struct stat /* should be stat64 */ - typedef int (lstat_t) (int ver, const char *file_name, - STAT_STRUCT *buf); + #define LINUX_WEIRD_LSTAT + #define STAT_STRUCT struct stat /* should be stat64 */ + typedef int (lstat_t) (int ver, const char *file_name, + STAT_STRUCT *buf); #else -#define STAT_STRUCT struct stat - typedef int (lstat_t) (const char *file_name, - STAT_STRUCT *buf); + #define STAT_STRUCT struct stat + typedef int (lstat_t) (const char *file_name, + STAT_STRUCT *buf); #endif } +typedef int (lstat_post_hook_t) (int old_ret, const char *file_name, + struct stat *buf); + void intercept_setup_error(const char *filename, unsigned int errorafter, int errortoreturn, int syscalltoerror); void intercept_setup_delay(const char *filename, unsigned int delay_after, @@ -42,6 +45,10 @@ bool intercept_triggered(); void intercept_setup_readdir_hook(const char *dirname, readdir_t hookfn); void intercept_setup_lstat_hook (const char *filename, lstat_t hookfn); +void intercept_setup_lstat_post_hook(lstat_post_hook_t hookfn); +void intercept_setup_stat_post_hook (lstat_post_hook_t hookfn); + +void intercept_clear_setup(); #endif // !PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE #endif // !INTERCEPT_H diff --git a/lib/raidfile/RaidFileController.cpp b/lib/raidfile/RaidFileController.cpp index 0cc2ede7..2cc6976b 100644 --- a/lib/raidfile/RaidFileController.cpp +++ b/lib/raidfile/RaidFileController.cpp @@ -70,11 +70,14 @@ void RaidFileController::Initialise(const std::string& rConfigFilename) static const ConfigurationVerifyKey verifykeys[] = { - {"SetNumber", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"BlockSize", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"Dir0", 0, ConfigTest_Exists, 0}, - {"Dir1", 0, ConfigTest_Exists, 0}, - {"Dir2", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("SetNumber", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("BlockSize", + ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("Dir0", ConfigTest_Exists), + ConfigurationVerifyKey("Dir1", ConfigTest_Exists), + ConfigurationVerifyKey("Dir2", + ConfigTest_Exists | ConfigTest_LastEntry) }; static const ConfigurationVerify subverify = @@ -105,6 +108,10 @@ void RaidFileController::Initialise(const std::string& rConfigFilename) BOX_ERROR("RaidFile configuration file errors: " << err); THROW_EXCEPTION(RaidFileException, BadConfigFile) } + + // Allow reinitializing the controller by remove any existing + // disc sets. Used by Boxi unit tests. + mSetList.clear(); // Use the values int expectedSetNum = 0; diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp index 187270f9..0a79be57 100644 --- a/lib/raidfile/RaidFileRead.cpp +++ b/lib/raidfile/RaidFileRead.cpp @@ -25,10 +25,11 @@ #include #endif -#include -#include -#include +#include +#include +#include #include +#include #include "RaidFileRead.h" #include "RaidFileException.h" @@ -1021,6 +1022,7 @@ std::auto_ptr RaidFileRead::Open(int SetNumber, const std::string RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, Filename, &startDisc, &existingFiles, pRevisionID); if(existance == RaidFileUtil::NoFile) { + BOX_ERROR("Expected raidfile " << Filename << " does not exist"); THROW_EXCEPTION(RaidFileException, RaidFileDoesntExist) } else if(existance == RaidFileUtil::NonRaid) diff --git a/lib/raidfile/RaidFileUtil.cpp b/lib/raidfile/RaidFileUtil.cpp index da23cef5..7c6299ec 100644 --- a/lib/raidfile/RaidFileUtil.cpp +++ b/lib/raidfile/RaidFileUtil.cpp @@ -22,17 +22,21 @@ // -------------------------------------------------------------------------- // // Function -// Name: RaidFileUtil::RaidFileExists(RaidFileDiscSet &, const std::string &) -// Purpose: Check to see the state of a RaidFile on disc (doesn't look at contents, -// just at existense of files) +// Name: RaidFileUtil::RaidFileExists(RaidFileDiscSet &, +// const std::string &, int *, int *, int64_t *) +// Purpose: Check to see the state of a RaidFile on disc +// (doesn't look at contents, just at existence of +// files) // Created: 2003/07/11 // // -------------------------------------------------------------------------- -RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, const std::string &rFilename, int *pStartDisc, int *pExisitingFiles, int64_t *pRevisionID) +RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, + const std::string &rFilename, int *pStartDisc, int *pExistingFiles, + int64_t *pRevisionID) { - if(pExisitingFiles) + if(pExistingFiles) { - *pExisitingFiles = 0; + *pExistingFiles = 0; } // For stat call, although the results are not examined @@ -53,11 +57,20 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, // Get unique ID if(pRevisionID != 0) { - (*pRevisionID) = FileModificationTime(st); -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC - // On linux, the time resolution is very low for modification times. - // So add the size to it to give a bit more chance of it changing. + #ifdef WIN32 + *pRevisionID = st.st_mtime; + #else + *pRevisionID = FileModificationTime(st); + #endif + +#ifdef BOX_RELEASE_BUILD + // The resolution of timestamps may be very + // low, e.g. 1 second. So add the size to it + // to give a bit more chance of it changing. // TODO: Make this better. + // Disabled in debug mode, to simulate + // filesystem with 1-second timestamp + // resolution, e.g. MacOS X HFS, Linux. (*pRevisionID) += st.st_size; #endif } @@ -71,10 +84,10 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, int64_t revisionID = 0; int setSize = rDiscSet.size(); int rfCount = 0; -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC + // TODO: replace this with better linux revision ID detection int64_t revisionIDplus = 0; -#endif + for(int f = 0; f < setSize; ++f) { std::string componentFile(RaidFileUtil::MakeRaidComponentName(rDiscSet, rFilename, (f + startDisc) % setSize)); @@ -83,25 +96,34 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, // Component file exists, add to count rfCount++; // Set flags for existance? - if(pExisitingFiles) + if(pExistingFiles) { - (*pExisitingFiles) |= (1 << f); + (*pExistingFiles) |= (1 << f); } // Revision ID if(pRevisionID != 0) { - int64_t rid = FileModificationTime(st); + #ifdef WIN32 + int64_t rid = st.st_mtime; + #else + int64_t rid = FileModificationTime(st); + #endif + if(rid > revisionID) revisionID = rid; -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC revisionIDplus += st.st_size; -#endif } } } if(pRevisionID != 0) { (*pRevisionID) = revisionID; -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC +#ifdef BOX_RELEASE_BUILD + // The resolution of timestamps may be very low, e.g. + // 1 second. So add the size to it to give a bit more + // chance of it changing. + // TODO: Make this better. + // Disabled in debug mode, to simulate filesystem with + // 1-second timestamp resolution, e.g. MacOS X HFS, Linux. (*pRevisionID) += revisionIDplus; #endif } diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp index 66ab81c8..efec43a2 100644 --- a/lib/raidfile/RaidFileWrite.cpp +++ b/lib/raidfile/RaidFileWrite.cpp @@ -96,7 +96,8 @@ void RaidFileWrite::Open(bool AllowOverwrite) RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, mFilename); if(existance != RaidFileUtil::NoFile) { - TRACE2("Trying to overwrite raidfile %d %s\n", mSetNumber, mFilename.c_str()); + BOX_ERROR("Attempted to overwrite raidfile " << + mSetNumber << " " << mFilename); THROW_EXCEPTION(RaidFileException, CannotOverwriteExistingFile) } } @@ -178,7 +179,8 @@ void RaidFileWrite::Write(const void *pBuffer, int Length) int written = ::write(mOSFileHandle, pBuffer, Length); if(written != Length) { - TRACE3("RaidFileWrite::Write: Write failure, Length = %d, written = %d, errno = %d\n", Length, written, errno); + BOX_LOG_SYS_ERROR("RaidFileWrite failed, Length = " << + Length << ", written = " << written); THROW_EXCEPTION(RaidFileException, OSError) } } @@ -272,6 +274,7 @@ void RaidFileWrite::Commit(bool ConvertToRaidNow) if(::unlink(renameTo.c_str()) != 0 && GetLastError() != ERROR_FILE_NOT_FOUND) { + BOX_LOG_WIN_ERROR("failed to delete file: " << renameTo); THROW_EXCEPTION(RaidFileException, OSError) } #endif @@ -387,7 +390,7 @@ void RaidFileWrite::TransformToRaidStorage() // // DEBUG MODE -- check file system size block size is same as block size for files // // doesn't really apply, as space benefits of using fragment size are worth efficiency, // // and anyway, it'll be buffered eventually so it won't matter. -// #ifndef NDEBUG +// #ifndef BOX_RELEASE_BUILD // { // if(writeFileStat.st_blksize != blockSize) // { @@ -779,7 +782,7 @@ int RaidFileWrite::Read(void *pBuffer, int NBytes, int Timeout) // -------------------------------------------------------------------------- void RaidFileWrite::Close() { - TRACE0("Warning: RaidFileWrite::Close() called, discarding file\n"); + BOX_WARNING("RaidFileWrite::Close() called, discarding file"); if(mOSFileHandle != -1) { Discard(); diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp index bdbbb433..d868774f 100644 --- a/lib/server/Daemon.cpp +++ b/lib/server/Daemon.cpp @@ -47,22 +47,21 @@ Daemon *Daemon::spDaemon = 0; // // -------------------------------------------------------------------------- Daemon::Daemon() - : mpConfiguration(NULL), - mReloadConfigWanted(false), + : mReloadConfigWanted(false), mTerminateWanted(false), + #ifdef WIN32 + mSingleProcess(true), + mRunInForeground(true), + mKeepConsoleOpenAfterFork(true), + #else mSingleProcess(false), mRunInForeground(false), mKeepConsoleOpenAfterFork(false), + #endif mHaveConfigFile(false), mAppName(DaemonName()) { - if(spDaemon != NULL) - { - THROW_EXCEPTION(ServerException, AlreadyDaemonConstructed) - } - spDaemon = this; - - // And in debug builds, we'll switch on assert failure logging to syslog + // In debug builds, switch on assert failure logging to syslog ASSERT_FAILS_TO_SYSLOG_ON // And trace goes to syslog too TRACE_TO_SYSLOG(true) @@ -78,14 +77,6 @@ Daemon::Daemon() // -------------------------------------------------------------------------- Daemon::~Daemon() { - if(mpConfiguration) - { - delete mpConfiguration; - mpConfiguration = 0; - } - - ASSERT(spDaemon == this); - spDaemon = NULL; } // -------------------------------------------------------------------------- @@ -104,9 +95,9 @@ std::string Daemon::GetOptionString() { return "c:" #ifndef WIN32 - "DFk" + "DFK" #endif - "hqvVt:T"; + "hkPqQt:TUvVW:"; } void Daemon::Usage() @@ -123,13 +114,20 @@ void Daemon::Usage() #ifndef WIN32 " -D Debugging mode, do not fork, one process only, one client only\n" " -F Do not fork into background, but fork to serve multiple clients\n" +#endif " -k Keep console open after fork, keep writing log messages to it\n" +#ifndef WIN32 + " -K Stop writing log messages to console while daemon is running\n" + " -P Show process ID (PID) in console output\n" #endif " -q Run more quietly, reduce verbosity level by one, can repeat\n" + " -Q Run at minimum verbosity, log nothing\n" " -v Run more verbosely, increase verbosity level by one, can repeat\n" - " -V Run at maximum verbosity\n" + " -V Run at maximum verbosity, log everything\n" + " -W Set verbosity to error/warning/notice/info/trace/everything\n" " -t Tag console output with specified marker\n" - " -T Timestamp console output\n"; + " -T Timestamp console output\n" + " -U Timestamp console output with microseconds\n"; } // -------------------------------------------------------------------------- @@ -166,13 +164,19 @@ int Daemon::ProcessOption(signed int option) mRunInForeground = true; } break; +#endif // !WIN32 case 'k': { mKeepConsoleOpenAfterFork = true; } break; -#endif + + case 'K': + { + mKeepConsoleOpenAfterFork = false; + } + break; case 'h': { @@ -181,6 +185,12 @@ int Daemon::ProcessOption(signed int option) } break; + case 'P': + { + Console::SetShowPID(true); + } + break; + case 'q': { if(mLogLevel == Log::NOTHING) @@ -194,6 +204,13 @@ int Daemon::ProcessOption(signed int option) } break; + case 'Q': + { + mLogLevel = Log::NOTHING; + } + break; + + case 'v': { if(mLogLevel == Log::EVERYTHING) @@ -213,9 +230,21 @@ int Daemon::ProcessOption(signed int option) } break; + case 'W': + { + mLogLevel = Logging::GetNamedLevel(optarg); + if (mLogLevel == Log::INVALID) + { + BOX_FATAL("Invalid logging level"); + return 2; + } + } + break; + case 't': { - Console::SetTag(optarg); + Logging::SetProgramName(optarg); + Console::SetShowTag(true); } break; @@ -225,6 +254,13 @@ int Daemon::ProcessOption(signed int option) } break; + case 'U': + { + Console::SetShowTime(true); + Console::SetShowTimeMicros(true); + } + break; + case '?': { BOX_FATAL("Unknown option on command line: " @@ -260,7 +296,7 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) mConfigFileName = DefaultConfigFile; mAppName = argv[0]; - #ifdef NDEBUG + #ifdef BOX_RELEASE_BUILD mLogLevel = Log::NOTICE; // need an int to do math with #else mLogLevel = Log::INFO; // need an int to do math with @@ -277,7 +313,7 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) // reset getopt, just in case anybody used it before. // unfortunately glibc and BSD differ on this point! // http://www.ussg.iu.edu/hypermail/linux/kernel/0305.3/0262.html - #if HAVE_DECL_OPTRESET == 1 + #if HAVE_DECL_OPTRESET == 1 || defined WIN32 optind = 1; optreset = 1; #elif defined __GLIBC__ @@ -314,11 +350,104 @@ int Daemon::Main(const char *DefaultConfigFile, int argc, const char *argv[]) return 2; } - Logging::SetGlobalLevel((Log::Level)mLogLevel); + Logging::FilterConsole((Log::Level)mLogLevel); + Logging::FilterSyslog ((Log::Level)mLogLevel); return Main(mConfigFileName); } +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Configure(const std::string& rConfigFileName) +// Purpose: Loads daemon configuration. Useful when you have +// a local Daemon object and don't intend to fork() +// or call Main(). +// Created: 2008/04/19 +// +// -------------------------------------------------------------------------- + +bool Daemon::Configure(const std::string& rConfigFileName) +{ + // Load the configuration file. + std::string errors; + std::auto_ptr apConfig; + + try + { + apConfig = Configuration::LoadAndVerify(rConfigFileName, + GetConfigVerify(), errors); + } + catch(BoxException &e) + { + if(e.GetType() == CommonException::ExceptionType && + e.GetSubType() == CommonException::OSFileOpenError) + { + BOX_ERROR("Failed to open configuration file: " << + rConfigFileName); + return false; + } + + throw; + } + + // Got errors? + if(apConfig.get() == 0) + { + BOX_ERROR("Failed to load or verify configuration file"); + return false; + } + + if(!Configure(*apConfig)) + { + BOX_ERROR("Failed to verify configuration file"); + return false; + } + + // Store configuration + mConfigFileName = rConfigFileName; + mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Daemon::Configure(const Configuration& rConfig) +// Purpose: Loads daemon configuration. Useful when you have +// a local Daemon object and don't intend to fork() +// or call Main(). +// Created: 2008/08/12 +// +// -------------------------------------------------------------------------- + +bool Daemon::Configure(const Configuration& rConfig) +{ + std::string errors; + + // Verify() may modify the configuration, e.g. adding default values + // for required keys, so need to make a copy here + std::auto_ptr apConf(new Configuration(rConfig)); + apConf->Verify(*GetConfigVerify(), errors); + + // Got errors? + if(!errors.empty()) + { + BOX_ERROR("Configuration errors: " << errors); + return false; + } + + // Store configuration + mapConfiguration = apConf; + + // Let the derived class have a go at setting up stuff + // in the initial process + SetupInInitialProcess(); + + return true; +} + // -------------------------------------------------------------------------- // // Function @@ -331,69 +460,38 @@ int Daemon::Main(const std::string &rConfigFileName) { // Banner (optional) { - #ifndef NDEBUG - BOX_NOTICE(DaemonBanner()); - #endif + BOX_SYSLOG(Log::NOTICE, DaemonBanner()); } std::string pidFileName; - mConfigFileName = rConfigFileName; - - bool asDaemon = !mSingleProcess && !mRunInForeground; + bool asDaemon = !mSingleProcess && !mRunInForeground; try { - // Load the configuration file. - std::string errors; - std::auto_ptr pconfig; - - try - { - pconfig = Configuration::LoadAndVerify( - mConfigFileName.c_str(), - GetConfigVerify(), errors); - } - catch(BoxException &e) - { - if(e.GetType() == CommonException::ExceptionType && - e.GetSubType() == CommonException::OSFileOpenError) - { - BOX_FATAL("Failed to start: failed to open " - "configuration file: " - << mConfigFileName); - return 1; - } - - throw; - } - - // Got errors? - if(pconfig.get() == 0 || !errors.empty()) + if (!Configure(rConfigFileName)) { - // Tell user about errors - BOX_FATAL("Failed to start: errors in configuration " - "file: " << mConfigFileName << ": " << errors); - // And give up + BOX_FATAL("Failed to start: failed to load " + "configuration file: " << rConfigFileName); return 1; } - // Store configuration - mpConfiguration = pconfig.release(); - mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); - - // Let the derived class have a go at setting up stuff in the initial process - SetupInInitialProcess(); - // Server configuration const Configuration &serverConfig( - mpConfiguration->GetSubConfiguration("Server")); + mapConfiguration->GetSubConfiguration("Server")); + + if(serverConfig.KeyExists("LogFacility")) + { + std::string facility = + serverConfig.GetKeyValue("LogFacility"); + Logging::SetFacility(Syslog::GetNamedFacility(facility)); + } // Open PID file for writing pidFileName = serverConfig.GetKeyValue("PidFile"); FileHandleGuard<(O_WRONLY | O_CREAT | O_TRUNC), (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)> pidFile(pidFileName.c_str()); -#ifndef WIN32 +#ifndef WIN32 // Handle changing to a different user if(serverConfig.KeyExists("User")) { @@ -409,7 +507,7 @@ int Daemon::Main(const std::string &rConfigFileName) // Change the process ID daemonUser.ChangeProcessUser(); } - + if(asDaemon) { // Let's go... Daemonise... @@ -436,8 +534,7 @@ int Daemon::Main(const std::string &rConfigFileName) // Set new session if(::setsid() == -1) { - BOX_ERROR("Failed to setsid(): " << - strerror(errno)); + BOX_LOG_SYS_ERROR("Failed to setsid()"); THROW_EXCEPTION(ServerException, DaemoniseFailed) } @@ -446,6 +543,7 @@ int Daemon::Main(const std::string &rConfigFileName) { case -1: // error + BOX_LOG_SYS_ERROR("Failed to fork() a child"); THROW_EXCEPTION(ServerException, DaemoniseFailed) break; @@ -460,7 +558,17 @@ int Daemon::Main(const std::string &rConfigFileName) break; } } - +#endif // !WIN32 + + // Must set spDaemon before installing signal handler, + // otherwise the handler will crash if invoked too soon. + if(spDaemon != NULL) + { + THROW_EXCEPTION(ServerException, AlreadyDaemonConstructed) + } + spDaemon = this; + +#ifndef WIN32 // Set signal handler // Don't do this in the parent, since it might be anything // (e.g. test/bbackupd) @@ -468,29 +576,24 @@ int Daemon::Main(const std::string &rConfigFileName) struct sigaction sa; sa.sa_handler = SignalHandler; sa.sa_flags = 0; - sigemptyset(&sa.sa_mask); // macro - if(::sigaction(SIGHUP, &sa, NULL) != 0 || ::sigaction(SIGTERM, &sa, NULL) != 0) + sigemptyset(&sa.sa_mask); // macro + if(::sigaction(SIGHUP, &sa, NULL) != 0 || + ::sigaction(SIGTERM, &sa, NULL) != 0) { + BOX_LOG_SYS_ERROR("Failed to set signal handlers"); THROW_EXCEPTION(ServerException, DaemoniseFailed) } #endif // !WIN32 - // Log the start message - BOX_NOTICE("Starting daemon, version " << BOX_VERSION - << ", config: " << mConfigFileName); - // Write PID to file char pid[32]; -#ifdef WIN32 - int pidsize = sprintf(pid, "%d", (int)GetCurrentProcessId()); -#else int pidsize = sprintf(pid, "%d", (int)getpid()); -#endif if(::write(pidFile, pid, pidsize) != pidsize) { - BOX_FATAL("can't write pid file"); + BOX_LOG_SYS_FATAL("Failed to write PID file: " << + pidFileName); THROW_EXCEPTION(ServerException, DaemoniseFailed) } @@ -515,6 +618,7 @@ int Daemon::Main(const std::string &rConfigFileName) int devnull = ::open(PLATFORM_DEV_NULL, O_RDWR, 0); if(devnull == -1) { + BOX_LOG_SYS_ERROR("Failed to open /dev/null"); THROW_EXCEPTION(CommonException, OSFileError); } // Then duplicate them to all three handles @@ -530,9 +634,13 @@ int Daemon::Main(const std::string &rConfigFileName) // And definitely don't try and send anything to those file descriptors // -- this has in the past sent text to something which isn't expecting it. TRACE_TO_STDOUT(false); - Logging::ToConsole(false); #endif // ! WIN32 - } + Logging::ToConsole(false); + } + + // Log the start message + BOX_NOTICE("Starting daemon, version: " << BOX_VERSION); + BOX_NOTICE("Using configuration file: " << mConfigFileName); } catch(BoxException &e) { @@ -581,10 +689,10 @@ int Daemon::Main(const std::string &rConfigFileName) BOX_NOTICE("Reloading configuration file: " << mConfigFileName); std::string errors; - std::auto_ptr pconfig = + std::auto_ptr pconfig( Configuration::LoadAndVerify( mConfigFileName.c_str(), - GetConfigVerify(), errors); + GetConfigVerify(), errors)); // Got errors? if(pconfig.get() == 0 || !errors.empty()) @@ -598,12 +706,8 @@ int Daemon::Main(const std::string &rConfigFileName) break; } - // delete old configuration - delete mpConfiguration; - mpConfiguration = 0; - // Store configuration - mpConfiguration = pconfig.release(); + mapConfiguration = pconfig; mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); @@ -638,8 +742,21 @@ int Daemon::Main(const std::string &rConfigFileName) #ifdef WIN32 WSACleanup(); +#else + // Should clean up here, but it breaks memory leak tests. + /* + if(asDaemon) + { + // we are running in the child by now, and should not return + mapConfiguration.reset(); + exit(0); + } + */ #endif - + + ASSERT(spDaemon == this); + spDaemon = NULL; + return retcode; } @@ -647,7 +764,8 @@ int Daemon::Main(const std::string &rConfigFileName) // // Function // Name: Daemon::EnterChild() -// Purpose: Sets up for a child task of the main server. Call just after fork() +// Purpose: Sets up for a child task of the main server. Call +// just after fork(). // Created: 2003/07/31 // // -------------------------------------------------------------------------- @@ -789,13 +907,13 @@ const ConfigurationVerify *Daemon::GetConfigVerify() const // -------------------------------------------------------------------------- const Configuration &Daemon::GetConfiguration() const { - if(mpConfiguration == 0) + if(mapConfiguration.get() == 0) { // Shouldn't get anywhere near this if a configuration file can't be loaded THROW_EXCEPTION(ServerException, Internal) } - return *mpConfiguration; + return *mapConfiguration; } @@ -803,8 +921,10 @@ const Configuration &Daemon::GetConfiguration() const // // Function // Name: Daemon::SetupInInitialProcess() -// Purpose: A chance for the daemon to do something initial setting up in the process which -// initiates everything, and after the configuration file has been read and verified. +// Purpose: A chance for the daemon to do something initial +// setting up in the process which initiates +// everything, and after the configuration file has +// been read and verified. // Created: 2003/08/20 // // -------------------------------------------------------------------------- @@ -849,14 +969,16 @@ void Daemon::SetProcessTitle(const char *format, ...) box_time_t Daemon::GetConfigFileModifiedTime() const { - struct stat st; + EMU_STRUCT_STAT st; - if(::stat(GetConfigFileName().c_str(), &st) != 0) + if(EMU_STAT(GetConfigFileName().c_str(), &st) != 0) { if (errno == ENOENT) { return 0; } + BOX_LOG_SYS_ERROR("Failed to stat configuration file: " << + GetConfigFileName()); THROW_EXCEPTION(CommonException, OSFileError) } diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h index 482f926e..a3212a00 100644 --- a/lib/server/Daemon.h +++ b/lib/server/Daemon.h @@ -19,8 +19,8 @@ #include #include "BoxTime.h" +#include "Configuration.h" -class Configuration; class ConfigurationVerify; // -------------------------------------------------------------------------- @@ -40,7 +40,8 @@ private: Daemon(const Daemon &rToCopy); public: - int Main(const char *DefaultConfigFile, int argc, const char *argv[]); + virtual int Main(const char *DefaultConfigFile, int argc, + const char *argv[]); /* override this Main() if you want custom option processing: */ virtual int Main(const std::string &rConfigFile); @@ -53,7 +54,10 @@ public: virtual std::string DaemonBanner() const; virtual const ConfigurationVerify *GetConfigVerify() const; virtual void Usage(); - + + virtual bool Configure(const std::string& rConfigFileName); + virtual bool Configure(const Configuration& rConfig); + bool StopRun() {return mReloadConfigWanted | mTerminateWanted;} bool IsReloadConfigWanted() {return mReloadConfigWanted;} bool IsTerminateWanted() {return mTerminateWanted;} @@ -62,12 +66,20 @@ public: void SetReloadConfigWanted() {mReloadConfigWanted = true;} void SetTerminateWanted() {mTerminateWanted = true;} - virtual void SetupInInitialProcess(); virtual void EnterChild(); static void SetProcessTitle(const char *format, ...); + void SetRunInForeground(bool foreground) + { + mRunInForeground = foreground; + } + void SetSingleProcess(bool value) + { + mSingleProcess = value; + } protected: + virtual void SetupInInitialProcess(); box_time_t GetLoadedConfigModifiedTime() const; bool IsSingleProcess() { return mSingleProcess; } virtual std::string GetOptionString(); @@ -78,7 +90,7 @@ private: box_time_t GetConfigFileModifiedTime() const; std::string mConfigFileName; - Configuration *mpConfiguration; + std::auto_ptr mapConfiguration; box_time_t mLoadedConfigModifiedTime; bool mReloadConfigWanted; bool mTerminateWanted; @@ -91,8 +103,10 @@ private: std::string mAppName; }; -#define DAEMON_VERIFY_SERVER_KEYS {"PidFile", 0, ConfigTest_Exists, 0}, \ - {"User", 0, ConfigTest_LastEntry, 0} +#define DAEMON_VERIFY_SERVER_KEYS \ + ConfigurationVerifyKey("PidFile", ConfigTest_Exists), \ + ConfigurationVerifyKey("LogFacility", 0), \ + ConfigurationVerifyKey("User", ConfigTest_LastEntry) #endif // DAEMON__H diff --git a/lib/server/OverlappedIO.h b/lib/server/OverlappedIO.h new file mode 100644 index 00000000..12495053 --- /dev/null +++ b/lib/server/OverlappedIO.h @@ -0,0 +1,42 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: OverlappedIO.h +// Purpose: Windows overlapped IO handle guard +// Created: 2008/09/30 +// +// -------------------------------------------------------------------------- + +#ifndef OVERLAPPEDIO__H +#define OVERLAPPEDIO__H + +class OverlappedIO +{ +public: + OVERLAPPED mOverlapped; + + OverlappedIO() + { + ZeroMemory(&mOverlapped, sizeof(mOverlapped)); + mOverlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, + NULL); + if (mOverlapped.hEvent == INVALID_HANDLE_VALUE) + { + BOX_LOG_WIN_ERROR("Failed to create event for " + "overlapped I/O"); + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + } + + ~OverlappedIO() + { + if (CloseHandle(mOverlapped.hEvent) != TRUE) + { + BOX_LOG_WIN_ERROR("Failed to delete event for " + "overlapped I/O"); + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + } +}; + +#endif // !OVERLAPPEDIO__H diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp index 4398a58f..5dc5d0b1 100644 --- a/lib/server/Protocol.cpp +++ b/lib/server/Protocol.cpp @@ -26,7 +26,7 @@ #include "MemLeakFindOn.h" -#ifdef NDEBUG +#ifdef BOX_RELEASE_BUILD #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 1024 #else // #define PROTOCOL_ALLOCATE_SEND_BLOCK_CHUNK 1024 @@ -670,9 +670,18 @@ std::auto_ptr Protocol::ReceiveStream() InformStreamReceiving(streamSize); // Return a stream object - return std::auto_ptr((streamSize == ProtocolStream_SizeUncertain)? - ((IOStream*)(new ProtocolUncertainStream(mrStream))) - :((IOStream*)(new PartialReadStream(mrStream, streamSize)))); + if(streamSize == ProtocolStream_SizeUncertain) + { + BOX_TRACE("Receiving stream, size uncertain"); + return std::auto_ptr( + new ProtocolUncertainStream(mrStream)); + } + else + { + BOX_TRACE("Receiving stream, size " << streamSize << " bytes"); + return std::auto_ptr( + new PartialReadStream(mrStream, streamSize)); + } } // -------------------------------------------------------------------------- @@ -749,8 +758,10 @@ void Protocol::SendStream(IOStream &rStream) } // Send final byte to finish the stream + BOX_TRACE("Sending end of stream byte"); uint8_t endOfStream = ProtocolStreamHeader_EndOfStream; mrStream.Write(&endOfStream, 1); + BOX_TRACE("Sent end of stream byte"); } catch(...) { @@ -788,6 +799,7 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) // Quick sanity check if(BytesInBlock == 0) { + BOX_TRACE("Zero size block, not sending anything"); return 0; } @@ -813,6 +825,8 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) } } ASSERT(header > 0); + BOX_TRACE("Sending header byte " << (int)header << " plus " << + writeSize << " bytes to stream"); // Store the header Block[-1] = header; @@ -820,6 +834,7 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) // Write everything out mrStream.Write(Block - 1, writeSize + 1); + BOX_TRACE("Sent " << (writeSize+1) << " bytes to stream"); // move the remainer to the beginning of the block for the next time round if(writeSize != BytesInBlock) { diff --git a/lib/server/ProtocolUncertainStream.cpp b/lib/server/ProtocolUncertainStream.cpp index 60c1fa1d..84a213a8 100644 --- a/lib/server/ProtocolUncertainStream.cpp +++ b/lib/server/ProtocolUncertainStream.cpp @@ -41,7 +41,8 @@ ProtocolUncertainStream::~ProtocolUncertainStream() { if(!mFinished) { - TRACE0("ProtocolUncertainStream::~ProtocolUncertainStream() destroyed when stream not complete\n"); + BOX_WARNING("ProtocolUncertainStream destroyed before " + "stream finished"); } } @@ -76,11 +77,15 @@ int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) toRead = mBytesLeftInCurrentBlock; } + BOX_TRACE("Reading " << toRead << " bytes from stream"); + // Read it int r = mrSource.Read(((uint8_t*)pBuffer) + read, toRead, Timeout); // Give up now if it didn't return anything if(r == 0) { + BOX_TRACE("Read " << r << " bytes from " + "stream, returning"); return read; } @@ -91,16 +96,22 @@ int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) // stop now if the stream returned less than we asked for -- avoid blocking if(r != toRead) { + BOX_TRACE("Read " << r << " bytes from " + "stream, returning"); return read; } } else { - // Read the header byte to find out how much there is in the next block + // Read the header byte to find out how much there is + // in the next block uint8_t header; if(mrSource.Read(&header, 1, Timeout) == 0) { // Didn't get the byte, return now + BOX_TRACE("Read 0 bytes of block header, " + "returning with " << read << " bytes " + "read this time"); return read; } @@ -109,6 +120,8 @@ int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) { // All done. mFinished = true; + BOX_TRACE("Stream finished, returning with " << + read << " bytes read this time"); return read; } else if(header <= Protocol::ProtocolStreamHeader_MaxEncodedSizeValue) @@ -126,6 +139,10 @@ int ProtocolUncertainStream::Read(void *pBuffer, int NBytes, int Timeout) // Bad. It used the reserved values. THROW_EXCEPTION(ServerException, ProtocolUncertainStreamBadBlockHeader) } + + BOX_TRACE("Read header byte " << (int)header << ", " + "next block has " << + mBytesLeftInCurrentBlock << " bytes"); } } diff --git a/lib/server/SSLLib.cpp b/lib/server/SSLLib.cpp index e9c990b9..de7a941b 100644 --- a/lib/server/SSLLib.cpp +++ b/lib/server/SSLLib.cpp @@ -14,12 +14,16 @@ #include #include +#ifdef WIN32 + #include +#endif + #include "SSLLib.h" #include "ServerException.h" #include "MemLeakFindOn.h" -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD bool SSLLib__TraceErrors = false; #endif @@ -35,7 +39,7 @@ void SSLLib::Initialise() { if(!::SSL_library_init()) { - LogError("Initialisation"); + LogError("initialising OpenSSL"); THROW_EXCEPTION(ServerException, SSLLibraryInitialisationError) } @@ -43,7 +47,37 @@ void SSLLib::Initialise() ::SSL_load_error_strings(); // Extra seeding over and above what's already done by the library -#ifdef HAVE_RANDOM_DEVICE +#ifdef WIN32 + HCRYPTPROV provider; + if(!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT)) + { + BOX_LOG_WIN_ERROR("Failed to acquire crypto context"); + BOX_WARNING("No random device -- additional seeding of " + "random number generator not performed."); + } + else + { + // must free provider + BYTE buf[1024]; + + if(!CryptGenRandom(provider, sizeof(buf), buf)) + { + BOX_LOG_WIN_ERROR("Failed to get random data"); + BOX_WARNING("No random device -- additional seeding of " + "random number generator not performed."); + } + else + { + RAND_seed(buf, sizeof(buf)); + } + + if(!CryptReleaseContext(provider, 0)) + { + BOX_LOG_WIN_ERROR("Failed to release crypto context"); + } + } +#elif HAVE_RANDOM_DEVICE if(::RAND_load_file(RANDOM_DEVICE, 1024) != 1024) { THROW_EXCEPTION(ServerException, SSLRandomInitFailed) @@ -63,14 +97,14 @@ void SSLLib::Initialise() // Created: 2003/08/06 // // -------------------------------------------------------------------------- -void SSLLib::LogError(const char *ErrorDuringAction) +void SSLLib::LogError(const std::string& rErrorDuringAction) { unsigned long errcode; char errname[256]; // SSL docs say at least 120 bytes while((errcode = ERR_get_error()) != 0) { ::ERR_error_string_n(errcode, errname, sizeof(errname)); - BOX_ERROR("SSL error during " << ErrorDuringAction << ": " << + BOX_ERROR("SSL error while " << rErrorDuringAction << ": " << errname); } } diff --git a/lib/server/SSLLib.h b/lib/server/SSLLib.h index cdff4f04..ff4aab19 100644 --- a/lib/server/SSLLib.h +++ b/lib/server/SSLLib.h @@ -10,7 +10,7 @@ #ifndef SSLLIB__H #define SSLLIB__H -#ifndef NDEBUG +#ifndef BOX_RELEASE_BUILD extern bool SSLLib__TraceErrors; #define SET_DEBUG_SSLLIB_TRACE_ERRORS {SSLLib__TraceErrors = true;} #else @@ -29,7 +29,7 @@ namespace SSLLib { void Initialise(); - void LogError(const char *ErrorDuringAction); + void LogError(const std::string& rErrorDuringAction); }; #endif // SSLLIB__H diff --git a/lib/server/ServerControl.cpp b/lib/server/ServerControl.cpp new file mode 100644 index 00000000..c4668b57 --- /dev/null +++ b/lib/server/ServerControl.cpp @@ -0,0 +1,227 @@ +#include "Box.h" + +#include + +#ifdef HAVE_SYS_TYPES_H + #include +#endif + +#ifdef HAVE_SYS_WAIT_H + #include +#endif + +#ifdef HAVE_SIGNAL_H + #include +#endif + +#include "ServerControl.h" +#include "Test.h" + +#ifdef WIN32 + +#include "WinNamedPipeStream.h" +#include "IOStreamGetLine.h" +#include "BoxPortsAndFiles.h" + +static std::string sPipeName; + +void SetNamedPipeName(const std::string& rPipeName) +{ + sPipeName = rPipeName; +} + +bool SendCommands(const std::string& rCmd) +{ + WinNamedPipeStream connection; + + try + { + connection.Connect(sPipeName); + } + catch(...) + { + BOX_ERROR("Failed to connect to daemon control socket"); + return false; + } + + // For receiving data + IOStreamGetLine getLine(connection); + + // Wait for the configuration summary + std::string configSummary; + if(!getLine.GetLine(configSummary)) + { + BOX_ERROR("Failed to receive configuration summary from daemon"); + return false; + } + + // Was the connection rejected by the server? + if(getLine.IsEOF()) + { + BOX_ERROR("Server rejected the connection"); + return false; + } + + // Decode it + int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait; + if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", + &autoBackup, &updateStoreInterval, + &minimumFileAge, &maxUploadWait) != 4) + { + BOX_ERROR("Config summary didn't decode"); + return false; + } + + std::string cmds; + bool expectResponse; + + if (rCmd != "") + { + cmds = rCmd; + cmds += "\nquit\n"; + expectResponse = true; + } + else + { + cmds = "quit\n"; + expectResponse = false; + } + + connection.Write(cmds.c_str(), cmds.size()); + + // Read the response + std::string line; + bool statusOk = !expectResponse; + + while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line)) + { + // Is this an OK or error line? + if (line == "ok") + { + statusOk = true; + } + else if (line == "error") + { + BOX_ERROR(rCmd); + break; + } + else + { + BOX_WARNING("Unexpected response to command '" << + rCmd << "': " << line) + } + } + + return statusOk; +} + +bool HUPServer(int pid) +{ + return SendCommands("reload"); +} + +bool KillServerInternal(int pid) +{ + HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, false, pid); + if (hProcess == NULL) + { + BOX_ERROR("Failed to open process " << pid << ": " << + GetErrorMessage(GetLastError())); + return false; + } + + if (!TerminateProcess(hProcess, 1)) + { + BOX_ERROR("Failed to terminate process " << pid << ": " << + GetErrorMessage(GetLastError())); + CloseHandle(hProcess); + return false; + } + + CloseHandle(hProcess); + return true; +} + +#else // !WIN32 + +bool HUPServer(int pid) +{ + if(pid == 0) return false; + return ::kill(pid, SIGHUP) == 0; +} + +bool KillServerInternal(int pid) +{ + if(pid == 0 || pid == -1) return false; + bool killed = (::kill(pid, SIGTERM) == 0); + if (!killed) + { + BOX_LOG_SYS_ERROR("Failed to kill process " << pid); + } + TEST_THAT(killed); + return killed; +} + +#endif // WIN32 + +bool KillServer(int pid, bool WaitForProcess) +{ + if (!KillServerInternal(pid)) + { + return false; + } + + #ifdef HAVE_WAITPID + if (WaitForProcess) + { + int status, result; + + result = waitpid(pid, &status, 0); + if (result != pid) + { + BOX_LOG_SYS_ERROR("waitpid failed"); + } + TEST_THAT(result == pid); + + TEST_THAT(WIFEXITED(status)); + if (WIFEXITED(status)) + { + if (WEXITSTATUS(status) != 0) + { + BOX_WARNING("process exited with code " << + WEXITSTATUS(status)); + } + TEST_THAT(WEXITSTATUS(status) == 0); + } + } + #endif + + for (int i = 0; i < 30; i++) + { + if (i == 0) + { + printf("Waiting for server to die (pid %d): ", pid); + } + + printf("."); + fflush(stdout); + + if (!ServerIsAlive(pid)) break; + ::sleep(1); + if (!ServerIsAlive(pid)) break; + } + + if (!ServerIsAlive(pid)) + { + printf(" done.\n"); + } + else + { + printf(" failed!\n"); + } + + fflush(stdout); + + return !ServerIsAlive(pid); +} + diff --git a/lib/server/ServerControl.h b/lib/server/ServerControl.h index ce5620c2..b2e51864 100644 --- a/lib/server/ServerControl.h +++ b/lib/server/ServerControl.h @@ -3,188 +3,16 @@ #include "Test.h" -#ifdef WIN32 - -#include "WinNamedPipeStream.h" -#include "IOStreamGetLine.h" -#include "BoxPortsAndFiles.h" - -static std::string sPipeName; - -static void SetNamedPipeName(const std::string& rPipeName) -{ - sPipeName = rPipeName; -} - -static bool SendCommands(const std::string& rCmd) -{ - WinNamedPipeStream connection; - - try - { - connection.Connect(sPipeName); - } - catch(...) - { - BOX_ERROR("Failed to connect to daemon control socket"); - return false; - } - - // For receiving data - IOStreamGetLine getLine(connection); - - // Wait for the configuration summary - std::string configSummary; - if(!getLine.GetLine(configSummary)) - { - BOX_ERROR("Failed to receive configuration summary from daemon"); - return false; - } - - // Was the connection rejected by the server? - if(getLine.IsEOF()) - { - BOX_ERROR("Server rejected the connection"); - return false; - } - - // Decode it - int autoBackup, updateStoreInterval, minimumFileAge, maxUploadWait; - if(::sscanf(configSummary.c_str(), "bbackupd: %d %d %d %d", - &autoBackup, &updateStoreInterval, - &minimumFileAge, &maxUploadWait) != 4) - { - BOX_ERROR("Config summary didn't decode"); - return false; - } - - std::string cmds; - bool expectResponse; - - if (rCmd != "") - { - cmds = rCmd; - cmds += "\nquit\n"; - expectResponse = true; - } - else - { - cmds = "quit\n"; - expectResponse = false; - } - - connection.Write(cmds.c_str(), cmds.size()); - - // Read the response - std::string line; - bool statusOk = !expectResponse; - - while (expectResponse && !getLine.IsEOF() && getLine.GetLine(line)) - { - // Is this an OK or error line? - if (line == "ok") - { - statusOk = true; - } - else if (line == "error") - { - BOX_ERROR(rCmd); - break; - } - else - { - BOX_WARNING("Unexpected response to command '" << - rCmd << "': " << line) - } - } - - return statusOk; -} - -inline bool HUPServer(int pid) -{ - return SendCommands("reload"); -} +bool HUPServer(int pid); +bool KillServer(int pid, bool WaitForProcess = false); -inline bool KillServerInternal(int pid) -{ - HANDLE hProcess = OpenProcess(PROCESS_TERMINATE, false, pid); - if (hProcess == NULL) - { - BOX_ERROR("Failed to open process " << pid << ": " << - GetErrorMessage(GetLastError())); - return false; - } - - if (!TerminateProcess(hProcess, 1)) - { - BOX_ERROR("Failed to terminate process " << pid << ": " << - GetErrorMessage(GetLastError())); - CloseHandle(hProcess); - return false; - } - - CloseHandle(hProcess); - return true; -} - -#else // !WIN32 - -inline bool HUPServer(int pid) -{ - if(pid == 0) return false; - return ::kill(pid, SIGHUP) == 0; -} - -inline bool KillServerInternal(int pid) -{ - if(pid == 0 || pid == -1) return false; - bool killed = (::kill(pid, SIGTERM) == 0); - if (!killed) - { - BOX_ERROR("Failed to kill process " << pid << ": " << - strerror(errno)); - } - TEST_THAT(killed); - return killed; -} +#ifdef WIN32 + #include "WinNamedPipeStream.h" + #include "IOStreamGetLine.h" + #include "BoxPortsAndFiles.h" + void SetNamedPipeName(const std::string& rPipeName); + // bool SendCommands(const std::string& rCmd); #endif // WIN32 -inline bool KillServer(int pid) -{ - if (!KillServerInternal(pid)) - { - return false; - } - - for (int i = 0; i < 30; i++) - { - if (i == 0) - { - printf("Waiting for server to die: "); - } - - printf("."); - fflush(stdout); - - if (!ServerIsAlive(pid)) break; - ::sleep(1); - if (!ServerIsAlive(pid)) break; - } - - if (!ServerIsAlive(pid)) - { - printf(" done.\n"); - } - else - { - printf(" failed!\n"); - } - - fflush(stdout); - - return !ServerIsAlive(pid); -} - #endif // SERVER_CONTROL_H diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h index 32a57bac..34de7def 100644 --- a/lib/server/ServerStream.h +++ b/lib/server/ServerStream.h @@ -108,7 +108,7 @@ public: { // Child task, dump leaks to trace, which we make sure is on #ifdef BOX_MEMORY_LEAK_TESTING - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD TRACE_TO_SYSLOG(true); TRACE_TO_STDOUT(true); #endif @@ -120,13 +120,19 @@ public: } } +protected: + virtual void NotifyListenerIsReady() { } + +public: virtual void Run2(bool &rChildExit) { try { - // Wait object with a timeout of 10 seconds, which is a reasonable time to wait before - // cleaning up finished child processes. - WaitForEvent connectionWait(10000); + // Wait object with a timeout of 1 second, which + // is a reasonable time to wait before cleaning up + // finished child processes, and allows the daemon + // to terminate reasonably quickly on request. + WaitForEvent connectionWait(1000); // BLOCK { @@ -218,6 +224,8 @@ public: } } } + + NotifyListenerIsReady(); while(!StopRun()) { @@ -273,7 +281,7 @@ public: } // Log it - BOX_WARNING("Message from child process " << pid << ": " << logMessage); + BOX_NOTICE("Message from child process " << pid << ": " << logMessage); } else { @@ -365,8 +373,8 @@ private: }; #define SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ - {"ListenAddresses", DEFAULT_ADDRESSES, 0, 0}, \ - DAEMON_VERIFY_SERVER_KEYS + ConfigurationVerifyKey("ListenAddresses", 0, DEFAULT_ADDRESSES), \ + DAEMON_VERIFY_SERVER_KEYS #include "MemLeakFindOff.h" diff --git a/lib/server/ServerTLS.h b/lib/server/ServerTLS.h index 71d35380..a74a671e 100644 --- a/lib/server/ServerTLS.h +++ b/lib/server/ServerTLS.h @@ -55,7 +55,8 @@ public: mContext.Initialise(true /* as server */, certFile.c_str(), keyFile.c_str(), caFile.c_str()); // Then do normal stream server stuff - ServerStream::Run2(rChildExit); + ServerStream::Run2(rChildExit); } virtual void HandleConnection(SocketStreamTLS &rStream) @@ -70,11 +71,10 @@ private: }; #define SERVERTLS_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) \ - {"CertificateFile", 0, ConfigTest_Exists, 0}, \ - {"PrivateKeyFile", 0, ConfigTest_Exists, 0}, \ - {"TrustedCAsFile", 0, ConfigTest_Exists, 0}, \ - SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) - + ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), \ + ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), \ + ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), \ + SERVERSTREAM_VERIFY_SERVER_KEYS(DEFAULT_ADDRESSES) #endif // SERVERTLS__H diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp index 28dae69f..4a83bdb0 100644 --- a/lib/server/Socket.cpp +++ b/lib/server/Socket.cpp @@ -32,12 +32,15 @@ // -------------------------------------------------------------------------- // // Function -// Name: Socket::NameLookupToSockAddr(SocketAllAddr &, int, char *, int) +// Name: Socket::NameLookupToSockAddr(SocketAllAddr &, int, +// char *, int) // Purpose: Sets up a sockaddr structure given a name and type // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type, const char *Name, int Port, int &rSockAddrLenOut) +void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, + enum Type Type, const std::string& rName, int Port, + int &rSockAddrLenOut) { int sockAddrLen = 0; @@ -47,7 +50,7 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type sockDomain = AF_INET; { // Lookup hostname - struct hostent *phost = ::gethostbyname(Name); + struct hostent *phost = ::gethostbyname(rName.c_str()); if(phost != NULL) { if(phost->h_addr_list[0] != 0) @@ -81,7 +84,7 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type sockDomain = AF_UNIX; { // Check length of name is OK - unsigned int nameLen = ::strlen(Name); + unsigned int nameLen = rName.length(); if(nameLen >= (sizeof(addr.sa_unix.sun_path) - 1)) { THROW_EXCEPTION(ServerException, SocketNameUNIXPathTooLong); @@ -91,7 +94,9 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type addr.sa_unix.sun_len = sockAddrLen; #endif addr.sa_unix.sun_family = PF_UNIX; - ::strcpy(addr.sa_unix.sun_path, Name); + ::strncpy(addr.sa_unix.sun_path, rName.c_str(), + sizeof(addr.sa_unix.sun_path) - 1); + addr.sa_unix.sun_path[sizeof(addr.sa_unix.sun_path)-1] = 0; } break; #endif diff --git a/lib/server/Socket.h b/lib/server/Socket.h index 057e4cad..5034dbd8 100644 --- a/lib/server/Socket.h +++ b/lib/server/Socket.h @@ -39,13 +39,15 @@ typedef union { // -------------------------------------------------------------------------- namespace Socket { - enum + enum Type { TypeINET = 1, TypeUNIX = 2 }; - void NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, int Type, const char *Name, int Port, int &rSockAddrLenOut); + void NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, + enum Type type, const std::string& rName, int Port, + int &rSockAddrLenOut); void LogIncomingConnection(const struct sockaddr *addr, socklen_t addrlen); std::string IncomingConnectionLogMessage(const struct sockaddr *addr, socklen_t addrlen); }; diff --git a/lib/server/SocketListen.h b/lib/server/SocketListen.h index ff08fb8f..586adf22 100644 --- a/lib/server/SocketListen.h +++ b/lib/server/SocketListen.h @@ -108,45 +108,57 @@ public: if(::close(mSocketHandle) == -1) #endif { - THROW_EXCEPTION(ServerException, SocketCloseError) + BOX_LOG_SYS_ERROR("Failed to close network " + "socket"); + THROW_EXCEPTION(ServerException, + SocketCloseError) } } mSocketHandle = -1; } - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ // // Function // Name: SocketListen::Listen(int, char*, int) // Purpose: Initialises, starts the socket listening. // Created: 2003/07/31 // - // -------------------------------------------------------------------------- - void Listen(int Type, const char *Name, int Port = 0) + // ------------------------------------------------------------------ + void Listen(Socket::Type Type, const char *Name, int Port = 0) { - if(mSocketHandle != -1) {THROW_EXCEPTION(ServerException, SocketAlreadyOpen)} + if(mSocketHandle != -1) + { + THROW_EXCEPTION(ServerException, SocketAlreadyOpen); + } // Setup parameters based on type, looking up names if required int sockDomain = 0; SocketAllAddr addr; int addrLen = 0; - Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, Port, addrLen); + Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, + Port, addrLen); // Create the socket - mSocketHandle = ::socket(sockDomain, SOCK_STREAM, 0 /* let OS choose protocol */); + mSocketHandle = ::socket(sockDomain, SOCK_STREAM, + 0 /* let OS choose protocol */); if(mSocketHandle == -1) { + BOX_LOG_SYS_ERROR("Failed to create a network socket"); THROW_EXCEPTION(ServerException, SocketOpenError) } // Set an option to allow reuse (useful for -HUP situations!) #ifdef WIN32 - if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, "", 0) == -1) + if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, "", + 0) == -1) #else int option = true; - if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, &option, sizeof(option)) == -1) + if(::setsockopt(mSocketHandle, SOL_SOCKET, SO_REUSEADDR, + &option, sizeof(option)) == -1) #endif { + BOX_LOG_SYS_ERROR("Failed to set socket options"); THROW_EXCEPTION(ServerException, SocketOpenError) } @@ -161,19 +173,25 @@ public: } } - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ // // Function // Name: SocketListen::Accept(int) - // Purpose: Accepts a connection, returning a pointer to a class of - // the specified type. May return a null pointer if a signal happens, - // or there's a timeout. Timeout specified in milliseconds, defaults to infinite time. + // Purpose: Accepts a connection, returning a pointer to + // a class of the specified type. May return a + // null pointer if a signal happens, or there's + // a timeout. Timeout specified in + // milliseconds, defaults to infinite time. // Created: 2003/07/31 // - // -------------------------------------------------------------------------- - std::auto_ptr Accept(int Timeout = INFTIM, std::string *pLogMsg = 0) + // ------------------------------------------------------------------ + std::auto_ptr Accept(int Timeout = INFTIM, + std::string *pLogMsg = 0) { - if(mSocketHandle == -1) {THROW_EXCEPTION(ServerException, BadSocketHandle)} + if(mSocketHandle == -1) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } // Do the accept, using the supplied locking type int sock; @@ -185,8 +203,10 @@ public: if(!socklock.HaveLock()) { - // Didn't get the lock for some reason. Wait a while, then - // return nothing. + // Didn't get the lock for some reason. + // Wait a while, then return nothing. + BOX_ERROR("Failed to get a lock on incoming " + "connection"); ::sleep(1); return std::auto_ptr(); } @@ -202,12 +222,18 @@ public: // signal? if(errno == EINTR) { + BOX_ERROR("Failed to accept " + "connection: interrupted by " + "signal"); // return nothing return std::auto_ptr(); } else { - THROW_EXCEPTION(ServerException, SocketPollError) + BOX_LOG_SYS_ERROR("Failed to poll " + "connection"); + THROW_EXCEPTION(ServerException, + SocketPollError) } break; case 0: // timed out @@ -220,16 +246,19 @@ public: sock = ::accept(mSocketHandle, &addr, &addrlen); } - // Got socket (or error), unlock (implcit in destruction) + + // Got socket (or error), unlock (implicit in destruction) if(sock == -1) { + BOX_LOG_SYS_ERROR("Failed to accept connection"); THROW_EXCEPTION(ServerException, SocketAcceptError) } // Log it if(pLogMsg) { - *pLogMsg = Socket::IncomingConnectionLogMessage(&addr, addrlen); + *pLogMsg = Socket::IncomingConnectionLogMessage(&addr, + addrlen); } else { @@ -243,27 +272,29 @@ public: // Functions to allow adding to WaitForEvent class, for efficient waiting // on multiple sockets. #ifdef HAVE_KQUEUE - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ // // Function // Name: SocketListen::FillInKEevent // Purpose: Fills in a kevent structure for this socket // Created: 9/3/04 // - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ void FillInKEvent(struct kevent &rEvent, int Flags = 0) const { - EV_SET(&rEvent, mSocketHandle, EVFILT_READ, 0, 0, 0, (void*)this); + EV_SET(&rEvent, mSocketHandle, EVFILT_READ, 0, 0, 0, + (void*)this); } #else - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ // // Function // Name: SocketListen::FillInPoll - // Purpose: Fills in the data necessary for a poll operation + // Purpose: Fills in the data necessary for a poll + // operation // Created: 9/3/04 // - // -------------------------------------------------------------------------- + // ------------------------------------------------------------------ void FillInPoll(int &fd, short &events, int Flags = 0) const { fd = mSocketHandle; diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp index 5cb252bd..95b4b4f4 100644 --- a/lib/server/SocketStream.cpp +++ b/lib/server/SocketStream.cpp @@ -18,7 +18,11 @@ #include #ifndef WIN32 -#include + #include +#endif + +#ifdef HAVE_UCRED_H + #include #endif #include "SocketStream.h" @@ -123,20 +127,23 @@ void SocketStream::Attach(int socket) THROW_EXCEPTION(ServerException, SocketAlreadyOpen) } - mSocketHandle = socket; ResetCounters(); + + mSocketHandle = socket; + mReadClosed = false; + mWriteClosed = false; } // -------------------------------------------------------------------------- // // Function -// Name: SocketStream::Open(int, char *, int) +// Name: SocketStream::Open(Socket::Type, char *, int) // Purpose: Opens a connection to a listening socket (INET or UNIX) // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void SocketStream::Open(int Type, const char *Name, int Port) +void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) { if(mSocketHandle != INVALID_SOCKET_VALUE) { @@ -147,12 +154,14 @@ void SocketStream::Open(int Type, const char *Name, int Port) int sockDomain = 0; SocketAllAddr addr; int addrLen = 0; - Socket::NameLookupToSockAddr(addr, sockDomain, Type, Name, Port, addrLen); + Socket::NameLookupToSockAddr(addr, sockDomain, Type, rName, Port, addrLen); // Create the socket - mSocketHandle = ::socket(sockDomain, SOCK_STREAM, 0 /* let OS choose protocol */); + mSocketHandle = ::socket(sockDomain, SOCK_STREAM, + 0 /* let OS choose protocol */); if(mSocketHandle == INVALID_SOCKET_VALUE) { + BOX_LOG_SYS_ERROR("Failed to create a network socket"); THROW_EXCEPTION(ServerException, SocketOpenError) } @@ -163,28 +172,24 @@ void SocketStream::Open(int Type, const char *Name, int Port) #ifdef WIN32 DWORD err = WSAGetLastError(); ::closesocket(mSocketHandle); -#else - int err = errno; + BOX_LOG_WIN_ERROR_NUMBER("Failed to connect to socket " + "(type " << Type << ", name " << rName << + ", port " << Port << ")", err); +#else // !WIN32 + BOX_LOG_SYS_ERROR("Failed to connect to socket (type " << + Type << ", name " << rName << ", port " << Port << + ")"); ::close(mSocketHandle); -#endif - -#ifdef WIN32 - BOX_ERROR("Failed to connect to socket (type " << Type << - ", name " << Name << ", port " << Port << "): " << - GetErrorMessage(err) - ); -#else - BOX_ERROR("Failed to connect to socket (type " << Type << - ", name " << Name << ", port " << Port << "): " << - strerror(err) << " (" << err << ")" - ); -#endif +#endif // WIN32 mSocketHandle = INVALID_SOCKET_VALUE; THROW_EXCEPTION(ConnectionException, Conn_SocketConnectError) } ResetCounters(); + + mReadClosed = false; + mWriteClosed = false; } // -------------------------------------------------------------------------- @@ -220,7 +225,9 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) else { // Bad! - THROW_EXCEPTION(ServerException, SocketPollError) + BOX_LOG_SYS_ERROR("Failed to poll socket"); + THROW_EXCEPTION(ServerException, + SocketPollError) } break; @@ -250,9 +257,12 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) else { // Other error - THROW_EXCEPTION(ConnectionException, Conn_SocketReadError) + BOX_LOG_SYS_ERROR("Failed to read from socket"); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError); } } + // Closed for reading? if(r == 0) { @@ -297,7 +307,9 @@ void SocketStream::Write(const void *pBuffer, int NBytes) { // Error. mWriteClosed = true; // assume can't write again - THROW_EXCEPTION(ConnectionException, Conn_SocketWriteError) + BOX_LOG_SYS_ERROR("Failed to write to socket"); + THROW_EXCEPTION(ConnectionException, + Conn_SocketWriteError); } // Knock off bytes sent @@ -310,7 +322,9 @@ void SocketStream::Write(const void *pBuffer, int NBytes) // Need to wait until it can send again? if(bytesLeft > 0) { - TRACE3("Waiting to send data on socket %d, (%d to send of %d)\n", mSocketHandle, bytesLeft, NBytes); + BOX_TRACE("Waiting to send data on socket " << + mSocketHandle << " (" << bytesLeft << + " of " << NBytes << " bytes left)"); // Wait for data to send. struct pollfd p; @@ -323,7 +337,10 @@ void SocketStream::Write(const void *pBuffer, int NBytes) // Don't exception if it's just a signal if(errno != EINTR) { - THROW_EXCEPTION(ServerException, SocketPollError) + BOX_LOG_SYS_ERROR("Failed to poll " + "socket"); + THROW_EXCEPTION(ServerException, + SocketPollError) } } } @@ -350,7 +367,9 @@ void SocketStream::Close() if(::close(mSocketHandle) == -1) #endif { - THROW_EXCEPTION(ServerException, SocketCloseError) + BOX_LOG_SYS_ERROR("Failed to close socket"); + // don't throw an exception here, assume that the socket was + // already closed or closing. } mSocketHandle = INVALID_SOCKET_VALUE; } @@ -380,6 +399,7 @@ void SocketStream::Shutdown(bool Read, bool Write) // Shut it down! if(::shutdown(mSocketHandle, how) == -1) { + BOX_LOG_SYS_ERROR("Failed to shutdown socket"); THROW_EXCEPTION(ConnectionException, Conn_SocketShutdownError) } } @@ -458,18 +478,37 @@ bool SocketStream::GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut) struct ucred cred; socklen_t credLen = sizeof(cred); - if(::getsockopt(mSocketHandle, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) + if(::getsockopt(mSocketHandle, SOL_SOCKET, SO_PEERCRED, &cred, + &credLen) == 0) { rUidOut = cred.uid; rGidOut = cred.gid; return true; } + + BOX_LOG_SYS_ERROR("Failed to get peer credentials on socket"); +#endif + +#if defined HAVE_UCRED_H && HAVE_GETPEERUCRED + ucred_t *pucred = NULL; + if(::getpeerucred(mSocketHandle, &pucred) == 0) + { + rUidOut = ucred_geteuid(pucred); + rGidOut = ucred_getegid(pucred); + ucred_free(pucred); + if (rUidOut == -1 || rGidOut == -1) + { + BOX_ERROR("Failed to get peer credentials on " + "socket: insufficient information"); + return false; + } + return true; + } + + BOX_LOG_SYS_ERROR("Failed to get peer credentials on socket"); #endif // Not available return false; } - - - diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h index 51f2e306..2b582f21 100644 --- a/lib/server/SocketStream.h +++ b/lib/server/SocketStream.h @@ -11,6 +11,7 @@ #define SOCKETSTREAM__H #include "IOStream.h" +#include "Socket.h" #ifdef WIN32 typedef SOCKET tOSSocketHandle; @@ -36,7 +37,7 @@ public: SocketStream(const SocketStream &rToCopy); ~SocketStream(); - void Open(int Type, const char *Name, int Port = 0); + void Open(Socket::Type Type, const std::string& rName, int Port = 0); void Attach(int socket); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); diff --git a/lib/server/SocketStreamTLS.cpp b/lib/server/SocketStreamTLS.cpp index 58dc5754..19fdadd4 100644 --- a/lib/server/SocketStreamTLS.cpp +++ b/lib/server/SocketStreamTLS.cpp @@ -99,9 +99,10 @@ SocketStreamTLS::~SocketStreamTLS() // Created: 2003/08/06 // // -------------------------------------------------------------------------- -void SocketStreamTLS::Open(const TLSContext &rContext, int Type, const char *Name, int Port) +void SocketStreamTLS::Open(const TLSContext &rContext, Socket::Type Type, + const std::string& rName, int Port) { - SocketStream::Open(Type, Name, Port); + SocketStream::Open(Type, rName, Port); Handshake(rContext); ResetCounters(); } @@ -123,7 +124,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) mpBIO = ::BIO_new(::BIO_s_socket()); if(mpBIO == 0) { - SSLLib::LogError("Create socket bio"); + SSLLib::LogError("creating socket bio"); THROW_EXCEPTION(ServerException, TLSAllocationFailed) } @@ -134,7 +135,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) mpSSL = ::SSL_new(rContext.GetRawContext()); if(mpSSL == 0) { - SSLLib::LogError("Create ssl"); + SSLLib::LogError("creating SSL object"); THROW_EXCEPTION(ServerException, TLSAllocationFailed) } @@ -202,12 +203,12 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) // Error occured if(IsServer) { - SSLLib::LogError("Accept"); + SSLLib::LogError("accepting connection"); THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) } else { - SSLLib::LogError("Connect"); + SSLLib::LogError("connecting"); THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) } } @@ -334,7 +335,7 @@ int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) break; default: - SSLLib::LogError("Read"); + SSLLib::LogError("reading"); THROW_EXCEPTION(ConnectionException, Conn_TLSReadFailed) break; } @@ -390,7 +391,7 @@ void SocketStreamTLS::Write(const void *pBuffer, int NBytes) case SSL_ERROR_WANT_WRITE: // wait for the requried data { - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD bool conditionmet = #endif WaitWhenRetryRequired(se, IOStream::TimeOutInfinite); @@ -399,7 +400,7 @@ void SocketStreamTLS::Write(const void *pBuffer, int NBytes) break; default: - SSLLib::LogError("Write"); + SSLLib::LogError("writing"); THROW_EXCEPTION(ConnectionException, Conn_TLSWriteFailed) break; } @@ -441,7 +442,7 @@ void SocketStreamTLS::Shutdown(bool Read, bool Write) if(::SSL_shutdown(mpSSL) < 0) { - SSLLib::LogError("Shutdown"); + SSLLib::LogError("shutting down"); THROW_EXCEPTION(ConnectionException, Conn_TLSShutdownFailed) } diff --git a/lib/server/SocketStreamTLS.h b/lib/server/SocketStreamTLS.h index 64e52833..bb40ed10 100644 --- a/lib/server/SocketStreamTLS.h +++ b/lib/server/SocketStreamTLS.h @@ -38,7 +38,8 @@ private: SocketStreamTLS(const SocketStreamTLS &rToCopy); public: - void Open(const TLSContext &rContext, int Type, const char *Name, int Port = 0); + void Open(const TLSContext &rContext, Socket::Type Type, + const std::string& rName, int Port = 0); void Handshake(const TLSContext &rContext, bool IsServer = false); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp index 49143801..ebc7384a 100644 --- a/lib/server/TLSContext.cpp +++ b/lib/server/TLSContext.cpp @@ -75,19 +75,25 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c // Setup our identity if(::SSL_CTX_use_certificate_chain_file(mpContext, CertificatesFile) != 1) { - SSLLib::LogError("Load certificates"); + std::string msg = "loading certificates from "; + msg += CertificatesFile; + SSLLib::LogError(msg); THROW_EXCEPTION(ServerException, TLSLoadCertificatesFailed) } if(::SSL_CTX_use_PrivateKey_file(mpContext, PrivateKeyFile, SSL_FILETYPE_PEM) != 1) { - SSLLib::LogError("Load private key"); + std::string msg = "loading private key from "; + msg += PrivateKeyFile; + SSLLib::LogError(msg); THROW_EXCEPTION(ServerException, TLSLoadPrivateKeyFailed) } // Setup the identify of CAs we trust if(::SSL_CTX_load_verify_locations(mpContext, TrustedCAsFile, NULL) != 1) { - SSLLib::LogError("Load CA cert"); + std::string msg = "loading CA cert from "; + msg += TrustedCAsFile; + SSLLib::LogError(msg); THROW_EXCEPTION(ServerException, TLSLoadTrustedCAsFailed) } @@ -99,7 +105,7 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c // Setup allowed ciphers if(::SSL_CTX_set_cipher_list(mpContext, CIPHER_LIST) != 1) { - SSLLib::LogError("Set cipher list"); + SSLLib::LogError("setting cipher list to " CIPHER_LIST); THROW_EXCEPTION(ServerException, TLSSetCiphersFailed) } } diff --git a/lib/server/WinNamedPipeListener.h b/lib/server/WinNamedPipeListener.h new file mode 100644 index 00000000..26e76e3d --- /dev/null +++ b/lib/server/WinNamedPipeListener.h @@ -0,0 +1,232 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: WinNamedPipeListener.h +// Purpose: Windows named pipe socket connection listener +// for server +// Created: 2008/09/30 +// +// -------------------------------------------------------------------------- + +#ifndef WINNAMEDPIPELISTENER__H +#define WINNAMEDPIPELISTENER__H + +#include +#include + +#include "ServerException.h" + +#include "MemLeakFindOn.h" + +// -------------------------------------------------------------------------- +// +// Class +// Name: WinNamedPipeListener +// Purpose: +// Created: 2008/09/30 +// +// -------------------------------------------------------------------------- +template +class WinNamedPipeListener +{ +private: + std::auto_ptr mapPipeName; + std::auto_ptr mapOverlapConnect; + HANDLE mPipeHandle; + +public: + // Initialise + WinNamedPipeListener() + : mPipeHandle(INVALID_HANDLE_VALUE) + { } + +private: + WinNamedPipeListener(const WinNamedPipeListener &rToCopy) + { /* forbidden */ } + + HANDLE CreatePipeHandle(const std::string& rName) + { + std::string socket = WinNamedPipeStream::sPipeNamePrefix + + rName; + + HANDLE handle = CreateNamedPipeA( + socket.c_str(), // pipe name + PIPE_ACCESS_DUPLEX | // read/write access + FILE_FLAG_OVERLAPPED, // enabled overlapped I/O + PIPE_TYPE_BYTE | // message type pipe + PIPE_READMODE_BYTE | // message-read mode + PIPE_WAIT, // blocking mode + ListenBacklog + 1, // max. instances + 4096, // output buffer size + 4096, // input buffer size + NMPWAIT_USE_DEFAULT_WAIT, // client time-out + NULL); // default security attribute + + if (handle == INVALID_HANDLE_VALUE) + { + BOX_LOG_WIN_ERROR("Failed to create named pipe " << + socket); + THROW_EXCEPTION(ServerException, SocketOpenError) + } + + return handle; + } + +public: + ~WinNamedPipeListener() + { + Close(); + } + + void Close() + { + if (mPipeHandle != INVALID_HANDLE_VALUE) + { + if (mapOverlapConnect.get()) + { + // outstanding connect in progress + if (CancelIo(mPipeHandle) != TRUE) + { + BOX_LOG_WIN_ERROR("Failed to cancel " + "outstanding connect request " + "on named pipe"); + } + + mapOverlapConnect.reset(); + } + + if (CloseHandle(mPipeHandle) != TRUE) + { + BOX_LOG_WIN_ERROR("Failed to close named pipe " + "handle"); + } + + mPipeHandle = INVALID_HANDLE_VALUE; + } + } + + // ------------------------------------------------------------------ + // + // Function + // Name: WinNamedPipeListener::Listen(std::string name) + // Purpose: Initialises socket name + // Created: 2003/07/31 + // + // ------------------------------------------------------------------ + void Listen(const std::string& rName) + { + Close(); + mapPipeName.reset(new std::string(rName)); + mPipeHandle = CreatePipeHandle(rName); + } + + // ------------------------------------------------------------------ + // + // Function + // Name: WinNamedPipeListener::Accept(int) + // Purpose: Accepts a connection, returning a pointer to + // a class of the specified type. May return a + // null pointer if a signal happens, or there's + // a timeout. Timeout specified in + // milliseconds, defaults to infinite time. + // Created: 2003/07/31 + // + // ------------------------------------------------------------------ + std::auto_ptr Accept(int Timeout = INFTIM, + const char* pLogMsgOut = NULL) + { + if(!mapPipeName.get()) + { + THROW_EXCEPTION(ServerException, BadSocketHandle); + } + + BOOL connected = FALSE; + std::auto_ptr mapStream; + + if (!mapOverlapConnect.get()) + { + // start a new connect operation + mapOverlapConnect.reset(new OverlappedIO()); + connected = ConnectNamedPipe(mPipeHandle, + &mapOverlapConnect->mOverlapped); + + if (connected == FALSE) + { + if (GetLastError() == ERROR_PIPE_CONNECTED) + { + connected = TRUE; + } + else if (GetLastError() != ERROR_IO_PENDING) + { + BOX_LOG_WIN_ERROR("Failed to connect " + "named pipe"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + } + } + + if (connected == FALSE) + { + // wait for connection + DWORD result = WaitForSingleObject( + mapOverlapConnect->mOverlapped.hEvent, + (Timeout == INFTIM) ? INFINITE : Timeout); + + if (result == WAIT_OBJECT_0) + { + DWORD dummy; + + if (!GetOverlappedResult(mPipeHandle, + &mapOverlapConnect->mOverlapped, + &dummy, TRUE)) + { + BOX_LOG_WIN_ERROR("Failed to get " + "overlapped connect result"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + + connected = TRUE; + } + else if (result == WAIT_TIMEOUT) + { + return mapStream; // contains NULL + } + else if (result == WAIT_ABANDONED) + { + BOX_ERROR("Wait for named pipe connection " + "was abandoned by the system"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + else if (result == WAIT_FAILED) + { + BOX_LOG_WIN_ERROR("Failed to wait for named " + "pipe connection"); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + else + { + BOX_ERROR("Failed to wait for named pipe " + "connection: unknown return code " << + result); + THROW_EXCEPTION(ServerException, + SocketAcceptError); + } + } + + ASSERT(connected == TRUE); + + mapStream.reset(new WinNamedPipeStream(mPipeHandle)); + mPipeHandle = CreatePipeHandle(*mapPipeName); + mapOverlapConnect.reset(); + + return mapStream; + } +}; + +#include "MemLeakFindOff.h" + +#endif // WINNAMEDPIPELISTENER__H diff --git a/lib/server/WinNamedPipeStream.cpp b/lib/server/WinNamedPipeStream.cpp index fedb57d8..1179516e 100644 --- a/lib/server/WinNamedPipeStream.cpp +++ b/lib/server/WinNamedPipeStream.cpp @@ -44,7 +44,55 @@ WinNamedPipeStream::WinNamedPipeStream() mWriteClosed(false), mIsServer(false), mIsConnected(false) -{ +{ } + +// -------------------------------------------------------------------------- +// +// Function +// Name: WinNamedPipeStream::WinNamedPipeStream(HANDLE) +// Purpose: Constructor (with already-connected pipe handle) +// Created: 2008/10/01 +// +// -------------------------------------------------------------------------- +WinNamedPipeStream::WinNamedPipeStream(HANDLE hNamedPipe) + : mSocketHandle(hNamedPipe), + mReadableEvent(INVALID_HANDLE_VALUE), + mBytesInBuffer(0), + mReadClosed(false), + mWriteClosed(false), + mIsServer(true), + mIsConnected(true) +{ + // create the Readable event + mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + + if (mReadableEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to create the Readable event: " << + GetErrorMessage(GetLastError())); + Close(); + THROW_EXCEPTION(CommonException, Internal) + } + + // initialise the OVERLAPPED structure + memset(&mReadOverlap, 0, sizeof(mReadOverlap)); + mReadOverlap.hEvent = mReadableEvent; + + // start the first overlapped read + if (!ReadFile(mSocketHandle, mReadBuffer, sizeof(mReadBuffer), + NULL, &mReadOverlap)) + { + DWORD err = GetLastError(); + + if (err != ERROR_IO_PENDING) + { + BOX_ERROR("Failed to start overlapped read: " << + GetErrorMessage(err)); + Close(); + THROW_EXCEPTION(ConnectionException, + Conn_SocketReadError) + } + } } // -------------------------------------------------------------------------- @@ -80,33 +128,17 @@ WinNamedPipeStream::~WinNamedPipeStream() // Created: 2005/12/07 // // -------------------------------------------------------------------------- -void WinNamedPipeStream::Accept(const std::string& rName) +/* +void WinNamedPipeStream::Accept() { - if (mSocketHandle != INVALID_HANDLE_VALUE || mIsConnected) + if (mSocketHandle == INVALID_HANDLE_VALUE) { - THROW_EXCEPTION(ServerException, SocketAlreadyOpen) + THROW_EXCEPTION(ServerException, BadSocketHandle); } - std::string socket = sPipeNamePrefix + rName; - - mSocketHandle = CreateNamedPipeA( - socket.c_str(), // pipe name - PIPE_ACCESS_DUPLEX | // read/write access - FILE_FLAG_OVERLAPPED, // enabled overlapped I/O - PIPE_TYPE_BYTE | // message type pipe - PIPE_READMODE_BYTE | // message-read mode - PIPE_WAIT, // blocking mode - 1, // max. instances - 4096, // output buffer size - 4096, // input buffer size - NMPWAIT_USE_DEFAULT_WAIT, // client time-out - NULL); // default security attribute - - if (mSocketHandle == INVALID_HANDLE_VALUE) + if (mIsConnected) { - BOX_ERROR("Failed to CreateNamedPipeA(" << socket << "): " << - GetErrorMessage(GetLastError())); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_EXCEPTION(ServerException, SocketAlreadyOpen); } bool connected = ConnectNamedPipe(mSocketHandle, (LPOVERLAPPED) NULL); @@ -156,6 +188,7 @@ void WinNamedPipeStream::Accept(const std::string& rName) } } } +*/ // -------------------------------------------------------------------------- // @@ -217,7 +250,7 @@ void WinNamedPipeStream::Connect(const std::string& rName) int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) { // TODO no support for timeouts yet - if (Timeout != IOStream::TimeOutInfinite) + if (!mIsServer && Timeout != IOStream::TimeOutInfinite) { THROW_EXCEPTION(CommonException, AssertFailed) } @@ -249,8 +282,29 @@ int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) { // overlapped I/O completed successfully? // (wait if needed) + DWORD waitResult = WaitForSingleObject( + mReadOverlap.hEvent, Timeout); - if (GetOverlappedResult(mSocketHandle, + if (waitResult == WAIT_ABANDONED) + { + BOX_ERROR("Wait for command socket read " + "abandoned by system"); + THROW_EXCEPTION(ServerException, + BadSocketHandle); + } + else if (waitResult == WAIT_TIMEOUT) + { + // wait timed out, nothing to read + NumBytesRead = 0; + } + else if (waitResult != WAIT_OBJECT_0) + { + BOX_ERROR("Failed to wait for command " + "socket read: unknown result " << + waitResult); + } + // object is ready to read from + else if (GetOverlappedResult(mSocketHandle, &mReadOverlap, &NumBytesRead, TRUE)) { needAnotherRead = true; @@ -267,7 +321,7 @@ int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) { if (err == ERROR_BROKEN_PIPE) { - BOX_ERROR("Control client " + BOX_NOTICE("Control client " "disconnected"); } else @@ -342,29 +396,6 @@ int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) Conn_SocketReadError) } } - - // If the read succeeded immediately, leave the event - // signaled, so that we will be called again to process - // the newly read data and start another overlapped read. - if (needAnotherRead && !mReadClosed) - { - // leave signalled - } - else if (!needAnotherRead && mBytesInBuffer > 0) - { - // leave signalled - } - else - { - // nothing left to read, reset the event - ResetEvent(mReadableEvent); - // FIXME: a pending read could have signalled - // the event (again) while we were busy reading. - // that signal would be lost, and the reading - // thread would block. Should be pretty obvious - // if this happens in practice: control client - // hangs. - } } else { @@ -440,23 +471,21 @@ void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) if (!Success) { - DWORD err = GetLastError(); - BOX_ERROR("Failed to write to control socket: " << - GetErrorMessage(err)); - Close(); - // ERROR_NO_DATA is a strange name for - // "The pipe is being closed". No exception wanted. + // "The pipe is being closed". - if (err == ERROR_NO_DATA) - { - return; - } - else + DWORD err = GetLastError(); + + if (err != ERROR_NO_DATA) { - THROW_EXCEPTION(ConnectionException, - Conn_SocketWriteError) + BOX_ERROR("Failed to write to control " + "socket: " << GetErrorMessage(err)); } + + Close(); + + THROW_EXCEPTION(ConnectionException, + Conn_SocketWriteError) } NumBytesWrittenTotal += NumBytesWrittenThisTime; diff --git a/lib/server/WinNamedPipeStream.h b/lib/server/WinNamedPipeStream.h index 6acd48f6..386ff7e3 100644 --- a/lib/server/WinNamedPipeStream.h +++ b/lib/server/WinNamedPipeStream.h @@ -24,10 +24,11 @@ class WinNamedPipeStream : public IOStream { public: WinNamedPipeStream(); + WinNamedPipeStream(HANDLE hNamedPipe); ~WinNamedPipeStream(); // server side - create the named pipe and listen for connections - void Accept(const std::string& rName); + // use WinNamedPipeListener to do this instead. // client side - connect to a waiting server void Connect(const std::string& rName); @@ -40,9 +41,6 @@ public: virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); - bool IsConnected() { return mIsConnected; } - HANDLE GetSocketHandle() { return mSocketHandle; } - HANDLE GetReadableEvent() { return mReadableEvent; } protected: void MarkAsReadClosed() {mReadClosed = true;} @@ -62,6 +60,7 @@ private: bool mIsServer; bool mIsConnected; +public: static std::string sPipeNamePrefix; }; diff --git a/lib/server/makeprotocol.pl.in b/lib/server/makeprotocol.pl.in index 269ff3f5..91ba55b0 100755 --- a/lib/server/makeprotocol.pl.in +++ b/lib/server/makeprotocol.pl.in @@ -178,6 +178,9 @@ print CPP <<__E; // Auto-generated file -- do not edit #include "Box.h" + +#include + #include "$h_filename" #include "IOStream.h" @@ -273,7 +276,7 @@ __E if($derive_objects_from ne 'ProtocolObject') { - # output a definition for the protocol object derviced class + # output a definition for the protocol object derived class print H <<__E; class ${protocol_name}ProtocolServer; @@ -338,6 +341,7 @@ __E if(obj_is_type($cmd,'IsError')) { print H "\tbool IsError(int &rTypeOut, int &rSubTypeOut) const;\n"; + print H "\tstd::string GetMessage() const;\n"; } if($type eq 'Server' && obj_is_type($cmd, 'Command')) { @@ -498,6 +502,27 @@ bool ${class}IsError(int &rTypeOut, int &rSubTypeOut) const rSubTypeOut = m$mem_subtype; return true; } +std::string ${class}GetMessage() const +{ + switch(m$mem_subtype) + { +__E + foreach my $const (@{$cmd_constants{$cmd}}) + { + next unless $const =~ /^Err_(.*)/; + my $shortname = $1; + $const =~ s/ = .*//; + print CPP <<__E; + case $const: return "$shortname"; +__E + } + print CPP <<__E; + default: + std::ostringstream out; + out << "Unknown subtype " << m$mem_subtype; + return out.str(); + } +} __E } @@ -513,11 +538,13 @@ __E } if($implement_filelog) { - my ($format,$args) = make_log_strings($cmd); + my ($log) = make_log_strings_framework($cmd); print CPP <<__E; void ${class}LogFile(const char *Action, FILE *File) const { - ::fprintf(File,"%s $format\\n",Action$args); + std::ostringstream oss; + oss << $log; + ::fprintf(File, "%s\\n", oss.str().c_str()); ::fflush(File); } __E @@ -620,16 +647,16 @@ protected: __E -my $construtor_extra = ''; -$construtor_extra .= ', mLogToSysLog(false)' if $implement_syslog; -$construtor_extra .= ', mLogToFile(0)' if $implement_filelog; +my $constructor_extra = ''; +$constructor_extra .= ', mLogToSysLog(false)' if $implement_syslog; +$constructor_extra .= ', mLogToFile(0)' if $implement_filelog; my $destructor_extra = ($type eq 'Server')?"\n\tDeleteStreamsToSend();":''; my $prefix = $classname_base.'::'; print CPP <<__E; $prefix$classname_base(IOStream &rStream) - : Protocol(rStream)$construtor_extra + : Protocol(rStream)$constructor_extra { } $prefix~$classname_base() @@ -661,7 +688,7 @@ print CPP <<__E; } } __E -# write receieve and send functions +# write receive and send functions print CPP <<__E; std::auto_ptr<$derive_objects_from> ${prefix}Receive() { @@ -734,51 +761,9 @@ void ${prefix}DoServer($context_class &rContext) // Get an object from the conversation std::auto_ptr<${derive_objects_from}> pobj(Receive()); -__E - if($implement_syslog) - { - print CPP <<__E; - if(mLogToSysLog) - { - pobj->LogSysLog("Receive"); - } -__E - } - if($implement_filelog) - { - print CPP <<__E; - if(mLogToFile != 0) - { - pobj->LogFile("Receive", mLogToFile); - } -__E - } - print CPP <<__E; - // Run the command std::auto_ptr<${derive_objects_from}> preply((${derive_objects_from}*)(pobj->DoCommand(*this, rContext).release())); -__E - if($implement_syslog) - { - print CPP <<__E; - if(mLogToSysLog) - { - preply->LogSysLog("Send"); - } -__E - } - if($implement_filelog) - { - print CPP <<__E; - if(mLogToFile != 0) - { - preply->LogFile("Send", mLogToFile); - } -__E - } - print CPP <<__E; - // Send the reply Send(*(preply.get())); @@ -824,13 +809,57 @@ if($implement_filelog || $implement_syslog) if($implement_syslog) { - $fR .= qq~\tif(mLogToSysLog) { ::syslog(LOG_INFO, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain":"Receiving stream, size %d", Size); }\n~; - $fS .= qq~\tif(mLogToSysLog) { ::syslog(LOG_INFO, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain":"Sending stream, size %d", Size); }\n~; + $fR .= <<__E; + if(mLogToSysLog) + { + if(Size==Protocol::ProtocolStream_SizeUncertain) + { + BOX_TRACE("Receiving stream, size uncertain"); + } + else + { + BOX_TRACE("Receiving stream, size " << Size); + } } +__E + + $fS .= <<__E; + if(mLogToSysLog) + { + if(Size==Protocol::ProtocolStream_SizeUncertain) + { + BOX_TRACE("Sending stream, size uncertain"); + } + else + { + BOX_TRACE("Sending stream, size " << Size); + } + } +__E + } + if($implement_filelog) { - $fR .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Receiving stream, size uncertain\\n":"Receiving stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; - $fS .= qq~\tif(mLogToFile) { ::fprintf(mLogToFile, (Size==Protocol::ProtocolStream_SizeUncertain)?"Sending stream, size uncertain\\n":"Sending stream, size %d\\n", Size); ::fflush(mLogToFile); }\n~; + $fR .= <<__E; + if(mLogToFile) + { + ::fprintf(mLogToFile, + (Size==Protocol::ProtocolStream_SizeUncertain) + ?"Receiving stream, size uncertain\\n" + :"Receiving stream, size %d\\n", Size); + ::fflush(mLogToFile); + } +__E + $fS .= <<__E; + if(mLogToFile) + { + ::fprintf(mLogToFile, + (Size==Protocol::ProtocolStream_SizeUncertain) + ?"Sending stream, size uncertain\\n" + :"Sending stream, size %d\\n", Size); + ::fflush(mLogToFile); + } +__E } print CPP <<__E; @@ -888,11 +917,15 @@ std::auto_ptr<$classname_base$reply> ${classname_base}::Query(const $classname_b if(preply->IsError(type, subType)) { SetError(type, subType); - TRACE2("Protocol: Error received %d/%d\\n", type, subType); + BOX_WARNING("$cmd command failed: received error " << + ((${classname_base}Error&)*preply).GetMessage()); } else { SetError(Protocol::UnknownError, Protocol::UnknownError); + BOX_WARNING("$cmd command failed: received " + "unexpected response type " << + preply->GetType()); } // Throw an exception @@ -1020,10 +1053,22 @@ sub make_log_strings_framework my ($format,$arg) = @{$log_display_types{$ty}}; $arg =~ s/VAR/m$nm/g; - if ($format =~ m'x$') + if ($format eq '\\"%s\\"') + { + $arg = "\"\\\"\" << $arg << \"\\\"\""; + } + elsif ($format =~ m'x$') { - $arg = "std::hex << std::showbase " . - "<< $arg << std::dec"; + # my $width = 0; + # $ty =~ /^int(\d+)$/ and $width = $1 / 4; + $arg = "($arg == 0 ? \"0x\" : \"\") " . + "<< std::hex " . + "<< std::showbase " . + # "<< std::setw($width) " . + # "<< std::setfill('0') " . + # "<< std::internal " . + "<< $arg " . + "<< std::dec"; } push @args, $arg; diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index 071dc788..ad8c3041 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -21,173 +21,36 @@ #include // message resource definitions for syslog() - #include "messages.h" -// our implementation for a timer, based on a -// simple thread which sleeps for a period of time - -static bool gTimerInitialised = false; -static bool gFinishTimer; -static CRITICAL_SECTION gLock; - -typedef struct -{ - int countDown; - int interval; -} -Timer_t; - -std::list gTimerList; -static void (__cdecl *gTimerFunc) (int) = NULL; - -int setitimer(int type, struct itimerval *timeout, void *arg) -{ - assert(gTimerInitialised); - - if (ITIMER_REAL != type) - { - errno = ENOSYS; - return -1; - } - - EnterCriticalSection(&gLock); - - // we only need seconds for the mo! - if (timeout->it_value.tv_sec == 0 && - timeout->it_value.tv_usec == 0) - { - gTimerList.clear(); - } - else - { - Timer_t ourTimer; - ourTimer.countDown = timeout->it_value.tv_sec; - ourTimer.interval = timeout->it_interval.tv_sec; - gTimerList.push_back(ourTimer); - } - - LeaveCriticalSection(&gLock); - - // indicate success - return 0; -} - -static unsigned int WINAPI RunTimer(LPVOID lpParameter) -{ - gFinishTimer = false; - - while (!gFinishTimer) - { - std::list::iterator it; - EnterCriticalSection(&gLock); - - for (it = gTimerList.begin(); it != gTimerList.end(); it++) - { - Timer_t& rTimer(*it); - - rTimer.countDown --; - if (rTimer.countDown == 0) - { - if (gTimerFunc != NULL) - { - gTimerFunc(0); - } - if (rTimer.interval) - { - rTimer.countDown = rTimer.interval; - } - else - { - // mark for deletion - rTimer.countDown = -1; - } - } - } - - for (it = gTimerList.begin(); it != gTimerList.end(); it++) - { - Timer_t& rTimer(*it); - - if (rTimer.countDown == -1) - { - gTimerList.erase(it); - - // the iterator is now invalid, so restart search - it = gTimerList.begin(); - - // if the list is now empty, don't try to increment - // the iterator again - if (it == gTimerList.end()) break; - } - } - - LeaveCriticalSection(&gLock); - // we only need to have a 1 second resolution - Sleep(1000); - } - - return 0; -} - -int SetTimerHandler(void (__cdecl *func ) (int)) -{ - gTimerFunc = func; - return 0; -} - -void InitTimer(void) -{ - assert(!gTimerInitialised); - - InitializeCriticalSection(&gLock); - - // create our thread - HANDLE ourThread = (HANDLE)_beginthreadex(NULL, 0, RunTimer, 0, - CREATE_SUSPENDED, NULL); - SetThreadPriority(ourThread, THREAD_PRIORITY_LOWEST); - ResumeThread(ourThread); - - gTimerInitialised = true; -} - -void FiniTimer(void) -{ - assert(gTimerInitialised); - gFinishTimer = true; - EnterCriticalSection(&gLock); - DeleteCriticalSection(&gLock); - gTimerInitialised = false; -} - -//Our constants we need to keep track of -//globals +DWORD winerrno; struct passwd gTempPasswd; -bool EnableBackupRights( void ) +bool EnableBackupRights() { HANDLE hToken; TOKEN_PRIVILEGES token_priv; //open current process to adjust privileges - if( !OpenProcessToken( GetCurrentProcess( ), - TOKEN_ADJUST_PRIVILEGES, - &hToken )) + if(!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, + &hToken)) { - printf( "Cannot open process token: error %d\n", - (int)GetLastError() ); + ::syslog(LOG_ERR, "Failed to open process token: %s", + GetErrorMessage(GetLastError()).c_str()); return false; } //let's build the token privilege struct - //first, look up the LUID for the backup privilege - if( !LookupPrivilegeValue( NULL, //this system + if (!LookupPrivilegeValue( + NULL, //this system SE_BACKUP_NAME, //the name of the privilege - &( token_priv.Privileges[0].Luid )) ) //result + &( token_priv.Privileges[0].Luid ))) //result { - printf( "Cannot lookup backup privilege: error %d\n", - (int)GetLastError( ) ); + ::syslog(LOG_ERR, "Failed to lookup backup privilege: %s", + GetErrorMessage(GetLastError()).c_str()); + CloseHandle(hToken); return false; } @@ -198,24 +61,25 @@ bool EnableBackupRights( void ) // because we're going exit right after dumping the streams, there isn't // any need to save current state - if( !AdjustTokenPrivileges( hToken, //our process token + if (!AdjustTokenPrivileges( + hToken, //our process token false, //we're not disabling everything &token_priv, //address of structure - sizeof( token_priv ), //size of structure - NULL, NULL )) //don't save current state + sizeof(token_priv), //size of structure + NULL, NULL)) //don't save current state { //this function is a little tricky - if we were adjusting //more than one privilege, it could return success but not //adjust them all - in the general case, you need to trap this - printf( "Could not enable backup privileges: error %d\n", - (int)GetLastError( ) ); + ::syslog(LOG_ERR, "Failed to enable backup privilege: %s", + GetErrorMessage(GetLastError()).c_str()); + CloseHandle(hToken); return false; } - else - { - return true; - } + + CloseHandle(hToken); + return true; } // forward declaration @@ -267,7 +131,8 @@ std::string GetDefaultConfigFilePath(const std::string& rName) // Created: 4th February 2006 // // -------------------------------------------------------------------------- -WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) +WCHAR* ConvertToWideString(const char* pString, unsigned int codepage, + bool logErrors) { int len = MultiByteToWideChar ( @@ -282,9 +147,13 @@ WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) if (len == 0) { - ::syslog(LOG_WARNING, - "Failed to convert string to wide string: " - "error %d", GetLastError()); + winerrno = GetLastError(); + if (logErrors) + { + ::syslog(LOG_WARNING, + "Failed to convert string to wide string: " + "%s", GetErrorMessage(winerrno).c_str()); + } errno = EINVAL; return NULL; } @@ -293,9 +162,13 @@ WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) if (buffer == NULL) { - ::syslog(LOG_WARNING, - "Failed to convert string to wide string: " - "out of memory"); + if (logErrors) + { + ::syslog(LOG_WARNING, + "Failed to convert string to wide string: " + "out of memory"); + } + winerrno = ERROR_OUTOFMEMORY; errno = ENOMEM; return NULL; } @@ -312,9 +185,13 @@ WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) if (len == 0) { - ::syslog(LOG_WARNING, - "Failed to convert string to wide string: " - "error %i", GetLastError()); + winerrno = GetLastError(); + if (logErrors) + { + ::syslog(LOG_WARNING, + "Failed to convert string to wide string: " + "%s", GetErrorMessage(winerrno).c_str()); + } errno = EACCES; delete [] buffer; return NULL; @@ -336,7 +213,7 @@ WCHAR* ConvertToWideString(const char* pString, unsigned int codepage) // -------------------------------------------------------------------------- WCHAR* ConvertUtf8ToWideString(const char* pString) { - return ConvertToWideString(pString, CP_UTF8); + return ConvertToWideString(pString, CP_UTF8, true); } // -------------------------------------------------------------------------- @@ -425,7 +302,8 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) bool ConvertEncoding(const std::string& rSource, int sourceCodePage, std::string& rDest, int destCodePage) { - WCHAR* pWide = ConvertToWideString(rSource.c_str(), sourceCodePage); + WCHAR* pWide = ConvertToWideString(rSource.c_str(), sourceCodePage, + true); if (pWide == NULL) { ::syslog(LOG_ERR, "Failed to convert string '%s' from " @@ -450,24 +328,26 @@ bool ConvertEncoding(const std::string& rSource, int sourceCodePage, return true; } -bool ConvertToUtf8(const char* pString, std::string& rDest, int sourceCodePage) +bool ConvertToUtf8(const std::string& rSource, std::string& rDest, + int sourceCodePage) { - return ConvertEncoding(pString, sourceCodePage, rDest, CP_UTF8); + return ConvertEncoding(rSource, sourceCodePage, rDest, CP_UTF8); } -bool ConvertFromUtf8(const char* pString, std::string& rDest, int destCodePage) +bool ConvertFromUtf8(const std::string& rSource, std::string& rDest, + int destCodePage) { - return ConvertEncoding(pString, CP_UTF8, rDest, destCodePage); + return ConvertEncoding(rSource, CP_UTF8, rDest, destCodePage); } -bool ConvertConsoleToUtf8(const char* pString, std::string& rDest) +bool ConvertConsoleToUtf8(const std::string& rSource, std::string& rDest) { - return ConvertEncoding(pString, GetConsoleCP(), rDest, CP_UTF8); + return ConvertToUtf8(rSource, rDest, GetConsoleCP()); } -bool ConvertUtf8ToConsole(const char* pString, std::string& rDest) +bool ConvertUtf8ToConsole(const std::string& rSource, std::string& rDest) { - return ConvertEncoding(pString, CP_UTF8, rDest, GetConsoleOutputCP()); + return ConvertFromUtf8(rSource, rDest, GetConsoleOutputCP()); } // -------------------------------------------------------------------------- @@ -506,6 +386,7 @@ std::string ConvertPathToAbsoluteUnicode(const char *pFileName) "Failed to open '%s': path too long", pFileName); errno = ENAMETOOLONG; + winerrno = ERROR_INVALID_NAME; tmpStr = ""; return tmpStr; } @@ -581,6 +462,8 @@ std::string GetErrorMessage(DWORD errorCode) // -------------------------------------------------------------------------- HANDLE openfile(const char *pFileName, int flags, int mode) { + winerrno = ERROR_INVALID_FUNCTION; + std::string AbsPathWithUnicode = ConvertPathToAbsoluteUnicode(pFileName); @@ -654,12 +537,37 @@ HANDLE openfile(const char *pFileName, int flags, int mode) if (hdir == INVALID_HANDLE_VALUE) { + winerrno = GetLastError(); + switch(winerrno) + { + case ERROR_SHARING_VIOLATION: + errno = EBUSY; + break; + + default: + errno = EINVAL; + } + ::syslog(LOG_WARNING, "Failed to open file '%s': " "%s", pFileName, GetErrorMessage(GetLastError()).c_str()); + return INVALID_HANDLE_VALUE; } + if (flags & O_APPEND) + { + if (SetFilePointer(hdir, 0, NULL, FILE_END) == + INVALID_SET_FILE_POINTER) + { + winerrno = GetLastError(); + errno = EINVAL; + CloseHandle(hdir); + return INVALID_HANDLE_VALUE; + } + } + + winerrno = NO_ERROR; return hdir; } @@ -667,11 +575,13 @@ HANDLE openfile(const char *pFileName, int flags, int mode) // // Function // Name: emu_fstat -// Purpose: replacement for fstat supply a windows handle +// Purpose: replacement for fstat. Supply a windows handle. +// Returns a struct emu_stat to have room for 64-bit +// file identifier in st_ino (mingw allows only 16!) // Created: 25th October 2004 // // -------------------------------------------------------------------------- -int emu_fstat(HANDLE hdir, struct stat * st) +int emu_fstat(HANDLE hdir, struct emu_stat * st) { if (hdir == INVALID_HANDLE_VALUE) { @@ -702,8 +612,8 @@ int emu_fstat(HANDLE hdir, struct stat * st) // This is how we get our INODE (equivalent) information ULARGE_INTEGER conv; conv.HighPart = fi.nFileIndexHigh; - conv.LowPart = fi.nFileIndexLow; - st->st_ino = (_ino_t)conv.QuadPart; + conv.LowPart = fi.nFileIndexLow; + st->st_ino = conv.QuadPart; // get the time information st->st_ctime = ConvertFileTimeToTime_t(&fi.ftCreationTime); @@ -716,20 +626,8 @@ int emu_fstat(HANDLE hdir, struct stat * st) } else { - // size of the file - LARGE_INTEGER st_size; - memset(&st_size, 0, sizeof(st_size)); - - if (!GetFileSizeEx(hdir, &st_size)) - { - ::syslog(LOG_WARNING, "Failed to get file size: " - "%s", GetErrorMessage(GetLastError()).c_str()); - errno = EACCES; - return -1; - } - - conv.HighPart = st_size.HighPart; - conv.LowPart = st_size.LowPart; + conv.HighPart = fi.nFileSizeHigh; + conv.LowPart = fi.nFileSizeLow; st->st_size = (_off_t)conv.QuadPart; } @@ -862,12 +760,14 @@ HANDLE OpenFileByNameUtf8(const char* pFileName, DWORD flags) // // Function // Name: emu_stat -// Purpose: replacement for the lstat and stat functions, -// works with unicode filenames supplied in utf8 format +// Purpose: replacement for the lstat and stat functions. +// Works with unicode filenames supplied in utf8. +// Returns a struct emu_stat to have room for 64-bit +// file identifier in st_ino (mingw allows only 16!) // Created: 25th October 2004 // // -------------------------------------------------------------------------- -int emu_stat(const char * pName, struct stat * st) +int emu_stat(const char * pName, struct emu_stat * st) { HANDLE handle = OpenFileByNameUtf8(pName, FILE_READ_ATTRIBUTES | FILE_READ_EA); @@ -1078,7 +978,7 @@ DIR *opendir(const char *name) } pDir->name = ConvertUtf8ToWideString(dirName.c_str()); - // We are responsible for freeing dir->name + // We are responsible for freeing dir->name with delete[] if (pDir->name == NULL) { @@ -1396,16 +1296,20 @@ static bool sHaveWarnedEventLogFull = false; void openlog(const char * daemonName, int, int) { + std::string nameStr = "Box Backup ("; + nameStr += daemonName; + nameStr += ")"; + // register a default event source, so that we can // log errors with the process of adding or registering our own. gSyslogH = RegisterEventSource( NULL, // uses local computer - daemonName); // source name + nameStr.c_str()); // source name if (gSyslogH == NULL) { } - char* name = strdup(daemonName); + char* name = strdup(nameStr.c_str()); BOOL success = AddEventSource(name, 0); free(name); @@ -1415,7 +1319,7 @@ void openlog(const char * daemonName, int, int) return; } - HANDLE newSyslogH = RegisterEventSource(NULL, daemonName); + HANDLE newSyslogH = RegisterEventSource(NULL, nameStr.c_str()); if (newSyslogH == NULL) { ::syslog(LOG_ERR, "Failed to register our own event source: " @@ -1484,8 +1388,6 @@ void syslog(int loglevel, const char *frmt, ...) va_end(args); - LPCSTR strings[] = { buffer, NULL }; - if (gSyslogH == 0) { printf("%s\r\n", buffer); @@ -1493,17 +1395,45 @@ void syslog(int loglevel, const char *frmt, ...) return; } - if (!ReportEvent(gSyslogH, // event log handle - errinfo, // event type - 0, // category zero - MSG_ERR, // event identifier - - // we will call them all the same - NULL, // no user security identifier - 1, // one substitution string - 0, // no data - strings, // pointer to string array - NULL)) // pointer to data + WCHAR* pWide = ConvertToWideString(buffer, CP_UTF8, false); + // must delete[] pWide + DWORD result; + + if (pWide == NULL) + { + std::string buffer2 = buffer; + buffer2 += " (failed to convert string encoding)"; + LPCSTR strings[] = { buffer2.c_str(), NULL }; + + result = ReportEventA(gSyslogH, // event log handle + errinfo, // event type + 0, // category zero + MSG_ERR, // event identifier - + // we will call them all the same + NULL, // no user security identifier + 1, // one substitution string + 0, // no data + strings, // pointer to string array + NULL); // pointer to data + } + else + { + LPCWSTR strings[] = { pWide, NULL }; + result = ReportEventW(gSyslogH, // event log handle + errinfo, // event type + 0, // category zero + MSG_ERR, // event identifier - + // we will call them all the same + NULL, // no user security identifier + 1, // one substitution string + 0, // no data + strings, // pointer to string array + NULL); // pointer to data + delete [] pWide; + } + + if (result == 0) { DWORD err = GetLastError(); if (err == ERROR_LOG_FILE_FULL) @@ -1527,9 +1457,6 @@ void syslog(int loglevel, const char *frmt, ...) { sHaveWarnedEventLogFull = false; } - - // printf("%s\r\n", buffer); - // fflush(stdout); } int emu_chdir(const char* pDirName) diff --git a/lib/win32/emu.h b/lib/win32/emu.h index 8ab74130..811e6495 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -1,5 +1,17 @@ // emulates unix syscalls to win32 functions +#ifdef WIN32 + #define EMU_STRUCT_STAT struct emu_stat + #define EMU_STAT emu_stat + #define EMU_FSTAT emu_fstat + #define EMU_LSTAT emu_stat +#else + #define EMU_STRUCT_STAT struct stat + #define EMU_STAT ::stat + #define EMU_FSTAT ::fstat + #define EMU_LSTAT ::lstat +#endif + #if ! defined EMU_INCLUDE && defined WIN32 #define EMU_INCLUDE @@ -28,11 +40,6 @@ #else typedef unsigned int mode_t; typedef unsigned int pid_t; - - // must define _INO_T_DEFINED before including - // to replace it with our own. - typedef u_int64_t _ino_t; - #define _INO_T_DEFINED #endif // set up to include the necessary parts of Windows headers @@ -78,11 +85,6 @@ #define fileno(struct_file) _fileno(struct_file) #endif -int SetTimerHandler(void (__cdecl *func ) (int)); -int setitimer(int type, struct itimerval *timeout, void *arg); -void InitTimer(void); -void FiniTimer(void); - struct passwd { char *pw_name; char *pw_passwd; @@ -184,13 +186,6 @@ inline int geteuid(void) #define timespec timeval -//not available in win32 -struct itimerval -{ - timeval it_interval; - timeval it_value; -}; - //win32 deals in usec not nsec - so need to ensure this follows through #define tv_nsec tv_usec @@ -205,6 +200,7 @@ struct itimerval //again need to verify these #define S_IFLNK 1 +#define S_IFSOCK 0 #define S_ISLNK(x) ( false ) @@ -244,6 +240,7 @@ int closedir(DIR *dp); // local constant to open file exclusively without shared access #define O_LOCK 0x10000 +extern DWORD winerrno; /* used to report errors from openfile() */ HANDLE openfile(const char *filename, int flags, int mode); #define LOG_DEBUG LOG_INFO @@ -260,6 +257,15 @@ void openlog (const char * daemonName, int, int); void closelog(void); void syslog (int loglevel, const char *fmt, ...); +#define LOG_LOCAL0 0 +#define LOG_LOCAL1 0 +#define LOG_LOCAL2 0 +#define LOG_LOCAL3 0 +#define LOG_LOCAL4 0 +#define LOG_LOCAL5 0 +#define LOG_LOCAL6 0 +#define LOG_DAEMON 0 + #ifndef __MINGW32__ #define strtoll _strtoi64 #endif @@ -292,6 +298,11 @@ inline int ioctl(SOCKET sock, int flag, int * something) return 0; } +extern "C" inline int getpid() +{ + return (int)GetCurrentProcessId(); +} + inline int waitpid(pid_t pid, int *status, int) { return 0; @@ -303,27 +314,19 @@ struct statfs TCHAR f_mntonname[MAX_PATH]; }; -#if 0 -// I think this should get us going -// Although there is a warning about -// mount points in win32 can now exists - which means inode number can be -// duplicated, so potential of a problem - perhaps this needs to be -// implemented with a little more thought... TODO - -struct stat { - //_dev_t st_dev; - u_int64_t st_ino; +struct emu_stat { + int st_dev; + uint64_t st_ino; DWORD st_mode; short st_nlink; short st_uid; short st_gid; //_dev_t st_rdev; - u_int64_t st_size; + uint64_t st_size; time_t st_atime; time_t st_mtime; time_t st_ctime; }; -#endif // 0 // need this for conversions time_t ConvertFileTimeToTime_t(FILETIME *fileTime); @@ -332,8 +335,8 @@ bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo); int emu_chdir (const char* pDirName); int emu_mkdir (const char* pPathName); int emu_unlink (const char* pFileName); -int emu_fstat (HANDLE file, struct stat* st); -int emu_stat (const char* pName, struct stat* st); +int emu_fstat (HANDLE file, struct emu_stat* st); +int emu_stat (const char* pName, struct emu_stat* st); int emu_utimes (const char* pName, const struct timeval[]); int emu_chmod (const char* pName, mode_t mode); char* emu_getcwd (char* pBuffer, int BufSize); @@ -342,14 +345,22 @@ int emu_rename (const char* pOldName, const char* pNewName); #define chdir(directory) emu_chdir (directory) #define mkdir(path, mode) emu_mkdir (path) #define unlink(file) emu_unlink (file) -#define stat(filename, struct) emu_stat (filename, struct) -#define lstat(filename, struct) emu_stat (filename, struct) -#define fstat(handle, struct) emu_fstat (handle, struct) #define utimes(buffer, times) emu_utimes (buffer, times) #define chmod(file, mode) emu_chmod (file, mode) #define getcwd(buffer, size) emu_getcwd (buffer, size) #define rename(oldname, newname) emu_rename (oldname, newname) +// Not safe to replace stat/fstat/lstat on mingw at least, as struct stat +// has a 16-bit st_ino and we need a 64-bit one. +// +// #define stat(filename, struct) emu_stat (filename, struct) +// #define lstat(filename, struct) emu_stat (filename, struct) +// #define fstat(handle, struct) emu_fstat (handle, struct) +// +// But lstat doesn't exist on Windows, so we have to provide something: + +#define lstat(filename, struct) stat(filename, struct) + int statfs(const char * name, struct statfs * s); int poll(struct pollfd *ufds, unsigned long nfds, int timeout); @@ -374,8 +385,8 @@ bool ConvertToUtf8 (const std::string& rSource, std::string& rDest, int sourceCodePage); bool ConvertFromUtf8 (const std::string& rSource, std::string& rDest, int destCodePage); -bool ConvertUtf8ToConsole(const char* pString, std::string& rDest); -bool ConvertConsoleToUtf8(const char* pString, std::string& rDest); +bool ConvertUtf8ToConsole(const std::string& rSource, std::string& rDest); +bool ConvertConsoleToUtf8(const std::string& rSource, std::string& rDest); // Utility function which returns a default config file name, // based on the path of the current executable. diff --git a/lib/win32/getopt_long.cpp b/lib/win32/getopt_long.cpp new file mode 100755 index 00000000..5d910e1b --- /dev/null +++ b/lib/win32/getopt_long.cpp @@ -0,0 +1,550 @@ +/* $OpenBSD: getopt_long.c,v 1.20 2005/10/25 15:49:37 jmc Exp $ */ +/* $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $ */ +// Adapted for Box Backup by Chris Wilson + +/* + * Copyright (c) 2002 Todd C. Miller + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * Sponsored in part by the Defense Advanced Research Projects + * Agency (DARPA) and Air Force Research Laboratory, Air Force + * Materiel Command, USAF, under agreement number F39502-99-1-0512. + */ +/*- + * Copyright (c) 2000 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Dieter Baron and Thomas Klausner. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the NetBSD + * Foundation, Inc. and its contributors. + * 4. Neither the name of The NetBSD Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +// #include "Box.h" + +#include +#include +#include +#include +#include + +#include "getopt.h" + +#if defined _MSC_VER || defined __MINGW32__ +#define REPLACE_GETOPT /* use this getopt as the system getopt(3) */ + +#ifdef REPLACE_GETOPT +int opterr = 1; /* if error message should be printed */ +int optind = 1; /* index into parent argv vector */ +int optopt = '?'; /* character checked for validity */ +int optreset; /* reset getopt */ +char *optarg; /* argument associated with option */ +#endif + +#define PRINT_ERROR ((opterr) && (*options != ':')) + +#define FLAG_PERMUTE 0x01 /* permute non-options to the end of argv */ +#define FLAG_ALLARGS 0x02 /* treat non-options as args to option "-1" */ +#define FLAG_LONGONLY 0x04 /* operate as getopt_long_only */ + +/* return values */ +#define BADCH (int)'?' +#define BADARG ((*options == ':') ? (int)':' : (int)'?') +#define INORDER (int)1 + +#define EMSG "" + +static void warnx(const char* fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, "\n"); +} + +static int getopt_internal(int, char * const *, const char *, + const struct option *, int *, int); +static int parse_long_options(char * const *, const char *, + const struct option *, int *, int); +static int gcd(int, int); +static void permute_args(int, int, int, char * const *); + +static char *place = EMSG; /* option letter processing */ + +/* XXX: set optreset to 1 rather than these two */ +static int nonopt_start = -1; /* first non option argument (for permute) */ +static int nonopt_end = -1; /* first option after non options (for permute) */ + +/* Error messages */ +static const char recargchar[] = "option requires an argument -- %c"; +static const char recargstring[] = "option requires an argument -- %s"; +static const char ambig[] = "ambiguous option -- %.*s"; +static const char noarg[] = "option doesn't take an argument -- %.*s"; +static const char illoptchar[] = "unknown option -- %c"; +static const char illoptstring[] = "unknown option -- %s"; + +/* + * Compute the greatest common divisor of a and b. + */ +static int +gcd(int a, int b) +{ + int c; + + c = a % b; + while (c != 0) { + a = b; + b = c; + c = a % b; + } + + return (b); +} + +/* + * Exchange the block from nonopt_start to nonopt_end with the block + * from nonopt_end to opt_end (keeping the same order of arguments + * in each block). + */ +static void +permute_args(int panonopt_start, int panonopt_end, int opt_end, + char * const *nargv) +{ + int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos; + char *swap; + + /* + * compute lengths of blocks and number and size of cycles + */ + nnonopts = panonopt_end - panonopt_start; + nopts = opt_end - panonopt_end; + ncycle = gcd(nnonopts, nopts); + cyclelen = (opt_end - panonopt_start) / ncycle; + + for (i = 0; i < ncycle; i++) { + cstart = panonopt_end+i; + pos = cstart; + for (j = 0; j < cyclelen; j++) { + if (pos >= panonopt_end) + pos -= nnonopts; + else + pos += nopts; + swap = nargv[pos]; + /* LINTED const cast */ + ((char **) nargv)[pos] = nargv[cstart]; + /* LINTED const cast */ + ((char **)nargv)[cstart] = swap; + } + } +} + +/* + * parse_long_options -- + * Parse long options in argc/argv argument vector. + * Returns -1 if short_too is set and the option does not match long_options. + */ +static int +parse_long_options(char * const *nargv, const char *options, + const struct option *long_options, int *idx, int short_too) +{ + char *current_argv, *has_equal; + size_t current_argv_len; + int i, match; + + current_argv = place; + match = -1; + + optind++; + + if ((has_equal = strchr(current_argv, '=')) != NULL) { + /* argument found (--option=arg) */ + current_argv_len = has_equal - current_argv; + has_equal++; + } else + current_argv_len = strlen(current_argv); + + for (i = 0; long_options[i].name; i++) { + /* find matching long option */ + if (strncmp(current_argv, long_options[i].name, + current_argv_len)) + continue; + + if (strlen(long_options[i].name) == current_argv_len) { + /* exact match */ + match = i; + break; + } + /* + * If this is a known short option, don't allow + * a partial match of a single character. + */ + if (short_too && current_argv_len == 1) + continue; + + if (match == -1) /* partial match */ + match = i; + else { + /* ambiguous abbreviation */ + if (PRINT_ERROR) + warnx(ambig, (int)current_argv_len, + current_argv); + optopt = 0; + return (BADCH); + } + } + if (match != -1) { /* option found */ + if (long_options[match].has_arg == no_argument + && has_equal) { + if (PRINT_ERROR) + warnx(noarg, (int)current_argv_len, + current_argv); + /* + * XXX: GNU sets optopt to val regardless of flag + */ + if (long_options[match].flag == NULL) + optopt = long_options[match].val; + else + optopt = 0; + return (BADARG); + } + if (long_options[match].has_arg == required_argument || + long_options[match].has_arg == optional_argument) { + if (has_equal) + optarg = has_equal; + else if (long_options[match].has_arg == + required_argument) { + /* + * optional argument doesn't use next nargv + */ + optarg = nargv[optind++]; + } + } + if ((long_options[match].has_arg == required_argument) + && (optarg == NULL)) { + /* + * Missing argument; leading ':' indicates no error + * should be generated. + */ + if (PRINT_ERROR) + warnx(recargstring, + current_argv); + /* + * XXX: GNU sets optopt to val regardless of flag + */ + if (long_options[match].flag == NULL) + optopt = long_options[match].val; + else + optopt = 0; + --optind; + return (BADARG); + } + } else { /* unknown option */ + if (short_too) { + --optind; + return (-1); + } + if (PRINT_ERROR) + warnx(illoptstring, current_argv); + optopt = 0; + return (BADCH); + } + if (idx) + *idx = match; + if (long_options[match].flag) { + *long_options[match].flag = long_options[match].val; + return (0); + } else + return (long_options[match].val); +} + +/* + * getopt_internal -- + * Parse argc/argv argument vector. Called by user level routines. + */ +static int +getopt_internal(int nargc, char * const *nargv, const char *options, + const struct option *long_options, int *idx, int flags) +{ + const char * oli; /* option letter list index */ + int optchar, short_too; + static int posixly_correct = -1; + + if (options == NULL) + return (-1); + + /* + * Disable GNU extensions if POSIXLY_CORRECT is set or options + * string begins with a '+'. + */ + if (posixly_correct == -1) + posixly_correct = (getenv("POSIXLY_CORRECT") != NULL); + if (posixly_correct || *options == '+') + flags &= ~FLAG_PERMUTE; + else if (*options == '-') + flags |= FLAG_ALLARGS; + if (*options == '+' || *options == '-') + options++; + + /* + * XXX Some GNU programs (like cvs) set optind to 0 instead of + * XXX using optreset. Work around this braindamage. + */ + if (optind == 0) + optind = optreset = 1; + + optarg = NULL; + if (optreset) + nonopt_start = nonopt_end = -1; +start: + if (optreset || !*place) { /* update scanning pointer */ + optreset = 0; + if (optind >= nargc) { /* end of argument vector */ + place = EMSG; + if (nonopt_end != -1) { + /* do permutation, if we have to */ + permute_args(nonopt_start, nonopt_end, + optind, nargv); + optind -= nonopt_end - nonopt_start; + } + else if (nonopt_start != -1) { + /* + * If we skipped non-options, set optind + * to the first of them. + */ + optind = nonopt_start; + } + nonopt_start = nonopt_end = -1; + return (-1); + } + if (*(place = nargv[optind]) != '-' || + (place[1] == '\0' && strchr(options, '-') == NULL)) { + place = EMSG; /* found non-option */ + if (flags & FLAG_ALLARGS) { + /* + * GNU extension: + * return non-option as argument to option 1 + */ + optarg = nargv[optind++]; + return (INORDER); + } + if (!(flags & FLAG_PERMUTE)) { + /* + * If no permutation wanted, stop parsing + * at first non-option. + */ + return (-1); + } + /* do permutation */ + if (nonopt_start == -1) + nonopt_start = optind; + else if (nonopt_end != -1) { + permute_args(nonopt_start, nonopt_end, + optind, nargv); + nonopt_start = optind - + (nonopt_end - nonopt_start); + nonopt_end = -1; + } + optind++; + /* process next argument */ + goto start; + } + if (nonopt_start != -1 && nonopt_end == -1) + nonopt_end = optind; + + /* + * If we have "-" do nothing, if "--" we are done. + */ + if (place[1] != '\0' && *++place == '-' && place[1] == '\0') { + optind++; + place = EMSG; + /* + * We found an option (--), so if we skipped + * non-options, we have to permute. + */ + if (nonopt_end != -1) { + permute_args(nonopt_start, nonopt_end, + optind, nargv); + optind -= nonopt_end - nonopt_start; + } + nonopt_start = nonopt_end = -1; + return (-1); + } + } + + /* + * Check long options if: + * 1) we were passed some + * 2) the arg is not just "-" + * 3) either the arg starts with -- we are getopt_long_only() + */ + if (long_options != NULL && place != nargv[optind] && + (*place == '-' || (flags & FLAG_LONGONLY))) { + short_too = 0; + if (*place == '-') + place++; /* --foo long option */ + else if (*place != ':' && strchr(options, *place) != NULL) + short_too = 1; /* could be short option too */ + + optchar = parse_long_options(nargv, options, long_options, + idx, short_too); + if (optchar != -1) { + place = EMSG; + return (optchar); + } + } + + if ((optchar = (int)*place++) == (int)':' || + optchar == (int)'-' && *place != '\0' || + (oli = strchr(options, optchar)) == NULL) { + /* + * If the user specified "-" and '-' isn't listed in + * options, return -1 (non-option) as per POSIX. + * Otherwise, it is an unknown option character (or ':'). + */ + if (optchar == (int)'-' && *place == '\0') + return (-1); + if (!*place) + ++optind; + if (PRINT_ERROR) + warnx(illoptchar, optchar); + optopt = optchar; + return (BADCH); + } + if (long_options != NULL && optchar == 'W' && oli[1] == ';') { + /* -W long-option */ + if (*place) /* no space */ + /* NOTHING */; + else if (++optind >= nargc) { /* no arg */ + place = EMSG; + if (PRINT_ERROR) + warnx(recargchar, optchar); + optopt = optchar; + return (BADARG); + } else /* white space */ + place = nargv[optind]; + optchar = parse_long_options(nargv, options, long_options, + idx, 0); + place = EMSG; + return (optchar); + } + if (*++oli != ':') { /* doesn't take argument */ + if (!*place) + ++optind; + } else { /* takes (optional) argument */ + optarg = NULL; + if (*place) /* no white space */ + optarg = place; + /* XXX: disable test for :: if PC? (GNU doesn't) */ + else if (oli[1] != ':') { /* arg not optional */ + if (++optind >= nargc) { /* no arg */ + place = EMSG; + if (PRINT_ERROR) + warnx(recargchar, optchar); + optopt = optchar; + return (BADARG); + } else + optarg = nargv[optind]; + } else if (!(flags & FLAG_PERMUTE)) { + /* + * If permutation is disabled, we can accept an + * optional arg separated by whitespace so long + * as it does not start with a dash (-). + */ + if (optind + 1 < nargc && *nargv[optind + 1] != '-') + optarg = nargv[++optind]; + } + place = EMSG; + ++optind; + } + /* dump back option letter */ + return (optchar); +} + +#ifdef REPLACE_GETOPT +/* + * getopt -- + * Parse argc/argv argument vector. + * + * [eventually this will replace the BSD getopt] + */ +int +getopt(int nargc, char * const *nargv, const char *options) +{ + + /* + * We don't pass FLAG_PERMUTE to getopt_internal() since + * the BSD getopt(3) (unlike GNU) has never done this. + * + * Furthermore, since many privileged programs call getopt() + * before dropping privileges it makes sense to keep things + * as simple (and bug-free) as possible. + */ + return (getopt_internal(nargc, nargv, options, NULL, NULL, 0)); +} +#endif /* REPLACE_GETOPT */ + +/* + * getopt_long -- + * Parse argc/argv argument vector. + */ +int +getopt_long(int nargc, char * const *nargv, const char *options, + const struct option *long_options, int *idx) +{ + + return (getopt_internal(nargc, nargv, options, long_options, idx, + FLAG_PERMUTE)); +} + +/* + * getopt_long_only -- + * Parse argc/argv argument vector. + */ +int +getopt_long_only(int nargc, char * const *nargv, const char *options, + const struct option *long_options, int *idx) +{ + + return (getopt_internal(nargc, nargv, options, long_options, idx, + FLAG_PERMUTE|FLAG_LONGONLY)); +} + +#endif // defined _MSC_VER || defined __MINGW32__ diff --git a/lib/win32/getopt_long.cxx b/lib/win32/getopt_long.cxx deleted file mode 100755 index a24930aa..00000000 --- a/lib/win32/getopt_long.cxx +++ /dev/null @@ -1,550 +0,0 @@ -/* $OpenBSD: getopt_long.c,v 1.20 2005/10/25 15:49:37 jmc Exp $ */ -/* $NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $ */ -// Adapted for Box Backup by Chris Wilson - -/* - * Copyright (c) 2002 Todd C. Miller - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - * Sponsored in part by the Defense Advanced Research Projects - * Agency (DARPA) and Air Force Research Laboratory, Air Force - * Materiel Command, USAF, under agreement number F39502-99-1-0512. - */ -/*- - * Copyright (c) 2000 The NetBSD Foundation, Inc. - * All rights reserved. - * - * This code is derived from software contributed to The NetBSD Foundation - * by Dieter Baron and Thomas Klausner. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the NetBSD - * Foundation, Inc. and its contributors. - * 4. Neither the name of The NetBSD Foundation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS - * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -// #include "Box.h" - -#include -#include -#include -#include -#include - -#include "getopt.h" - -#if defined _MSC_VER || defined __MINGW32__ -#define REPLACE_GETOPT /* use this getopt as the system getopt(3) */ -#endif - -#ifdef REPLACE_GETOPT -int opterr = 1; /* if error message should be printed */ -int optind = 1; /* index into parent argv vector */ -int optopt = '?'; /* character checked for validity */ -int optreset; /* reset getopt */ -char *optarg; /* argument associated with option */ -#endif - -#define PRINT_ERROR ((opterr) && (*options != ':')) - -#define FLAG_PERMUTE 0x01 /* permute non-options to the end of argv */ -#define FLAG_ALLARGS 0x02 /* treat non-options as args to option "-1" */ -#define FLAG_LONGONLY 0x04 /* operate as getopt_long_only */ - -/* return values */ -#define BADCH (int)'?' -#define BADARG ((*options == ':') ? (int)':' : (int)'?') -#define INORDER (int)1 - -#define EMSG "" - -static void warnx(const char* fmt, ...) -{ - va_list ap; - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - fprintf(stderr, "\n"); -} - -static int getopt_internal(int, char * const *, const char *, - const struct option *, int *, int); -static int parse_long_options(char * const *, const char *, - const struct option *, int *, int); -static int gcd(int, int); -static void permute_args(int, int, int, char * const *); - -static char *place = EMSG; /* option letter processing */ - -/* XXX: set optreset to 1 rather than these two */ -static int nonopt_start = -1; /* first non option argument (for permute) */ -static int nonopt_end = -1; /* first option after non options (for permute) */ - -/* Error messages */ -static const char recargchar[] = "option requires an argument -- %c"; -static const char recargstring[] = "option requires an argument -- %s"; -static const char ambig[] = "ambiguous option -- %.*s"; -static const char noarg[] = "option doesn't take an argument -- %.*s"; -static const char illoptchar[] = "unknown option -- %c"; -static const char illoptstring[] = "unknown option -- %s"; - -/* - * Compute the greatest common divisor of a and b. - */ -static int -gcd(int a, int b) -{ - int c; - - c = a % b; - while (c != 0) { - a = b; - b = c; - c = a % b; - } - - return (b); -} - -/* - * Exchange the block from nonopt_start to nonopt_end with the block - * from nonopt_end to opt_end (keeping the same order of arguments - * in each block). - */ -static void -permute_args(int panonopt_start, int panonopt_end, int opt_end, - char * const *nargv) -{ - int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos; - char *swap; - - /* - * compute lengths of blocks and number and size of cycles - */ - nnonopts = panonopt_end - panonopt_start; - nopts = opt_end - panonopt_end; - ncycle = gcd(nnonopts, nopts); - cyclelen = (opt_end - panonopt_start) / ncycle; - - for (i = 0; i < ncycle; i++) { - cstart = panonopt_end+i; - pos = cstart; - for (j = 0; j < cyclelen; j++) { - if (pos >= panonopt_end) - pos -= nnonopts; - else - pos += nopts; - swap = nargv[pos]; - /* LINTED const cast */ - ((char **) nargv)[pos] = nargv[cstart]; - /* LINTED const cast */ - ((char **)nargv)[cstart] = swap; - } - } -} - -/* - * parse_long_options -- - * Parse long options in argc/argv argument vector. - * Returns -1 if short_too is set and the option does not match long_options. - */ -static int -parse_long_options(char * const *nargv, const char *options, - const struct option *long_options, int *idx, int short_too) -{ - char *current_argv, *has_equal; - size_t current_argv_len; - int i, match; - - current_argv = place; - match = -1; - - optind++; - - if ((has_equal = strchr(current_argv, '=')) != NULL) { - /* argument found (--option=arg) */ - current_argv_len = has_equal - current_argv; - has_equal++; - } else - current_argv_len = strlen(current_argv); - - for (i = 0; long_options[i].name; i++) { - /* find matching long option */ - if (strncmp(current_argv, long_options[i].name, - current_argv_len)) - continue; - - if (strlen(long_options[i].name) == current_argv_len) { - /* exact match */ - match = i; - break; - } - /* - * If this is a known short option, don't allow - * a partial match of a single character. - */ - if (short_too && current_argv_len == 1) - continue; - - if (match == -1) /* partial match */ - match = i; - else { - /* ambiguous abbreviation */ - if (PRINT_ERROR) - warnx(ambig, (int)current_argv_len, - current_argv); - optopt = 0; - return (BADCH); - } - } - if (match != -1) { /* option found */ - if (long_options[match].has_arg == no_argument - && has_equal) { - if (PRINT_ERROR) - warnx(noarg, (int)current_argv_len, - current_argv); - /* - * XXX: GNU sets optopt to val regardless of flag - */ - if (long_options[match].flag == NULL) - optopt = long_options[match].val; - else - optopt = 0; - return (BADARG); - } - if (long_options[match].has_arg == required_argument || - long_options[match].has_arg == optional_argument) { - if (has_equal) - optarg = has_equal; - else if (long_options[match].has_arg == - required_argument) { - /* - * optional argument doesn't use next nargv - */ - optarg = nargv[optind++]; - } - } - if ((long_options[match].has_arg == required_argument) - && (optarg == NULL)) { - /* - * Missing argument; leading ':' indicates no error - * should be generated. - */ - if (PRINT_ERROR) - warnx(recargstring, - current_argv); - /* - * XXX: GNU sets optopt to val regardless of flag - */ - if (long_options[match].flag == NULL) - optopt = long_options[match].val; - else - optopt = 0; - --optind; - return (BADARG); - } - } else { /* unknown option */ - if (short_too) { - --optind; - return (-1); - } - if (PRINT_ERROR) - warnx(illoptstring, current_argv); - optopt = 0; - return (BADCH); - } - if (idx) - *idx = match; - if (long_options[match].flag) { - *long_options[match].flag = long_options[match].val; - return (0); - } else - return (long_options[match].val); -} - -/* - * getopt_internal -- - * Parse argc/argv argument vector. Called by user level routines. - */ -static int -getopt_internal(int nargc, char * const *nargv, const char *options, - const struct option *long_options, int *idx, int flags) -{ - const char * oli; /* option letter list index */ - int optchar, short_too; - static int posixly_correct = -1; - - if (options == NULL) - return (-1); - - /* - * Disable GNU extensions if POSIXLY_CORRECT is set or options - * string begins with a '+'. - */ - if (posixly_correct == -1) - posixly_correct = (getenv("POSIXLY_CORRECT") != NULL); - if (posixly_correct || *options == '+') - flags &= ~FLAG_PERMUTE; - else if (*options == '-') - flags |= FLAG_ALLARGS; - if (*options == '+' || *options == '-') - options++; - - /* - * XXX Some GNU programs (like cvs) set optind to 0 instead of - * XXX using optreset. Work around this braindamage. - */ - if (optind == 0) - optind = optreset = 1; - - optarg = NULL; - if (optreset) - nonopt_start = nonopt_end = -1; -start: - if (optreset || !*place) { /* update scanning pointer */ - optreset = 0; - if (optind >= nargc) { /* end of argument vector */ - place = EMSG; - if (nonopt_end != -1) { - /* do permutation, if we have to */ - permute_args(nonopt_start, nonopt_end, - optind, nargv); - optind -= nonopt_end - nonopt_start; - } - else if (nonopt_start != -1) { - /* - * If we skipped non-options, set optind - * to the first of them. - */ - optind = nonopt_start; - } - nonopt_start = nonopt_end = -1; - return (-1); - } - if (*(place = nargv[optind]) != '-' || - (place[1] == '\0' && strchr(options, '-') == NULL)) { - place = EMSG; /* found non-option */ - if (flags & FLAG_ALLARGS) { - /* - * GNU extension: - * return non-option as argument to option 1 - */ - optarg = nargv[optind++]; - return (INORDER); - } - if (!(flags & FLAG_PERMUTE)) { - /* - * If no permutation wanted, stop parsing - * at first non-option. - */ - return (-1); - } - /* do permutation */ - if (nonopt_start == -1) - nonopt_start = optind; - else if (nonopt_end != -1) { - permute_args(nonopt_start, nonopt_end, - optind, nargv); - nonopt_start = optind - - (nonopt_end - nonopt_start); - nonopt_end = -1; - } - optind++; - /* process next argument */ - goto start; - } - if (nonopt_start != -1 && nonopt_end == -1) - nonopt_end = optind; - - /* - * If we have "-" do nothing, if "--" we are done. - */ - if (place[1] != '\0' && *++place == '-' && place[1] == '\0') { - optind++; - place = EMSG; - /* - * We found an option (--), so if we skipped - * non-options, we have to permute. - */ - if (nonopt_end != -1) { - permute_args(nonopt_start, nonopt_end, - optind, nargv); - optind -= nonopt_end - nonopt_start; - } - nonopt_start = nonopt_end = -1; - return (-1); - } - } - - /* - * Check long options if: - * 1) we were passed some - * 2) the arg is not just "-" - * 3) either the arg starts with -- we are getopt_long_only() - */ - if (long_options != NULL && place != nargv[optind] && - (*place == '-' || (flags & FLAG_LONGONLY))) { - short_too = 0; - if (*place == '-') - place++; /* --foo long option */ - else if (*place != ':' && strchr(options, *place) != NULL) - short_too = 1; /* could be short option too */ - - optchar = parse_long_options(nargv, options, long_options, - idx, short_too); - if (optchar != -1) { - place = EMSG; - return (optchar); - } - } - - if ((optchar = (int)*place++) == (int)':' || - optchar == (int)'-' && *place != '\0' || - (oli = strchr(options, optchar)) == NULL) { - /* - * If the user specified "-" and '-' isn't listed in - * options, return -1 (non-option) as per POSIX. - * Otherwise, it is an unknown option character (or ':'). - */ - if (optchar == (int)'-' && *place == '\0') - return (-1); - if (!*place) - ++optind; - if (PRINT_ERROR) - warnx(illoptchar, optchar); - optopt = optchar; - return (BADCH); - } - if (long_options != NULL && optchar == 'W' && oli[1] == ';') { - /* -W long-option */ - if (*place) /* no space */ - /* NOTHING */; - else if (++optind >= nargc) { /* no arg */ - place = EMSG; - if (PRINT_ERROR) - warnx(recargchar, optchar); - optopt = optchar; - return (BADARG); - } else /* white space */ - place = nargv[optind]; - optchar = parse_long_options(nargv, options, long_options, - idx, 0); - place = EMSG; - return (optchar); - } - if (*++oli != ':') { /* doesn't take argument */ - if (!*place) - ++optind; - } else { /* takes (optional) argument */ - optarg = NULL; - if (*place) /* no white space */ - optarg = place; - /* XXX: disable test for :: if PC? (GNU doesn't) */ - else if (oli[1] != ':') { /* arg not optional */ - if (++optind >= nargc) { /* no arg */ - place = EMSG; - if (PRINT_ERROR) - warnx(recargchar, optchar); - optopt = optchar; - return (BADARG); - } else - optarg = nargv[optind]; - } else if (!(flags & FLAG_PERMUTE)) { - /* - * If permutation is disabled, we can accept an - * optional arg separated by whitespace so long - * as it does not start with a dash (-). - */ - if (optind + 1 < nargc && *nargv[optind + 1] != '-') - optarg = nargv[++optind]; - } - place = EMSG; - ++optind; - } - /* dump back option letter */ - return (optchar); -} - -#ifdef REPLACE_GETOPT -/* - * getopt -- - * Parse argc/argv argument vector. - * - * [eventually this will replace the BSD getopt] - */ -int -getopt(int nargc, char * const *nargv, const char *options) -{ - - /* - * We don't pass FLAG_PERMUTE to getopt_internal() since - * the BSD getopt(3) (unlike GNU) has never done this. - * - * Furthermore, since many privileged programs call getopt() - * before dropping privileges it makes sense to keep things - * as simple (and bug-free) as possible. - */ - return (getopt_internal(nargc, nargv, options, NULL, NULL, 0)); -} -#endif /* REPLACE_GETOPT */ - -/* - * getopt_long -- - * Parse argc/argv argument vector. - */ -int -getopt_long(int nargc, char * const *nargv, const char *options, - const struct option *long_options, int *idx) -{ - - return (getopt_internal(nargc, nargv, options, long_options, idx, - FLAG_PERMUTE)); -} - -/* - * getopt_long_only -- - * Parse argc/argv argument vector. - */ -int -getopt_long_only(int nargc, char * const *nargv, const char *options, - const struct option *long_options, int *idx) -{ - - return (getopt_internal(nargc, nargv, options, long_options, idx, - FLAG_PERMUTE|FLAG_LONGONLY)); -} - diff --git a/modules.txt b/modules.txt index d0d0b76b..8932f014 100644 --- a/modules.txt +++ b/modules.txt @@ -41,5 +41,9 @@ test/backupstorepatch bin/bbstored bin/bbstoreaccounts lib/backupstore lib/raidf test/backupdiff lib/backupclient test/bbackupd bin/bbackupd bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupctl lib/server lib/backupstore lib/backupclient lib/intercept +# HTTP server system +lib/httpserver lib/server +test/httpserver lib/httpserver + # END_IF_DISTRIBUTION diff --git a/parcels.txt b/parcels.txt index bf480cf8..fb708188 100644 --- a/parcels.txt +++ b/parcels.txt @@ -8,7 +8,12 @@ backup-client bin bbackupquery bin bbackupctl script bin/bbackupd/bbackupd-config - script LICENSE.txt + noinstall script LICENSE.txt + man bbackupd.8 + man bbackupquery.8 + man bbackupctl.8 + man bbackupd-config.8 + man bbackupd.conf.5 ONLY:mingw32,mingw32msvc script bin/bbackupd/win32/installer.iss @@ -17,9 +22,7 @@ ONLY:mingw32,mingw32msvc END-ONLY ONLY:mingw32 - script /bin/mgwz.dll script /bin/mingwm10.dll - optional script /bin/pcreposix.dll END-ONLY ONLY:SunOS @@ -27,14 +30,30 @@ ONLY:SunOS script contrib/solaris/bbackupd-smf-method lib/svc/method END-ONLY +ONLY:Darwin + script contrib/mac_osx/org.boxbackup.bbackupd.plist /Library/LaunchDaemons +END-ONLY + backup-server bin bbstored bin bbstoreaccounts script bin/bbstored/bbstored-certs script bin/bbstored/bbstored-config script lib/raidfile/raidfile-config + noinstall script LICENSE.txt + man bbstored.8 + man bbstoreaccounts.8 + man bbstored-certs.8 + man bbstored-config.8 + man raidfile-config.8 + man bbstored.conf.5 + man raidfile.conf.5 ONLY:SunOS script contrib/solaris/bbstored-manifest.xml lib/svc/manifest script contrib/solaris/bbstored-smf-method lib/svc/method END-ONLY + +ONLY:Darwin + script contrib/mac_osx/org.boxbackup.bbstored.plist /Library/LaunchDaemons +END-ONLY diff --git a/runtest.pl.in b/runtest.pl.in index 0d4381a6..42407378 100755 --- a/runtest.pl.in +++ b/runtest.pl.in @@ -71,6 +71,26 @@ else # report results print "--------\n",join("\n",@results),"\n"; +if ($exit_code != 0) +{ + print <<__E; + +One or more tests have failed. Please check the following common causes: + +* Check that no instances of bbstored or bbackupd are already running + on this machine. +* Make sure there isn't a firewall blocking incoming or outgoing connections + on port 2201. +* Check that there is sufficient space in the filesystem that the tests + are being run from (at least 1 GB free). +* The backupdiff test fails if it takes too long, so it's sensitive to + the speed of the host and your connection to it. + +After checking all the above, if you still have problems please contact +us on the mailing list, boxbackup\@boxbackup.org. Thanks! +__E +} + exit $exit_code; sub runtest @@ -104,6 +124,7 @@ sub runtest close RESULTS; chomp $last; + $last =~ s/\r//; push @results, "$t: $last"; if ($last ne "PASSED") diff --git a/test/backupdiff/testbackupdiff.cpp b/test/backupdiff/testbackupdiff.cpp index a91d6dfe..816f50d1 100644 --- a/test/backupdiff/testbackupdiff.cpp +++ b/test/backupdiff/testbackupdiff.cpp @@ -64,7 +64,7 @@ bool files_identical(const char *file1, const char *file2) return true; } -void make_file_of_zeros(const char *filename, size_t size) +bool make_file_of_zeros(const char *filename, size_t size) { #ifdef WIN32 HANDLE handle = openfile(filename, O_WRONLY | O_CREAT | O_EXCL, 0); @@ -75,7 +75,9 @@ void make_file_of_zeros(const char *filename, size_t size) BOOL result = SetEndOfFile(handle); if (result != TRUE) { - printf("Error %u\n", GetLastError()); + BOX_ERROR("Failed to create large file " << filename << + " (" << (size >> 20) << " MB): " << + GetErrorMessage(GetLastError())); } TEST_THAT(result == TRUE); TEST_THAT(CloseHandle(handle) == TRUE); @@ -87,7 +89,16 @@ void make_file_of_zeros(const char *filename, size_t size) TEST_THAT(close(fd) == 0); #endif - TEST_THAT((size_t)TestGetFileSize(filename) == size); + bool correct_size = ((size_t)TestGetFileSize(filename) == size); + TEST_THAT(correct_size); + if (!correct_size) + { + BOX_ERROR("Failed to create large file " << filename << + " (" << (size >> 20) << " MB): " << + "got " << (TestGetFileSize(filename) >> 20) << + " MB instead"); + } + return correct_size; } @@ -110,8 +121,10 @@ void check_encoded_file(const char *filename, int64_t OtherFileID, int new_block TEST_THAT((uint64_t)box_ntoh64(hdr.mOtherFileID) == (uint64_t)OtherFileID); // number of blocks int64_t nblocks = box_ntoh64(hdr.mNumBlocks); - TRACE2("Reading index from '%s', has %lld blocks\n", filename, nblocks); - TRACE0("======== ===== ========== ======== ========\n Index Where EncSz/Idx Size WChcksm\n"); + BOX_TRACE("Reading index from '" << filename << "', has " << + nblocks << " blocks"); + BOX_TRACE("======== ===== ========== ======== ========"); + BOX_TRACE(" Index Where EncSz/Idx Size WChcksm"); // Read them all in int64_t nnew = 0, nold = 0; for(int64_t b = 0; b < nblocks; ++b) @@ -119,35 +132,36 @@ void check_encoded_file(const char *filename, int64_t OtherFileID, int new_block file_BlockIndexEntry en; TEST_THAT(enc.ReadFullBuffer(&en, sizeof(en), 0)); int64_t s = box_ntoh64(en.mEncodedSize); + + // Decode the rest + uint64_t iv = box_ntoh64(hdr.mEntryIVBase); + iv += b; + sBlowfishDecryptBlockEntry.SetIV(&iv); + file_BlockIndexEntryEnc entryEnc; + sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, + sizeof(entryEnc), en.mEnEnc, sizeof(en.mEnEnc)); + + if(s > 0) { nnew++; - #ifdef WIN32 - TRACE2("%8I64d this s=%8I64d", b, s); - #else - TRACE2("%8lld this s=%8lld", b, s); - #endif + BOX_TRACE(std::setw(8) << b << " this s=" << + std::setw(8) << s << " " << + std::setw(8) << ntohl(entryEnc.mSize) << " " << + std::setw(8) << std::setfill('0') << + std::hex << ntohl(entryEnc.mWeakChecksum)); } else { nold++; - #ifdef WIN32 - TRACE2("%8I64d other i=%8I64d", b, 0 - s); - #else - TRACE2("%8lld other i=%8lld", b, 0 - s); - #endif + BOX_TRACE(std::setw(8) << b << " other i=" << + std::setw(8) << (0-s) << " " << + std::setw(8) << ntohl(entryEnc.mSize) << " " << + std::setw(8) << std::setfill('0') << + std::hex << ntohl(entryEnc.mWeakChecksum)); } - // Decode the rest - uint64_t iv = box_ntoh64(hdr.mEntryIVBase); - iv += b; - sBlowfishDecryptBlockEntry.SetIV(&iv); - file_BlockIndexEntryEnc entryEnc; - sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), - en.mEnEnc, sizeof(en.mEnEnc)); - TRACE2(" %8d %08x\n", ntohl(entryEnc.mSize), ntohl(entryEnc.mWeakChecksum)); - } - TRACE0("======== ===== ========== ======== ========\n"); + BOX_TRACE("======== ===== ========== ======== ========"); TEST_THAT(new_blocks_expected == nnew); TEST_THAT(old_blocks_expected == nold); } @@ -374,7 +388,7 @@ void test_combined_diffs() int test(int argc, const char *argv[]) { // Want to trace out all the details - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD #ifndef WIN32 BackupStoreFile::TraceDetailsOfDiffProcess = true; #endif @@ -512,13 +526,20 @@ int test(int argc, const char *argv[]) // found. Check this out! #ifdef WIN32 - ::fprintf(stdout, "Testing diffing two large streams, " - "may take a while!\n"); - ::fflush(stdout); + BOX_WARNING("Testing diffing two large streams, may take a while!"); + ::fflush(stderr); #endif - make_file_of_zeros("testfiles/zero.0", 20*1024*1024); - make_file_of_zeros("testfiles/zero.1", 200*1024*1024); + if (!make_file_of_zeros("testfiles/zero.0", 20*1024*1024)) + { + return 1; + } + + if (!make_file_of_zeros("testfiles/zero.1", 200*1024*1024)) + { + remove("testfiles/zero.0"); + return 1; + } // Generate a first encoded file { diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp index a5eacc7d..0266d097 100644 --- a/test/backupstore/testbackupstore.cpp +++ b/test/backupstore/testbackupstore.cpp @@ -72,7 +72,7 @@ static const char *ens_filenames[] = {"obj1ertewt", "obj2", "obj3", "obj4dfedfg4 typedef struct { - char *fnextra; + const char *fnextra; BackupStoreFilenameClear name; int seed; int size; @@ -232,7 +232,8 @@ int test1(int argc, const char *argv[]) TEST_THAT(fn1 == fn3); // Check that it's been encrypted - TEST_THAT(fn2.find("name") == fn2.npos); + std::string name(fn2.GetEncodedFilename()); + TEST_THAT(name.find("name") == name.npos); // Bung it in a stream, get it out in a Clear filename { @@ -974,6 +975,9 @@ int test_server(const char *hostname) test_server_1(protocol, protocolReadOnly); + // sleep to ensure that the timestamp on the file will change + ::safe_sleep(1); + // Create and upload some test files int64_t maxID = 0; for(int t = 0; t < UPLOAD_NUM; ++t) @@ -1048,11 +1052,16 @@ int test_server(const char *hostname) StreamableMemBlock attrtest(attr3, sizeof(attr3)); // Use the read only connection to verify that the directory is as we expect + printf("\n\n==== Reading directory using read-only connection\n"); check_dir_after_uploads(protocolReadOnly, attrtest); + printf("done.\n\n"); // And on the read/write one check_dir_after_uploads(protocol, attrtest); } + // sleep to ensure that the timestamp on the file will change + ::safe_sleep(1); + // Check diffing and rsync like stuff... // Build a modified file { @@ -1149,6 +1158,8 @@ int test_server(const char *hostname) *upload)); subdirfileid = stored->GetObjectID(); } + + printf("\n==== Checking upload using read-only connection\n"); // Check the directories on the read only connection { // Command @@ -1181,6 +1192,7 @@ int test_server(const char *hostname) TEST_THAT(en->GetObjectID() == subdirid); TEST_THAT(en->GetModificationTime() == 0); // dirs don't have modification times. } + { // Command std::auto_ptr dirreply(protocolReadOnly.QueryListDirectory( @@ -1211,6 +1223,7 @@ int test_server(const char *hostname) StreamableMemBlock attr(attr1, sizeof(attr1)); TEST_THAT(dir.GetAttributes() == attr); } + printf("done.\n\n"); // Check that we don't get attributes if we don't ask for them { @@ -1258,6 +1271,9 @@ int test_server(const char *hostname) TEST_THAT(dir.GetAttributes() == attrtest); } + // sleep to ensure that the timestamp on the file will change + ::safe_sleep(1); + // Test moving a file { BackupStoreFilenameClear newName("moved-files"); @@ -1281,9 +1297,6 @@ int test_server(const char *hostname) ConnectionException, Conn_Protocol_UnexpectedReply); } - // sleep to ensure that the timestamp on the file will change - ::safe_sleep(1); - // Rename within a directory { BackupStoreFilenameClear newName("moved-files-x"); @@ -1342,6 +1355,9 @@ int test_server(const char *hostname) TEST_THAT(foundOld); } + // sleep to ensure that the timestamp on the file will change + ::safe_sleep(1); + // make a little bit more of a thing to look at int64_t subsubdirid = 0; int64_t subsubfileid = 0; @@ -1763,9 +1779,12 @@ int test3(int argc, const char *argv[]) TEST_THAT(KillServer(pid)); ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); -#ifndef WIN32 - TestRemoteProcessMemLeaks("bbstored.memleaks"); -#endif + + #ifdef WIN32 + TEST_THAT(unlink("testfiles/bbstored.pid") == 0); + #else + TestRemoteProcessMemLeaks("bbstored.memleaks"); + #endif // Set a new limit on the account -- leave the hard limit // high to make sure the target for freeing space is the @@ -1857,9 +1876,11 @@ int test3(int argc, const char *argv[]) ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); -#ifndef WIN32 - TestRemoteProcessMemLeaks("bbstored.memleaks"); -#endif + #ifdef WIN32 + TEST_THAT(unlink("testfiles/bbstored.pid") == 0); + #else + TestRemoteProcessMemLeaks("bbstored.memleaks"); + #endif } return 0; @@ -1896,9 +1917,12 @@ int multi_server() TEST_THAT(KillServer(pid)); ::sleep(1); TEST_THAT(!ServerIsAlive(pid)); -#ifndef WIN32 - TestRemoteProcessMemLeaks("bbstored.memleaks"); -#endif + + #ifdef WIN32 + TEST_THAT(unlink("testfiles/bbstored.pid") == 0); + #else + TestRemoteProcessMemLeaks("bbstored.memleaks"); + #endif } diff --git a/test/backupstorefix/testbackupstorefix.cpp b/test/backupstorefix/testbackupstorefix.cpp index 34565bd5..2d4ce052 100644 --- a/test/backupstorefix/testbackupstorefix.cpp +++ b/test/backupstorefix/testbackupstorefix.cpp @@ -338,7 +338,7 @@ int test(int argc, const char *argv[]) } // Generate a list of all the object IDs - TEST_THAT_ABORTONFAIL(::system(BBACKUPQUERY " -q " + TEST_THAT_ABORTONFAIL(::system(BBACKUPQUERY " -Wwarning " "-c testfiles/bbackupd.conf \"list -r\" quit " "> testfiles/initial-listing.txt") == 0); @@ -598,10 +598,13 @@ int test(int argc, const char *argv[]) " testfiles/testbackupstorefix.pl reroot 6") == 0); - // ------------------------------------------------------------------------------------------------ + // --------------------------------------------------------- // Stop server TEST_THAT(KillServer(pid)); - #ifndef WIN32 + + #ifdef WIN32 + TEST_THAT(unlink("testfiles/bbstored.pid") == 0); + #else TestRemoteProcessMemLeaks("bbstored.memleaks"); #endif } diff --git a/test/backupstorefix/testfiles/testbackupstorefix.pl.in b/test/backupstorefix/testfiles/testbackupstorefix.pl.in index e64474f0..d27c1106 100755 --- a/test/backupstorefix/testfiles/testbackupstorefix.pl.in +++ b/test/backupstorefix/testfiles/testbackupstorefix.pl.in @@ -93,8 +93,12 @@ elsif($ARGV[0] eq 'check') } # read in the new listing, and compare - open LISTING,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf \"list -r\" quit |" or die "Can't open list utility"; - open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' or die "can't open copy listing file"; + open LISTING,"../../bin/bbackupquery/bbackupquery -Wwarning " . + "-c testfiles/bbackupd.conf " . + "\"list -r\" quit |" + or die "Can't open list utility"; + open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' + or die "can't open copy listing file"; my $err = 0; while() { @@ -125,8 +129,12 @@ elsif($ARGV[0] eq 'check') } elsif($ARGV[0] eq 'reroot') { - open LISTING,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf \"list -r\" quit |" or die "Can't open list utility"; - open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' or die "can't open copy listing file"; + open LISTING,"../../bin/bbackupquery/bbackupquery -Wwarning " . + "-c testfiles/bbackupd.conf " . + "\"list -r\" quit |" + or die "Can't open list utility"; + open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' + or die "can't open copy listing file"; my $err = 0; my $count = 0; while() diff --git a/test/basicserver/testbasicserver.cpp b/test/basicserver/testbasicserver.cpp index 18329441..18bc0aa8 100644 --- a/test/basicserver/testbasicserver.cpp +++ b/test/basicserver/testbasicserver.cpp @@ -185,7 +185,7 @@ const ConfigurationVerify *testserver::GetConfigVerify() const { static ConfigurationVerifyKey verifyserverkeys[] = { - SERVERSTREAM_VERIFY_SERVER_KEYS(0) // no default addresses + SERVERSTREAM_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default listen addresses }; static ConfigurationVerify verifyserver[] = @@ -258,7 +258,7 @@ const ConfigurationVerify *testTLSserver::GetConfigVerify() const { static ConfigurationVerifyKey verifyserverkeys[] = { - SERVERTLS_VERIFY_SERVER_KEYS(0) // no default listen addresses + SERVERTLS_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default listen addresses }; static ConfigurationVerify verifyserver[] = diff --git a/test/bbackupd/Makefile.extra b/test/bbackupd/Makefile.extra index 1d3f5103..0ae56bd1 100644 --- a/test/bbackupd/Makefile.extra +++ b/test/bbackupd/Makefile.extra @@ -1 +1,14 @@ -link-extra: ../../bin/bbackupd/autogen_ClientException.o ../../bin/bbackupd/BackupClientContext.o ../../bin/bbackupd/BackupClientDeleteList.o ../../bin/bbackupd/BackupClientDirectoryRecord.o ../../bin/bbackupd/Win32BackupService.o ../../bin/bbackupd/BackupClientInodeToIDMap.o ../../bin/bbackupd/Win32ServiceFunctions.o ../../bin/bbackupd/BackupDaemon.o +link-extra: ../../bin/bbackupd/autogen_ClientException.o \ + ../../bin/bbackupd/BackupClientContext.o \ + ../../bin/bbackupd/BackupClientDeleteList.o \ + ../../bin/bbackupd/BackupClientDirectoryRecord.o \ + ../../bin/bbackupd/Win32BackupService.o \ + ../../bin/bbackupd/BackupClientInodeToIDMap.o \ + ../../bin/bbackupd/Win32ServiceFunctions.o \ + ../../bin/bbackupd/BackupDaemon.o \ + ../../bin/bbstored/BackupStoreContext.o \ + ../../bin/bbstored/BBStoreDHousekeeping.o \ + ../../bin/bbstored/HousekeepStoreAccount.o \ + ../../bin/bbstored/autogen_BackupProtocolServer.o \ + ../../bin/bbstored/BackupCommands.o \ + ../../bin/bbstored/BackupStoreDaemon.o diff --git a/test/bbackupd/testbbackupd.cpp b/test/bbackupd/testbbackupd.cpp index a0732c21..9d82a11b 100644 --- a/test/bbackupd/testbbackupd.cpp +++ b/test/bbackupd/testbbackupd.cpp @@ -32,18 +32,26 @@ #include #endif +#ifdef HAVE_SIGNAL_H + #include +#endif + #include #ifdef HAVE_SYSCALL #include #endif +#include "autogen_BackupProtocolServer.h" #include "BackupClientCryptoKeys.h" #include "BackupClientFileAttributes.h" #include "BackupClientRestore.h" #include "BackupDaemon.h" #include "BackupDaemonConfigVerify.h" +#include "BackupQueries.h" #include "BackupStoreConstants.h" +#include "BackupStoreContext.h" +#include "BackupStoreDaemon.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BoxPortsAndFiles.h" @@ -79,27 +87,6 @@ // two cycles and a bit #define TIME_TO_WAIT_FOR_BACKUP_OPERATION 12 -// utility macro for comparing two strings in a line -#define TEST_EQUAL(expected, found, line) \ -{ \ - std::string exp_str = expected; \ - std::string found_str = found; \ - TEST_THAT(exp_str == found_str); \ - if(exp_str != found_str) \ - { \ - printf("Expected <%s> but found <%s> in <%s>\n", \ - exp_str.c_str(), found_str.c_str(), line.c_str()); \ - } \ -} - -// utility macro for testing a line -#define TEST_LINE(condition, line) \ - TEST_THAT(condition); \ - if (!(condition)) \ - { \ - printf("Test failed on <%s>\n", line.c_str()); \ - } - void wait_for_backup_operation(int seconds = TIME_TO_WAIT_FOR_BACKUP_OPERATION) { wait_for_operation(seconds); @@ -271,9 +258,9 @@ void finish_with_write_xattr_test() bool attrmatch(const char *f1, const char *f2) { - struct stat s1, s2; - TEST_THAT(::lstat(f1, &s1) == 0); - TEST_THAT(::lstat(f2, &s2) == 0); + EMU_STRUCT_STAT s1, s2; + TEST_THAT(EMU_LSTAT(f1, &s1) == 0); + TEST_THAT(EMU_LSTAT(f2, &s2) == 0); #ifdef HAVE_SYS_XATTR_H { @@ -336,7 +323,11 @@ int test_basics() #endif BackupClientFileAttributes t3; - TEST_CHECK_THROWS(t3.ReadAttributes("doesn't exist"), CommonException, OSFileError); + { + Logging::Guard guard(Log::ERROR); + TEST_CHECK_THROWS(t3.ReadAttributes("doesn't exist"), + CommonException, OSFileError); + } // Create some more files FILE *f = fopen("testfiles/test1_n", "w"); @@ -353,8 +344,13 @@ int test_basics() #endif #ifndef WIN32 - TEST_CHECK_THROWS(t1.WriteAttributes("testfiles/test1_nXX"), CommonException, OSFileError); - TEST_CHECK_THROWS(t3.WriteAttributes("doesn't exist"), BackupStoreException, AttributesNotLoaded); + { + Logging::Guard guard(Log::ERROR); + TEST_CHECK_THROWS(t1.WriteAttributes("testfiles/test1_nXX"), + CommonException, OSFileError); + TEST_CHECK_THROWS(t3.WriteAttributes("doesn't exist"), + BackupStoreException, AttributesNotLoaded); + } // Test that attributes are vaguely similar TEST_THAT(attrmatch("testfiles/test1", "testfiles/test1_n")); @@ -440,7 +436,8 @@ int test_setupaccount() int test_run_bbstored() { - std::string cmd = BBSTORED + bbstored_args + " testfiles/bbstored.conf"; + std::string cmd = BBSTORED " " + bbstored_args + + " testfiles/bbstored.conf"; bbstored_pid = LaunchServer(cmd, "testfiles/bbstored.pid"); TEST_THAT(bbstored_pid != -1 && bbstored_pid != 0); @@ -455,13 +452,19 @@ int test_run_bbstored() return 1; } -int test_kill_bbstored() +int test_kill_bbstored(bool wait_for_process = false) { - TEST_THAT(KillServer(bbstored_pid)); + TEST_THAT(KillServer(bbstored_pid, wait_for_process)); ::safe_sleep(1); TEST_THAT(!ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbstored_pid)) + { + bbstored_pid = 0; + } - #ifndef WIN32 + #ifdef WIN32 + TEST_THAT(unlink("testfiles/bbstored.pid") == 0); + #else TestRemoteProcessMemLeaks("bbstored.memleaks"); #endif @@ -511,7 +514,8 @@ void do_interrupted_restore(const TLSContext &context, int64_t restoredirid) { // connect and log in SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED); + conn.Open(context, Socket::TypeINET, "localhost", + 22011); BackupProtocolClient protocol(conn); protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION); std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, BackupProtocolClientLogin::Flags_ReadOnly)); @@ -601,10 +605,10 @@ int64_t SearchDir(BackupStoreDirectory& rDir, SocketStreamTLS sSocket; -std::auto_ptr Connect(TLSContext& rContext, int flags) +std::auto_ptr Connect(TLSContext& rContext) { sSocket.Open(rContext, Socket::TypeINET, - "localhost", BOX_PORT_BBSTORED); + "localhost", 22011); std::auto_ptr connection; connection.reset(new BackupProtocolClient(sSocket)); connection->Handshake(); @@ -617,10 +621,17 @@ std::auto_ptr Connect(TLSContext& rContext, int flags) THROW_EXCEPTION(BackupStoreException, WrongServerVersion); } - connection->QueryLogin(0x01234567, flags); return connection; } +std::auto_ptr ConnectAndLogin(TLSContext& rContext, + int flags) +{ + std::auto_ptr connection(Connect(rContext)); + connection->QueryLogin(0x01234567, flags); + return connection; +} + std::auto_ptr ReadDirectory ( BackupProtocolClient& rClient, @@ -639,6 +650,7 @@ int start_internal_daemon() { // ensure that no child processes end up running tests! int own_pid = getpid(); + BOX_TRACE("Test PID is " << own_pid); // this is a quick hack to allow passing some options to the daemon const char* argv[] = { @@ -658,18 +670,14 @@ int start_internal_daemon() result = daemon.Main("testfiles/bbackupd.conf", 1, argv); } - TEST_THAT(result == 0); - if (result != 0) - { - printf("Daemon exited with code %d\n", result); - } + TEST_EQUAL_LINE(0, result, "Daemon exit code"); // ensure that no child processes end up running tests! - TEST_THAT(getpid() == own_pid); if (getpid() != own_pid) { // abort! - _exit(1); + BOX_INFO("Daemon child finished, exiting now."); + _exit(0); } TEST_THAT(TestFileExists("testfiles/bbackupd.pid")); @@ -703,20 +711,9 @@ int start_internal_daemon() bool stop_internal_daemon(int pid) { - bool killed_server = KillServer(pid); + bool killed_server = KillServer(pid, false); TEST_THAT(killed_server); return killed_server; - - /* - int status; - TEST_THAT(waitpid(pid, &status, 0) == pid); - TEST_THAT(WIFEXITED(status)); - - if (WIFEXITED(status)) - { - TEST_THAT(WEXITSTATUS(status) == 0); - } - */ } static struct dirent readdir_test_dirent; @@ -728,15 +725,15 @@ static char stat_hook_filename[512]; // This will not match the directory on the store, so a sync will start. // We set up the next intercept for the same directory by passing NULL. -struct dirent *readdir_test_hook_2(DIR *dir); +extern "C" struct dirent *readdir_test_hook_2(DIR *dir); #ifdef LINUX_WEIRD_LSTAT -int lstat_test_hook(int ver, const char *file_name, struct stat *buf); +extern "C" int lstat_test_hook(int ver, const char *file_name, struct stat *buf); #else -int lstat_test_hook(const char *file_name, struct stat *buf); +extern "C" int lstat_test_hook(const char *file_name, struct stat *buf); #endif -struct dirent *readdir_test_hook_1(DIR *dir) +extern "C" struct dirent *readdir_test_hook_1(DIR *dir) { #ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE intercept_setup_readdir_hook(NULL, readdir_test_hook_2); @@ -747,7 +744,7 @@ struct dirent *readdir_test_hook_1(DIR *dir) // Second test hook, during the directory sync stage, keeps returning // new filenames until the timer expires, then disables the intercept. -struct dirent *readdir_test_hook_2(DIR *dir) +extern "C" struct dirent *readdir_test_hook_2(DIR *dir) { if (time(NULL) >= readdir_stop_time) { @@ -783,17 +780,45 @@ struct dirent *readdir_test_hook_2(DIR *dir) } #ifdef LINUX_WEIRD_LSTAT -int lstat_test_hook(int ver, const char *file_name, struct stat *buf) +extern "C" int lstat_test_hook(int ver, const char *file_name, struct stat *buf) #else -int lstat_test_hook(const char *file_name, struct stat *buf) +extern "C" int lstat_test_hook(const char *file_name, struct stat *buf) #endif { - // TRACE1("lstat hook triggered for %s", file_name); + // TRACE1("lstat hook triggered for %s", file_name); memset(buf, 0, sizeof(*buf)); buf->st_mode = S_IFREG; return 0; } +// Simulate a symlink that is on a different device than the file +// that it points to. +int lstat_test_post_hook(int old_ret, const char *file_name, struct stat *buf) +{ + BOX_TRACE("lstat post hook triggered for " << file_name); + if (old_ret == 0 && + strcmp(file_name, "testfiles/symlink-to-TestDir1") == 0) + { + buf->st_dev ^= 0xFFFF; + } + return old_ret; +} + +bool test_entry_deleted(BackupStoreDirectory& rDir, + const std::string& rName) +{ + BackupStoreDirectory::Iterator i(rDir); + + BackupStoreDirectory::Entry *en = i.FindMatchingClearName( + BackupStoreFilenameClear(rName)); + TEST_THAT(en != 0); + if (en == 0) return false; + + int16_t flags = en->GetFlags(); + TEST_THAT(flags && BackupStoreDirectory::Entry::Flags_Deleted); + return flags && BackupStoreDirectory::Entry::Flags_Deleted; +} + int test_bbackupd() { // First, wait for a normal period to make sure the last changes @@ -807,6 +832,23 @@ int test_bbackupd() "testfiles/clientPrivKey.pem", "testfiles/clientTrustedCAs.pem"); + printf("\n==== Testing that ReadDirectory on nonexistent directory " + "does not crash\n"); + { + std::auto_ptr client = ConnectAndLogin( + context, 0 /* read-write */); + + { + Logging::Guard guard(Log::ERROR); + TEST_CHECK_THROWS(ReadDirectory(*client, 0x12345678), + ConnectionException, + Conn_Protocol_UnexpectedReply); + } + + client->QueryFinished(); + sSocket.Close(); + } + // unpack the files for the initial test TEST_THAT(::system("rm -rf testfiles/TestDir1") == 0); TEST_THAT(::mkdir("testfiles/TestDir1", 0777) == 0); @@ -818,7 +860,7 @@ int test_bbackupd() TEST_THAT(::system("gzip -d < testfiles/spacetest1.tgz " "| ( cd testfiles/TestDir1 && tar xf - )") == 0); #endif - + #ifdef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE printf("\n==== Skipping intercept-based KeepAlive tests " "on this platform.\n"); @@ -841,7 +883,9 @@ int test_bbackupd() char buffer[10000]; memset(buffer, 0, sizeof(buffer)); - TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_EQUAL_LINE(sizeof(buffer), + write(fd, buffer, sizeof(buffer)), + "Buffer write"); TEST_THAT(close(fd) == 0); int pid = start_internal_daemon(); @@ -856,11 +900,14 @@ int test_bbackupd() TEST_THAT(unlink("testfiles/bbackupd.log") == 0); pid = start_internal_daemon(); + intercept_clear_setup(); fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); TEST_THAT(fd > 0); // write again, to update the file's timestamp - TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_EQUAL_LINE(sizeof(buffer), + write(fd, buffer, sizeof(buffer)), + "Buffer write"); TEST_THAT(close(fd) == 0); wait_for_backup_operation(); @@ -892,20 +939,22 @@ int test_bbackupd() std::string line; TEST_THAT(reader.GetLine(line)); std::string comp = "Receive Success(0x"; - TEST_EQUAL(comp, line.substr(0, comp.size()), line); + TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), + line); TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receiving stream, size 124", line, line); + TEST_EQUAL("Receiving stream, size 124", line); TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Send GetIsAlive()", line, line); + TEST_EQUAL("Send GetIsAlive()", line); TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive IsAlive()", line, line); + TEST_EQUAL("Receive IsAlive()", line); TEST_THAT(reader.GetLine(line)); comp = "Send StoreFile(0x3,"; - TEST_EQUAL(comp, line.substr(0, comp.size()), line); + TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), + line); comp = ",\"f1\")"; std::string sub = line.substr(line.size() - comp.size()); - TEST_EQUAL(comp, sub, line); + TEST_EQUAL_LINE(comp, sub, line); std::string comp2 = ",0x0,"; sub = line.substr(line.size() - comp.size() - comp2.size() + 1, comp2.size()); @@ -926,11 +975,14 @@ int test_bbackupd() intercept_setup_delay("testfiles/TestDir1/spacetest/f1", 0, 4000, SYS_read, 1); pid = start_internal_daemon(); + intercept_clear_setup(); fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); TEST_THAT(fd > 0); // write again, to update the file's timestamp - TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_EQUAL_LINE(sizeof(buffer), + write(fd, buffer, sizeof(buffer)), + "Buffer write"); TEST_THAT(close(fd) == 0); wait_for_backup_operation(); @@ -959,9 +1011,10 @@ int test_bbackupd() std::string line; TEST_THAT(reader.GetLine(line)); std::string comp = "Receive Success(0x"; - TEST_THAT(line.substr(0, comp.size()) == comp); + TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), + line); TEST_THAT(reader.GetLine(line)); - TEST_THAT(line == "Receiving stream, size 124"); + TEST_EQUAL("Receiving stream, size 124", line); // delaying for 4 seconds in one step means that // the diff timer and the keepalive timer will @@ -970,10 +1023,11 @@ int test_bbackupd() TEST_THAT(reader.GetLine(line)); comp = "Send StoreFile(0x3,"; - TEST_EQUAL(comp, line.substr(0, comp.size()), line); + TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), + line); comp = ",0x0,\"f1\")"; std::string sub = line.substr(line.size() - comp.size()); - TEST_EQUAL(comp, sub, line); + TEST_EQUAL_LINE(comp, sub, line); } if (failures > 0) @@ -986,11 +1040,14 @@ int test_bbackupd() intercept_setup_delay("testfiles/TestDir1/spacetest/f1", 0, 1000, SYS_read, 3); pid = start_internal_daemon(); + intercept_clear_setup(); fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); TEST_THAT(fd > 0); // write again, to update the file's timestamp - TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_EQUAL_LINE(sizeof(buffer), + write(fd, buffer, sizeof(buffer)), + "Buffer write"); TEST_THAT(close(fd) == 0); wait_for_backup_operation(); @@ -1019,9 +1076,10 @@ int test_bbackupd() std::string line; TEST_THAT(reader.GetLine(line)); std::string comp = "Receive Success(0x"; - TEST_THAT(line.substr(0, comp.size()) == comp); + TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), + line); TEST_THAT(reader.GetLine(line)); - TEST_THAT(line == "Receiving stream, size 124"); + TEST_EQUAL("Receiving stream, size 124", line); // delaying for 3 seconds in steps of 1 second // means that the keepalive timer will expire 3 times, @@ -1030,23 +1088,24 @@ int test_bbackupd() // only two keepalives. TEST_THAT(reader.GetLine(line)); - TEST_THAT(line == "Send GetIsAlive()"); + TEST_EQUAL("Send GetIsAlive()", line); TEST_THAT(reader.GetLine(line)); - TEST_THAT(line == "Receive IsAlive()"); + TEST_EQUAL("Receive IsAlive()", line); TEST_THAT(reader.GetLine(line)); - TEST_THAT(line == "Send GetIsAlive()"); + TEST_EQUAL("Send GetIsAlive()", line); TEST_THAT(reader.GetLine(line)); - TEST_THAT(line == "Receive IsAlive()"); + TEST_EQUAL("Receive IsAlive()", line); // but two matching blocks should have been found // already, so the upload should be a diff. TEST_THAT(reader.GetLine(line)); comp = "Send StoreFile(0x3,"; - TEST_EQUAL(comp, line.substr(0, comp.size()), line); + TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), + line); comp = ",\"f1\")"; std::string sub = line.substr(line.size() - comp.size()); - TEST_EQUAL(comp, sub, line); + TEST_EQUAL_LINE(comp, sub, line); std::string comp2 = ",0x0,"; sub = line.substr(line.size() - comp.size() - comp2.size() + 1, comp2.size()); @@ -1067,6 +1126,7 @@ int test_bbackupd() readdir_stop_time = time(NULL) + 12 + 2; pid = start_internal_daemon(); + intercept_clear_setup(); std::string touchfile = "testfiles/TestDir1/spacetest/d1/touch-me"; @@ -1074,7 +1134,9 @@ int test_bbackupd() fd = open(touchfile.c_str(), O_CREAT | O_WRONLY); TEST_THAT(fd > 0); // write again, to update the file's timestamp - TEST_THAT(write(fd, buffer, sizeof(buffer)) == sizeof(buffer)); + TEST_EQUAL_LINE(sizeof(buffer), + write(fd, buffer, sizeof(buffer)), + "Buffer write"); TEST_THAT(close(fd) == 0); wait_for_backup_operation(); @@ -1091,7 +1153,7 @@ int test_bbackupd() { std::string line; TEST_THAT(reader.GetLine(line)); - if (line == "Send ListDirectory(0x3,0xffffffff,0xc,true)") + if (line == "Send ListDirectory(0x3,0xffff,0xc,true)") { found1 = true; break; @@ -1119,17 +1181,17 @@ int test_bbackupd() { std::string line; TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive Success(0x3)", line, line); + TEST_EQUAL("Receive Success(0x3)", line); TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receiving stream, size 425", line, line); + TEST_EQUAL("Receiving stream, size 425", line); TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Send GetIsAlive()", line, line); + TEST_EQUAL("Send GetIsAlive()", line); TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive IsAlive()", line, line); + TEST_EQUAL("Receive IsAlive()", line); TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Send GetIsAlive()", line, line); + TEST_EQUAL("Send GetIsAlive()", line); TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive IsAlive()", line, line); + TEST_EQUAL("Receive IsAlive()", line); } if (failures > 0) @@ -1146,8 +1208,11 @@ int test_bbackupd() } #endif // PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + // Check that no read error has been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + std::string cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd-temploc.conf"; + " testfiles/bbackupd.conf"; bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); @@ -1158,6 +1223,553 @@ int test_bbackupd() if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + if(bbackupd_pid > 0) + { + printf("\n==== Testing that backup pauses when " + "store is full\n"); + + // wait for files to be uploaded + BOX_TRACE("Waiting for all outstanding files to be uploaded") + wait_for_sync_end(); + BOX_TRACE("done.") + + // Set limit to something very small + // 26 blocks will be used at this point. + // (12 files + location * 2 for raidfile) + // 20 is what we'll need in a minute + // set soft limit to 0 to ensure that all deleted files + // are deleted immediately by housekeeping + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " + "testfiles/bbstored.conf setlimit 01234567 0B 20B") + == 0); + TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + + // Unpack some more files + #ifdef WIN32 + TEST_THAT(::system("tar xzvf testfiles/spacetest2.tgz " + "-C testfiles/TestDir1") == 0); + #else + TEST_THAT(::system("gzip -d < testfiles/spacetest2.tgz " + "| ( cd testfiles/TestDir1 && tar xf - )") == 0); + #endif + + // Delete a file and a directory + TEST_THAT(::unlink("testfiles/TestDir1/spacetest/f1") == 0); + TEST_THAT(::system("rm -rf testfiles/TestDir1/spacetest/d7") == 0); + + // The following files should be on the server: + // 00000001 -d---- 00002 (root) + // 00000002 -d---- 00002 Test1 + // 00000003 -d---- 00002 Test1/spacetest + // 00000004 f-X--- 00002 Test1/spacetest/f1 + // 00000005 f----- 00002 Test1/spacetest/f2 + // 00000006 -d---- 00002 Test1/spacetest/d1 + // 00000007 f----- 00002 Test1/spacetest/d1/f3 + // 00000008 f----- 00002 Test1/spacetest/d1/f4 + // 00000009 -d---- 00002 Test1/spacetest/d2 + // 0000000a -d---- 00002 Test1/spacetest/d3 + // 0000000b -d---- 00002 Test1/spacetest/d3/d4 + // 0000000c f----- 00002 Test1/spacetest/d3/d4/f5 + // 0000000d -d---- 00002 Test1/spacetest/d6 + // 0000000e -dX--- 00002 Test1/spacetest/d7 + // This is 28 blocks total, of which 2 in deleted files + // and 18 in directories. Note that f1 and d7 may or may + // not be deleted yet. + // + // spacetest1 + spacetest2 = 16 files = 32 blocks with raidfile + // minus one file and one dir is 28 blocks + // + // d2/f6, d6/d8 and d6/d8/f7 are new + // even if the client marks f1 and d7 as deleted, and + // housekeeping deleted them, the backup cannot complete + // if the limit is 20 blocks. + + BOX_TRACE("Waiting for bbackupd to notice that the " + "store is full"); + wait_for_sync_end(); + BOX_TRACE("done."); + + BOX_TRACE("Compare to check that there are differences"); + int compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-l testfiles/query0a.log " + "-Werror \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + BOX_TRACE("done."); + + // Check that the notify script was run + TEST_THAT(TestFileExists("testfiles/notifyran.store-full.1")); + // But only once! + TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2")); + + // Kill the daemon + terminate_bbackupd(bbackupd_pid); + + BOX_TRACE("Wait for housekeeping to remove the deleted files"); + wait_for_backup_operation(5); + BOX_TRACE("done."); + + // This removes f1 and d7, which were previously marked + // as deleted, so total usage drops by 4 blocks to 24. + + // BLOCK + { + std::auto_ptr client = + ConnectAndLogin(context, 0 /* read-write */); + + std::auto_ptr usage( + client->QueryGetAccountUsage()); + TEST_EQUAL_LINE(24, usage->GetBlocksUsed(), + "blocks used"); + TEST_EQUAL_LINE(0, usage->GetBlocksInDeletedFiles(), + "deleted blocks"); + TEST_EQUAL_LINE(16, usage->GetBlocksInDirectories(), + "directory blocks"); + + client->QueryFinished(); + sSocket.Close(); + } + + if (failures > 0) + { + // stop early to make debugging easier + return 1; + } + + // ensure time is different to refresh the cache + ::safe_sleep(1); + + BOX_TRACE("Restart bbackupd with more exclusions"); + // Start again with a new config that excludes d3 and f2, + // and hence also d3/d4 and d3/d4/f5. bbackupd should mark + // them as deleted and housekeeping should clean up, + // making space to upload the new files. + // total required: (13-2-4+3)*2 = 20 blocks + /* + cmd = BBACKUPD " " + bbackupd_args + + " testfiles/bbackupd-exclude.conf"; + bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); + TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + */ + + BackupDaemon bbackupd; + bbackupd.Configure("testfiles/bbackupd-exclude.conf"); + bbackupd.InitCrypto(); + BOX_TRACE("done."); + + // Should be marked as deleted by this run + // wait_for_sync_end(); + { + // Logging::Guard guard(Log::ERROR); + bbackupd.RunSyncNow(); + } + + TEST_THAT(bbackupd.StorageLimitExceeded()); + + // Check that the notify script was run + // TEST_THAT(TestFileExists("testfiles/notifyran.store-full.2")); + // But only twice! + // TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.3")); + + // All these should be marked as deleted but hopefully + // not removed by housekeeping yet: + // f1 deleted + // f2 excluded + // d1 excluded (why?) + // d1/f3 excluded (why?) + // d3 excluded + // d3/d4 excluded + // d3/d4/f5 excluded + // d7 deleted + // Careful with timing here, these files will be removed by + // housekeeping the next time it runs. On Win32, housekeeping + // runs immediately after disconnect, but only if enough time + // has elapsed since the last housekeeping. Since the + // backup run closely follows the last one, housekeeping + // should not run afterwards. On other platforms, we want to + // get in immediately after the backup and hope that + // housekeeping doesn't beat us to it. + + BOX_TRACE("Find out whether bbackupd marked files as deleted"); + { + std::auto_ptr client = + ConnectAndLogin(context, 0 /* read-write */); + + std::auto_ptr rootDir = + ReadDirectory(*client, + BackupProtocolClientListDirectory::RootDirectory); + + int64_t testDirId = SearchDir(*rootDir, "Test1"); + TEST_THAT(testDirId != 0); + + std::auto_ptr Test1_dir = + ReadDirectory(*client, testDirId); + + int64_t spacetestDirId = SearchDir(*Test1_dir, + "spacetest"); + TEST_THAT(spacetestDirId != 0); + + std::auto_ptr spacetest_dir = + ReadDirectory(*client, spacetestDirId); + + // these files were deleted before, they should be + // long gone by now + + TEST_THAT(SearchDir(*spacetest_dir, "f1") == 0); + TEST_THAT(SearchDir(*spacetest_dir, "d7") == 0); + + // these files have just been deleted, because + // they are excluded by the new configuration. + // but housekeeping should not have run yet + + TEST_THAT(test_entry_deleted(*spacetest_dir, "f2")); + TEST_THAT(test_entry_deleted(*spacetest_dir, "d3")); + + int64_t d3_id = SearchDir(*spacetest_dir, "d3"); + TEST_THAT(d3_id != 0); + + std::auto_ptr d3_dir = + ReadDirectory(*client, d3_id); + TEST_THAT(test_entry_deleted(*d3_dir, "d4")); + + int64_t d4_id = SearchDir(*d3_dir, "d4"); + TEST_THAT(d4_id != 0); + + std::auto_ptr d4_dir = + ReadDirectory(*client, d4_id); + TEST_THAT(test_entry_deleted(*d4_dir, "f5")); + + std::auto_ptr usage( + client->QueryGetAccountUsage()); + TEST_EQUAL_LINE(24, usage->GetBlocksUsed(), + "blocks used"); + TEST_EQUAL_LINE(4, usage->GetBlocksInDeletedFiles(), + "deleted blocks"); + TEST_EQUAL_LINE(16, usage->GetBlocksInDirectories(), + "directory blocks"); + // d1/f3 and d1/f4 are the only two files on the + // server which are not deleted, they use 2 blocks + // each, the rest is directories and 2 deleted files + // (f1 and d3/d4/f5) + + // Log out. + client->QueryFinished(); + sSocket.Close(); + } + BOX_TRACE("done."); + + if (failures > 0) + { + // stop early to make debugging easier + return 1; + } + + BOX_TRACE("Wait for housekeeping to remove the deleted files"); + wait_for_backup_operation(5); + BOX_TRACE("done."); + + BOX_TRACE("Check that the files were removed"); + { + std::auto_ptr client = + ConnectAndLogin(context, 0 /* read-write */); + + std::auto_ptr rootDir = + ReadDirectory(*client, + BackupProtocolClientListDirectory::RootDirectory); + + int64_t testDirId = SearchDir(*rootDir, "Test1"); + TEST_THAT(testDirId != 0); + + std::auto_ptr Test1_dir = + ReadDirectory(*client, testDirId); + + int64_t spacetestDirId = SearchDir(*Test1_dir, + "spacetest"); + TEST_THAT(spacetestDirId != 0); + + std::auto_ptr spacetest_dir = + ReadDirectory(*client, spacetestDirId); + + TEST_THAT(SearchDir(*spacetest_dir, "f1") == 0); + TEST_THAT(SearchDir(*spacetest_dir, "f2") == 0); + TEST_THAT(SearchDir(*spacetest_dir, "d3") == 0); + TEST_THAT(SearchDir(*spacetest_dir, "d7") == 0); + + std::auto_ptr usage( + client->QueryGetAccountUsage()); + TEST_EQUAL_LINE(16, usage->GetBlocksUsed(), + "blocks used"); + TEST_EQUAL_LINE(0, usage->GetBlocksInDeletedFiles(), + "deleted blocks"); + TEST_EQUAL_LINE(12, usage->GetBlocksInDirectories(), + "directory blocks"); + // d1/f3 and d1/f4 are the only two files on the + // server, they use 2 blocks each, the rest is + // directories. + + // Log out. + client->QueryFinished(); + sSocket.Close(); + } + + if (failures > 0) + { + // stop early to make debugging easier + return 1; + } + + // Need 22 blocks free to upload everything + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " + "testfiles/bbstored.conf setlimit 01234567 0B 22B") + == 0); + TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + + // Run another backup, now there should be enough space + // for everything we want to upload. + { + Logging::Guard guard(Log::ERROR); + bbackupd.RunSyncNow(); + } + TEST_THAT(!bbackupd.StorageLimitExceeded()); + + // Check that the contents of the store are the same + // as the contents of the disc + // (-a = all, -c = give result in return code) + BOX_TRACE("Check that all files were uploaded successfully"); + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd-exclude.conf " + "-l testfiles/query1.log " + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + BOX_TRACE("done."); + + // BLOCK + { + std::auto_ptr client = + ConnectAndLogin(context, 0 /* read-write */); + + std::auto_ptr usage( + client->QueryGetAccountUsage()); + TEST_EQUAL_LINE(22, usage->GetBlocksUsed(), + "blocks used"); + TEST_EQUAL_LINE(0, usage->GetBlocksInDeletedFiles(), + "deleted blocks"); + TEST_EQUAL_LINE(14, usage->GetBlocksInDirectories(), + "directory blocks"); + // d2/f6, d6/d8 and d6/d8/f7 are new + // i.e. 2 new files, 1 new directory + + client->QueryFinished(); + sSocket.Close(); + } + + if (failures > 0) + { + // stop early to make debugging easier + return 1; + } + + // Put the limit back + TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " + "testfiles/bbstored.conf setlimit 01234567 " + "1000B 2000B") == 0); + TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + + // Start again with the old config + BOX_TRACE("Restart bbackupd with original configuration"); + // terminate_bbackupd(); + cmd = BBACKUPD " " + bbackupd_args + + " testfiles/bbackupd.conf"; + bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); + TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + ::safe_sleep(1); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + BOX_TRACE("done."); + + // unpack the initial files again + #ifdef WIN32 + TEST_THAT(::system("tar xzvf testfiles/test_base.tgz " + "-C testfiles") == 0); + #else + TEST_THAT(::system("gzip -d < testfiles/test_base.tgz " + "| ( cd testfiles && tar xf - )") == 0); + #endif + + BOX_TRACE("Wait for bbackupd to upload more files"); + wait_for_backup_operation(); + BOX_TRACE("done."); + + // Check that the contents of the store are the same + // as the contents of the disc + // (-a = all, -c = give result in return code) + BOX_TRACE("Check that all files were uploaded successfully"); + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-l testfiles/query1.log " + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + BOX_TRACE("done."); + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + if (failures > 0) + { + // stop early to make debugging easier + return 1; + } + } + + // Check that no read error has been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + + #ifndef WIN32 // requires fork + printf("\n==== Testing that bbackupd responds correctly to " + "connection failure\n"); + + { + // Kill the daemons + terminate_bbackupd(bbackupd_pid); + test_kill_bbstored(); + + // create a new file to force an upload + + const char* new_file = "testfiles/TestDir1/force-upload-2"; + int fd = open(new_file, + O_CREAT | O_EXCL | O_WRONLY, 0700); + if (fd <= 0) + { + perror(new_file); + } + TEST_THAT(fd > 0); + + const char* control_string = "whee!\n"; + TEST_THAT(write(fd, control_string, + strlen(control_string)) == + (int)strlen(control_string)); + close(fd); + + // sleep to make it old enough to upload + safe_sleep(4); + + class MyHook : public BackupStoreContext::TestHook + { + virtual std::auto_ptr StartCommand( + BackupProtocolObject& rCommand) + { + if (rCommand.GetType() == + BackupProtocolServerStoreFile::TypeID) + { + // terminate badly + THROW_EXCEPTION(CommonException, + Internal); + } + return std::auto_ptr(); + } + }; + MyHook hook; + + bbstored_pid = fork(); + + if (bbstored_pid < 0) + { + BOX_LOG_SYS_ERROR("failed to fork()"); + return 1; + } + + if (bbstored_pid == 0) + { + // in fork child + TEST_THAT(setsid() != -1); + + if (!Logging::IsEnabled(Log::TRACE)) + { + Logging::SetGlobalLevel(Log::NOTHING); + } + + // BackupStoreDaemon must be destroyed before exit(), + // to avoid memory leaks being reported. + { + BackupStoreDaemon bbstored; + bbstored.SetTestHook(hook); + bbstored.SetRunInForeground(true); + bbstored.Main("testfiles/bbstored.conf"); + } + + Timers::Cleanup(); // avoid memory leaks + exit(0); + } + + // in fork parent + bbstored_pid = WaitForServerStartup("testfiles/bbstored.pid", + bbstored_pid); + + TEST_THAT(::system("rm -f testfiles/notifyran.store-full.*") == 0); + + // Ignore SIGPIPE so that when the connection is broken, + // the daemon doesn't terminate. + ::signal(SIGPIPE, SIG_IGN); + + { + Log::Level newLevel = Logging::GetGlobalLevel(); + + if (!Logging::IsEnabled(Log::TRACE)) + { + newLevel = Log::NOTHING; + } + + Logging::Guard guard(newLevel); + + BackupDaemon bbackupd; + bbackupd.Configure("testfiles/bbackupd.conf"); + bbackupd.InitCrypto(); + bbackupd.RunSyncNowWithExceptionHandling(); + } + + ::signal(SIGPIPE, SIG_DFL); + + TEST_THAT(TestFileExists("testfiles/notifyran.backup-error.1")); + TEST_THAT(!TestFileExists("testfiles/notifyran.backup-error.2")); + TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.1")); + + test_kill_bbstored(true); + + if (failures > 0) + { + // stop early to make debugging easier + return 1; + } + + test_run_bbstored(); + + cmd = BBACKUPD " " + bbackupd_args + + " testfiles/bbackupd.conf"; + bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); + TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + ::safe_sleep(1); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + } + #endif // !WIN32 + + #ifndef WIN32 printf("\n==== Testing that absolute symlinks are not followed " "during restore\n"); @@ -1179,7 +1791,7 @@ int test_bbackupd() fclose(fp); char buf[PATH_MAX]; - TEST_THAT(getcwd(buf, sizeof(buf)) != NULL); + TEST_THAT(getcwd(buf, sizeof(buf)) == buf); std::string path = buf; path += DIRECTORY_SEPARATOR SYM_DIR DIRECTORY_SEPARATOR "a" @@ -1188,15 +1800,19 @@ int test_bbackupd() DIRECTORY_SEPARATOR "b" DIRECTORY_SEPARATOR "link") == 0); + // also test symlink-to-self loop does not break restore + TEST_THAT(symlink("self", SYM_DIR "/self") == 0); + ::wait_for_operation(4); ::sync_and_wait(); // Check that the backup was successful, i.e. no differences - int compareReturnValue = ::system(BBACKUPQUERY " -q " + int compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query1.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // now stop bbackupd and update the test file, @@ -1215,9 +1831,10 @@ int test_bbackupd() // check that we can restore it compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " - "-q \"restore Test1 testfiles/restore-symlink\" " + "-Wwarning \"restore Test1 testfiles/restore-symlink\" " "quit"); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); // make it accessible again TEST_THAT(chmod(SYM_DIR, 0755) == 0); @@ -1228,10 +1845,52 @@ int test_bbackupd() std::string line; TEST_THAT(gl.GetLine(line)); TEST_THAT(line != "before"); - TEST_THAT(line == "after"); + TEST_EQUAL("after", line); #undef SYM_DIR + /* + // This is not worth testing or fixing. + // + #ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + printf("\n==== Testing that symlinks to other filesystems " + "can be backed up as roots\n"); + + intercept_setup_lstat_post_hook(lstat_test_post_hook); + TEST_THAT(symlink("TestDir1", "testfiles/symlink-to-TestDir1") + == 0); + + struct stat stat_st, lstat_st; + TEST_THAT(stat("testfiles/symlink-to-TestDir1", &stat_st) == 0); + TEST_THAT(lstat("testfiles/symlink-to-TestDir1", &lstat_st) == 0); + TEST_EQUAL_LINE((stat_st.st_dev ^ 0xFFFF), lstat_st.st_dev, + "stat vs lstat"); + + BackupDaemon bbackupd; + bbackupd.Configure("testfiles/bbackupd-symlink.conf"); + bbackupd.InitCrypto(); + bbackupd.RunSyncNow(); + intercept_clear_setup(); + + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-l testfiles/query0a.log " + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // and again using the symlink during compare + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd-symlink.conf " + "-l testfiles/query0a.log " + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + #endif + */ + bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); ::safe_sleep(1); @@ -1241,31 +1900,65 @@ int test_bbackupd() if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; } + #endif // !WIN32 + + // Check that no read error has been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); printf("\n==== Testing that redundant locations are deleted on time\n"); + // unpack the files for the redundant location test + TEST_THAT(::system("rm -rf testfiles/TestDir2") == 0); + TEST_THAT(::mkdir("testfiles/TestDir2", 0777) == 0); + + #ifdef WIN32 + TEST_THAT(::system("tar xzvf testfiles/spacetest1.tgz " + "-C testfiles/TestDir2") == 0); + #else + TEST_THAT(::system("gzip -d < testfiles/spacetest1.tgz " + "| ( cd testfiles/TestDir2 && tar xf - )") == 0); + #endif + + // BLOCK { - std::auto_ptr client = Connect( - context, - BackupProtocolClientLogin::Flags_ReadOnly); - - std::auto_ptr dir = ReadDirectory( - *client, - BackupProtocolClientListDirectory::RootDirectory); + // Kill the daemon + terminate_bbackupd(bbackupd_pid); + + // Start it with a config that has a temporary location + // that will be created on the server + std::string cmd = BBACKUPD " " + bbackupd_args + + " testfiles/bbackupd-temploc.conf"; + + bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); + TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + ::safe_sleep(1); - // int64_t testDirId = SearchDir(*dir, "Test2"); - // TEST_THAT(testDirId == 0); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; sync_and_wait(); - dir = ReadDirectory(*client, - BackupProtocolClientListDirectory::RootDirectory); - int64_t testDirId = SearchDir(*dir, "Test2"); - TEST_THAT(testDirId != 0); + { + std::auto_ptr client = + ConnectAndLogin(context, + BackupProtocolClientLogin::Flags_ReadOnly); + + std::auto_ptr dir = + ReadDirectory(*client, + BackupProtocolClientListDirectory::RootDirectory); + int64_t testDirId = SearchDir(*dir, "Test2"); + TEST_THAT(testDirId != 0); + + client->QueryFinished(); + sSocket.Close(); + } // Kill the daemon terminate_bbackupd(bbackupd_pid); + // Start it again with the normal config (no Test2) cmd = BBACKUPD " " + bbackupd_args + " testfiles/bbackupd.conf"; bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); @@ -1284,116 +1977,58 @@ int test_bbackupd() wait_for_sync_end(); wait_for_sync_end(); - dir = ReadDirectory(*client, - BackupProtocolClientListDirectory::RootDirectory); - testDirId = SearchDir(*dir, "Test2"); - TEST_THAT(testDirId != 0); - - wait_for_sync_end(); - - dir = ReadDirectory(*client, - BackupProtocolClientListDirectory::RootDirectory); - testDirId = SearchDir(*dir, "Test2"); - TEST_THAT(testDirId != 0); - - BackupStoreDirectory::Iterator i(*dir); - BackupStoreFilenameClear dirname("Test2"); - BackupStoreDirectory::Entry *en = i.FindMatchingClearName(dirname); - TEST_THAT(en != 0); - int16_t en_flags = en->GetFlags(); - TEST_THAT(en_flags && BackupStoreDirectory::Entry::Flags_Deleted); - - // Log out. - client->QueryFinished(); - sSocket.Close(); - } - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; + // not yet! should still be there - if(bbackupd_pid > 0) - { - printf("\n==== Testing that backup pauses when store is full\n"); - - // wait for files to be uploaded - wait_for_backup_operation(); + { + std::auto_ptr client = + ConnectAndLogin(context, + BackupProtocolClientLogin::Flags_ReadOnly); + + std::auto_ptr dir = + ReadDirectory(*client, + BackupProtocolClientListDirectory::RootDirectory); + int64_t testDirId = SearchDir(*dir, "Test2"); + TEST_THAT(testDirId != 0); - // Set limit to something very small - // About 28 blocks will be used at this point. bbackupd - // will only pause if the size used is greater than - // soft limit + 1/3 of (hard - soft). Set small values - // for limits accordingly. - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " - "testfiles/bbstored.conf setlimit 01234567 9B 10B") - == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + client->QueryFinished(); + sSocket.Close(); + } - // Unpack some more files - #ifdef WIN32 - TEST_THAT(::system("tar xzvf testfiles/spacetest2.tgz " - "-C testfiles/TestDir1") == 0); - #else - TEST_THAT(::system("gzip -d < testfiles/spacetest2.tgz " - "| ( cd testfiles/TestDir1 && tar xf - )") == 0); - #endif + wait_for_sync_end(); - // Delete a file and a directory - TEST_THAT(::unlink("testfiles/TestDir1/spacetest/d1/f3") == 0); - TEST_THAT(::system("rm -rf testfiles/TestDir1/spacetest/d3/d4") == 0); - wait_for_backup_operation(); + // NOW it should be gone - // Make sure there are some differences - int compareReturnValue = ::system(BBACKUPQUERY " -q " - "-c testfiles/bbackupd.conf " - "-l testfiles/query0a.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 2); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + { + std::auto_ptr client = + ConnectAndLogin(context, + BackupProtocolClientLogin::Flags_ReadOnly); + + std::auto_ptr root_dir = + ReadDirectory(*client, + BackupProtocolClientListDirectory::RootDirectory); - // Put the limit back - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " - "testfiles/bbstored.conf setlimit 01234567 " - "1000B 2000B") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - - // Check that the notify script was run - TEST_THAT(TestFileExists("testfiles/notifyran.store-full.1")); - // But only once! - TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2")); - - // unpack the initial files again - #ifdef WIN32 - TEST_THAT(::system("tar xzvf testfiles/test_base.tgz " - "-C testfiles") == 0); - #else - TEST_THAT(::system("gzip -d < testfiles/test_base.tgz " - "| ( cd testfiles && tar xf - )") == 0); - #endif + TEST_THAT(test_entry_deleted(*root_dir, "Test2")); - // wait for it to do it's stuff - wait_for_backup_operation(); - - // Check that the contents of the store are the same - // as the contents of the disc - // (-a = all, -c = give result in return code) - compareReturnValue = ::system(BBACKUPQUERY " -q " - "-c testfiles/bbackupd.conf " - "-l testfiles/query1.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + client->QueryFinished(); + sSocket.Close(); + } + } - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + if(bbackupd_pid > 0) + { + // Check that no read error has been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); printf("\n==== Check that read-only directories and " "their contents can be restored.\n"); + int compareReturnValue; + { #ifdef WIN32 TEST_THAT(::system("chmod 0555 testfiles/" @@ -1407,35 +2042,45 @@ int test_bbackupd() wait_for_sync_end(); // should be backed up now compareReturnValue = ::system(BBACKUPQUERY " " + "-Wwarning " "-c testfiles/bbackupd.conf " - "-q \"compare -cEQ Test1 testfiles/TestDir1\" " + "\"compare -cEQ Test1 testfiles/TestDir1\" " "quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // check that we can restore it compareReturnValue = ::system(BBACKUPQUERY " " + "-Wwarning " "-c testfiles/bbackupd.conf " - "-q \"restore Test1 testfiles/restore1\" " + "\"restore Test1 testfiles/restore1\" " "quit"); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // check that it restored properly compareReturnValue = ::system(BBACKUPQUERY " " + "-Wwarning " "-c testfiles/bbackupd.conf " - "-q \"compare -cEQ Test1 testfiles/restore1\" " + "\"compare -cEQ Test1 testfiles/restore1\" " "quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // put the permissions back to sensible values #ifdef WIN32 TEST_THAT(::system("chmod 0755 testfiles/" "TestDir1/x1") == 0); + TEST_THAT(::system("chmod 0755 testfiles/" + "restore1/x1") == 0); #else TEST_THAT(chmod("testfiles/TestDir1/x1", 0755) == 0); + TEST_THAT(chmod("testfiles/restore1/x1", + 0755) == 0); #endif } @@ -1512,58 +2157,68 @@ int test_bbackupd() // test that bbackupd will let us lcd into the local // directory using a relative path - std::string command = BBACKUPQUERY " -q " + std::string command = BBACKUPQUERY " " + "-Wwarning " "-c testfiles/bbackupd.conf " "\"lcd testfiles/TestDir1/" + systemDirName + "\" " "quit"; compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); // and back out again - command = BBACKUPQUERY " -q " + command = BBACKUPQUERY " " + "-Wwarning " "-c testfiles/bbackupd.conf " "\"lcd testfiles/TestDir1/" + systemDirName + "\" " "\"lcd ..\" quit"; compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); // and using an absolute path - command = BBACKUPQUERY " -q " + command = BBACKUPQUERY " " + "-Wwarning " "-c testfiles/bbackupd.conf " "\"lcd " + cwd + "/testfiles/TestDir1/" + systemDirName + "\" quit"; compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); // and back out again - command = BBACKUPQUERY " -q " + command = BBACKUPQUERY " " + "-Wwarning " "-c testfiles/bbackupd.conf " "\"lcd " + cwd + "/testfiles/TestDir1/" + systemDirName + "\" " "\"lcd ..\" quit"; compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); { FileStream fs(filepath.c_str(), O_CREAT | O_RDWR); std::string data("hello world\n"); fs.Write(data.c_str(), data.size()); - TEST_THAT(fs.GetPosition() == 12); + TEST_EQUAL_LINE(12, fs.GetPosition(), + "FileStream position"); fs.Close(); } wait_for_backup_operation(); // Compare to check that the file was uploaded - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " -Wwarning " "-c testfiles/bbackupd.conf \"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // Check that we can find it in directory listing { std::auto_ptr client = - Connect(context, 0); + ConnectAndLogin(context, 0); std::auto_ptr dir = ReadDirectory( *client, @@ -1573,19 +2228,18 @@ int test_bbackupd() TEST_THAT(baseDirId != 0); dir = ReadDirectory(*client, baseDirId); - int64_t testDirId = SearchDir(dir, dirname.c_str()); + int64_t testDirId = SearchDir(*dir, dirname.c_str()); TEST_THAT(testDirId != 0); dir = ReadDirectory(*client, testDirId); - TEST_THAT(SearchDir(dir, filename.c_str()) != 0); + TEST_THAT(SearchDir(*dir, filename.c_str()) != 0); // Log out client->QueryFinished(); sSocket.Close(); } - // Check that bbackupquery shows the dir in console encoding - command = BBACKUPQUERY " -q " + command = BBACKUPQUERY " -Wwarning " "-c testfiles/bbackupd.conf " "-q \"list Test1\" quit"; pid_t bbackupquery_pid; @@ -1615,7 +2269,7 @@ int test_bbackupd() // on the command line in system encoding, and shows // the file in console encoding command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-q \"list Test1/" + systemDirName + "\" quit"; + "-Wwarning \"list Test1/" + systemDirName + "\" quit"; queryout = LocalProcessStream(command.c_str(), bbackupquery_pid); TEST_THAT(queryout.get() != NULL); @@ -1639,31 +2293,34 @@ int test_bbackupd() // Check that bbackupquery can compare the dir when given // on the command line in system encoding. command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-q \"compare -cEQ Test1/" + systemDirName + + "-Wwarning \"compare -cEQ Test1/" + systemDirName + " testfiles/TestDir1/" + systemDirName + "\" quit"; compareReturnValue = ::system(command.c_str()); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); // Check that bbackupquery can restore the dir when given // on the command line in system encoding. command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-q \"restore Test1/" + systemDirName + + "-Wwarning \"restore Test1/" + systemDirName + " testfiles/restore-" + systemDirName + "\" quit"; compareReturnValue = ::system(command.c_str()); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); // Compare to make sure it was restored properly. command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-q \"compare -cEQ Test1/" + systemDirName + + "-Wwarning \"compare -cEQ Test1/" + systemDirName + " testfiles/restore-" + systemDirName + "\" quit"; compareReturnValue = ::system(command.c_str()); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); std::string fileToUnlink = "testfiles/restore-" + dirname + "/" + filename; @@ -1672,22 +2329,25 @@ int test_bbackupd() // Check that bbackupquery can get the file when given // on the command line in system encoding. command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-q \"get Test1/" + systemDirName + "/" + + "-Wwarning \"get Test1/" + systemDirName + "/" + systemFileName + " " + "testfiles/restore-" + systemDirName + "/" + systemFileName + "\" quit"; compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // And after changing directory to a relative path - command = BBACKUPQUERY " -c testfiles/bbackupd.conf -q " + command = BBACKUPQUERY " -c testfiles/bbackupd.conf " + "-Wwarning " "\"lcd testfiles\" " "\"cd Test1/" + systemDirName + "\" " + "\"get " + systemFileName + "\" quit"; compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks"); // cannot overwrite a file that exists, so delete it @@ -1695,36 +2355,43 @@ int test_bbackupd() TEST_THAT(::unlink(tmp.c_str()) == 0); // And after changing directory to an absolute path - command = BBACKUPQUERY " -c testfiles/bbackupd.conf -q " + command = BBACKUPQUERY " -c testfiles/bbackupd.conf -Wwarning " "\"lcd " + cwd + "/testfiles\" " "\"cd Test1/" + systemDirName + "\" " + "\"get " + systemFileName + "\" quit"; compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, 0); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Command_OK); TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks"); // Compare to make sure it was restored properly. // The Get command does not restore attributes, so // we must compare without them (-A) to succeed. - command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-q \"compare -cAEQ Test1/" + systemDirName + + command = BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-Wwarning \"compare -cAEQ Test1/" + systemDirName + " testfiles/restore-" + systemDirName + "\" quit"; compareReturnValue = ::system(command.c_str()); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); // Compare without attributes. This should fail. - command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-q \"compare -cEQ Test1/" + systemDirName + + command = BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-Werror \"compare -cEQ Test1/" + systemDirName + " testfiles/restore-" + systemDirName + "\" quit"; - compareReturnValue = ::system(command.c_str()); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, 2); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); #endif // WIN32 + // Check that no read error has been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + TEST_THAT(ServerIsAlive(bbackupd_pid)); TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; @@ -1739,7 +2406,7 @@ int test_bbackupd() // we now have 3 seconds before bbackupd // runs the SyncAllowScript again. - char* sync_control_file = "testfiles" + const char* sync_control_file = "testfiles" DIRECTORY_SEPARATOR "syncallowscript.control"; int fd = open(sync_control_file, O_CREAT | O_EXCL | O_WRONLY, 0700); @@ -1749,7 +2416,7 @@ int test_bbackupd() } TEST_THAT(fd > 0); - char* control_string = "10\n"; + const char* control_string = "10\n"; TEST_THAT(write(fd, control_string, strlen(control_string)) == (int)strlen(control_string)); @@ -1758,7 +2425,7 @@ int test_bbackupd() // this will pause backups, bbackupd will check // every 10 seconds to see if they are allowed again. - char* new_test_file = "testfiles" + const char* new_test_file = "testfiles" DIRECTORY_SEPARATOR "TestDir1" DIRECTORY_SEPARATOR "Added_During_Pause"; fd = open(new_test_file, @@ -1799,11 +2466,13 @@ int test_bbackupd() long start_time = time(NULL); // check that no backup has run (compare fails) - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " + "-Werror " "-c testfiles/bbackupd.conf " "-l testfiles/query3.log " "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 2); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(unlink(sync_control_file) == 0); @@ -1819,11 +2488,13 @@ int test_bbackupd() wait_for_sync_end(); // check that backup has run (compare succeeds) - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " + "-Wwarning " "-c testfiles/bbackupd.conf " "-l testfiles/query3a.log " "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); if (failures > 0) @@ -1833,6 +2504,9 @@ int test_bbackupd() } } + // Check that no read error has been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + TEST_THAT(ServerIsAlive(bbackupd_pid)); TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; @@ -1865,22 +2539,24 @@ int test_bbackupd() TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") > 1024); } - + // wait for backup daemon to do it's stuff, and compare again wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " -Wwarning " "-c testfiles/bbackupd.conf " "-l testfiles/query2.log " "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // Try a quick compare, just for fun - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query2q.log " - "\"compare -acqQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acqQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -1890,12 +2566,17 @@ int test_bbackupd() // Check that store errors are reported neatly printf("\n==== Create store error\n"); + TEST_THAT(system("rm -f testfiles/notifyran.backup-error.*") + == 0); + + // break the store TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf", "testfiles/0_0/backup/01234567/info.rf.bak") == 0); TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf", "testfiles/0_1/backup/01234567/info.rf.bak") == 0); TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf", "testfiles/0_2/backup/01234567/info.rf.bak") == 0); + // Create a file to trigger an upload { int fd1 = open("testfiles/TestDir1/force-upload", @@ -1903,50 +2584,195 @@ int test_bbackupd() TEST_THAT(fd1 > 0); TEST_THAT(write(fd1, "just do it", 10) == 10); TEST_THAT(close(fd1) == 0); - wait_for_backup_operation(4); } - // Wait and test... - wait_for_backup_operation(); - // Check that it was reported correctly + + wait_for_backup_operation(4); + // Check that an error was reported just once TEST_THAT(TestFileExists("testfiles/notifyran.backup-error.1")); - // Check that the error was only reported once TEST_THAT(!TestFileExists("testfiles/notifyran.backup-error.2")); - // Fix the store + // Now kill bbackupd and start one that's running in + // snapshot mode, check that it automatically syncs after + // an error, without waiting for another sync command. + terminate_bbackupd(bbackupd_pid); + std::string cmd = BBACKUPD " " + bbackupd_args + + " testfiles/bbackupd-snapshot.conf"; + bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); + TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + ::safe_sleep(1); + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + + sync_and_wait(); + + // Check that the error was reported once more + TEST_THAT(TestFileExists("testfiles/notifyran.backup-error.2")); + TEST_THAT(!TestFileExists("testfiles/notifyran.backup-error.3")); + + // Fix the store (so that bbackupquery compare works) TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf.bak", "testfiles/0_0/backup/01234567/info.rf") == 0); TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf.bak", "testfiles/0_1/backup/01234567/info.rf") == 0); TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf.bak", "testfiles/0_2/backup/01234567/info.rf") == 0); + + int store_fixed_time = time(NULL); + + // Check that we DO get errors on compare (cannot do this + // until after we fix the store, which creates a race) + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3b.log " + "-Werror \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // Test initial state + TEST_THAT(!TestFileExists("testfiles/" + "notifyran.backup-start.wait-snapshot.1")); + + // Set a tag for the notify script to distinguish from + // previous runs. + { + int fd1 = open("testfiles/notifyscript.tag", + O_CREAT | O_EXCL | O_WRONLY, 0700); + TEST_THAT(fd1 > 0); + TEST_THAT(write(fd1, "wait-snapshot", 13) == 13); + TEST_THAT(close(fd1) == 0); + } + + // bbackupd should pause for about 90 seconds from store_fixed_time, + // so check that it hasn't run after 85 seconds from store_fixed_time + wait_for_backup_operation(85 - time(NULL) + store_fixed_time); + TEST_THAT(!TestFileExists("testfiles/" + "notifyran.backup-start.wait-snapshot.1")); + + // Should not have backed up, should still get errors + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3b.log " + "-Werror \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // wait another 10 seconds, bbackup should have run + wait_for_backup_operation(10); + TEST_THAT(TestFileExists("testfiles/" + "notifyran.backup-start.wait-snapshot.1")); - // Check that we DO get errors on compare - compareReturnValue = ::system(BBACKUPQUERY " -q " + // Check that it did get uploaded, and we have no more errors + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3b.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 2); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(::unlink("testfiles/notifyscript.tag") == 0); + + // Stop the snapshot bbackupd + terminate_bbackupd(bbackupd_pid); + + // Break the store again + TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf", + "testfiles/0_0/backup/01234567/info.rf.bak") == 0); + TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf", + "testfiles/0_1/backup/01234567/info.rf.bak") == 0); + TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf", + "testfiles/0_2/backup/01234567/info.rf.bak") == 0); + + // Modify a file to trigger an upload + { + int fd1 = open("testfiles/TestDir1/force-upload", + O_WRONLY, 0700); + TEST_THAT(fd1 > 0); + TEST_THAT(write(fd1, "and again", 9) == 9); + TEST_THAT(close(fd1) == 0); + } + + // Restart the old bbackupd, in automatic mode + cmd = BBACKUPD " " + bbackupd_args + + " testfiles/bbackupd.conf"; + bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); + TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + ::safe_sleep(1); TEST_THAT(ServerIsAlive(bbackupd_pid)); TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; - // Wait until bbackupd recovers from the exception - wait_for_backup_operation(100); - - // Ensure that the force-upload file gets uploaded, - // meaning that bbackupd recovered sync_and_wait(); + // Fix the store again + TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf.bak", + "testfiles/0_0/backup/01234567/info.rf") == 0); + TEST_THAT(::rename("testfiles/0_1/backup/01234567/info.rf.bak", + "testfiles/0_1/backup/01234567/info.rf") == 0); + TEST_THAT(::rename("testfiles/0_2/backup/01234567/info.rf.bak", + "testfiles/0_2/backup/01234567/info.rf") == 0); + + store_fixed_time = time(NULL); + + // Check that we DO get errors on compare (cannot do this + // until after we fix the store, which creates a race) + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3b.log " + "-Werror \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // Test initial state + TEST_THAT(!TestFileExists("testfiles/" + "notifyran.backup-start.wait-automatic.1")); + + // Set a tag for the notify script to distinguist from + // previous runs. + { + int fd1 = open("testfiles/notifyscript.tag", + O_CREAT | O_EXCL | O_WRONLY, 0700); + TEST_THAT(fd1 > 0); + TEST_THAT(write(fd1, "wait-automatic", 14) == 14); + TEST_THAT(close(fd1) == 0); + } + + // bbackupd should pause for about 90 seconds from store_fixed_time, + // so check that it hasn't run after 85 seconds from store_fixed_time + wait_for_backup_operation(85 - time(NULL) + store_fixed_time); + TEST_THAT(!TestFileExists("testfiles/" + "notifyran.backup-start.wait-automatic.1")); + + // Should not have backed up, should still get errors + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " + "-l testfiles/query3b.log " + "-Werror \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); + TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + + // wait another 10 seconds, bbackup should have run + wait_for_backup_operation(10); + TEST_THAT(TestFileExists("testfiles/" + "notifyran.backup-start.wait-automatic.1")); + // Check that it did get uploaded, and we have no more errors - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3b.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(::unlink("testfiles/notifyscript.tag") == 0); + TEST_THAT(ServerIsAlive(bbackupd_pid)); TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; @@ -1976,11 +2802,12 @@ int test_bbackupd() #endif wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3c.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2004,11 +2831,12 @@ int test_bbackupd() #endif wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3d.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2035,11 +2863,12 @@ int test_bbackupd() #endif wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3e.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2066,11 +2895,12 @@ int test_bbackupd() #endif wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3f.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2097,11 +2927,12 @@ int test_bbackupd() wait_for_operation(5); // back up both files wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3g.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); #ifdef WIN32 @@ -2114,11 +2945,12 @@ int test_bbackupd() TEST_THAT(!TestFileExists("testfiles/TestDir1/untracked-1")); TEST_THAT( TestFileExists("testfiles/TestDir1/untracked-2")); wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3g.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2148,11 +2980,12 @@ int test_bbackupd() wait_for_operation(5); // back up both files wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3h.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); #ifdef WIN32 @@ -2165,11 +2998,12 @@ int test_bbackupd() TEST_THAT(!TestFileExists("testfiles/TestDir1/tracked-1")); TEST_THAT( TestFileExists("testfiles/TestDir1/tracked-2")); wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3i.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2185,13 +3019,17 @@ int test_bbackupd() "testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3j.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + // Check that no read error has been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + TEST_THAT(ServerIsAlive(bbackupd_pid)); TEST_THAT(ServerIsAlive(bbstored_pid)); if (!ServerIsAlive(bbackupd_pid)) return 1; @@ -2219,11 +3057,12 @@ int test_bbackupd() // Wait and test wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3k.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2231,6 +3070,9 @@ int test_bbackupd() if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + // Check that no read error has been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + // Check that modifying files with old timestamps // still get added printf("\n==== Modify existing file, but change timestamp " @@ -2277,11 +3119,12 @@ int test_bbackupd() wait_for_sync_end(); // files too new wait_for_sync_end(); // should (not) be backed up this time - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3l.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2289,6 +3132,9 @@ int test_bbackupd() if (!ServerIsAlive(bbackupd_pid)) return 1; if (!ServerIsAlive(bbstored_pid)) return 1; + // Check that no read error has been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + // Add some files and directories which are marked as excluded printf("\n==== Add files and dirs for exclusion test\n"); #ifdef WIN32 @@ -2305,19 +3151,21 @@ int test_bbackupd() wait_for_sync_end(); // compare with exclusions, should not find differences - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3m.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // compare without exclusions, should find differences - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3n.log " - "\"compare -acEQ\" quit"); - TEST_RETURN(compareReturnValue, 2); + "-Werror \"compare -acEQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2331,8 +3179,8 @@ int test_bbackupd() "actually work\n"); { - std::auto_ptr client = Connect( - context, + std::auto_ptr client = + ConnectAndLogin(context, BackupProtocolClientLogin::Flags_ReadOnly); std::auto_ptr dir = ReadDirectory( @@ -2373,6 +3221,9 @@ int test_bbackupd() // These tests only work as non-root users. if(::getuid() != 0) { + // Check that the error has not been reported yet + TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + // Check that read errors are reported neatly printf("\n==== Add unreadable files\n"); @@ -2389,13 +3240,14 @@ int test_bbackupd() // Wait and test... wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3o.log " - "\"compare -acQ\" quit"); + "-Werror \"compare -acQ\" quit"); // should find differences - TEST_RETURN(compareReturnValue, 3); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Error); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // Check that it was reported correctly @@ -2492,11 +3344,12 @@ int test_bbackupd() // Wait and test wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query4.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Werror \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); printf("\n==== Restore files and directories\n"); @@ -2504,8 +3357,8 @@ int test_bbackupd() int64_t restoredirid = 0; { // connect and log in - std::auto_ptr client = Connect( - context, + std::auto_ptr client = + ConnectAndLogin(context, BackupProtocolClientLogin::Flags_ReadOnly); // Find the ID of the Test1 directory @@ -2528,13 +3381,6 @@ int test_bbackupd() true /* print progress dots */) == Restore_TargetExists); - // Make sure you can't restore to a nonexistant path - printf("Try to restore to a path that doesn't exist\n"); - TEST_THAT(BackupClientRestore(*client, restoredirid, - "testfiles/no-such-path/subdir", - true /* print progress dots */) - == Restore_TargetPathNotFound); - // Find ID of the deleted directory deldirid = GetDirID(*client, "x1", restoredirid); TEST_THAT(deldirid != 0); @@ -2547,18 +3393,34 @@ int test_bbackupd() true /* deleted files */) == Restore_Complete); + // Make sure you can't restore to a nonexistant path + printf("\n==== Try to restore to a path " + "that doesn't exist\n"); + fflush(stdout); + + { + Logging::Guard guard(Log::FATAL); + TEST_THAT(BackupClientRestore(*client, + restoredirid, + "testfiles/no-such-path/subdir", + true /* print progress dots */) + == Restore_TargetPathNotFound); + } + // Log out client->QueryFinished(); sSocket.Close(); } // Compare the restored files - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query10.log " + "-Wwarning " "\"compare -cEQ Test1 testfiles/restore-Test1\" " "quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2572,12 +3434,14 @@ int test_bbackupd() "testfiles\\restore-Test1\\f1.dat"); TEST_RETURN(compareReturnValue, 0); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query10a.log " + "-Werror " "\"compare -cEQ Test1 testfiles/restore-Test1\" " "quit"); - TEST_RETURN(compareReturnValue, 2); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // set it back, expect no failures @@ -2585,11 +3449,13 @@ int test_bbackupd() "testfiles\\restore-Test1\\f1.dat"); TEST_RETURN(compareReturnValue, 0); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf -l testfiles/query10a.log " + "-Wwarning " "\"compare -cEQ Test1 testfiles/restore-Test1\" " "quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // change the timestamp on a file, expect a compare failure @@ -2609,46 +3475,54 @@ int test_bbackupd() // a compare failure TEST_THAT(set_file_time(testfile, dummyTime, lastModTime, lastAccessTime)); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query10a.log " + "-Werror " "\"compare -cEQ Test1 testfiles/restore-Test1\" " "quit"); - TEST_RETURN(compareReturnValue, 2); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // last access time is not backed up, so it cannot be compared TEST_THAT(set_file_time(testfile, creationTime, lastModTime, dummyTime)); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query10a.log " + "-Wwarning " "\"compare -cEQ Test1 testfiles/restore-Test1\" " "quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // last write time is backed up, so changing it should cause // a compare failure TEST_THAT(set_file_time(testfile, creationTime, dummyTime, lastAccessTime)); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query10a.log " + "-Werror " "\"compare -cEQ Test1 testfiles/restore-Test1\" " "quit"); - TEST_RETURN(compareReturnValue, 2); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // set back to original values, check that compare succeeds TEST_THAT(set_file_time(testfile, creationTime, lastModTime, lastAccessTime)); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query10a.log " + "-Wwarning " "\"compare -cEQ Test1 testfiles/restore-Test1\" " "quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); #endif // WIN32 @@ -2671,11 +3545,12 @@ int test_bbackupd() // Wait and test wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query5.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2688,19 +3563,21 @@ int test_bbackupd() TEST_THAT(rename("testfiles/TestDir1/sub23/dhsfdss", "testfiles/TestDir1/renamed-dir") == 0); wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query6.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // and again, but with quick flag - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query6q.log " - "\"compare -acqQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acqQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // Rename some files -- one under the threshold, others above @@ -2712,11 +3589,12 @@ int test_bbackupd() TEST_THAT(rename("testfiles/TestDir1/sub23/find2perl", "testfiles/TestDir1/find2perl-ren") == 0); wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query6.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2754,11 +3632,12 @@ int test_bbackupd() // Wait and test wait_for_backup_operation(); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query3e.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2777,23 +3656,23 @@ int test_bbackupd() { try { - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, - "localhost", BOX_PORT_BBSTORED); - BackupProtocolClient protocol(conn); - protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION); - std::auto_ptr loginConf(protocol.QueryLogin(0x01234567, 0)); // read-write - // Make sure the marker isn't zero, because that's the default, and it should have changed + std::auto_ptr + protocol = Connect(context); + // Make sure the marker isn't zero, + // because that's the default, and + // it should have changed + std::auto_ptr loginConf(protocol->QueryLogin(0x01234567, 0)); TEST_THAT(loginConf->GetClientStoreMarker() != 0); // Change it to something else - protocol.QuerySetClientStoreMarker(12); + protocol->QuerySetClientStoreMarker(12); // Success! done = true; // Log out - protocol.QueryFinished(); + protocol->QueryFinished(); + sSocket.Close(); } catch(...) { @@ -2822,11 +3701,12 @@ int test_bbackupd() // Wait and test that there *are* differences wait_for_backup_operation((TIME_TO_WAIT_FOR_BACKUP_OPERATION * 3) / 2); // little bit longer than usual - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query6.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 2); + "-Werror \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Different); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); TEST_THAT(ServerIsAlive(bbackupd_pid)); @@ -2856,8 +3736,8 @@ int test_bbackupd() printf("\n==== Resume restore\n"); - std::auto_ptr client = Connect( - context, + std::auto_ptr client = + ConnectAndLogin(context, BackupProtocolClientLogin::Flags_ReadOnly); // Check that the restore fn returns resume possible, @@ -2880,12 +3760,13 @@ int test_bbackupd() sSocket.Close(); // Then check it has restored the correct stuff - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query14.log " - "\"compare -cEQ Test1 " + "-Wwarning \"compare -cEQ Test1 " "testfiles/restore-interrupt\" quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); } #endif // !WIN32 @@ -2898,8 +3779,8 @@ int test_bbackupd() printf("\n==== Check restore deleted files\n"); { - std::auto_ptr client = Connect( - context, 0 /* read-write */); + std::auto_ptr client = + ConnectAndLogin(context, 0 /* read-write */); // Do restore and undelete TEST_THAT(BackupClientRestore(*client, deldirid, @@ -2913,12 +3794,14 @@ int test_bbackupd() sSocket.Close(); // Do a compare with the now undeleted files - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query11.log " + "-Wwarning " "\"compare -cEQ Test1/x1 " "testfiles/restore-Test1-x1-2\" quit"); - TEST_RETURN(compareReturnValue, 1); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); } @@ -2937,7 +3820,7 @@ int test_bbackupd() // Test that locked files cannot be backed up, // and the appropriate error is reported. // Wait for the sync to finish, so that we have time to work - wait_for_sync_start(); + wait_for_sync_end(); // Now we have about three seconds to work handle = openfile("testfiles/TestDir1/lockedfile", @@ -2950,7 +3833,15 @@ int test_bbackupd() wait_for_sync_end(); TEST_THAT(!TestFileExists("testfiles/" "notifyran.read-error.1")); + } + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + if (handle != 0) + { // this sync should try to back up the file, // and fail, because it's locked wait_for_sync_end(); @@ -2958,45 +3849,82 @@ int test_bbackupd() "notifyran.read-error.1")); TEST_THAT(!TestFileExists("testfiles/" "notifyran.read-error.2")); + } + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + if (handle != 0) + { // now close the file and check that it is // backed up on the next run. CloseHandle(handle); wait_for_sync_end(); + + // still no read errors? TEST_THAT(!TestFileExists("testfiles/" "notifyran.read-error.2")); + } + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + if (handle != 0) + { // compare, and check that it works // reports the correct error message (and finishes) - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query15a.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + } + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; + if (handle != 0) + { // open the file again, compare and check that compare // reports the correct error message (and finishes) handle = openfile("testfiles/TestDir1/lockedfile", O_LOCK, 0); TEST_THAT(handle != INVALID_HANDLE_VALUE); - compareReturnValue = ::system(BBACKUPQUERY - " -q -c testfiles/bbackupd.conf " + compareReturnValue = ::system(BBACKUPQUERY " " + "-c testfiles/bbackupd.conf " "-l testfiles/query15.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 3); + "-Werror \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Error); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // close the file again, check that compare // works again CloseHandle(handle); + } + + TEST_THAT(ServerIsAlive(bbackupd_pid)); + TEST_THAT(ServerIsAlive(bbstored_pid)); + if (!ServerIsAlive(bbackupd_pid)) return 1; + if (!ServerIsAlive(bbstored_pid)) return 1; - compareReturnValue = ::system(BBACKUPQUERY " -q " + if (handle != 0) + { + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query15a.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); } #endif @@ -3021,11 +3949,12 @@ int test_bbackupd() // Wait and compare (a little bit longer than usual) wait_for_backup_operation( (TIME_TO_WAIT_FOR_BACKUP_OPERATION*3) / 2); - compareReturnValue = ::system(BBACKUPQUERY " -q " + compareReturnValue = ::system(BBACKUPQUERY " " "-c testfiles/bbackupd.conf " "-l testfiles/query4a.log " - "\"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, 1); + "-Wwarning \"compare -acQ\" quit"); + TEST_RETURN(compareReturnValue, + BackupQueries::ReturnCode::Compare_Same); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); // Kill it again @@ -3033,10 +3962,12 @@ int test_bbackupd() } } - // List the files on the server + /* + // List the files on the server - why? ::system(BBACKUPQUERY " -q -c testfiles/bbackupd.conf " "-l testfiles/queryLIST.log \"list -rotdh\" quit"); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + */ #ifndef WIN32 if(::getuid() == 0) @@ -3080,8 +4011,14 @@ int test(int argc, const char *argv[]) r = test_bbackupd(); if(r != 0) { - KillServer(bbackupd_pid); - KillServer(bbstored_pid); + if (bbackupd_pid) + { + KillServer(bbackupd_pid); + } + if (bbstored_pid) + { + KillServer(bbstored_pid); + } return r; } diff --git a/test/bbackupd/testfiles/bbackupd-exclude.conf.in b/test/bbackupd/testfiles/bbackupd-exclude.conf.in new file mode 100644 index 00000000..4c08753f --- /dev/null +++ b/test/bbackupd/testfiles/bbackupd-exclude.conf.in @@ -0,0 +1,47 @@ + +CertificateFile = testfiles/clientCerts.pem +PrivateKeyFile = testfiles/clientPrivKey.pem +TrustedCAsFile = testfiles/clientTrustedCAs.pem + +KeysFile = testfiles/bbackupd.keys + +DataDirectory = testfiles/bbackupd-data + +StoreHostname = localhost +StorePort = 22011 +AccountNumber = 0x01234567 + +UpdateStoreInterval = 3 +MinimumFileAge = 4 +MaxUploadWait = 24 +DeleteRedundantLocationsAfter = 10 + +FileTrackingSizeThreshold = 1024 +DiffingUploadSizeThreshold = 1024 + +MaximumDiffingTime = 3 +KeepAliveTime = 1 + +ExtendedLogging = no +ExtendedLogFile = testfiles/bbackupd.log + +CommandSocket = testfiles/bbackupd.sock + +NotifyScript = @TARGET_PERL@ testfiles/notifyscript.pl +SyncAllowScript = @TARGET_PERL@ testfiles/syncallowscript.pl + +Server +{ + PidFile = testfiles/bbackupd.pid +} + +BackupLocations +{ + Test1 + { + Path = testfiles/TestDir1 + ExcludeDir = testfiles/TestDir1/spacetest/d3 + ExcludeFile = testfiles/TestDir1/spacetest/f2 + } +} + diff --git a/test/bbackupd/testfiles/bbackupd-snapshot.conf.in b/test/bbackupd/testfiles/bbackupd-snapshot.conf.in new file mode 100644 index 00000000..d245d077 --- /dev/null +++ b/test/bbackupd/testfiles/bbackupd-snapshot.conf.in @@ -0,0 +1,56 @@ + +CertificateFile = testfiles/clientCerts.pem +PrivateKeyFile = testfiles/clientPrivKey.pem +TrustedCAsFile = testfiles/clientTrustedCAs.pem + +KeysFile = testfiles/bbackupd.keys + +DataDirectory = testfiles/bbackupd-data + +StoreHostname = localhost +StorePort = 22011 +AccountNumber = 0x01234567 + +AutomaticBackup = no +UpdateStoreInterval = 0 +MinimumFileAge = 4 +MaxUploadWait = 24 +DeleteRedundantLocationsAfter = 10 + +FileTrackingSizeThreshold = 1024 +DiffingUploadSizeThreshold = 1024 + +MaximumDiffingTime = 3 +KeepAliveTime = 1 + +ExtendedLogging = no +ExtendedLogFile = testfiles/bbackupd.log + +CommandSocket = testfiles/bbackupd.sock + +NotifyScript = @TARGET_PERL@ testfiles/notifyscript.pl +SyncAllowScript = @TARGET_PERL@ testfiles/syncallowscript.pl + +Server +{ + PidFile = testfiles/bbackupd.pid +} + +BackupLocations +{ + Test1 + { + Path = testfiles/TestDir1 + + ExcludeFile = testfiles/TestDir1/excluded_1 + ExcludeFile = testfiles/TestDir1/excluded_2 + ExcludeFilesRegex = \.excludethis$ + ExcludeFilesRegex = EXCLUDE + AlwaysIncludeFile = testfiles/TestDir1/dont.excludethis + ExcludeDir = testfiles/TestDir1/exclude_dir + ExcludeDir = testfiles/TestDir1/exclude_dir_2 + ExcludeDirsRegex = not_this_dir + AlwaysIncludeDirsRegex = ALWAYSINCLUDE + } +} + diff --git a/test/bbackupd/testfiles/bbackupd-symlink.conf.in b/test/bbackupd/testfiles/bbackupd-symlink.conf.in new file mode 100644 index 00000000..33bb6157 --- /dev/null +++ b/test/bbackupd/testfiles/bbackupd-symlink.conf.in @@ -0,0 +1,55 @@ + +CertificateFile = testfiles/clientCerts.pem +PrivateKeyFile = testfiles/clientPrivKey.pem +TrustedCAsFile = testfiles/clientTrustedCAs.pem + +KeysFile = testfiles/bbackupd.keys + +DataDirectory = testfiles/bbackupd-data + +StoreHostname = localhost +StorePort = 22011 +AccountNumber = 0x01234567 + +UpdateStoreInterval = 3 +MinimumFileAge = 4 +MaxUploadWait = 24 +DeleteRedundantLocationsAfter = 10 + +FileTrackingSizeThreshold = 1024 +DiffingUploadSizeThreshold = 1024 + +MaximumDiffingTime = 3 +KeepAliveTime = 1 + +ExtendedLogging = no +ExtendedLogFile = testfiles/bbackupd.log + +CommandSocket = testfiles/bbackupd.sock + +NotifyScript = @TARGET_PERL@ testfiles/notifyscript.pl +SyncAllowScript = @TARGET_PERL@ testfiles/syncallowscript.pl + +Server +{ + PidFile = testfiles/bbackupd.pid +} + +BackupLocations +{ + Test1 + { + Path = testfiles/symlink-to-TestDir1 + + ExcludeFile = testfiles/TestDir1/excluded_1 + ExcludeFile = testfiles/TestDir1/excluded_2 + ExcludeFilesRegex = \.excludethis$ + ExcludeFilesRegex = EXCLUDE + AlwaysIncludeFile = testfiles/TestDir1/dont.excludethis + ExcludeDir = testfiles/TestDir1/exclude_dir + ExcludeDir = testfiles/TestDir1/exclude_dir_2 + ExcludeDirsRegex = not_this_dir + AlwaysIncludeDirsRegex = ALWAYSINCLUDE + } +} + diff --git a/test/bbackupd/testfiles/bbackupd-temploc.conf b/test/bbackupd/testfiles/bbackupd-temploc.conf index 86901298..07cbdcd1 100644 --- a/test/bbackupd/testfiles/bbackupd-temploc.conf +++ b/test/bbackupd/testfiles/bbackupd-temploc.conf @@ -8,6 +8,7 @@ KeysFile = testfiles/bbackupd.keys DataDirectory = testfiles/bbackupd-data StoreHostname = localhost +StorePort = 22011 AccountNumber = 0x01234567 UpdateStoreInterval = 3 @@ -48,7 +49,7 @@ BackupLocations } Test2 { - Path = testfiles/TestDir1 + Path = testfiles/TestDir2 } } diff --git a/test/bbackupd/testfiles/bbackupd.conf.in b/test/bbackupd/testfiles/bbackupd.conf.in index aecb3884..712b58b2 100644 --- a/test/bbackupd/testfiles/bbackupd.conf.in +++ b/test/bbackupd/testfiles/bbackupd.conf.in @@ -8,6 +8,7 @@ KeysFile = testfiles/bbackupd.keys DataDirectory = testfiles/bbackupd-data StoreHostname = localhost +StorePort = 22011 AccountNumber = 0x01234567 UpdateStoreInterval = 3 diff --git a/test/bbackupd/testfiles/bbstored.conf b/test/bbackupd/testfiles/bbstored.conf index 18c73a40..87f4fe6b 100644 --- a/test/bbackupd/testfiles/bbstored.conf +++ b/test/bbackupd/testfiles/bbstored.conf @@ -9,7 +9,7 @@ TimeBetweenHousekeeping = 5 Server { PidFile = testfiles/bbstored.pid - ListenAddresses = inet:localhost + ListenAddresses = inet:localhost:22011 CertificateFile = testfiles/serverCerts.pem PrivateKeyFile = testfiles/serverPrivKey.pem TrustedCAsFile = testfiles/serverTrustedCAs.pem diff --git a/test/bbackupd/testfiles/extcheck1.pl.in b/test/bbackupd/testfiles/extcheck1.pl.in index 4d0f2157..74884dd8 100755 --- a/test/bbackupd/testfiles/extcheck1.pl.in +++ b/test/bbackupd/testfiles/extcheck1.pl.in @@ -3,7 +3,10 @@ use strict; my $flags = $ARGV[0] or ""; -unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac$flags\" quit 2>&1 |") +unless(open IN,"../../bin/bbackupquery/bbackupquery -Wwarning " . + "-c testfiles/bbackupd.conf " . + "-l testfiles/query4.log " . + "\"compare -ac$flags\" quit 2>&1 |") { print "Couldn't open compare utility\n"; exit 2; diff --git a/test/bbackupd/testfiles/extcheck2.pl.in b/test/bbackupd/testfiles/extcheck2.pl.in index 074defc0..3671ad93 100755 --- a/test/bbackupd/testfiles/extcheck2.pl.in +++ b/test/bbackupd/testfiles/extcheck2.pl.in @@ -3,7 +3,10 @@ use strict; my $flags = $ARGV[0] or ""; -unless(open IN,"../../bin/bbackupquery/bbackupquery -q -c testfiles/bbackupd.conf -l testfiles/query4.log \"compare -ac$flags\" quit 2>&1 |") +unless(open IN,"../../bin/bbackupquery/bbackupquery -Wwarning " . + "-c testfiles/bbackupd.conf " . + "-l testfiles/query4.log " . + "\"compare -ac$flags\" quit 2>&1 |") { print "Couldn't open compare utility\n"; exit 2; diff --git a/test/bbackupd/testfiles/notifyscript.pl.in b/test/bbackupd/testfiles/notifyscript.pl.in index 89741e46..d3e324e9 100755 --- a/test/bbackupd/testfiles/notifyscript.pl.in +++ b/test/bbackupd/testfiles/notifyscript.pl.in @@ -1,7 +1,16 @@ #!@TARGET_PERL@ - my $f = 'testfiles/notifyran.'.$ARGV[0].'.'; + +if (-e 'testfiles/notifyscript.tag') +{ + open FILE, '< testfiles/notifyscript.tag' or die $!; + my $tag = ; + chomp $tag; + $f .= "$tag."; + close FILE; +} + my $n = 1; while(-e $f.$n) diff --git a/test/common/testcommon.cpp b/test/common/testcommon.cpp index eb057228..da2133cd 100644 --- a/test/common/testcommon.cpp +++ b/test/common/testcommon.cpp @@ -57,15 +57,15 @@ void test_conversions() ConfigurationVerifyKey verifykeys1_1_1[] = { - {"bing", 0, ConfigTest_Exists, 0}, - {"carrots", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"terrible", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("bing", ConfigTest_Exists), + ConfigurationVerifyKey("carrots", ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("terrible", ConfigTest_Exists | ConfigTest_LastEntry) }; ConfigurationVerifyKey verifykeys1_1_2[] = { - {"fish", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"string", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("fish", ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("string", ConfigTest_Exists | ConfigTest_LastEntry) }; @@ -89,15 +89,15 @@ ConfigurationVerify verifysub1_1[] = ConfigurationVerifyKey verifykeys1_1[] = { - {"value", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"string1", 0, ConfigTest_Exists, 0}, - {"string2", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("value", ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("string1", ConfigTest_Exists), + ConfigurationVerifyKey("string2", ConfigTest_Exists | ConfigTest_LastEntry) }; ConfigurationVerifyKey verifykeys1_2[] = { - {"carrots", 0, ConfigTest_Exists | ConfigTest_IsInt, 0}, - {"string", 0, ConfigTest_Exists | ConfigTest_LastEntry, 0} + ConfigurationVerifyKey("carrots", ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("string", ConfigTest_Exists | ConfigTest_LastEntry) }; ConfigurationVerify verifysub1[] = @@ -120,14 +120,15 @@ ConfigurationVerify verifysub1[] = ConfigurationVerifyKey verifykeys1[] = { - {"notExpected", 0, 0, 0}, - {"HasDefaultValue", "Lovely default value", 0, 0}, - {"MultiValue", 0, ConfigTest_MultiValueAllowed, 0}, - {"BoolTrue1", 0, ConfigTest_IsBool, 0}, - {"BoolTrue2", 0, ConfigTest_IsBool, 0}, - {"BoolFalse1", 0, ConfigTest_IsBool, 0}, - {"BoolFalse2", 0, ConfigTest_IsBool, 0}, - {"TOPlevel", 0, ConfigTest_LastEntry | ConfigTest_Exists, 0} + ConfigurationVerifyKey("notExpected", 0), + ConfigurationVerifyKey("HasDefaultValue", 0, "Lovely default value"), + ConfigurationVerifyKey("MultiValue", ConfigTest_MultiValueAllowed), + ConfigurationVerifyKey("BoolTrue1", ConfigTest_IsBool), + ConfigurationVerifyKey("BoolTrue2", ConfigTest_IsBool), + ConfigurationVerifyKey("BoolFalse1", ConfigTest_IsBool), + ConfigurationVerifyKey("BoolFalse2", ConfigTest_IsBool), + ConfigurationVerifyKey("TOPlevel", + ConfigTest_LastEntry | ConfigTest_Exists) }; ConfigurationVerify verify = @@ -297,7 +298,7 @@ int test(int argc, const char *argv[]) // Check that using timer methods without initialisation // throws an assertion failure. Can only do this in debug mode - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD TEST_CHECK_THROWS(Timers::Add(*(Timer*)NULL), CommonException, AssertFailed); TEST_CHECK_THROWS(Timers::Remove(*(Timer*)NULL), @@ -305,7 +306,7 @@ int test(int argc, const char *argv[]) #endif // TEST_CHECK_THROWS(Timers::Signal(), CommonException, AssertFailed); - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD TEST_CHECK_THROWS(Timers::Cleanup(), CommonException, AssertFailed); #endif @@ -314,7 +315,7 @@ int test(int argc, const char *argv[]) Timers::Init(); // Check that double initialisation throws an exception - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD TEST_CHECK_THROWS(Timers::Init(), CommonException, AssertFailed); #endif @@ -323,17 +324,17 @@ int test(int argc, const char *argv[]) Timers::Cleanup(); // Check that double cleanup throws an exception - #ifndef NDEBUG + #ifndef BOX_RELEASE_BUILD TEST_CHECK_THROWS(Timers::Cleanup(), CommonException, AssertFailed); #endif Timers::Init(); - Timer t0(0); // should never expire - Timer t1(1); - Timer t2(2); - Timer t3(3); + Timer t0(0, "t0"); // should never expire + Timer t1(1, "t1"); + Timer t2(2, "t2"); + Timer t3(3, "t3"); TEST_THAT(!t0.HasExpired()); TEST_THAT(!t1.HasExpired()); @@ -352,8 +353,8 @@ int test(int argc, const char *argv[]) TEST_THAT(t2.HasExpired()); TEST_THAT(!t3.HasExpired()); - t1 = Timer(1); - t2 = Timer(2); + t1 = Timer(1, "t1a"); + t2 = Timer(2, "t2a"); TEST_THAT(!t0.HasExpired()); TEST_THAT(!t1.HasExpired()); TEST_THAT(!t2.HasExpired()); diff --git a/test/compress/testcompress.cpp b/test/compress/testcompress.cpp index 592dd641..4a522d31 100644 --- a/test/compress/testcompress.cpp +++ b/test/compress/testcompress.cpp @@ -90,7 +90,8 @@ int test_stream() // Check sizes TEST_THAT(poutput->GetSize() < source.GetSize()); - TRACE2("compressed size = %d, source size = %d\n", poutput->GetSize(), source.GetSize()); + BOX_TRACE("compressed size = " << poutput->GetSize() << + ", source size = " << source.GetSize()); // Decompress the data { diff --git a/test/httpserver/testfiles/httpserver.conf b/test/httpserver/testfiles/httpserver.conf new file mode 100644 index 00000000..1a1c4644 --- /dev/null +++ b/test/httpserver/testfiles/httpserver.conf @@ -0,0 +1,8 @@ + +AddressPrefix = http://localhost:1080 + +Server +{ + PidFile = testfiles/httpserver.pid + ListenAddresses = inet:localhost:1080 +} diff --git a/test/httpserver/testfiles/photos/puppy.jpg b/test/httpserver/testfiles/photos/puppy.jpg new file mode 100644 index 00000000..a326a6a7 --- /dev/null +++ b/test/httpserver/testfiles/photos/puppy.jpg @@ -0,0 +1 @@ +omgpuppies! diff --git a/test/httpserver/testfiles/testrequests.pl b/test/httpserver/testfiles/testrequests.pl new file mode 100755 index 00000000..85380ee0 --- /dev/null +++ b/test/httpserver/testfiles/testrequests.pl @@ -0,0 +1,143 @@ +#!/usr/bin/perl +use strict; +use LWP::UserAgent; + +my $url_base = 'http://localhost:1080'; + +my $ua = LWP::UserAgent->new(env_proxy => 0, keep_alive => 1, timeout => 30); + +print "GET request...\n"; + +my $response1 = $ua->get("$url_base/test-one/34/341s/234?p1=vOne&p2=vTwo"); +die $response1->content unless $response1->is_success(); + +my $content = $response1->content(); + +check_url($content, '/test-one/34/341s/234'); +check_params($content, 'p1'=>'vOne','p2'=>'vTwo'); + +print "POST request...\n"; + +my %post = ('sdfgksjhdfsd'=>'dfvsiufy2e3434','sxciuhwf8723e4'=>'238947829334', + '&sfsfsfskfhs'=>'?hdkfjhsjfds','fdsf=sdf2342'=>'3984sajhksda'); + +my $response2 = $ua->post("$url_base/tdskjhfsjdkhf2943734?p1=vOne&p2=vTwo", \%post); + +my $content2 = $response2->content(); + +check_url($content2, '/tdskjhfsjdkhf2943734'); +check_params($content2, %post); + +print "HEAD request...\n"; + +my $response3 = $ua->head("$url_base/tdskjhfsdfkjhs"); + +if($response3->content() ne '') +{ + print "Content not zero length\n"; + exit(1); +} + +if($response3->code() != 200) +{ + print "Wrong response code\n"; + exit(1); +} + +print "Redirected GET request...\n"; + +my $response4 = $ua->get("$url_base/redirect?key=value"); +exit 4 unless $response4->is_success(); + +my $content4 = $response4->content(); + +check_url($content4, '/redirected'); +check_params($content4); + +print "Cookie tests...\n"; + +# from examples in specs +test_cookies('CUSTOMER=WILE_E_COYOTE', 'CUSTOMER=WILE_E_COYOTE'); +test_cookies('CUSTOMER="WILE_E_COYOTE"; C2="pants"', 'CUSTOMER=WILE_E_COYOTE', 'C2=pants'); +test_cookies('CUSTOMER=WILE_E_COYOTE; PART_NUMBER=ROCKET_LAUNCHER_0001', 'CUSTOMER=WILE_E_COYOTE', 'PART_NUMBER=ROCKET_LAUNCHER_0001'); +test_cookies('CUSTOMER=WILE_E_COYOTE; PART_NUMBER=ROCKET_LAUNCHER_0001; SHIPPING=FEDEX', 'CUSTOMER=WILE_E_COYOTE', 'PART_NUMBER=ROCKET_LAUNCHER_0001', 'SHIPPING=FEDEX'); +test_cookies('$Version="1"; Customer="WILE_E_COYOTE"; $Path="/acme"', 'Customer=WILE_E_COYOTE'); +test_cookies('$Version="1"; Customer="WILE_E_COYOTE"; $Path="/acme"; Part_Number="Rocket_Launcher_0001"; $Path="/acme" ', + 'Customer=WILE_E_COYOTE', 'Part_Number=Rocket_Launcher_0001'); +test_cookies(qq!\$Version="1"; Customer="WILE_E_COYOTE"; \$Path="/acme"; Part_Number="Rocket_Launcher_0001"; \$Path="/acme"; Shipping="FedEx"; \t \$Path="/acme"!, + 'Customer=WILE_E_COYOTE', 'Part_Number=Rocket_Launcher_0001', 'Shipping=FedEx'); + +# test the server setting cookies in the UA +require HTTP::Cookies; +$ua->cookie_jar(HTTP::Cookies->new()); +$ua->get("$url_base/set-cookie"); +test_cookies('', 'SetByServer=Value1'); + +sub test_cookies +{ + my ($c_str, @cookies) = @_; + test_cookies2($c_str, @cookies); + $c_str =~ s/;/,/g; + test_cookies2($c_str, @cookies); +} + +sub test_cookies2 +{ + my ($c_str, @cookies) = @_; + my $r; + if($c_str ne '') + { + $r = $ua->get("$url_base/cookie", 'Cookie' => $c_str); + } + else + { + $r = $ua->get("$url_base/cookie"); + } + my $c = $r->content(); + for(@cookies) + { + unless($c =~ m/COOKIE:$_
/) + { + print "Cookie $_ not found\n"; + exit(1); + } + } +} + + +sub check_url +{ + my ($c,$url) = @_; + unless($c =~ m~URI: (.+?)

~) + { + print "URI not found\n"; + exit(1); + } + if($url ne $1) + { + print "Wrong URI in content\n"; + exit(1); + } +} + +sub check_params +{ + my ($c,%params) = @_; + + while($c =~ m/^PARAM:(.+)=(.+?)
/mg) + { + if($params{$1} ne $2) + { + print "$1=$2 not found in response\n"; + exit(1); + } + delete $params{$1} + } + + my @k = keys %params; + if($#k != -1) + { + print "Didn't find all params\n"; + exit(1); + } +} diff --git a/test/httpserver/testhttpserver.cpp b/test/httpserver/testhttpserver.cpp new file mode 100644 index 00000000..c38ef8fe --- /dev/null +++ b/test/httpserver/testhttpserver.cpp @@ -0,0 +1,661 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: testhttpserver.cpp +// Purpose: Test code for HTTP server class +// Created: 26/3/04 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include +#include + +#ifdef HAVE_SIGNAL_H + #include +#endif + +#include + +#include "autogen_HTTPException.h" +#include "HTTPRequest.h" +#include "HTTPResponse.h" +#include "HTTPServer.h" +#include "IOStreamGetLine.h" +#include "S3Client.h" +#include "ServerControl.h" +#include "Test.h" +#include "decode.h" +#include "encode.h" + +#include "MemLeakFindOn.h" + +class TestWebServer : public HTTPServer +{ +public: + TestWebServer(); + ~TestWebServer(); + + virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse); + +}; + +// Build a nice HTML response, so this can also be tested neatly in a browser +void TestWebServer::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + // Test redirection mechanism + if(rRequest.GetRequestURI() == "/redirect") + { + rResponse.SetAsRedirect("/redirected"); + return; + } + + // Set a cookie? + if(rRequest.GetRequestURI() == "/set-cookie") + { + rResponse.SetCookie("SetByServer", "Value1"); + } + + #define DEFAULT_RESPONSE_1 "\nTEST SERVER RESPONSE\n

Test response

\n

URI: " + #define DEFAULT_RESPONSE_3 "

\n

Query string: " + #define DEFAULT_RESPONSE_4 "

\n

Method: " + #define DEFAULT_RESPONSE_5 "

\n

Decoded query:
" + #define DEFAULT_RESPONSE_6 "

\n

Content type: " + #define DEFAULT_RESPONSE_7 "

\n

Content length: " + #define DEFAULT_RESPONSE_8 "

\n

Cookies:
\n" + #define DEFAULT_RESPONSE_2 "

\n\n\n" + + rResponse.SetResponseCode(HTTPResponse::Code_OK); + rResponse.SetContentType("text/html"); + rResponse.Write(DEFAULT_RESPONSE_1, sizeof(DEFAULT_RESPONSE_1) - 1); + const std::string &ruri(rRequest.GetRequestURI()); + rResponse.Write(ruri.c_str(), ruri.size()); + rResponse.Write(DEFAULT_RESPONSE_3, sizeof(DEFAULT_RESPONSE_3) - 1); + const std::string &rquery(rRequest.GetQueryString()); + rResponse.Write(rquery.c_str(), rquery.size()); + rResponse.Write(DEFAULT_RESPONSE_4, sizeof(DEFAULT_RESPONSE_4) - 1); + { + const char *m = "????"; + switch(rRequest.GetMethod()) + { + case HTTPRequest::Method_GET: m = "GET "; break; + case HTTPRequest::Method_HEAD: m = "HEAD"; break; + case HTTPRequest::Method_POST: m = "POST"; break; + default: m = "UNKNOWN"; + } + rResponse.Write(m, 4); + } + rResponse.Write(DEFAULT_RESPONSE_5, sizeof(DEFAULT_RESPONSE_5) - 1); + { + const HTTPRequest::Query_t &rquery(rRequest.GetQuery()); + for(HTTPRequest::Query_t::const_iterator i(rquery.begin()); i != rquery.end(); ++i) + { + rResponse.Write("\nPARAM:", 7); + rResponse.Write(i->first.c_str(), i->first.size()); + rResponse.Write("=", 1); + rResponse.Write(i->second.c_str(), i->second.size()); + rResponse.Write("
\n", 4); + } + } + rResponse.Write(DEFAULT_RESPONSE_6, sizeof(DEFAULT_RESPONSE_6) - 1); + const std::string &rctype(rRequest.GetContentType()); + rResponse.Write(rctype.c_str(), rctype.size()); + rResponse.Write(DEFAULT_RESPONSE_7, sizeof(DEFAULT_RESPONSE_7) - 1); + { + char l[32]; + rResponse.Write(l, ::sprintf(l, "%d", rRequest.GetContentLength())); + } + rResponse.Write(DEFAULT_RESPONSE_8, sizeof(DEFAULT_RESPONSE_8) - 1); + const HTTPRequest::CookieJar_t *pcookies = rRequest.GetCookies(); + if(pcookies != 0) + { + HTTPRequest::CookieJar_t::const_iterator i(pcookies->begin()); + for(; i != pcookies->end(); ++i) + { + char t[512]; + rResponse.Write(t, ::sprintf(t, "COOKIE:%s=%s
\n", i->first.c_str(), i->second.c_str())); + } + } + rResponse.Write(DEFAULT_RESPONSE_2, sizeof(DEFAULT_RESPONSE_2) - 1); +} + +TestWebServer::TestWebServer() {} +TestWebServer::~TestWebServer() {} + +class S3Simulator : public HTTPServer +{ +public: + S3Simulator() { } + ~S3Simulator() { } + + virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse); + virtual void HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse); +}; + +void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + // if anything goes wrong, return a 500 error + rResponse.SetResponseCode(HTTPResponse::Code_InternalServerError); + rResponse.SetContentType("text/plain"); + + try + { + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAuthentication.html + std::string access_key = "0PN5J17HBGZHT7JJ3X82"; + std::string secret_key = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"; + + std::string md5, date, bucket; + rRequest.GetHeader("Content-MD5", &md5); + rRequest.GetHeader("Date", &date); + + std::string host = rRequest.GetHostName(); + std::string s3suffix = ".s3.amazonaws.com"; + if (host.size() > s3suffix.size()) + { + std::string suffix = host.substr(host.size() - + s3suffix.size(), s3suffix.size()); + if (suffix == s3suffix) + { + bucket = host.substr(0, host.size() - + s3suffix.size()); + } + } + + std::ostringstream data; + data << rRequest.GetVerb() << "\n"; + data << md5 << "\n"; + data << rRequest.GetContentType() << "\n"; + data << date << "\n"; + + std::vector headers = rRequest.GetHeaders(); + + for (std::vector::iterator + i = headers.begin(); i != headers.end(); i++) + { + std::string& rHeaderName = i->first; + + for (std::string::iterator c = rHeaderName.begin(); + c != rHeaderName.end() && *c != ':'; c++) + { + *c = tolower(*c); + } + } + + sort(headers.begin(), headers.end()); + + for (std::vector::iterator + i = headers.begin(); i != headers.end(); i++) + { + if (i->first.substr(0, 5) == "x-amz") + { + data << i->first << ":" << i->second << "\n"; + } + } + + if (! bucket.empty()) + { + data << "/" << bucket; + } + + data << rRequest.GetRequestURI(); + std::string data_string = data.str(); + + unsigned char digest_buffer[EVP_MAX_MD_SIZE]; + unsigned int digest_size = sizeof(digest_buffer); + /* unsigned char* mac = */ HMAC(EVP_sha1(), + secret_key.c_str(), secret_key.size(), + (const unsigned char*)data_string.c_str(), + data_string.size(), digest_buffer, &digest_size); + std::string digest((const char *)digest_buffer, digest_size); + + base64::encoder encoder; + std::string expectedAuth = "AWS " + access_key + ":" + + encoder.encode(digest); + + if (expectedAuth[expectedAuth.size() - 1] == '\n') + { + expectedAuth = expectedAuth.substr(0, + expectedAuth.size() - 1); + } + + std::string actualAuth; + if (!rRequest.GetHeader("Authorization", &actualAuth) || + actualAuth != expectedAuth) + { + rResponse.SetResponseCode(HTTPResponse::Code_Unauthorized); + } + else if (rRequest.GetMethod() == HTTPRequest::Method_GET) + { + HandleGet(rRequest, rResponse); + } + else if (rRequest.GetMethod() == HTTPRequest::Method_PUT) + { + HandlePut(rRequest, rResponse); + } + else + { + rResponse.SetResponseCode(HTTPResponse::Code_MethodNotAllowed); + } + } + catch (CommonException &ce) + { + rResponse.IOStream::Write(ce.what()); + } + catch (std::exception &e) + { + rResponse.IOStream::Write(e.what()); + } + catch (...) + { + rResponse.IOStream::Write("Unknown error"); + } + + return; +} + +void S3Simulator::HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + std::string path = "testfiles"; + path += rRequest.GetRequestURI(); + std::auto_ptr apFile; + + try + { + apFile.reset(new FileStream(path)); + } + catch (CommonException &ce) + { + if (ce.GetSubType() == CommonException::OSFileOpenError) + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + } + else if (ce.GetSubType() == CommonException::AccessDenied) + { + rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); + } + throw; + } + + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingRESTOperations.html + apFile->CopyStreamTo(rResponse); + rResponse.AddHeader("x-amz-id-2", "qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY"); + rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); + rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); + rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); + rResponse.AddHeader("Server", "AmazonS3"); + rResponse.SetResponseCode(HTTPResponse::Code_OK); +} + +void S3Simulator::HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse) +{ + std::string path = "testfiles"; + path += rRequest.GetRequestURI(); + std::auto_ptr apFile; + + try + { + apFile.reset(new FileStream(path, O_CREAT | O_WRONLY)); + } + catch (CommonException &ce) + { + if (ce.GetSubType() == CommonException::OSFileOpenError) + { + rResponse.SetResponseCode(HTTPResponse::Code_NotFound); + } + else if (ce.GetSubType() == CommonException::AccessDenied) + { + rResponse.SetResponseCode(HTTPResponse::Code_Forbidden); + } + throw; + } + + if (rRequest.IsExpectingContinue()) + { + rResponse.SendContinue(); + } + + rRequest.ReadContent(*apFile); + + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTObjectPUT.html + rResponse.AddHeader("x-amz-id-2", "LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7"); + rResponse.AddHeader("x-amz-request-id", "F2A8CCCA26B4B26D"); + rResponse.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + rResponse.AddHeader("Last-Modified", "Sun, 1 Jan 2006 12:00:00 GMT"); + rResponse.AddHeader("ETag", "\"828ef3fdfa96f00ad9f27c383fc9ac7f\""); + rResponse.SetContentType(""); + rResponse.AddHeader("Server", "AmazonS3"); + rResponse.SetResponseCode(HTTPResponse::Code_OK); +} + +int test(int argc, const char *argv[]) +{ + if(argc >= 2 && ::strcmp(argv[1], "server") == 0) + { + // Run a server + TestWebServer server; + return server.Main("doesnotexist", argc - 1, argv + 1); + } + + if(argc >= 2 && ::strcmp(argv[1], "s3server") == 0) + { + // Run a server + S3Simulator server; + return server.Main("doesnotexist", argc - 1, argv + 1); + } + + // Start the server + int pid = LaunchServer("./test server testfiles/httpserver.conf", "testfiles/httpserver.pid"); + TEST_THAT(pid != -1 && pid != 0); + if(pid <= 0) + { + return 0; + } + + // Run the request script + TEST_THAT(::system("perl testfiles/testrequests.pl") == 0); + + signal(SIGPIPE, SIG_IGN); + + SocketStream sock; + sock.Open(Socket::TypeINET, "localhost", 1080); + + for (int i = 0; i < 4; i++) + { + HTTPRequest request(HTTPRequest::Method_GET, + "/test-one/34/341s/234?p1=vOne&p2=vTwo"); + + if (i < 2) + { + // first set of passes has keepalive off by default, + // so when i == 1 the socket has already been closed + // by the server, and we'll get -EPIPE when we try + // to send the request. + request.SetClientKeepAliveRequested(false); + } + else + { + request.SetClientKeepAliveRequested(true); + } + + if (i == 1) + { + sleep(1); // need time for our process to realise + // that the peer has died, otherwise no SIGPIPE :( + TEST_CHECK_THROWS(request.Send(sock, + IOStream::TimeOutInfinite), + ConnectionException, SocketWriteError); + sock.Close(); + sock.Open(Socket::TypeINET, "localhost", 1080); + continue; + } + else + { + request.Send(sock, IOStream::TimeOutInfinite); + } + + HTTPResponse response; + response.Receive(sock); + + TEST_THAT(response.GetResponseCode() == HTTPResponse::Code_OK); + TEST_THAT(response.GetContentType() == "text/html"); + + IOStreamGetLine getline(response); + std::string line; + + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("TEST SERVER RESPONSE", + line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("

Test response

", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("

URI: /test-one/34/341s/234

", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("

Query string: p1=vOne&p2=vTwo

", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("

Method: GET

", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("

Decoded query:
", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("PARAM:p1=vOne
", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("PARAM:p2=vTwo

", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("

Content type:

", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("

Content length: -1

", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("

Cookies:
", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("

", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("", line); + TEST_THAT(getline.GetLine(line)); + TEST_EQUAL("", line); + } + + // Kill it + TEST_THAT(KillServer(pid)); + TestRemoteProcessMemLeaks("generic-httpserver.memleaks"); + + // correct, official signature should succeed + { + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAuthentication.html + HTTPRequest request(HTTPRequest::Method_GET, "/photos/puppy.jpg"); + request.SetHostName("johnsmith.s3.amazonaws.com"); + request.AddHeader("Date", "Tue, 27 Mar 2007 19:36:42 +0000"); + request.AddHeader("Authorization", + "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA="); + + S3Simulator simulator; + + CollectInBufferStream response_buffer; + HTTPResponse response(&response_buffer); + + simulator.Handle(request, response); + TEST_EQUAL(200, response.GetResponseCode()); + + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + TEST_EQUAL("omgpuppies!\n", response_data); + } + + // modified signature should fail + { + // http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAuthentication.html + HTTPRequest request(HTTPRequest::Method_GET, "/photos/puppy.jpg"); + request.SetHostName("johnsmith.s3.amazonaws.com"); + request.AddHeader("Date", "Tue, 27 Mar 2007 19:36:42 +0000"); + request.AddHeader("Authorization", + "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbB="); + + S3Simulator simulator; + + CollectInBufferStream response_buffer; + HTTPResponse response(&response_buffer); + + simulator.Handle(request, response); + TEST_EQUAL(401, response.GetResponseCode()); + + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + TEST_EQUAL("", response_data); + } + + // S3Client tests + { + S3Simulator simulator; + S3Client client(&simulator, "johnsmith.s3.amazonaws.com", + "0PN5J17HBGZHT7JJ3X82", + "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"); + + HTTPResponse response = client.GetObject("/photos/puppy.jpg"); + TEST_EQUAL(200, response.GetResponseCode()); + std::string response_data((const char *)response.GetBuffer(), + response.GetSize()); + TEST_EQUAL("omgpuppies!\n", response_data); + + // make sure that assigning to HTTPResponse does clear stream + response = client.GetObject("/photos/puppy.jpg"); + TEST_EQUAL(200, response.GetResponseCode()); + response_data = std::string((const char *)response.GetBuffer(), + response.GetSize()); + TEST_EQUAL("omgpuppies!\n", response_data); + + response = client.GetObject("/nonexist"); + TEST_EQUAL(404, response.GetResponseCode()); + + FileStream fs("testfiles/testrequests.pl"); + response = client.PutObject("/newfile", fs); + TEST_EQUAL(200, response.GetResponseCode()); + + response = client.GetObject("/newfile"); + TEST_EQUAL(200, response.GetResponseCode()); + TEST_THAT(fs.CompareWith(response)); + TEST_EQUAL(0, ::unlink("testfiles/newfile")); + } + + { + HTTPRequest request(HTTPRequest::Method_PUT, + "/newfile"); + request.SetHostName("quotes.s3.amazonaws.com"); + request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:XtMYZf0hdOo4TdPYQknZk0Lz7rw="); + request.AddHeader("Content-Type", "text/plain"); + + FileStream fs("testfiles/testrequests.pl"); + fs.CopyStreamTo(request); + request.SetForReading(); + + CollectInBufferStream response_buffer; + HTTPResponse response(&response_buffer); + + S3Simulator simulator; + simulator.Handle(request, response); + + TEST_EQUAL(200, response.GetResponseCode()); + TEST_EQUAL("LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7", response.GetHeaderValue("x-amz-id-2")); + TEST_EQUAL("F2A8CCCA26B4B26D", response.GetHeaderValue("x-amz-request-id")); + TEST_EQUAL("Wed, 01 Mar 2006 12:00:00 GMT", response.GetHeaderValue("Date")); + TEST_EQUAL("Sun, 1 Jan 2006 12:00:00 GMT", response.GetHeaderValue("Last-Modified")); + TEST_EQUAL("\"828ef3fdfa96f00ad9f27c383fc9ac7f\"", response.GetHeaderValue("ETag")); + TEST_EQUAL("", response.GetContentType()); + TEST_EQUAL("AmazonS3", response.GetHeaderValue("Server")); + TEST_EQUAL(0, response.GetSize()); + + FileStream f1("testfiles/testrequests.pl"); + FileStream f2("testfiles/newfile"); + TEST_THAT(f1.CompareWith(f2)); + TEST_EQUAL(0, ::unlink("testfiles/newfile")); + } + + // Start the S3Simulator server + pid = LaunchServer("./test s3server testfiles/httpserver.conf", + "testfiles/httpserver.pid"); + TEST_THAT(pid != -1 && pid != 0); + if(pid <= 0) + { + return 0; + } + + sock.Close(); + sock.Open(Socket::TypeINET, "localhost", 1080); + + { + HTTPRequest request(HTTPRequest::Method_GET, "/nonexist"); + request.SetHostName("quotes.s3.amazonaws.com"); + request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:0cSX/YPdtXua1aFFpYmH1tc0ajA="); + request.SetClientKeepAliveRequested(true); + request.Send(sock, IOStream::TimeOutInfinite); + + HTTPResponse response; + response.Receive(sock); + std::string value; + TEST_EQUAL(404, response.GetResponseCode()); + } + + // Make file inaccessible, should cause server to return a 403 error, + // unless of course the test is run as root :) + { + TEST_THAT(chmod("testfiles/testrequests.pl", 0) == 0); + HTTPRequest request(HTTPRequest::Method_GET, + "/testrequests.pl"); + request.SetHostName("quotes.s3.amazonaws.com"); + request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:qc1e8u8TVl2BpIxwZwsursIb8U8="); + request.SetClientKeepAliveRequested(true); + request.Send(sock, IOStream::TimeOutInfinite); + + HTTPResponse response; + response.Receive(sock); + std::string value; + TEST_EQUAL(403, response.GetResponseCode()); + TEST_THAT(chmod("testfiles/testrequests.pl", 0755) == 0); + } + + { + HTTPRequest request(HTTPRequest::Method_GET, + "/testrequests.pl"); + request.SetHostName("quotes.s3.amazonaws.com"); + request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:qc1e8u8TVl2BpIxwZwsursIb8U8="); + request.SetClientKeepAliveRequested(true); + request.Send(sock, IOStream::TimeOutInfinite); + + HTTPResponse response; + response.Receive(sock); + std::string value; + TEST_EQUAL(200, response.GetResponseCode()); + TEST_EQUAL("qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY", response.GetHeaderValue("x-amz-id-2")); + TEST_EQUAL("F2A8CCCA26B4B26D", response.GetHeaderValue("x-amz-request-id")); + TEST_EQUAL("Wed, 01 Mar 2006 12:00:00 GMT", response.GetHeaderValue("Date")); + TEST_EQUAL("Sun, 1 Jan 2006 12:00:00 GMT", response.GetHeaderValue("Last-Modified")); + TEST_EQUAL("\"828ef3fdfa96f00ad9f27c383fc9ac7f\"", response.GetHeaderValue("ETag")); + TEST_EQUAL("text/plain", response.GetContentType()); + TEST_EQUAL("AmazonS3", response.GetHeaderValue("Server")); + + FileStream file("testfiles/testrequests.pl"); + TEST_THAT(file.CompareWith(response)); + } + + { + HTTPRequest request(HTTPRequest::Method_PUT, + "/newfile"); + request.SetHostName("quotes.s3.amazonaws.com"); + request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); + request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:kfY1m6V3zTufRy2kj92FpQGKz4M="); + request.AddHeader("Content-Type", "text/plain"); + FileStream fs("testfiles/testrequests.pl"); + HTTPResponse response; + request.SendWithStream(sock, + IOStream::TimeOutInfinite /* or 10000 milliseconds */, + &fs, response); + std::string value; + TEST_EQUAL(200, response.GetResponseCode()); + TEST_EQUAL("LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7", response.GetHeaderValue("x-amz-id-2")); + TEST_EQUAL("F2A8CCCA26B4B26D", response.GetHeaderValue("x-amz-request-id")); + TEST_EQUAL("Wed, 01 Mar 2006 12:00:00 GMT", response.GetHeaderValue("Date")); + TEST_EQUAL("Sun, 1 Jan 2006 12:00:00 GMT", response.GetHeaderValue("Last-Modified")); + TEST_EQUAL("\"828ef3fdfa96f00ad9f27c383fc9ac7f\"", response.GetHeaderValue("ETag")); + TEST_EQUAL("", response.GetContentType()); + TEST_EQUAL("AmazonS3", response.GetHeaderValue("Server")); + TEST_EQUAL(0, response.GetSize()); + + FileStream f1("testfiles/testrequests.pl"); + FileStream f2("testfiles/newfile"); + TEST_THAT(f1.CompareWith(f2)); + } + + // Kill it + TEST_THAT(KillServer(pid)); + TestRemoteProcessMemLeaks("generic-httpserver.memleaks"); + + return 0; +} + diff --git a/test/raidfile/testraidfile.cpp b/test/raidfile/testraidfile.cpp index 40703de5..f15fec27 100644 --- a/test/raidfile/testraidfile.cpp +++ b/test/raidfile/testraidfile.cpp @@ -622,7 +622,8 @@ int test(int argc, const char *argv[]) TEST_THAT(n2 != n3); TEST_THAT(n1 != n3); TEST_THAT(n1 == n4); // ie wraps around - TRACE3("Gen paths= '%s','%s',%s'\n", n1.c_str(), n2.c_str(), n3.c_str()); + BOX_TRACE("Gen paths = '" << n1 << "', '" << n2 << + "', '" << n3); } // Create a RaidFile -- cgit v1.2.3