diff options
| author | Reinhard Tartler <siretart@tauware.de> | 2018-02-20 21:49:13 -0500 |
|---|---|---|
| committer | Reinhard Tartler <siretart@tauware.de> | 2018-02-20 21:49:13 -0500 |
| commit | f28f88e5e72ba1499409047a9d6985eb312c0232 (patch) | |
| tree | c9c267f18264b3dfe715a363935bb6ac20904492 | |
| parent | e19a5db232e1ef90e9a02159d2fbd9707ffe4373 (diff) | |
| parent | 6d7e9562e8485591a4888f1fc2d3c6c657dc7a01 (diff) | |
Merge tag 'BoxBackup-0.12.master.180102.6d7e956' into upstream
303 files changed, 18659 insertions, 19544 deletions
diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..11b61ab9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +# https://help.github.com/articles/dealing-with-line-endings/ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Declare files that will always have CRLF line endings on checkout. +*.sln text eol=crlf +*.vcxproj.user text eol=crlf + +# Denote all files that are truly binary and should not be modified. +*.png binary +*.jpg binary diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..ad9bfba2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,111 @@ +# built from .in files by autoconf/CMake +bin/bbackupd/bbackupd-config +bin/bbstored/bbstored-certs +bin/bbstored/bbstored-config +contrib/debian/bbackupd +contrib/debian/bbstored +contrib/mac_osx/org.boxbackup.bbackupd.plist +contrib/mac_osx/org.boxbackup.bbstored.plist +contrib/redhat/bbackupd +contrib/redhat/bbstored +contrib/suse/bbackupd +contrib/suse/bbstored +contrib/solaris/bbackupd-manifest.xml +contrib/solaris/bbackupd-smf-method +contrib/solaris/bbstored-manifest.xml +contrib/solaris/bbstored-smf-method +contrib/windows/installer/boxbackup.mpi +infrastructure/BoxPlatform.pm +infrastructure/cmake/build +infrastructure/makebuildenv.pl +infrastructure/makedistribution.pl +infrastructure/makeparcels.pl +lib/bbackupquery/makedocumentation.pl +lib/common/BoxConfig.h +lib/common/BoxConfig.h.in +lib/common/BoxConfig.cmake.h +lib/common/BoxPortsAndFiles.h +lib/common/BoxVersion.h +lib/common/makeexception.pl +lib/raidfile/raidfile-config +lib/server/makeprotocol.pl +runtest.pl +test/backupstorefix/testfiles/testbackupstorefix.pl +test/bbackupd/testfiles/bbackupd.conf +test/bbackupd/testfiles/bbackupd-exclude.conf +test/bbackupd/testfiles/bbackupd-snapshot.conf +test/bbackupd/testfiles/bbackupd-symlink.conf +test/bbackupd/testfiles/bbackupd-temploc.conf +test/bbackupd/testfiles/extcheck1.pl +test/bbackupd/testfiles/extcheck2.pl +test/bbackupd/testfiles/notifyscript.pl +test/bbackupd/testfiles/syncallowscript.pl + +# from svn:ignore +aclocal.m4 +autom4te.cache +config.log +config.log.features +config.status +config.env +configure +configure.lineno +debug +release +parcels +ExceptionCodes.txt +local +Makefile +.hg +*.log +test/*/*.memleaks + +# built by makebuildenv.pl +test/*/_main.cpp +test/*/t +test/*/t-gdb +bin/*/autogen_*.cpp +bin/*/autogen_*.h +lib/*/autogen_*.cpp +lib/*/autogen_*.h +test/*/autogen_*.cpp +test/*/autogen_*.h + +# qdbm +qdbm/*.o +qdbm/libqdbm.a +qdbm/qdbm.pc +qdbm/qdbm.spec +qdbm/LTmakefile +qdbm/cbcodec +qdbm/cbtest +qdbm/crmgr +qdbm/crtest +qdbm/crtsv +qdbm/dpmgr +qdbm/dptest +qdbm/dptsv +qdbm/hvmgr +qdbm/hvtest +qdbm/libqdbm.so +qdbm/libqdbm.so.14 +qdbm/libqdbm.so.14.13.0 +qdbm/odidx +qdbm/odmgr +qdbm/odtest +qdbm/qmttest +qdbm/rlmgr +qdbm/rltest +qdbm/vlmgr +qdbm/vltest +qdbm/vltsv + +# development +*.orig +*.patch +.*.swo +.*.swp + +# generated documentation +docs/htmlguide +docs/man diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..57bd36f3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,48 @@ +language: cpp + +compiler: + - gcc + - clang + +cache: + - apt + - ccache + +sudo: false + +addons: + apt: + # We need cmake > 2.8.7 + # https://github.com/travis-ci/travis-ci/issues/4631#issuecomment-191153634 + sources: + - george-edison55-precise-backports # cmake 3.2.3 / doxygen 1.8.3 + packages: + - cmake + - cmake-data + - libboost-dev + - libdb-dev + - libreadline-dev + - libssl-dev + - libwww-perl + - time + - xsltproc + - zlib1g-dev + +env: + - TEST_TARGET=debug + - TEST_TARGET=release + - BUILD=cmake TEST_TARGET=debug + - BUILD=cmake TEST_TARGET=release + +os: + - linux + - osx + +matrix: + exclude: + # No point testing both GCC and Clang on Mac OS X, as they're the same thing. + - os: osx + compiler: gcc + +script: + - env EXTRA_MAKE_ARGS=-j2 ./infrastructure/travis-build.sh @@ -13,3 +13,4 @@ Bugs * more automated tests for win32 * change off_t to box_off_t in preparation for win32 large file support * support large files on win32 by using native *i64 functions instead of posix +* when storage is implemented, ensure that no write operations can happen to an account that's opened read-only (BackupStoreCheck without fix) diff --git a/COPYING.txt b/COPYING.txt index 54d27b62..8adb524c 100644 --- a/COPYING.txt +++ b/COPYING.txt @@ -1,6 +1,6 @@ Box Backup, http://www.boxbackup.org/ -Copyright (c) 2003-2010, Ben Summers and contributors. +Copyright (c) 2003-2015, Ben Summers and contributors. All rights reserved. The license of the code was changed on 23-Jan-2010 in order to meet the diff --git a/LICENSE-DUAL.txt b/LICENSE-DUAL.txt index 6aa31da2..3e555cfc 100644 --- a/LICENSE-DUAL.txt +++ b/LICENSE-DUAL.txt @@ -1,6 +1,6 @@ Box Backup, http://www.boxbackup.org/ -Copyright (c) 2003-2010, Ben Summers and contributors. +Copyright (c) 2003-2015, Ben Summers and contributors. All rights reserved. Note that this project uses mixed licensing. Any file with this license diff --git a/LICENSE-GPL.txt b/LICENSE-GPL.txt index 3e84b646..77d7e79b 100644 --- a/LICENSE-GPL.txt +++ b/LICENSE-GPL.txt @@ -1,6 +1,6 @@ Box Backup, http://www.boxbackup.org/ -Copyright (c) 2003-2010, Ben Summers and contributors. +Copyright (c) 2003-2015, Ben Summers and contributors. All rights reserved. Note that this project uses mixed licensing. Any file with this license diff --git a/LICENSE.txt b/LICENSE.txt index 70b513ac..dd4e26e1 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ Box Backup, http://www.boxbackup.org/
-Copyright (c) 2003-2010, Ben Summers and contributors.
+Copyright (c) 2003-2015, Ben Summers and contributors.
All rights reserved.
The license of the code was changed on 23-Jan-2010 in order to meet the
@@ -76,4 +76,4 @@ distribution The dual license text may be found in the file
LICENSE-DUAL.txt, or online at:
-[https://www.boxbackup.org/svn/box/trunk/LICENSE-DUAL.txt]
\ No newline at end of file +[https://www.boxbackup.org/svn/box/trunk/LICENSE-DUAL.txt]
diff --git a/README.md b/README.md new file mode 100644 index 00000000..6d3b5261 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# Box Backup + +[](https://travis-ci.org/boxbackup/boxbackup) +[](https://ci.appveyor.com/project/qris/boxbackup/branch/master) + +Box Backup is an open source, completely automatic, secure, encrypted on-line backup system. + +It has the following key features: + +* All backed up data is stored on the server in files on a filesystem - no tape, archive or other special devices are required. +* The server is trusted only to make files available when they are required - all data is encrypted and can be decoded only by the original client. This makes it ideal for backing up over an untrusted network (such as the Internet), or where the server is in an uncontrolled environment. +* A backup daemon runs on systems to be backed up, and copies encrypted data to the server when it notices changes - so backups are continuous and up-to-date (although traditional snapshot backups are possible too). +* Only changes within files are sent to the server, just like rsync, minimising the bandwidth used between clients and server. This makes it particularly suitable for backing up between distant locations, or over the Internet. +* It behaves like tape - old file versions and deleted files are available. +* Old versions of files on the server are stored as changes from the current version, minimising the storage space required on the server. Files are the server are also compressed to minimise their size. +* Choice of backup behaviour - it can be optimised for document or server backup. +* It is designed to be easy and cheap to run a server. It has a portable implementation, and optional RAID implemented in userland for reliability without complex server setup or expensive hardware. + +Please see the [website](https://www.boxbackup.org) for more information, including installation instructions. + +Box Backup is distributed under a [mixed BSD/GPL license](https://github.com/boxbackup/boxbackup/blob/master/LICENSE.txt). + diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..d75eff77 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,109 @@ +version: 0.12.{branch}.appveyor_{build} + +clone_depth: 1 + +# Do not build on tags (GitHub only) +skip_tags: true + +os: Windows Server 2012 + +platform: + - Win32 # CMake sucks + - x64 + +configuration: + - Debug + - Release + +environment: + VisualStudioVersion: 11.0 + Generator_Base: Visual Studio 11 2012 + OPENSSL_VERSION: 1.1.0f + PCRE_VERSION: 8.38 + CMAKE_UNIBUILD_DIR: '%APPVEYOR_BUILD_FOLDER%\..\cmake' + BOXBACKUP_VERSION_BASE: 0.12 + +init: +# Uncomment the following two lines to enable RDP access to the virtual machine for debugging. +# - reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp" /v UserAuthentication /t REG_DWORD /d 0 /f +# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + - ps: $env:date_string = Get-Date -Format "yyMMdd" + - ps: $env:sane_platform = $($env:PLATFORM.replace("Win32", "x86")) + # http://help.appveyor.com/discussions/problems/2874-how-can-i-add-commit-id-to-build-version + - ps: $env:boxbackup_version_full="$env:BOXBACKUP_VERSION_BASE.$env:APPVEYOR_REPO_BRANCH.$env:date_string.$($env:APPVEYOR_REPO_COMMIT.substring(0,7))" + - ps: Update-AppveyorBuild -Version "$env:boxbackup_version_full" + - ps: $env:compiled_version="$($env:boxbackup_version_full)_Win_$($env:sane_platform)_$($env:CONFIGURATION)" + # The only way to switch between 32-bit and 64-bit compilers appears to be to append " Win64" + # to the generator name if you want a 64-bit build (x64 platform): + - ps: $env:generator_name="$($env:Generator_Base)$(if ($env:PLATFORM.equals('x64')) {' Win64'})" + +# scripts that run after cloning repository (before the build step, not after!) +install: + # test_bbackupd needs 7zip (or cmake -E tar) to extract tar archives on Windows: + - cinst -y --limit-output 7zip.commandline nsis.portable + - dir "c:\Program Files" + - dir "c:\Program Files (x86)" + # We don't need strawberryperl on AppVeyor because there is already a Perl in c:\Perl. + # If you are doing this on a fresh box for development, you would probably want to + # install Chocolatey and then run: + # cinst -y cmake.portable strawberryperl git vim visualstudio2012wdx + # We install cmake.portable instead of cmake, to get it on the path again + # <http://disq.us/p/xdknrt>. + + - '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x86' + - if not exist %CMAKE_UNIBUILD_DIR% md %CMAKE_UNIBUILD_DIR% + - cd %CMAKE_UNIBUILD_DIR% + # We need to specify the generator here, in case the user has more than one installed. + # CMake always seems to default to the latest version of Visual Studio, not the one on + # the current PATH. + - echo cmake -G "%generator_name%" + -DBOXBACKUP_VERSION=%compiled_version% + -DSUB_CMAKE_EXTRA_ARGS="-- /verbosity:minimal" + %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows + - cmake -G "%generator_name%" + -DBOXBACKUP_VERSION=%compiled_version% + -DSUB_CMAKE_EXTRA_ARGS="-- /verbosity:minimal" + %APPVEYOR_BUILD_FOLDER%\infrastructure\cmake\windows + + # Leave the current directory in the correct place to find the solution file using its relative path above. + +# Remove Xamarin to remove 500 lines of junk from build logs +# http://help.appveyor.com/discussions/problems/4569-the-target-_convertpdbfiles-listed-in-a-beforetargets-attribute-at-c-does-not-exist-in-the-project-and-will-be-ignored +before_build: + - del "C:\Program Files (x86)\MSBuild\4.0\Microsoft.Common.Targets\ImportAfter\Xamarin.Common.targets" + +build: + parallel: true + project: ..\cmake\BoxBackup_Windows.sln + verbosity: minimal + +test_script: + - cd %CMAKE_UNIBUILD_DIR%\Build\boxbackup + # - dir + # - dir bin_bbackupd.dir + # - dir %PLATFORM% + # - dir %PLATFORM%\%CONFIGURATION% + - ctest -C %CONFIGURATION% -V --interactive-debug-mode 0 + + # After running tests successfully, build the artifacts that we want to upload: + - cmake --build . --config %CONFIGURATION% --target package + + # AppVeyor refuses to package files outside of the project directory, so we need to push them: + - dir + - appveyor PushArtifact -path BoxBackup-%compiled_version%.zip -DeploymentName BoxBackup-%compiled_version% + - appveyor PushArtifact -path BoxBackup-%compiled_version%.exe -DeploymentName BoxBackup-%compiled_version% + +deploy: + - provider: GitHub + release: BoxBackup-$(boxbackup_version_full) + artifact: BoxBackup-$(compiled_version) + description: "Windows client binaries auto-built by AppVeyor" + draft: false + prerelease: true + auth_token: + secure: WZi3MJGA5zIIAAij0if4auYeltJlyWUOePTYlCGvrNrgEVjYRkqILHzvVKDnLn43 + on: + branch: + - master + - windows_binary_packages + diff --git a/bin/bbackupctl/bbackupctl.cpp b/bin/bbackupctl/bbackupctl.cpp index 39f15baf..0e0c1e9c 100644 --- a/bin/bbackupctl/bbackupctl.cpp +++ b/bin/bbackupctl/bbackupctl.cpp @@ -11,14 +11,15 @@ #include <cstdio> #include <cstdlib> +#include <iostream> #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -#include <cstdlib> - +#include "box_getopt.h" #include "MainHelper.h" +#include "BackupDaemon.h" #include "BoxPortsAndFiles.h" #include "BackupDaemonConfigVerify.h" #include "Socket.h" @@ -37,12 +38,19 @@ enum Command WaitForSyncStart, WaitForSyncEnd, SyncAndWaitForEnd, + NoCommand, }; -void PrintUsageAndExit() +void PrintUsageAndExit(int ret) { - printf("Usage: bbackupctl [-q] [-c config_file] <command>\n" + std::cout << + "Usage: bbackupctl [options] <command>\n" + "\n" + "Options:\n" << + Logging::OptionParser::GetUsageString() << + "\n" "Commands are:\n" + " status -- report daemon status without changing anything\n" " sync -- start a synchronisation (backup) run now\n" " force-sync -- force the start of a synchronisation run, " "even if SyncAllowScript says no\n" @@ -51,8 +59,8 @@ void PrintUsageAndExit() " wait-for-sync -- wait until the next sync starts, then exit\n" " wait-for-end -- wait until the next sync finishes, then exit\n" " sync-and-wait -- start sync, wait until it finishes, then exit\n" - ); - exit(1); + ; + exit(ret); } int main(int argc, const char *argv[]) @@ -69,28 +77,27 @@ int main(int argc, const char *argv[]) // Filename for configuration file? std::string configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; - // Quiet? - bool quiet = false; - // See if there's another entry on the command line int c; - while((c = getopt(argc, (char * const *)argv, "qc:l:")) != -1) + std::string options("c:"); + options += Logging::OptionParser::GetOptionString(); + Logging::OptionParser LogLevel; + + while((c = getopt(argc, (char * const *)argv, options.c_str())) != -1) { switch(c) { - case 'q': - // Quiet mode - quiet = true; - break; - case 'c': // store argument configFilename = optarg; break; - case '?': default: - PrintUsageAndExit(); + int ret = LogLevel.ProcessOption(c); + if(ret != 0) + { + PrintUsageAndExit(ret); + } } } // Adjust arguments @@ -100,11 +107,13 @@ int main(int argc, const char *argv[]) // Check there's a command if(argc != 1) { - PrintUsageAndExit(); + PrintUsageAndExit(2); } + Logging::FilterConsole(LogLevel.GetCurrentLevel()); + // Read in the configuration file - if(!quiet) BOX_NOTICE("Using configuration file " << configFilename); + BOX_INFO("Using configuration file " << configFilename); std::string errs; std::auto_ptr<Configuration> config( @@ -164,7 +173,7 @@ int main(int argc, const char *argv[]) // Wait for the configuration summary std::string configSummary; - if(!getLine.GetLine(configSummary)) + if(!getLine.GetLine(configSummary, false, PROTOCOL_DEFAULT_TIMEOUT)) { BOX_ERROR("Failed to receive configuration summary " "from daemon"); @@ -188,19 +197,16 @@ int main(int argc, const char *argv[]) return 1; } // Print summary? - if(!quiet) - { - BOX_INFO("Daemon configuration summary:\n" - " AutomaticBackup = " << - (autoBackup?"true":"false") << "\n" - " UpdateStoreInterval = " << updateStoreInterval << - " seconds\n" - " MinimumFileAge = " << minimumFileAge << " seconds\n" - " MaxUploadWait = " << maxUploadWait << " seconds"); - } + BOX_TRACE("Daemon configuration summary:\n" + " AutomaticBackup = " << + (autoBackup?"true":"false") << "\n" + " UpdateStoreInterval = " << updateStoreInterval << + " seconds\n" + " MinimumFileAge = " << minimumFileAge << " seconds\n" + " MaxUploadWait = " << maxUploadWait << " seconds"); std::string stateLine; - if(!getLine.GetLine(stateLine) || getLine.IsEOF()) + if(!getLine.GetLine(stateLine, false, PROTOCOL_DEFAULT_TIMEOUT) || getLine.IsEOF()) { BOX_ERROR("Failed to receive state line from daemon"); return 1; @@ -214,23 +220,32 @@ int main(int argc, const char *argv[]) return 1; } + BOX_TRACE("Current state: " << + BackupDaemon::GetStateName(currentState)); + Command command = Default; std::string commandName(argv[0]); - if (commandName == "wait-for-sync") + if(commandName == "wait-for-sync") { command = WaitForSyncStart; } - else if (commandName == "wait-for-end") + else if(commandName == "wait-for-end") { command = WaitForSyncEnd; } - else if (commandName == "sync-and-wait") + else if(commandName == "sync-and-wait") { command = SyncAndWaitForEnd; } + else if(commandName == "status") + { + BOX_NOTICE("state " << + BackupDaemon::GetStateName(currentState)); + command = NoCommand; + } - switch (command) + switch(command) { case WaitForSyncStart: case WaitForSyncEnd: @@ -244,7 +259,6 @@ int main(int argc, const char *argv[]) "sync will never start!"); return 1; } - } break; @@ -253,10 +267,10 @@ int main(int argc, const char *argv[]) // send a sync command commandName = "force-sync"; std::string cmd = commandName + "\n"; - connection.Write(cmd.c_str(), cmd.size()); + connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT); connection.WriteAllBuffered(); - if (currentState != 0) + if(currentState != 0) { BOX_INFO("Waiting for current sync/error state " "to finish..."); @@ -266,11 +280,19 @@ int main(int argc, const char *argv[]) default: { - // Normal case, just send the command given - // plus a quit command. - std::string cmd = commandName; - cmd += "\nquit\n"; - connection.Write(cmd.c_str(), cmd.size()); + // Normal case, just send the command given, plus a + // quit command. + std::string cmd = commandName + "\n"; + connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT); + } + // fall through + + case NoCommand: + { + // Normal case, just send the command given plus a + // quit command. + std::string cmd = "quit\n"; + connection.Write(cmd, PROTOCOL_DEFAULT_TIMEOUT); } } @@ -279,18 +301,33 @@ int main(int argc, const char *argv[]) bool syncIsRunning = false; bool finished = false; - while(!finished && !getLine.IsEOF() && getLine.GetLine(line)) + while(command != NoCommand && !finished && !getLine.IsEOF() && + getLine.GetLine(line, false, PROTOCOL_DEFAULT_TIMEOUT)) { - switch (command) + BOX_TRACE("Received line: " << line); + + if(line.substr(0, 6) == "state ") + { + std::string state_str = line.substr(6); + int state_num; + if(sscanf(state_str.c_str(), "%d", &state_num) == 1) + { + BOX_INFO("Daemon state changed to: " << + BackupDaemon::GetStateName(state_num)); + } + else + { + BOX_WARNING("Failed to parse line: " << line); + } + } + + switch(command) { case WaitForSyncStart: { // Need to wait for the state change... if(line == "start-sync") { - // Send a quit command to finish nicely - connection.Write("quit\n", 5); - // And we're done finished = true; } @@ -302,23 +339,20 @@ int main(int argc, const char *argv[]) { if(line == "start-sync") { - if (!quiet) BOX_INFO("Sync started..."); + BOX_TRACE("Sync started..."); syncIsRunning = true; } else if(line == "finish-sync") { if (syncIsRunning) { - if (!quiet) BOX_INFO("Sync finished."); - // Send a quit command to finish nicely - connection.Write("quit\n", 5); - // And we're done + BOX_TRACE("Sync finished."); finished = true; } else { - if (!quiet) BOX_INFO("Previous sync finished."); + BOX_TRACE("Previous sync finished."); } // daemon must still be busy } @@ -330,12 +364,9 @@ int main(int argc, const char *argv[]) // Is this an OK or error line? if(line == "ok") { - if(!quiet) - { - BOX_INFO("Control command " - "sent: " << - commandName); - } + BOX_TRACE("Control command " + "sent: " << + commandName); finished = true; } else if(line == "error") @@ -350,6 +381,9 @@ int main(int argc, const char *argv[]) } } + // Send a quit command to finish nicely + connection.Write("quit\n", 5, PROTOCOL_DEFAULT_TIMEOUT); + MAINHELPER_END #if defined WIN32 && ! defined BOX_RELEASE_BUILD diff --git a/bin/bbackupd/bbackupd.cpp b/bin/bbackupd/bbackupd.cpp index bb64f745..5977615f 100644 --- a/bin/bbackupd/bbackupd.cpp +++ b/bin/bbackupd/bbackupd.cpp @@ -45,10 +45,12 @@ int main(int argc, const char *argv[]) #else // !WIN32 - BackupDaemon daemon; - ExitCode = daemon.Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE, - argc, argv); - + { + BackupDaemon daemon; + ExitCode = daemon.Main(BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE, + argc, argv); + } + #endif // WIN32 MAINHELPER_END diff --git a/bin/bbackupquery/Makefile.extra b/bin/bbackupquery/Makefile.extra deleted file mode 100644 index e1049b6d..00000000 --- a/bin/bbackupquery/Makefile.extra +++ /dev/null @@ -1,6 +0,0 @@ - -# AUTOGEN SEEDING -autogen_Documentation.cpp: makedocumentation.pl documentation.txt - $(_PERL) makedocumentation.pl - - diff --git a/bin/bbackupquery/bbackupquery.cpp b/bin/bbackupquery/bbackupquery.cpp index 5493f49c..e10c48fe 100644 --- a/bin/bbackupquery/bbackupquery.cpp +++ b/bin/bbackupquery/bbackupquery.cpp @@ -66,11 +66,6 @@ void PrintUsageAndExit() "Usage: bbackupquery [options] [command]...\n" "\n" "Options:\n" - " -q Run more quietly, reduce verbosity level by one, can repeat\n" - " -Q Run at minimum verbosity, log nothing\n" - " -v Run more verbosely, increase verbosity level by one, can repeat\n" - " -V Run at maximum verbosity, log everything\n" - " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n" " -w Read/write mode, allow changes to store\n" #ifdef WIN32 " -u Enable Unicode console, requires font change to Lucida Console\n" @@ -80,11 +75,13 @@ void PrintUsageAndExit() #endif " -c <file> Use the specified configuration file. If -c is omitted, the last\n" " argument is the configuration file, or else the default \n" - " [" << BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE << - "]\n" + " [" << BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE << "]\n" " -o <file> Write logging output to specified file as well as console\n" " -O <level> Set file verbosity to error/warning/notice/info/trace/everything\n" " -l <file> Write protocol debugging logs to specified file\n" + << + Logging::OptionParser::GetUsageString() + << "\n" "Parameters: as many commands as you like. If commands are multiple words,\n" "remember to enclose the command in quotes. Remember to use the quit command\n" @@ -154,6 +151,7 @@ char * completion_generator(const char *text, int state) if(state < 0 || state >= (int) completions.size()) { rl_attempted_completion_over = 1; + sapCmd.reset(); return NULL; } @@ -219,12 +217,6 @@ int main(int argc, const char *argv[]) Logging::SetProgramName("bbackupquery"); - #ifdef BOX_RELEASE_BUILD - int masterLevel = Log::NOTICE; // need an int to do math with - #else - int masterLevel = Log::INFO; // need an int to do math with - #endif - #ifdef WIN32 #define WIN32_OPTIONS "u" bool unicodeConsole = false; @@ -239,60 +231,19 @@ int main(int argc, const char *argv[]) #define READLINE_OPTIONS #endif - const char* validOpts = "qvVwc:l:o:O:W:" WIN32_OPTIONS READLINE_OPTIONS; + std::string options("wc:l:o:O:" WIN32_OPTIONS READLINE_OPTIONS); + options += Logging::OptionParser::GetOptionString(); + Logging::OptionParser LogLevel; std::string fileLogFile; Log::Level fileLogLevel = Log::INVALID; // See if there's another entry on the command line int c; - while((c = getopt(argc, (char * const *)argv, validOpts)) != -1) + while((c = getopt(argc, (char * const *)argv, options.c_str())) != -1) { switch(c) { - case 'q': - { - if(masterLevel == Log::NOTHING) - { - BOX_FATAL("Too many '-q': " - "Cannot reduce logging " - "level any more"); - return 2; - } - masterLevel--; - } - break; - - case 'v': - { - if(masterLevel == Log::EVERYTHING) - { - BOX_FATAL("Too many '-v': " - "Cannot increase logging " - "level any more"); - return 2; - } - masterLevel++; - } - break; - - case 'V': - { - masterLevel = Log::EVERYTHING; - } - break; - - case 'W': - { - masterLevel = Logging::GetNamedLevel(optarg); - if (masterLevel == Log::INVALID) - { - BOX_FATAL("Invalid logging level"); - return 2; - } - } - break; - case 'w': // Read/write mode readWrite = true; @@ -341,36 +292,28 @@ int main(int argc, const char *argv[]) break; #endif - case '?': default: - PrintUsageAndExit(); + int ret = LogLevel.ProcessOption(c); + if (ret != 0) + { + PrintUsageAndExit(); + } } } // Adjust arguments argc -= optind; argv += optind; - Logging::SetGlobalLevel((Log::Level)masterLevel); + Logging::GetConsole().Filter(LogLevel.GetCurrentLevel()); std::auto_ptr<FileLogger> fileLogger; if (fileLogLevel != Log::INVALID) { - fileLogger.reset(new FileLogger(fileLogFile, fileLogLevel)); - } - - bool quiet = false; - if (masterLevel < Log::NOTICE) - { - // Quiet mode - quiet = true; + fileLogger.reset(new FileLogger(fileLogFile, fileLogLevel, + true)); // open in append mode } - // Print banner? - if(!quiet) - { - const char *banner = BANNER_TEXT("Backup Query Tool"); - BOX_NOTICE(banner); - } + BOX_NOTICE(BANNER_TEXT("Backup Query Tool")); #ifdef WIN32 if (unicodeConsole) @@ -398,7 +341,7 @@ int main(int argc, const char *argv[]) #endif // WIN32 // Read in the configuration file - if(!quiet) BOX_INFO("Using configuration file " << configFilename); + BOX_INFO("Using configuration file " << configFilename); std::string errs; std::auto_ptr<Configuration> config( @@ -427,16 +370,17 @@ int main(int argc, const char *argv[]) BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str()); // 2. Connect to server - if(!quiet) BOX_INFO("Connecting to store..."); - SocketStreamTLS socket; - socket.Open(tlsContext, Socket::TypeINET, + BOX_INFO("Connecting to store..."); + SocketStreamTLS *socket = new SocketStreamTLS; + std::auto_ptr<SocketStream> apSocket(socket); + socket->Open(tlsContext, Socket::TypeINET, conf.GetKeyValue("StoreHostname").c_str(), conf.GetKeyValueInt("StorePort")); // 3. Make a protocol, and handshake - if(!quiet) BOX_INFO("Handshake with store..."); + BOX_INFO("Handshake with store..."); std::auto_ptr<BackupProtocolClient> - apConnection(new BackupProtocolClient(socket)); + apConnection(new BackupProtocolClient(apSocket)); BackupProtocolClient& connection(*(apConnection.get())); connection.Handshake(); @@ -447,7 +391,7 @@ int main(int argc, const char *argv[]) } // 4. Log in to server - if(!quiet) BOX_INFO("Login to store..."); + BOX_INFO("Login to store..."); // Check the version of the server { std::auto_ptr<BackupProtocolVersion> serverVersion(connection.QueryVersion(BACKUP_STORE_SERVER_VERSION)); @@ -461,7 +405,8 @@ int main(int argc, const char *argv[]) (readWrite)?0:(BackupProtocolLogin::Flags_ReadOnly)); // 5. Tell user. - if(!quiet) printf("Login complete.\n\nType \"help\" for a list of commands.\n\n"); + BOX_INFO("Login complete."); + BOX_INFO("Type \"help\" for a list of commands."); // Set up a context for our work BackupQueries context(connection, conf, readWrite); @@ -525,6 +470,7 @@ int main(int argc, const char *argv[]) if(cmd_ptr == NULL) { // Ctrl-D pressed -- terminate now + puts(""); break; } @@ -569,9 +515,9 @@ int main(int argc, const char *argv[]) } // Done... stop nicely - if(!quiet) BOX_INFO("Logging off..."); + BOX_INFO("Logging off..."); connection.QueryFinished(); - if(!quiet) BOX_INFO("Session finished."); + BOX_INFO("Session finished."); // Return code returnCode = context.GetReturnCode(); diff --git a/bin/bbstoreaccounts/bbstoreaccounts.cpp b/bin/bbstoreaccounts/bbstoreaccounts.cpp index 6a5c2e33..6a173680 100644 --- a/bin/bbstoreaccounts/bbstoreaccounts.cpp +++ b/bin/bbstoreaccounts/bbstoreaccounts.cpp @@ -18,12 +18,7 @@ #include <sys/types.h> -#include <algorithm> -#include <cstring> -#include <iostream> -#include <ostream> -#include <vector> - +#include "box_getopt.h" #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreCheck.h" @@ -32,7 +27,6 @@ #include "BoxPortsAndFiles.h" #include "HousekeepStoreAccount.h" #include "MainHelper.h" -#include "NamedLock.h" #include "RaidFileController.h" #include "StoreStructure.h" #include "UnixUser.h" @@ -42,476 +36,6 @@ #include <cstring> -// max size of soft limit as percent of hard limit -#define MAX_SOFT_LIMIT_SIZE 97 - -bool sMachineReadableOutput = false; - -void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit) -{ - if(SoftLimit > HardLimit) - { - BOX_FATAL("Soft limit must be less than the hard limit."); - exit(1); - } - if(SoftLimit > ((HardLimit * MAX_SOFT_LIMIT_SIZE) / 100)) - { - BOX_WARNING("We recommend setting the soft limit below " << - MAX_SOFT_LIMIT_SIZE << "% of the hard limit, or " << - HumanReadableSize((HardLimit * MAX_SOFT_LIMIT_SIZE) - / 100) << " in this case."); - } -} - -int BlockSizeOfDiscSet(int discSetNum) -{ - // Get controller, check disc set number - RaidFileController &controller(RaidFileController::GetController()); - if(discSetNum < 0 || discSetNum >= controller.GetNumDiscSets()) - { - BOX_FATAL("Disc set " << discSetNum << " does not exist."); - exit(1); - } - - // Return block size - return controller.GetDiscSet(discSetNum).GetBlockSize(); -} - -std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int discSetNum) -{ - return FormatUsageBar(Blocks, Blocks * BlockSizeOfDiscSet(discSetNum), - MaxBlocks * BlockSizeOfDiscSet(discSetNum), - sMachineReadableOutput); -} - -int64_t SizeStringToBlocks(const char *string, int discSetNum) -{ - // Find block size - int blockSize = BlockSizeOfDiscSet(discSetNum); - - // Get number - char *endptr = (char*)string; - int64_t number = strtol(string, &endptr, 0); - if(endptr == string || number == LONG_MIN || number == LONG_MAX) - { - BOX_FATAL("'" << string << "' is not a valid number."); - exit(1); - } - - // Check units - switch(*endptr) - { - case 'M': - case 'm': - // Units: Mb - return (number * 1024*1024) / blockSize; - break; - - case 'G': - case 'g': - // Units: Gb - return (number * 1024*1024*1024) / blockSize; - break; - - case 'B': - case 'b': - // Units: Blocks - // Easy! Just return the number specified. - return number; - break; - - default: - BOX_FATAL(string << " has an invalid units specifier " - "(use B for blocks, M for MB, G for GB, eg 2GB)"); - exit(1); - break; - } -} - -bool OpenAccount(Configuration &rConfig, int32_t ID, std::string &rRootDirOut, - int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock); - -int SetLimit(Configuration &rConfig, int32_t ID, const char *SoftLimitStr, - const char *HardLimitStr) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr<UnixUser> user; // used to reset uid when we return - NamedLock writeLock; - - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " to change limits."); - return 1; - } - - // Load the info - std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, - discSetNum, false /* Read/Write */)); - - // Change the limits - int64_t softlimit = SizeStringToBlocks(SoftLimitStr, discSetNum); - int64_t hardlimit = SizeStringToBlocks(HardLimitStr, discSetNum); - CheckSoftHardLimits(softlimit, hardlimit); - info->ChangeLimits(softlimit, hardlimit); - - // Save - info->Save(); - - BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) << - " changed to " << softlimit << " soft, " << - hardlimit << " hard."); - - return 0; -} - -int SetAccountName(Configuration &rConfig, int32_t ID, - const std::string& rNewAccountName) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr<UnixUser> user; // used to reset uid when we return - NamedLock writeLock; - - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " to change name."); - return 1; - } - - // Load the info - std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, - rootDir, discSetNum, false /* Read/Write */)); - - info->SetAccountName(rNewAccountName); - - // Save - info->Save(); - - BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << - " name changed to " << rNewAccountName); - - return 0; -} - -int AccountInfo(Configuration &rConfig, int32_t ID) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr<UnixUser> user; // used to reset uid when we return - - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, - NULL /* no write lock needed for this read-only operation */)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " to display info."); - return 1; - } - - // Load it in - std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, - rootDir, discSetNum, true /* ReadOnly */)); - - // Then print out lots of info - std::cout << FormatUsageLineStart("Account ID", sMachineReadableOutput) << - BOX_FORMAT_ACCOUNT(ID) << std::endl; - std::cout << FormatUsageLineStart("Account Name", sMachineReadableOutput) << - info->GetAccountName() << std::endl; - std::cout << FormatUsageLineStart("Last object ID", sMachineReadableOutput) << - BOX_FORMAT_OBJECTID(info->GetLastObjectIDUsed()) << std::endl; - std::cout << FormatUsageLineStart("Used", sMachineReadableOutput) << - BlockSizeToString(info->GetBlocksUsed(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Current files", - sMachineReadableOutput) << - BlockSizeToString(info->GetBlocksInCurrentFiles(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Old files", sMachineReadableOutput) << - BlockSizeToString(info->GetBlocksInOldFiles(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Deleted files", sMachineReadableOutput) << - BlockSizeToString(info->GetBlocksInDeletedFiles(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Directories", sMachineReadableOutput) << - BlockSizeToString(info->GetBlocksInDirectories(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Soft limit", sMachineReadableOutput) << - BlockSizeToString(info->GetBlocksSoftLimit(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Hard limit", sMachineReadableOutput) << - BlockSizeToString(info->GetBlocksHardLimit(), - info->GetBlocksHardLimit(), discSetNum) << std::endl; - std::cout << FormatUsageLineStart("Client store marker", sMachineReadableOutput) << - info->GetLastObjectIDUsed() << std::endl; - std::cout << FormatUsageLineStart("Live Files", sMachineReadableOutput) << - info->GetNumFiles() << std::endl; - std::cout << FormatUsageLineStart("Old Files", sMachineReadableOutput) << - info->GetNumOldFiles() << std::endl; - std::cout << FormatUsageLineStart("Deleted Files", sMachineReadableOutput) << - info->GetNumDeletedFiles() << std::endl; - std::cout << FormatUsageLineStart("Directories", sMachineReadableOutput) << - info->GetNumDirectories() << std::endl; - std::cout << FormatUsageLineStart("Enabled", sMachineReadableOutput) << - (info->IsAccountEnabled() ? "yes" : "no") << std::endl; - - return 0; -} - -int SetAccountEnabled(Configuration &rConfig, int32_t ID, bool enabled) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr<UnixUser> user; // used to reset uid when we return - NamedLock writeLock; - - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " to change enabled flag."); - return 1; - } - - // Load it in - std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, - rootDir, discSetNum, false /* ReadOnly */)); - info->SetAccountEnabled(enabled); - info->Save(); - return 0; -} - -int DeleteAccount(Configuration &rConfig, int32_t ID, bool AskForConfirmation) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr<UnixUser> user; // used to reset uid when we return - NamedLock writeLock; - - // Obtain a write lock, as the daemon user - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " for deletion."); - return 1; - } - - // Check user really wants to do this - if(AskForConfirmation) - { - BOX_WARNING("Really delete account " << - BOX_FORMAT_ACCOUNT(ID) << "? (type 'yes' to confirm)"); - char response[256]; - if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0) - { - BOX_NOTICE("Deletion cancelled."); - return 0; - } - } - - // Back to original user, but write lock is maintained - user.reset(); - - std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); - - // Delete from account database - db->DeleteEntry(ID); - - // Write back to disc - db->Write(); - - // Remove the store files... - - // First, become the user specified in the config file - std::string username; - { - const Configuration &rserverConfig(rConfig.GetSubConfiguration("Server")); - if(rserverConfig.KeyExists("User")) - { - username = rserverConfig.GetKeyValue("User"); - } - } - - // Become the right user - if(!username.empty()) - { - // Username specified, change... - user.reset(new UnixUser(username)); - user->ChangeProcessUser(true /* temporary */); - // Change will be undone when user goes out of scope - } - - // Secondly, work out which directories need wiping - std::vector<std::string> toDelete; - RaidFileController &rcontroller(RaidFileController::GetController()); - RaidFileDiscSet discSet(rcontroller.GetDiscSet(discSetNum)); - for(RaidFileDiscSet::const_iterator i(discSet.begin()); i != discSet.end(); ++i) - { - if(std::find(toDelete.begin(), toDelete.end(), *i) == toDelete.end()) - { - toDelete.push_back((*i) + DIRECTORY_SEPARATOR + rootDir); - } - } - -#ifdef WIN32 - // Cannot remove files while holding a lock on them - writeLock.ReleaseLock(); -#endif - - int retcode = 0; - - // Thirdly, delete the directories... - for(std::vector<std::string>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) - { - BOX_NOTICE("Deleting store directory " << (*d) << "..."); - // Just use the rm command to delete the files - std::string cmd("rm -rf "); - cmd += *d; - // Run command - if(::system(cmd.c_str()) != 0) - { - BOX_ERROR("Failed to delete files in " << (*d) << - ", delete them manually."); - retcode = 1; - } - } - - // Success! - return retcode; -} - -bool OpenAccount(Configuration &rConfig, int32_t ID, std::string &rRootDirOut, - int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock) -{ - // Load in the account database - std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); - - // Exists? - if(!db->EntryExists(ID)) - { - BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << - " does not exist."); - return false; - } - - // Get info from the database - BackupStoreAccounts acc(*db); - acc.GetAccountRoot(ID, rRootDirOut, rDiscSetOut); - - // Get the user under which the daemon runs - std::string username; - { - const Configuration &rserverConfig(rConfig.GetSubConfiguration("Server")); - if(rserverConfig.KeyExists("User")) - { - username = rserverConfig.GetKeyValue("User"); - } - } - - // Become the right user - if(!username.empty()) - { - // Username specified, change... - apUser.reset(new UnixUser(username)); - apUser->ChangeProcessUser(true /* temporary */); - // Change will be undone when apUser goes out of scope - // in the caller. - } - - if(pLock) - { - acc.LockAccount(ID, *pLock); - } - - return true; -} - -int CheckAccount(Configuration &rConfig, int32_t ID, bool FixErrors, bool Quiet) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr<UnixUser> user; // used to reset uid when we return - NamedLock writeLock; - - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, &writeLock)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " for checking."); - return 1; - } - - // Check it - BackupStoreCheck check(rootDir, discSetNum, ID, FixErrors, Quiet); - check.Check(); - - return check.ErrorsFound()?1:0; -} - -int CreateAccount(Configuration &rConfig, int32_t ID, int32_t DiscNumber, - int32_t SoftLimit, int32_t HardLimit) -{ - // Load in the account database - std::auto_ptr<BackupStoreAccountDatabase> db(BackupStoreAccountDatabase::Read(rConfig.GetKeyValue("AccountDatabase").c_str())); - - // Already exists? - if(db->EntryExists(ID)) - { - BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << - " already exists."); - return 1; - } - - // Get the user under which the daemon runs - std::string username; - { - const Configuration &rserverConfig(rConfig.GetSubConfiguration("Server")); - if(rserverConfig.KeyExists("User")) - { - username = rserverConfig.GetKeyValue("User"); - } - } - - // Create it. - BackupStoreAccounts acc(*db); - acc.Create(ID, DiscNumber, SoftLimit, HardLimit, username); - - BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " created."); - - return 0; -} - -int HousekeepAccountNow(Configuration &rConfig, int32_t ID) -{ - std::string rootDir; - int discSetNum; - std::auto_ptr<UnixUser> user; // used to reset uid when we return - - if(!OpenAccount(rConfig, ID, rootDir, discSetNum, user, - NULL /* housekeeping locks the account itself */)) - { - BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) - << " for housekeeping."); - return 1; - } - - HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL); - bool success = housekeeping.DoHousekeeping(); - - if(!success) - { - BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID) - << " for housekeeping: perhaps a client is " - "still connected?"); - return 1; - } - else - { - BOX_TRACE("Finished housekeeping on account " << - BOX_FORMAT_ACCOUNT(ID)); - return 0; - } -} - void PrintUsageAndExit() { printf( @@ -562,8 +86,9 @@ int main(int argc, const char *argv[]) Logging::SetProgramName("bbstoreaccounts"); // Filename for configuration file? - std::string configFilename = BOX_GET_DEFAULT_BBACKUPD_CONFIG_FILE; + std::string configFilename = BOX_GET_DEFAULT_BBSTORED_CONFIG_FILE; int logLevel = Log::EVERYTHING; + bool machineReadableOutput = false; // See if there's another entry on the command line int c; @@ -587,7 +112,7 @@ int main(int argc, const char *argv[]) case 'm': // enable machine readable output - sMachineReadableOutput = true; + machineReadableOutput = true; break; case '?': @@ -614,7 +139,7 @@ int main(int argc, const char *argv[]) BOX_ERROR("Invalid configuration file " << configFilename << ":" << errs); } - + // Initialise the raid file controller RaidFileController &rcontroller(RaidFileController::GetController()); rcontroller.Initialise(config->GetKeyValue("RaidFileConf").c_str()); @@ -633,6 +158,7 @@ int main(int argc, const char *argv[]) } std::string command = argv[0]; + BackupStoreAccountsControl control(*config, machineReadableOutput); // Now do the command. if(command == "create") @@ -650,17 +176,18 @@ int main(int argc, const char *argv[]) } // Decode limits - softlimit = SizeStringToBlocks(argv[3], discnum); - hardlimit = SizeStringToBlocks(argv[4], discnum); - CheckSoftHardLimits(softlimit, hardlimit); + int blocksize = control.BlockSizeOfDiscSet(discnum); + softlimit = control.SizeStringToBlocks(argv[3], blocksize); + hardlimit = control.SizeStringToBlocks(argv[4], blocksize); + control.CheckSoftHardLimits(softlimit, hardlimit); // Create the account... - return CreateAccount(*config, id, discnum, softlimit, hardlimit); + return control.CreateAccount(id, discnum, softlimit, hardlimit); } else if(command == "info") { // Print information on this account - return AccountInfo(*config, id); + return control.PrintAccountInfo(id); } else if(command == "enabled") { @@ -685,7 +212,7 @@ int main(int argc, const char *argv[]) PrintUsageAndExit(); } - return SetAccountEnabled(*config, id, enabled); + return control.SetAccountEnabled(id, enabled); } else if(command == "setlimit") { @@ -696,7 +223,7 @@ int main(int argc, const char *argv[]) return 1; } - return SetLimit(*config, id, argv[2], argv[3]); + return control.SetLimit(id, argv[2], argv[3]); } else if(command == "name") { @@ -707,7 +234,7 @@ int main(int argc, const char *argv[]) return 1; } - return SetAccountName(*config, id, argv[2]); + return control.SetAccountName(id, argv[2]); } else if(command == "delete") { @@ -717,7 +244,7 @@ int main(int argc, const char *argv[]) { askForConfirmation = false; } - return DeleteAccount(*config, id, askForConfirmation); + return control.DeleteAccount(id, askForConfirmation); } else if(command == "check") { @@ -743,11 +270,11 @@ int main(int argc, const char *argv[]) } // Check the account - return CheckAccount(*config, id, fixErrors, quiet); + return control.CheckAccount(id, fixErrors, quiet); } else if(command == "housekeep") { - return HousekeepAccountNow(*config, id); + return control.HousekeepAccountNow(id); } else { diff --git a/bin/bbstored/bbstored-certs.in b/bin/bbstored/bbstored-certs.in index 85560748..00085662 100755 --- a/bin/bbstored/bbstored-certs.in +++ b/bin/bbstored/bbstored-certs.in @@ -288,7 +288,7 @@ sub get_csr_common_name my $subject; while(<CSRTEXT>) { - $subject = $1 if m/Subject:.+?CN=([-\.\w]+)/ + $subject = $1 if m/Subject:.+?CN\s?=\s?([-\.\w]+)/; } close CSRTEXT; @@ -3,3 +3,4 @@ aclocal -I infrastructure/m4 autoheader autoconf +(cd qdbm; autoconf) diff --git a/configure.ac b/configure.ac index c5a05fe4..f9f481d5 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,9 @@ AC_PREREQ(2.59) AC_INIT([Box Backup], 0.11, [boxbackup@boxbackup.org],[boxbackup]) AC_CONFIG_SRCDIR([lib/common/Box.h]) +AC_CONFIG_AUX_DIR([infrastructure]) AC_CONFIG_HEADERS([lib/common/BoxConfig.h]) +AC_CONFIG_SUBDIRS([qdbm]) touch install-sh AC_CANONICAL_SYSTEM @@ -93,7 +95,6 @@ AC_CONFIG_FILES([infrastructure/BoxPlatform.pm test/bbackupd/testfiles/bbackupd-temploc.conf ]) AX_CONFIG_SCRIPTS([bin/bbackupd/bbackupd-config - bin/bbackupquery/makedocumentation.pl bin/bbstored/bbstored-certs bin/bbstored/bbstored-config contrib/debian/bbackupd @@ -106,8 +107,8 @@ AX_CONFIG_SCRIPTS([bin/bbackupd/bbackupd-config contrib/solaris/bbstored-smf-method contrib/windows/installer/boxbackup.mpi infrastructure/makebuildenv.pl - infrastructure/makeparcels.pl infrastructure/makedistribution.pl + lib/bbackupquery/makedocumentation.pl lib/common/makeexception.pl lib/raidfile/raidfile-config lib/server/makeprotocol.pl @@ -122,27 +123,11 @@ AC_OUTPUT # Configure the Box build system echo -if ! $PERL ./infrastructure/makebuildenv.pl \ -|| ! $PERL ./infrastructure/makeparcels.pl; then +if ! $PERL ./infrastructure/makebuildenv.pl; then echo "Making infrastructure failed!" exit 1 fi -cat parcels.txt | sed -e 's/#.*//' | while read cmd subdir configure_args; do - if test "$cmd" = "subdir"; then - echo - export CC CXX CFLAGS CXXFLAGS LDFLAGS LIBS - args="$configure_args --target=$target_alias" - echo "Configuring $subdir with: $args" - - cd $subdir - if ! ./configure $args; then - echo "Configuring $subdir with $args failed!" >&2 - exit 1 - fi - fi -done || exit $? - # Write summary of important info tee config.log.features <<EOC A summary of the build configuration is below. Box Backup will function diff --git a/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt index 70a3a7bc..31e16d34 100644 --- a/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt +++ b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt @@ -37,8 +37,6 @@ docs/api-notes/raidfile docs/api-notes/raidfile/lib_raidfile.txt LICENSE GPL -docs/api-notes/backup - docs/images docs/htmlguide docs/htmlguide/adminguide diff --git a/distribution/boxbackup/VERSION.txt b/distribution/boxbackup/VERSION.txt index 0188cacd..89770ba5 100644 --- a/distribution/boxbackup/VERSION.txt +++ b/distribution/boxbackup/VERSION.txt @@ -1,2 +1,2 @@ -0.12 +0.13 boxbackup diff --git a/docs/api-notes/win32_build_on_cygwin_using_mingw.txt b/docs/api-notes/win32_build_on_cygwin_using_mingw.txt index 34dbac0b..5b413cd3 100644 --- a/docs/api-notes/win32_build_on_cygwin_using_mingw.txt +++ b/docs/api-notes/win32_build_on_cygwin_using_mingw.txt @@ -11,14 +11,14 @@ Start by installing Cygwin on your Windows machine from [http://www.cygwin.org/c Make sure to select the following packages during installation: * Archive/unzip - * Devel/automake * Devel/autoconf - * Devel/gcc-mingw - * Devel/gcc-mingw-g++ + * Devel/automake * Devel/make + * Devel/mingw64-x86_64-gcc + * Devel/mingw64-x86_64-gcc-g++ + * Devel/mingw64-x86_64-zlib * Libs/libxml2 * Libs/libxslt (for xsltproc) - * Mingw/mingw-zlib-devel * Perl/Perl If you already have Cygwin installed, please re-run the installer and @@ -45,16 +45,16 @@ your user's home directory in /etc/passwd to match. == OpenSSL == -Download OpenSSL from [http://www.openssl.org/source/openssl-1.0.0e.tar.gz] +Download OpenSSL from [http://www.openssl.org/source/openssl-1.0.1f.tar.gz] Open a Cygwin shell, go to the base directory, and unpack OpenSSL: - tar xzvf openssl-1.0.0e.tar.gz + tar xzvf openssl-1.0.1f.tar.gz Configure OpenSSL for MinGW compilation, and build and install it: - cd openssl-1.0.0e - ./Configure --prefix=/usr/i686-pc-mingw32/ mingw + cd openssl-1.0.1f + ./Configure --prefix=/usr/x86_64-w64-mingw32 mingw64 --cross-compile-prefix=x86_64-w64-mingw32- make make install_sw @@ -63,17 +63,15 @@ Configure OpenSSL for MinGW compilation, and build and install it: This step is only required to support regular expressions in including/excluding files from backups. However, this is a very useful feature. Download PCRE from -[http://prdownloads.sourceforge.net/pcre/pcre-8.12.tar.bz2?download]. +[http://prdownloads.sourceforge.net/pcre/pcre-8.34.tar.bz2?download]. Open a Cygwin shell, go to the base directory, and unpack, build and install PCRE: - tar xjvf pcre-8.12.tar.bz2 - cd pcre-8.12 - export CFLAGS="-mno-cygwin" - export CXXFLAGS="-mno-cygwin" - ./configure --prefix=/usr/i686-pc-mingw32 --disable-shared - make + tar xjvf pcre-8.34.tar.bz2 + cd pcre-8.34 + ./configure --prefix=/usr/x86_64-w64-mingw32 --disable-shared --host=x86_64-w64-mingw32 + make make install Note: we must disable shared libraries on Windows because otherwise @@ -97,7 +95,7 @@ unpack it, compile and install it: tar xzvf PDCurses-3.4.tar.gz cd PDCurses-3.4/win32 make -f gccwin32.mak CC="gcc -mno-cygwin" LINK="gcc -mno-cygwin" - cp pdcurses.a /usr/i686-pc-mingw32/lib/libpdcurses.a + cp pdcurses.a /usr/x86_64-w64-mingw32/lib/libpdcurses.a Download Readline version 6.2 from [ftp://ftp.cwru.edu/pub/bash/readline-6.2.tar.gz], and unpack @@ -105,12 +103,10 @@ and install it thus: tar xzvf readline-6.2.tar.gz cd readline-6.2 - ./configure --prefix=/usr/i686-pc-mingw32 \ - CFLAGS="-mno-cygwin" \ - CPPFLAGS="-mno-cygwin" \ - LDFLAGS="-mno-cygwin" \ - LIBS="-lpdcurses" \ - --with-curses --disable-shared + ./configure --prefix=/usr/x86_64-w64-mingw32 \ + --host=x86_64-w64-mingw32 \ + LIBS="-lpdcurses" --with-curses --disable-shared \ + CFLAGS="-DSIGHUP=0 -DSIGQUIT=0 -DS_ISUID=0 -DS_ISGID=0 -DS_IXGRP=0 -DS_IXOTH=0 -DS_IWOTH=0" make make install diff --git a/docs/images/box-alpha.png b/docs/images/box-alpha.png Binary files differnew file mode 100644 index 00000000..10137b7d --- /dev/null +++ b/docs/images/box-alpha.png diff --git a/docs/images/box-alpha.xcf b/docs/images/box-alpha.xcf Binary files differnew file mode 100644 index 00000000..8155a679 --- /dev/null +++ b/docs/images/box-alpha.xcf diff --git a/infrastructure/BoxPlatform.pm.in b/infrastructure/BoxPlatform.pm.in index b2b2ce43..325e56c3 100644 --- a/infrastructure/BoxPlatform.pm.in +++ b/infrastructure/BoxPlatform.pm.in @@ -1,24 +1,34 @@ package BoxPlatform; use Exporter; @ISA = qw/Exporter/; -@EXPORT = qw/$build_os $target_os $make_command $bsd_make $platform_define $platform_cpu $gcc_v3 $product_version $product_name $install_into_dir $sub_make_options $platform_compile_line_extra $platform_link_line_extra $platform_lib_files $platform_exe_ext $target_windows/; +@EXPORT = qw/$build_os $build_os_ver $ac_target $ac_target_cpu $ac_target_vendor $ac_target_os $make_command $bsd_make $platform_define $platform_cpu $gcc_v3 $product_version $product_name $install_into_dir $sub_make_options $platform_compile_line_extra $platform_link_line_extra $platform_lib_files $platform_exe_ext $target_windows $target_msvc/; BEGIN { # which OS are we building under? - $target_os = '@target_os@'; + $ac_target = '@target@'; + $ac_target_cpu = '@target_cpu@'; + $ac_target_vendor = '@target_vendor@'; + $ac_target_os = '@target_os@'; $target_windows = 0; - $target_windows = 1 if $target_os =~ m'^mingw32' - or $target_os eq "winnt"; if ($^O eq "MSWin32" and not -x "/usr/bin/uname") { + $target_windows = 1; + $target_msvc = 1; $build_os = "winnt"; + eval "use Win32"; + $build_os_ver = Win32::GetOSName(); } else { + $target_windows = 1 if $ac_target_os =~ m'^mingw32' + or $ac_target_os eq "winnt"; + $target_msvc = 0; $build_os = `uname`; + $build_os_ver = `uname -r`; chomp $build_os; + chomp $build_os_ver; } # Cygwin Builds usually something like CYGWIN_NT-5.0, CYGWIN_NT-5.1 @@ -30,11 +40,23 @@ BEGIN $xcode_ver = `xcodebuild -version | awk '/^Xcode/ {print \$2}'` } - $make_command = ($build_os eq 'Darwin' && $xcode_ver < 4) ? 'bsdmake' : ($build_os eq 'SunOS') ? 'gmake' : 'make'; - - $bsd_make = ($build_os ne 'Linux' && $build_os ne 'CYGWIN' && - $build_os ne "MINGW32" && $build_os ne "SunOS" - && $build_os ne 'GNU/kFreeBSD' && $xcode_ver < 4); + if ($build_os eq 'Darwin' and $xcode_ver < 4) + { + $make_command = 'bsdmake'; + $bsd_make = 1; + } + elsif ($build_os eq 'SunOS') + { + $make_command = 'gmake'; + $bsd_make = 0; + } + else + { + $make_command = 'make'; + $bsd_make = ($build_os ne 'Linux' && $build_os ne 'CYGWIN' && + $build_os ne "MINGW32" && $build_os ne 'GNU/kFreeBSD' && + $build_os ne 'Darwin'); + } # blank extra flags by default $platform_compile_line_extra = ''; @@ -56,33 +78,10 @@ BEGIN if($product_version =~ /USE_SVN_VERSION/) { - # for developers, use SVN version - my $svnversion = `svnversion .`; - chomp $svnversion; - $svnversion =~ tr/0-9A-Za-z/_/c; - open INFO,'svn info . |'; - my $svnurl; - while(<INFO>) - { - if(m/^URL: (.+?)[\n\r]+/) - { - $svnurl = $1 - } - } - close INFO; - - my $svndir; - if ($svnurl =~ m!/box/(.+)$!) - { - $svndir = $1; - } - elsif ($svnurl =~ m'/(boxi/.+)/boxi/boxbackup') - { - $svndir = $1; - } - - $svndir =~ tr/0-9A-Za-z/_/c; - $product_version =~ s/USE_SVN_VERSION/$svndir.'_'.$svnversion/e; + # for developers, use Git version (SVN is no more): + my $gitversion = `git rev-parse HEAD`; + chomp $gitversion; + $product_version =~ s/USE_SVN_VERSION/git_$gitversion/; } # where to put the files @@ -108,7 +107,7 @@ BEGIN print STDERR "$cpus processors detected, will set make to perform concurrent jobs\n"; $sub_make_options = ' -j '.($cpus + 1); } - + # if it's Darwin, if($build_os eq 'Darwin') { @@ -120,6 +119,11 @@ BEGIN $platform_link_line_extra = '-L/sw/lib '; } } + + if ($target_windows) + { + $platform_exe_ext = '.exe'; + } } sub make_flag @@ -133,7 +137,7 @@ sub make_flag sub parcel_root { - my $tos = $_[1] || $target_os; + my $tos = $_[1] || $ac_target; return $product_name.'-'.$product_version.'-'.$_[0].'-'.$tos; } diff --git a/infrastructure/buildenv-testmain-template.cpp b/infrastructure/buildenv-testmain-template.cpp index e5b1360d..d946c25d 100644 --- a/infrastructure/buildenv-testmain-template.cpp +++ b/infrastructure/buildenv-testmain-template.cpp @@ -23,10 +23,9 @@ #include <stdarg.h> #include <stdlib.h> #include <stdio.h> -#include <unistd.h> -#ifdef HAVE_GETOPT_H - #include <getopt.h> +#ifdef HAVE_NETDB_H +# include <netdb.h> #endif #ifdef HAVE_SYS_SOCKET_H @@ -41,8 +40,12 @@ #endif #include <exception> +#include <iostream> +#include <list> #include <string> +#include "box_getopt.h" +#include "depot.h" #include "Logging.h" #include "Test.h" #include "Timer.h" @@ -57,10 +60,6 @@ int test(int argc, const char *argv[]); #define MODE_TEXT "debug" #endif -int failures = 0; -int first_fail_line; -std::string first_fail_file; - #ifdef WIN32 #define QUIET_PROCESS "-Q" #else @@ -72,13 +71,13 @@ std::string bbackupd_args = QUIET_PROCESS, bbackupquery_args, test_args; -int filedes_open_at_beginning = -1; +bool filedes_initialised = false; #ifdef WIN32 // any way to check for open file descriptors on Win32? -inline bool check_filedes(bool x) { return 0; } -inline bool checkfilesleftopen() { return false; } +inline bool check_filedes(bool x) { return true; } +inline bool checkfilesleftopen() { return true; } #else // !WIN32 @@ -88,7 +87,9 @@ typedef enum { OPEN, CLOSED, - SYSLOG + SYSLOG, + STILLOPEN, + LEAKED, } filedes_t; @@ -98,23 +99,60 @@ bool check_filedes(bool report) { bool allOk = true; - // See how many file descriptors there are with values < 256 + // See how many file descriptors there are with values < 256. + // In order to avoid disturbing things, we scan the file descriptors + // first, marking the ones that were OPEN at startup (report == FALSE) + // as STILLOPEN and the ones that were not as LEAKED. Then we run + // through again and print the results. for(int d = 0; d < FILEDES_MAX; ++d) { if(::fcntl(d, F_GETFD) != -1) { // File descriptor obviously exists, but is it /dev/log? - - struct stat st; - bool stat_success = false; - bool is_syslog_socket = false; - - if(fstat(d, &st) == 0) + // Mark it as OPEN for now, and we'll find out later. + if(report) { - stat_success = true; + if(filedes_open[d] == OPEN) + { + filedes_open[d] = STILLOPEN; + } + else + { + filedes_open[d] = LEAKED; + } + } + else + { + filedes_open[d] = OPEN; } + } + else + { + filedes_open[d] = CLOSED; + } + } + + if(!report) + { + filedes_initialised = true; + return true; + } + + // Now loop again, reporting differences. + for(int d = 0; d < FILEDES_MAX; ++d) + { + if(filedes_open[d] != LEAKED) + { + continue; + } + + bool stat_success = false; + struct stat st; + if(fstat(d, &st) == 0) + { + stat_success = true; - if(stat_success && (st.st_mode & S_IFSOCK)) + if(st.st_mode & S_IFSOCK) { char buffer[256]; socklen_t addrlen = sizeof(buffer); @@ -122,7 +160,7 @@ bool check_filedes(bool report) #ifdef HAVE_GETPEERNAME if(getpeername(d, (sockaddr*)buffer, &addrlen) != 0) { - BOX_WARNING("Failed to getpeername(" << + BOX_LOG_SYS_WARNING("Failed to getpeername(" << d << "), cannot identify /dev/log"); } else @@ -132,90 +170,61 @@ bool check_filedes(bool report) if(sa->sun_family == PF_UNIX && !strcmp(sa->sun_path, "/dev/log")) { - is_syslog_socket = true; + // it's a syslog socket, ignore it + filedes_open[d] = SYSLOG; } } #endif // HAVE_GETPEERNAME } + } - if(report && filedes_open[d] != OPEN) - { - if(filedes_open[d] == SYSLOG) - { - // Different libcs have different ideas - // about when to open and close this - // socket, and it's not a leak, so - // ignore it. - } - else if(stat_success) - { - int m = st.st_mode; - #define flag(x) ((m & x) ? #x " " : "") - BOX_FATAL("File descriptor " << d << - " left open (type == " << - flag(S_IFIFO) << - flag(S_IFCHR) << - flag(S_IFDIR) << - flag(S_IFBLK) << - flag(S_IFREG) << - flag(S_IFLNK) << - flag(S_IFSOCK) << - " or " << m << ")"); - allOk = false; - } - else - { - BOX_FATAL("File descriptor " << d << - " left open (and stat failed)"); - allOk = false; - } - } - else if (!report) - { - filedes_open[d] = is_syslog_socket ? SYSLOG : OPEN; - } + if(filedes_open[d] == SYSLOG) + { + // Different libcs have different ideas + // about when to open and close this + // socket, and it's not a leak, so + // ignore it. } - else + else if(stat_success) { - if (report && filedes_open[d] != CLOSED) - { - if (filedes_open[d] == SYSLOG) - { - // Different libcs have different ideas - // about when to open and close this - // socket, and it's not a leak, so - // ignore it. - } - else if(filedes_open[d] == OPEN) - { - BOX_FATAL("File descriptor " << d << - " was open, now closed"); - allOk = false; - } - } - else - { - filedes_open[d] = CLOSED; - } + int m = st.st_mode; + #define flag(x) ((m & x) ? #x " " : "") + BOX_FATAL("File descriptor " << d << + " left open (type == " << + flag(S_IFIFO) << + flag(S_IFCHR) << + flag(S_IFDIR) << + flag(S_IFBLK) << + flag(S_IFREG) << + flag(S_IFLNK) << + flag(S_IFSOCK) << + " or " << m << ")"); + allOk = false; + } + else + { + BOX_FATAL("File descriptor " << d << + " left open (and stat failed)"); + allOk = false; } } if (!report && allOk) { - filedes_open_at_beginning = 0; + filedes_initialised = true; } - return !allOk; + return allOk; } bool checkfilesleftopen() { - if(filedes_open_at_beginning == -1) + if(!filedes_initialised) { // Not used correctly, pretend that there were things // left open so this gets investigated BOX_FATAL("File descriptor test was not initialised"); - return true; + return false; } // Count the file descriptors open @@ -224,6 +233,23 @@ bool checkfilesleftopen() #endif +int Usage(const std::string& ProgramName) +{ + std::cout << + "(built with QDBM " << dpversion << ")\n" + "\n" + "Usage: " << ProgramName << " [options]\n" + "\n" + "Options:\n" + " -c/--bbackupd-args <args> Arguments to pass to bbackupd/BackupDaemon\n" + " -s/--bbstored-args <args> Arguments to pass to bbstored/BackupStoreDaemon\n" + " -d/--test-daemon-args <args> Arguments to pass to TestDaemon\n" + " -e/--execute-only <test> Execute only specific named test, can repeat\n" + " -h/--help Show this command-line help\n" + << Logging::OptionParser::GetUsageString(); + return 0; +} + int main(int argc, char * const * argv) { // Start memory leak testing @@ -231,27 +257,25 @@ int main(int argc, char * const * argv) Logging::SetProgramName(BOX_MODULE); -#ifdef HAVE_GETOPT_H - #ifdef BOX_RELEASE_BUILD - int logLevel = Log::NOTICE; // need an int to do math with - #else - int logLevel = Log::INFO; // need an int to do math with - #endif - struct option longopts[] = { { "bbackupd-args", required_argument, NULL, 'c' }, { "bbstored-args", required_argument, NULL, 's' }, { "test-daemon-args", required_argument, NULL, 'd' }, + { "execute-only", required_argument, NULL, 'e' }, + { "help", no_argument, NULL, 'h' }, { NULL, 0, NULL, 0 } }; - - int ch; - - while ((ch = getopt_long(argc, argv, "c:d:qs:t:vPTUVW:", longopts, NULL)) + + int c; + std::string options("c:d:e:hs:"); + options += Logging::OptionParser::GetOptionString(); + Logging::OptionParser LogLevel; + + while ((c = getopt_long(argc, argv, options.c_str(), longopts, NULL)) != -1) { - switch(ch) + switch(c) { case 'c': { @@ -273,106 +297,43 @@ int main(int argc, char * const * argv) } break; - case 's': - { - bbstored_args += " "; - bbstored_args += optarg; - } - break; - - #ifndef WIN32 - case 'P': + case 'e': { - Console::SetShowPID(true); + run_only_named_tests.push_back(optarg); } break; - #endif - case 'q': + case 'h': { - if(logLevel == Log::NOTHING) - { - BOX_FATAL("Too many '-q': " - "Cannot reduce logging " - "level any more"); - return 2; - } - logLevel--; + return Usage(argv[0]); } break; - case 'v': - { - if(logLevel == Log::EVERYTHING) - { - BOX_FATAL("Too many '-v': " - "Cannot increase logging " - "level any more"); - return 2; - } - logLevel++; - } - break; - - case 'V': + case 's': { - logLevel = Log::EVERYTHING; + bbstored_args += " "; + bbstored_args += optarg; } break; - case 'W': + default: { - logLevel = Logging::GetNamedLevel(optarg); - if (logLevel == Log::INVALID) + int ret = LogLevel.ProcessOption(c); + if(ret != 0) { - BOX_FATAL("Invalid logging level: " << optarg); - return 2; + fprintf(stderr, "Unknown option code " + "'%c'\n", c); + exit(2); } } - break; - - case 't': - { - Logging::SetProgramName(optarg); - Console::SetShowTag(true); - } - break; - - case 'T': - { - Console::SetShowTime(true); - } - break; - - case 'U': - { - Console::SetShowTime(true); - Console::SetShowTimeMicros(true); - } - break; - - case '?': - { - fprintf(stderr, "Unknown option: '%c'\n", - optopt); - exit(2); - } - - default: - { - fprintf(stderr, "Unknown option code '%c'\n", - ch); - exit(2); - } } } - Logging::SetGlobalLevel((Log::Level)logLevel); - Logging::FilterConsole((Log::Level)logLevel); + Logging::FilterSyslog(Log::NOTHING); + Logging::FilterConsole(LogLevel.GetCurrentLevel()); argc -= optind - 1; argv += optind - 1; -#endif // HAVE_GETOPT_H // If there is more than one argument, then the test is doing something advanced, so leave it alone bool fulltestmode = (argc == 1); @@ -383,6 +344,14 @@ int main(int argc, char * const * argv) BOX_NOTICE("Running test TEST_NAME in " MODE_TEXT " mode..."); // Count open file descriptors for a very crude "files left open" test + Logging::GetSyslog().Shutdown(); + + // On NetBSD, gethostbyname() appears to open a kqueue socket + // and it's not clear how to close it again. So let's just do + // it once, before counting fds for the first time, so that it's + // already open and doesn't count as a leak. + ::gethostbyname("nonexistent"); + check_filedes(false); #ifdef WIN32 @@ -402,7 +371,7 @@ int main(int argc, char * const * argv) Timers::Init(); int returncode = test(argc, (const char **)argv); - Timers::Cleanup(); + Timers::Cleanup(false); fflush(stdout); fflush(stderr); @@ -411,7 +380,7 @@ int main(int argc, char * const * argv) #ifdef BOX_MEMORY_LEAK_TESTING if(memleakfinder_numleaks() != 0) { - failures++; + num_failures++; printf("FAILURE: Memory leaks detected in test code\n"); printf("==== MEMORY LEAKS =================================\n"); memleakfinder_reportleaks(); @@ -421,20 +390,22 @@ int main(int argc, char * const * argv) if(fulltestmode) { - bool filesleftopen = checkfilesleftopen(); + Logging::GetSyslog().Shutdown(); + + bool filesleftopen = !checkfilesleftopen(); fflush(stdout); fflush(stderr); if(filesleftopen) { - failures++; + num_failures++; printf("IMPLICIT TEST FAILED: Something left files open\n"); } - if(failures > 0) + if(num_failures > 0) { printf("FAILED: %d tests failed (first at " - "%s:%d)\n", failures, + "%s:%d)\n", num_failures, first_fail_file.c_str(), first_fail_line); } @@ -446,12 +417,6 @@ int main(int argc, char * const * argv) return returncode; } - catch(BoxException &e) - { - printf("FAILED: Exception caught: %s: %s\n", e.what(), - e.GetMessage().c_str()); - return 1; - } catch(std::exception &e) { printf("FAILED: Exception caught: %s\n", e.what()); @@ -464,7 +429,7 @@ int main(int argc, char * const * argv) } if(fulltestmode) { - if(checkfilesleftopen()) + if(!checkfilesleftopen()) { printf("WARNING: Files were left open\n"); } diff --git a/infrastructure/cmake/.gitignore b/infrastructure/cmake/.gitignore new file mode 100644 index 00000000..b27d6b9f --- /dev/null +++ b/infrastructure/cmake/.gitignore @@ -0,0 +1,6 @@ +*.vcxproj +*.vcxproj.filters +BoxBackup.sln +CMakeCache.txt +CMakeFiles +cmake_install.cmake diff --git a/infrastructure/cmake/CMakeLists.txt b/infrastructure/cmake/CMakeLists.txt new file mode 100644 index 00000000..65f59eb8 --- /dev/null +++ b/infrastructure/cmake/CMakeLists.txt @@ -0,0 +1,744 @@ +cmake_minimum_required(VERSION 2.6) + +find_program(CCACHE_PROGRAM ccache) +if(CCACHE_PROGRAM) + # Support Unix Makefiles and Ninja + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") +endif() + +project(BoxBackup) +enable_testing() + +set(base_dir ${CMAKE_SOURCE_DIR}/../..) + +set(files_to_configure + bin/bbackupd/bbackupd-config + bin/bbstored/bbstored-certs + bin/bbstored/bbstored-config + contrib/mac_osx/org.boxbackup.bbackupd.plist + contrib/mac_osx/org.boxbackup.bbstored.plist + contrib/solaris/bbackupd-manifest.xml + contrib/solaris/bbstored-manifest.xml + contrib/debian/bbackupd + contrib/debian/bbstored + contrib/redhat/bbackupd + contrib/redhat/bbstored + contrib/suse/bbackupd + contrib/suse/bbstored + contrib/solaris/bbackupd-smf-method + contrib/solaris/bbstored-smf-method + contrib/windows/installer/boxbackup.mpi + infrastructure/BoxPlatform.pm + infrastructure/makebuildenv.pl + infrastructure/makedistribution.pl + lib/bbackupquery/makedocumentation.pl + lib/common/BoxPortsAndFiles.h + lib/common/makeexception.pl + lib/raidfile/raidfile-config + lib/server/makeprotocol.pl + runtest.pl + test/backupstorefix/testfiles/testbackupstorefix.pl + test/bbackupd/testfiles/bbackupd.conf + test/bbackupd/testfiles/bbackupd-exclude.conf + test/bbackupd/testfiles/bbackupd-snapshot.conf + test/bbackupd/testfiles/bbackupd-symlink.conf + test/bbackupd/testfiles/bbackupd-temploc.conf + test/bbackupd/testfiles/extcheck1.pl + test/bbackupd/testfiles/extcheck2.pl + test/bbackupd/testfiles/notifyscript.pl + test/bbackupd/testfiles/syncallowscript.pl +) + +# We need to substitute TARGET_PERL in test/bbackupd/testfiles/bbackupd.conf, so define it +# as a variable before running configure_file(). +include(FindPerl) +set(TARGET_PERL ${PERL_EXECUTABLE}) + +function(replace_file_if_different dest_file source_file) + execute_process( + COMMAND "${CMAKE_COMMAND}" -E + copy_if_different "${source_file}" "${dest_file}") + execute_process( + COMMAND "${CMAKE_COMMAND}" -E + remove "${source_file}") +endfunction() + +function(move_file_if_exists source_file dest_file) + if(EXISTS "${source_file}") + execute_process( + COMMAND "${CMAKE_COMMAND}" -E + rename "${source_file}" "${dest_file}") + endif() +endfunction() + +foreach(file_to_configure ${files_to_configure}) + configure_file("${base_dir}/${file_to_configure}.in" "${base_dir}/${file_to_configure}.out" @ONLY) + replace_file_if_different( + "${base_dir}/${file_to_configure}" + "${base_dir}/${file_to_configure}.out") +endforeach() + +# If BOXBACKUP_VERSION is defined when running CMake (as the AppVeyor config does), use it +# as-is, since it contains the full version number, branch, and platform (Win32/Win64): +if(BOXBACKUP_VERSION) + set(boxbackup_version ${BOXBACKUP_VERSION}) + # Remove CPACK_SYSTEM_NAME from the default CPACK_PACKAGE_NAME, because it's already + # included in the CPACK_PACKAGE_VERSION: + set(CPACK_PACKAGE_FILE_NAME ${CMAKE_PROJECT_NAME}-${boxbackup_version}) +else() + # Work out the current Box version (requires Perl) and update lib/common/BoxVersion.h, + # but only if it has changed, to avoid unnecessary complete rebuilds due to timestamps. + execute_process( + COMMAND ${PERL_EXECUTABLE} ${base_dir}/infrastructure/cmake/getversion.pl + WORKING_DIRECTORY ${base_dir}/infrastructure + RESULT_VARIABLE status + OUTPUT_VARIABLE boxbackup_version + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_VARIABLE command_output) + if(NOT status EQUAL 0) + message(FATAL_ERROR "Failed to execute: " + "${PERL_EXECUTABLE} ${base_dir}/infrastructure/cmake/getversion.pl: " + "status ${status}: ${command_output}") + endif() +endif() + +file(WRITE "${base_dir}/lib/common/BoxVersion.h.new" + "#define BOX_VERSION \"${boxbackup_version}\"\n") +replace_file_if_different( + "${base_dir}/lib/common/BoxVersion.h" + "${base_dir}/lib/common/BoxVersion.h.new") + +add_definitions(-DBOX_CMAKE -DNEED_BOX_VERSION_H) + +if(WIN32) + add_definitions(-DWIN32) +endif() + +# Parsing Makefile.extra files in CMake script is a pain, so the relevant rules for +# code-generating Perl scripts are hard-coded here. + +set(exception_files + lib/backupclient/ClientException.txt + lib/backupstore/BackupStoreException.txt + lib/common/CommonException.txt + lib/common/ConversionException.txt + lib/compress/CompressException.txt + lib/crypto/CipherException.txt + lib/httpserver/HTTPException.txt + lib/raidfile/RaidFileException.txt + lib/server/ServerException.txt + lib/server/ConnectionException.txt +) + +foreach(exception_file ${exception_files}) + string(REGEX MATCH "(.*)/(.*).txt" valid_exception_file ${exception_file}) + if(NOT valid_exception_file) + message(FATAL_ERROR "invalid exception file: '${exception_file}'") + endif() + + set(output_file "${base_dir}/${CMAKE_MATCH_1}/autogen_${CMAKE_MATCH_2}.cpp") + add_custom_command(OUTPUT "${output_file}" + MAIN_DEPENDENCY "${base_dir}/${exception_file}" + COMMAND ${PERL_EXECUTABLE} "${base_dir}/lib/common/makeexception.pl" "${CMAKE_MATCH_2}.txt" + WORKING_DIRECTORY "${base_dir}/${CMAKE_MATCH_1}") + + string(REPLACE "/" "_" module_name ${CMAKE_MATCH_1}) + set(${module_name}_extra_files ${${module_name}_extra_files} ${output_file}) +endforeach() + +set(protocol_files + lib/backupstore/BackupProtocol.txt + test/basicserver/TestProtocol.txt +) + +foreach(protocol_file ${protocol_files}) + string(REGEX MATCH "(.*)/(.*).txt" valid_protocol_file ${protocol_file}) + if(NOT valid_protocol_file) + message(FATAL_ERROR "invalid protocol file: '${protocol_file}'") + endif() + + set(output_file "${base_dir}/${CMAKE_MATCH_1}/autogen_${CMAKE_MATCH_2}.cpp") + add_custom_command(OUTPUT "${output_file}" + MAIN_DEPENDENCY "${base_dir}/${protocol_file}" + COMMAND ${PERL_EXECUTABLE} "${base_dir}/lib/server/makeprotocol.pl" "${CMAKE_MATCH_2}.txt" + WORKING_DIRECTORY "${base_dir}/${CMAKE_MATCH_1}") + + string(REPLACE "/" "_" module_name ${CMAKE_MATCH_1}) + set(${module_name}_extra_files ${${module_name}_extra_files} ${output_file}) +endforeach() + +set(documentation_files + lib/bbackupquery/Documentation.txt +) + +foreach(documentation_file ${documentation_files}) + string(REGEX MATCH "(.*)/(.*).txt" valid_documentation_file ${documentation_file}) + if(NOT valid_documentation_file) + message(FATAL_ERROR "invalid documentation file: '${documentation_file}'") + endif() + + set(output_file "${base_dir}/${CMAKE_MATCH_1}/autogen_${CMAKE_MATCH_2}.cpp") + add_custom_command(OUTPUT "${output_file}" + MAIN_DEPENDENCY "${base_dir}/${documentation_file}" + COMMAND ${PERL_EXECUTABLE} "${base_dir}/lib/bbackupquery/makedocumentation.pl" + WORKING_DIRECTORY "${base_dir}/${CMAKE_MATCH_1}") + + string(REPLACE "/" "_" module_name ${CMAKE_MATCH_1}) + set(${module_name}_extra_files ${${module_name}_extra_files} ${output_file}) +endforeach() + +set(testmain_template_cpp + "${base_dir}/infrastructure/buildenv-testmain-template.cpp" +) +set(release_or_debug_dir + "${base_dir}/$<$<CONFIG:Debug>:debug>$<$<CONFIG:Release>:release>$<$<CONFIG:RelWithDebInfo>:release>" +) + +file(STRINGS ${base_dir}/modules.txt module_deps REGEX "^[^#]") +foreach(module_dep ${module_deps}) + + string(REGEX MATCH "([^ ]+)[ ]*(.*)" valid_module_line ${module_dep}) + if(valid_module_line) + if(DEBUG) + message(STATUS "found module: ${CMAKE_MATCH_1} -> ${CMAKE_MATCH_2}") + endif() + + set(module_dir ${CMAKE_MATCH_1}) + set(module_path ${base_dir}/${module_dir}) + string(REPLACE "/" "_" module_name ${CMAKE_MATCH_1}) + string(REPLACE "/" "_" dependencies "${CMAKE_MATCH_2}") + + # We are replacing QDBM's normal build system, and we only want to include + # the modules that we actually need, to avoid warnings about duplicate + # definitions, and potential conflicts with Box Backup code in future, so + # we specify the C files to compile in explicitly. + if(module_name STREQUAL "qdbm") + file(GLOB module_files ${module_path}/depot.c ${module_path}/myconf.c) + else() + file(GLOB module_files ${module_path}/*.cpp ${module_path}/*.h) + endif() + + set(module_files ${module_files} ${${module_name}_extra_files}) + + string(REGEX REPLACE "^ " "" dependencies "${dependencies}") + string(REGEX REPLACE " $" "" dependencies "${dependencies}") + + if(module_name MATCHES "^bin_") + string(REGEX MATCH "^bin_(.*)" valid_exe ${module_name}) + set(bin_name ${CMAKE_MATCH_1}) + if(DEBUG) + message(STATUS "add executable '${module_name}': '${module_files}'") + endif() + + add_executable(${module_name} ${module_files}) + # Rename the output executable from bin_bbackupd(.exe) to bbackupd(.exe): + set_target_properties(${module_name} PROPERTIES + OUTPUT_NAME "${bin_name}") + + # Use a custom post-build command instead of install(...) to install + # binaries ready for running tests, because we don't want "make install" to + # do this too, and absolute-path installations break the CPack generators. + add_custom_command(TARGET ${module_name} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + "$<TARGET_FILE:${module_name}>" + "${release_or_debug_dir}/${module_dir}/${bin_name}${CMAKE_EXECUTABLE_SUFFIX}" + VERBATIM) + + # For "make install" and CPack generators: + install(TARGETS ${module_name} RUNTIME + CONFIGURATIONS Debug;Release + DESTINATION "." + COMPONENT Applications) + elseif(module_name MATCHES "^test_") + string(REGEX MATCH "^test_(.*)" valid_test ${module_name}) + set(test_name ${CMAKE_MATCH_1}) + set(bin_name ${module_name}) + + if(DEBUG) + message(STATUS "add test '${module_name}': '${module_files}'") + endif() + + set(module_files ${module_files} "${testmain_template_cpp}") + add_executable(${module_name} ${module_files}) + + # Use a custom post-build command instead of install(...) to install + # binaries ready for running tests, because we don't want "make install" to + # do this too, and absolute-path installations break the CPack generators. + add_custom_command(TARGET ${module_name} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + "$<TARGET_FILE:${module_name}>" + "${release_or_debug_dir}/${module_dir}/${bin_name}${CMAKE_EXECUTABLE_SUFFIX}" + VERBATIM) + + if(${APPVEYOR_MODE}) + set(appveyor_runtest_pl_switch -a) + else() + set(appveyor_runtest_pl_switch) + endif() + + if(WIN32) + set(test_command_internal "$<TARGET_FILE_NAME:${module_name}>") + else() + set(test_command_internal "./$<TARGET_FILE_NAME:${module_name}>") + endif() + + target_compile_definitions(${module_name} PRIVATE + -DTEST_EXECUTABLE="${test_command_internal}") + add_test(NAME ${test_name} + COMMAND ${PERL_EXECUTABLE} ${base_dir}/runtest.pl + ${appveyor_runtest_pl_switch} -c ${test_name} + $<$<CONFIG:Debug>:DEBUG>$<$<CONFIG:Release>:RELEASE>$<$<CONFIG:RelWithDebInfo>:RELEASE> + "$<TARGET_FILE:${module_name}>" "${test_command_internal}" + WORKING_DIRECTORY ${base_dir}) + + if(${APPVEYOR_MODE}) + execute_process(COMMAND appveyor AddTest -Name ${test_name} + -Framework Custom -FileName "") + endif() + + # It helps with debugging if the test depends on another step which + # prepares the target directory, and is always out of date. + add_custom_target(${module_name}-prepare + COMMAND ${PERL_EXECUTABLE} ${base_dir}/runtest.pl + -n -c ${test_name} + $<$<CONFIG:Debug>:DEBUG>$<$<CONFIG:Release>:RELEASE>$<$<CONFIG:RelWithDebInfo>:RELEASE> + "$<TARGET_FILE:${module_name}>" "${test_command_internal}" + WORKING_DIRECTORY ${base_dir}) + elseif(module_name MATCHES "^(lib_.*|qdbm)$") + if(DEBUG) + message(STATUS "add library '${module_name}': '${module_files}'") + endif() + add_library(${module_name} STATIC ${module_files}) + else() + message(FATAL_ERROR "Unsupported module type: " ${module_name}) + endif() + + target_compile_definitions(${module_name} PRIVATE -DBOX_MODULE="${module_name}") + + if(dependencies) + string(REGEX REPLACE "[ ]+" ";" dependency_list "${dependencies}") + + foreach(dependency ${dependency_list}) + if(DEBUG) + message(STATUS "add dependency to '${module_name}': '${dependency}'") + endif() + add_dependencies(${module_name} ${dependency}) + if(dependency MATCHES "^(lib_.*|qdbm)$") + # message(STATUS "add link library to '${module_name}': '${dependency}'") + target_link_libraries(${module_name} PUBLIC ${dependency}) + endif() + + # We can't make a binary depend on another binary, so we need to + # add the dependency's directory directly to our include path. + if(dependency MATCHES "^bin_") + get_property(dep_include_dirs + TARGET ${dependency} + PROPERTY INTERFACE_INCLUDE_DIRECTORIES) + target_include_directories(${module_name} + PUBLIC ${dep_include_dirs}) + endif() + endforeach() + endif() + + target_include_directories(${module_name} PUBLIC ${module_path}) + endif() +endforeach() + +if(WIN32) + install(FILES ${base_dir}/bin/bbackupd/win32/NotifySysAdmin.vbs + DESTINATION "." COMPONENT Extras) + install(FILES ${base_dir}/bin/bbackupd/win32/bbackupd.conf + DESTINATION "." COMPONENT Extras) +else() + install(FILES ${base_dir}/bin/bbackupd/bbackupd-config + DESTINATION "." COMPONENT Extras) +endif() + +# We can't do anything conditional on CMAKE_BUILD_TYPE because that's not valid for multi-configuration +# generators such as MSVC. We need to use a generator expression instead. +target_compile_definitions(lib_common PUBLIC $<$<CONFIG:Release>:BOX_RELEASE_BUILD>) + +# Detect platform features and write BoxConfig.h.in. Reuse code from +# infrastructure/m4/boxbackup_tests.m4 where possible + +include(CheckCXXCompilerFlag) +include(CheckCXXSourceCompiles) +include(CheckFunctionExists) +include(CheckIncludeFiles) +include(CheckLibraryExists) +include(CheckSymbolExists) + +set(boxconfig_h_file "${CMAKE_BINARY_DIR}/BoxConfig.h.in") +file(REMOVE "${boxconfig_h_file}") +file(WRITE "${boxconfig_h_file}" "// Auto-generated by CMake. Do not edit.\n") + +if(WIN32) + target_link_libraries(lib_common PUBLIC ws2_32 gdi32) +endif() + +# On Windows we want to statically link zlib to make debugging and distribution easier, +# but FindZLIB.cmake doesn't offer that as an option, so we have to go through some +# contortions to "find" the correct library. ZLIB_ROOT is required in this case. +if(WIN32) + if(NOT DEFINED ZLIB_ROOT) + message(FATAL_ERROR "You must set ZLIB_ROOT to point to include/zlib.h and lib/zlibstatic[d].lib on Windows") + endif() + + message(STATUS "Searching for Zlib in: ${ZLIB_ROOT}") + find_path(ZLIB_INCLUDE_DIR zlib.h PATHS ${ZLIB_ROOT}/include NO_DEFAULT_PATH) + include_directories(${ZLIB_INCLUDE_DIR}) + message(STATUS "Found Zlib headers: ${ZLIB_INCLUDE_DIR}") + + # We must link against zlibstaticD if this is a debug build, otherwise + # we have a C runtime mismatch (/MD versus /MDd) and the application + # crashes at runtime. + find_library(ZLIB_LIBRARY_STATIC_DEBUG NAMES zlibstaticd + PATHS ${ZLIB_ROOT}/lib NO_DEFAULT_PATH) + find_library(ZLIB_LIBRARY_STATIC_RELEASE NAMES zlibstatic + PATHS ${ZLIB_ROOT}/lib NO_DEFAULT_PATH) + + target_link_libraries(lib_compress PUBLIC + debug ${ZLIB_LIBRARY_STATIC_DEBUG} + optimized ${ZLIB_LIBRARY_STATIC_RELEASE}) +else() + find_package(ZLIB REQUIRED) + include_directories(${ZLIB_INCLUDE_DIRS}) + target_link_libraries(lib_compress PUBLIC ${ZLIB_LIBRARIES}) +endif() + +# Link to OpenSSL +# Workaround for incorrect library suffixes searched by FindOpenSSL: +# https://gitlab.kitware.com/cmake/cmake/issues/17604 +if(WIN32 AND MSVC) + find_package(OpenSSL) + set(OPENSSL_SSL_LIBRARY ${SSL_EAY_RELEASE}) + set(OPENSSL_CRYPTO_LIBRARY ${LIB_EAY_RELEASE}) + set(OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY} crypt32) + find_package_handle_standard_args(OpenSSL + REQUIRED_VARS + OPENSSL_SSL_LIBRARY + OPENSSL_CRYPTO_LIBRARY + OPENSSL_INCLUDE_DIR + VERSION_VAR + OPENSSL_VERSION + FAIL_MESSAGE + "Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR" + ) +else() + find_package(OpenSSL REQUIRED) +endif() +include_directories(${OPENSSL_INCLUDE_DIR}) +target_link_libraries(lib_crypto PUBLIC ${OPENSSL_LIBRARIES}) + +# Link to PCRE +if (WIN32) + if(NOT DEFINED PCRE_ROOT) + message(FATAL_ERROR "You must set PCRE_ROOT to point to include/pcreposix.h and lib/pcreposix[d].lib on Windows") + endif() + + target_compile_definitions(lib_common PUBLIC -DPCRE_STATIC) + find_library(pcreposix_lib_path pcreposix ${PCRE_ROOT}/lib) + find_library(pcreposixd_lib_path pcreposixd ${PCRE_ROOT}/lib) + find_library(pcre_lib_path pcre ${PCRE_ROOT}/lib) + find_library(pcred_lib_path pcred ${PCRE_ROOT}/lib) + target_link_libraries(lib_common PUBLIC debug "${pcreposixd_lib_path}" optimized "${pcreposix_lib_path}") + target_link_libraries(lib_common PUBLIC debug "${pcred_lib_path}" optimized "${pcre_lib_path}") + include_directories(${PCRE_ROOT}/include) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(PCRE REQUIRED libpcreposix) + include_directories(${PCRE_INCLUDE_DIRS}) + target_link_libraries(lib_common PUBLIC ${PCRE_LIBRARIES}) + + if(DEBUG) + message(STATUS "Linking PCRE libraries from ${PCRE_LIBRARY_DIRS}: ${PCRE_LIBRARIES}") + endif() +endif() + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}") +find_package(Readline) +if(READLINE_FOUND) + include_directories(${Readline_INCLUDE_DIR}) + target_link_libraries(lib_common PUBLIC ${Readline_LIBRARY}) +endif() + +set(boxconfig_cmake_h_dir "${base_dir}/lib/common") +# Get the values of all directories added to the INCLUDE_DIRECTORIES property +# by include_directory() statements, and save it in CMAKE_REQUIRED_INCLUDES +# which check_include_files() uses to set the include file search path: +get_property(CMAKE_REQUIRED_INCLUDES DIRECTORY PROPERTY INCLUDE_DIRECTORIES) +list(APPEND CMAKE_REQUIRED_INCLUDES "${boxconfig_cmake_h_dir}") +# message(STATUS "CMAKE_REQUIRED_INCLUDES=${CMAKE_REQUIRED_INCLUDES}") + +# Save the original BoxConfig.cmake.h so that we can move it back later, +# and not need to recompile everything. +move_file_if_exists( + "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h" + "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h.bak") + +foreach(m4_filename boxbackup_tests.m4 ax_check_mount_point.m4 ax_func_syscall.m4) + file(STRINGS "${base_dir}/infrastructure/m4/${m4_filename}" m4_functions REGEX "^ *AC[_A-Z]+\\(.*\\)$") + foreach(m4_function ${m4_functions}) + if(DEBUG) + message(STATUS "Processing m4_function: ${m4_function}") + endif() + + string(REGEX MATCH .* ac_check_headers ${m4_function}) + if(m4_function MATCHES "^ *AC_CHECK_HEADERS?\\(\\[([a-z./ ]+)\\](.*)\\)$") + if(DEBUG) + message(STATUS "Processing ac_check_headers: ${CMAKE_MATCH_1}") + endif() + + # http://stackoverflow.com/questions/5272781/what-is-common-way-to-split-string-into-list-with-cmake + string(REPLACE " " ";" header_files ${CMAKE_MATCH_1}) + + foreach(header_file ${header_files}) + list(APPEND detect_header_files ${header_file}) + endforeach() + elseif(m4_function MATCHES "^ *AC_CHECK_FUNCS\\(\\[([a-z./_ ]+)\\](.*)\\)$") + if(DEBUG) + message(STATUS "Processing ac_check_funcs: ${CMAKE_MATCH_1}") + endif() + + # http://stackoverflow.com/questions/5272781/what-is-common-way-to-split-string-into-list-with-cmake + string(REPLACE " " ";" function_names ${CMAKE_MATCH_1}) + + foreach(function_name ${function_names}) + list(APPEND detect_functions ${function_name}) + endforeach() + elseif(m4_function MATCHES "^ *AC_CHECK_DECLS\\(\\[([A-Za-z._/ ]+)\\](,,, ..#include <([^>]+)>..)?\\)$") + if(DEBUG) + message(STATUS "Processing ac_check_decls: ${CMAKE_MATCH_1} in ${CMAKE_MATCH_3}") + endif() + + # http://stackoverflow.com/questions/5272781/what-is-common-way-to-split-string-into-list-with-cmake + string(REPLACE " " ";" decl_names "${CMAKE_MATCH_1}") + string(REPLACE " " ";" header_files "${CMAKE_MATCH_3}") + + foreach(decl_name ${decl_names}) + string(TOUPPER ${decl_name} platform_var_name) + string(REGEX REPLACE "[/.]" "_" platform_var_name ${platform_var_name}) + check_symbol_exists("${decl_name}" "${header_files}" HAVE_DECL_${platform_var_name}) + file(APPEND "${boxconfig_h_file}" "#cmakedefine01 HAVE_DECL_${platform_var_name}\n") + endforeach() + elseif(m4_function MATCHES "^ *AC_SEARCH_LIBS\\(\\[([A-Za-z._/ ]+)\\], \\[([A-Za-z._]+)\\]\\)$") + if(DEBUG) + message(STATUS "Processing ac_search_libs: ${CMAKE_MATCH_1} in ${CMAKE_MATCH_2}") + endif() + + set(function_name ${CMAKE_MATCH_1}) + # http://stackoverflow.com/questions/5272781/what-is-common-way-to-split-string-into-list-with-cmake + string(REPLACE " " ";" library_names "${CMAKE_MATCH_2}") + + foreach(library_name ${library_names}) + string(TOUPPER ${library_name} platform_var_name) + check_library_exists(${library_name} ${function_name} "" HAVE_LIB_${platform_var_name}) + if(HAVE_LIB_${platform_var_name}) + target_link_libraries(lib_common PUBLIC ${library_name}) + endif() + endforeach() + elseif(m4_function MATCHES "^ *AC_CHECK_MEMBERS\\(\\[([A-Za-z._/ ]+)\\.([[A-Za-z_]+)\\](,,, ..(#include <([^>]+)>)..)?\\)$") + if(DEBUG) + message(STATUS "Processing ac_check_members: ${CMAKE_MATCH_1}.${CMAKE_MATCH_2} in ${CMAKE_MATCH_5}") + endif() + + set(struct_name "${CMAKE_MATCH_1}") + set(member_name "${CMAKE_MATCH_2}") + set(include_file "${CMAKE_MATCH_5}") + + string(TOUPPER "${struct_name}_${member_name}" platform_var_name) + string(REGEX REPLACE "[/. ]" "_" platform_var_name ${platform_var_name}) + + CHECK_CXX_SOURCE_COMPILES([=[ + #include "BoxConfig.cmake.h" + #include <${include_file}> + int main() + { + ${struct_name} foo; + return sizeof(foo.${member_name}) > 0 ? 0 : 1; + } + ]=] "HAVE_${platform_var_name}") + file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_${platform_var_name}\n") + endif() + endforeach() + + # Build an intermediate version of BoxConfig.cmake.h for use in the following tests. + configure_file("${boxconfig_h_file}" "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h") +endforeach() + +list(APPEND detect_header_files mntent.h sys/mnttab.h sys/mount.h sys/param.h) + +foreach(header_file ${detect_header_files}) + list(APPEND detect_header_files ${header_file}) + string(TOUPPER ${header_file} platform_var_name) + string(REGEX REPLACE "[/.]" "_" platform_var_name ${platform_var_name}) + check_include_files(${header_file} HAVE_${platform_var_name}) + file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_${platform_var_name}\n") +endforeach() + +if(NOT HAVE_PCREPOSIX_H) + message(FATAL_ERROR "pcreposix.h not found at PCRE_ROOT/include: ${PCRE_ROOT}/include") +endif() + +# PCRE is required, so unconditionally define this: +set(HAVE_REGEX_SUPPORT 1) +file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_REGEX_SUPPORT\n") + +foreach(function_name ${detect_functions}) + string(TOUPPER ${function_name} platform_var_name) + string(REGEX REPLACE "[/.]" "_" platform_var_name ${platform_var_name}) + check_function_exists(${function_name} HAVE_${platform_var_name}) + file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_${platform_var_name}\n") +endforeach() + +check_symbol_exists(dirfd "dirent.h" HAVE_DECL_DIRFD) +file(APPEND "${boxconfig_h_file}" "#cmakedefine01 HAVE_DECL_DIRFD\n") + +# Emulate ax_check_mount_point.m4 +# These checks are run by multi-line M4 commands which are harder to parse/fake using +# regexps above, so we hard-code them here: +CHECK_CXX_SOURCE_COMPILES([=[ + #include "BoxConfig.cmake.h" + #ifdef HAVE_SYS_PARAM_H + # include <sys/param.h> + #endif + #include <sys/mount.h> + int main() + { + struct statfs foo; + return sizeof(foo.f_mntonname) > 0 ? 0 : 1; + } + ]=] "HAVE_STRUCT_STATFS_F_MNTONNAME") +file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_STRUCT_STATFS_F_MNTONNAME\n") +CHECK_CXX_SOURCE_COMPILES([=[ + #include "BoxConfig.cmake.h" + #ifdef HAVE_SYS_PARAM_H + # include <sys/param.h> + #endif + #include <sys/mount.h> + int main() + { + struct statvfs foo; + return sizeof(foo.f_mntonname) > 0 ? 0 : 1; + } + ]=] "HAVE_STRUCT_STATVFS_F_MNTONNAME") +file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_STRUCT_STATVFS_F_MNTONNAME\n") +if(HAVE_STRUCT_STATFS_F_MNTONNAME OR + HAVE_STRUCT_STATVFS_F_MNTONNAME OR + HAVE_STRUCT_MNTENT_MNT_DIR OR + HAVE_STRUCT_MNTTAB_MNT_MOUNTP) + + set(HAVE_MOUNTS 1) + file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_MOUNTS\n") +endif() + +# Emulate ax_random_device.m4 +if(EXISTS /dev/urandom) + set(RANDOM_DEVICE /dev/urandom) +elseif(EXISTS /dev/arandom) + set(RANDOM_DEVICE /dev/arandom) +elseif(EXISTS /dev/random) + set(RANDOM_DEVICE /dev/random) +endif() +if(RANDOM_DEVICE) + set(HAVE_RANDOM_DEVICE TRUE) +endif() +file(APPEND "${boxconfig_h_file}" "#cmakedefine RANDOM_DEVICE \"${RANDOM_DEVICE}\"\n") +file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_RANDOM_DEVICE\n") + +# Build an intermediate version of BoxConfig.cmake.h for use in the following tests: +configure_file("${boxconfig_h_file}" "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h") + +foreach(struct_member_name "struct ucred.uid" "struct ucred.cr_uid") + string(REGEX MATCH "(.*)\\.(.*)" dummy_var ${struct_member_name}) + set(struct_name "${CMAKE_MATCH_1}") + set(member_name "${CMAKE_MATCH_2}") + + string(TOUPPER "${struct_name}_${member_name}" platform_var_name) + string(REGEX REPLACE "[/. ]" "_" platform_var_name ${platform_var_name}) + + CHECK_CXX_SOURCE_COMPILES([=[ + #include "BoxConfig.cmake.h" + + #ifdef HAVE_UCRED_H + # include <ucred.h> + #endif + + #ifdef HAVE_SYS_PARAM_H + # include <sys/param.h> + #endif + + #ifdef HAVE_SYS_UCRED_H + # include <sys/ucred.h> + #endif + + #ifdef HAVE_SYS_SOCKET_H + # include <sys/socket.h> + #endif + + int main() + { + ${struct_name} foo; + return sizeof(foo.${member_name}) > 0 ? 0 : 1; + } + ]=] "HAVE_${platform_var_name}") + file(APPEND "${boxconfig_h_file}" "#cmakedefine HAVE_${platform_var_name}\n") +endforeach() +set(CMAKE_REQUIRED_INCLUDES "") + +# Build the final version of BoxConfig.cmake.h, as a temporary file. +configure_file("${boxconfig_h_file}" "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h.new") + +# Move the original back into place, and then replace it with the +# temporary one if different (which will force a rebuild of everything). +move_file_if_exists( + "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h.bak" + "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h") +replace_file_if_different( + "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h" + "${boxconfig_cmake_h_dir}/BoxConfig.cmake.h.new") + +# Tell QDBM not to build itself as a DLL, because we want to link statically to it. +target_compile_definitions(qdbm PUBLIC -DQDBM_STATIC) + +# Silence some less-useful warnings +if(MSVC) + add_definitions(/wd4291 /wd4710 /wd4820 /wd4996) + set_property(TARGET qdbm PROPERTY CMAKE_STATIC_LINKER_FLAGS /IGNORE:LNK4006) +endif(MSVC) + +# Define the location of the Perl executable, needed by testbackupstorefix +file(TO_NATIVE_PATH "${PERL_EXECUTABLE}" perl_executable_native) +string(REPLACE "\\" "\\\\" perl_path_escaped ${perl_executable_native}) +target_compile_definitions(test_backupstorefix PRIVATE -DPERL_EXECUTABLE="${perl_path_escaped}") + +# Configure test timeouts: +# I've set the timeout to 4 times as long as it took to run on a particular run on Appveyor: +# https://ci.appveyor.com/project/qris/boxbackup/build/job/xm10itascygtu93j +set_tests_properties(common PROPERTIES TIMEOUT 20) +set_tests_properties(crypto PROPERTIES TIMEOUT 10) +set_tests_properties(compress PROPERTIES TIMEOUT 80) +set_tests_properties(raidfile PROPERTIES TIMEOUT 32) +set_tests_properties(basicserver PROPERTIES TIMEOUT 80) +set_tests_properties(backupstore PROPERTIES TIMEOUT 1320) +set_tests_properties(backupstorefix PROPERTIES TIMEOUT 180) +set_tests_properties(backupstorepatch PROPERTIES TIMEOUT 320) +set_tests_properties(backupdiff PROPERTIES TIMEOUT 32) +set_tests_properties(bbackupd PROPERTIES TIMEOUT 1200) +set_tests_properties(s3store PROPERTIES TIMEOUT 20) +set_tests_properties(httpserver PROPERTIES TIMEOUT 40) + +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Box Backup is an open source, completely automatic, on-line backup system") +set(CPACK_PACKAGE_VENDOR "www.BoxBackup.org") +set(CPACK_PACKAGE_DESCRIPTION_FILE "${base_dir}/README.md") +set(CPACK_RESOURCE_FILE_LICENSE "${base_dir}/LICENSE.txt") +set(CPACK_PACKAGE_VERSION ${boxbackup_version}) +set(CPACK_PACKAGE_INSTALL_DIRECTORY "Box Backup") +set(CPACK_COMPONENTS_ALL Applications Extras) +set(CPACK_GENERATOR "ZIP;NSIS") +set(CPACK_NSIS_DISPLAY_NAME "Box Backup") +set(CPACK_NSIS_HELP_LINK "http://www.boxbackup.org/") +set(CPACK_NSIS_URL_INFO_ABOUT "http://www.boxbackup.org/") +set(CPACK_NSIS_CONTACT "boxbackup@boxbackup.org") +set(CPACK_NSIS_MODIFY_PATH ON) +include(CPack) diff --git a/infrastructure/cmake/FindReadline.cmake b/infrastructure/cmake/FindReadline.cmake new file mode 100644 index 00000000..3ba4d21d --- /dev/null +++ b/infrastructure/cmake/FindReadline.cmake @@ -0,0 +1,84 @@ +# https://github.com/bro/cmake/blob/master/FindReadline.cmake +# +# Copyright (c) 1995-2015, The Regents of the University of California +# through the Lawrence Berkeley National Laboratory and the +# International Computer Science Institute. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# (1) Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# (2) Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# (3) Neither the name of the University of California, Lawrence Berkeley +# National Laboratory, U.S. Dept. of Energy, International Computer +# Science Institute, nor the names of contributors may be used to endorse +# or promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# Note that some files in the distribution may carry their own copyright +# notices. + +# - Try to find readline include dirs and libraries +# +# Usage of this module as follows: +# +# find_package(Readline) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# Readline_ROOT_DIR Set this variable to the root installation of +# readline if the module has problems finding the +# proper installation path. +# +# Variables defined by this module: +# +# READLINE_FOUND System has readline, include and lib dirs found +# Readline_INCLUDE_DIR The readline include directories. +# Readline_LIBRARY The readline library. + +find_path(Readline_ROOT_DIR + NAMES include/readline/readline.h +) + +find_path(Readline_INCLUDE_DIR + NAMES readline/readline.h + HINTS ${Readline_ROOT_DIR}/include +) + +find_library(Readline_LIBRARY + NAMES readline + HINTS ${Readline_ROOT_DIR}/lib +) + +if(Readline_INCLUDE_DIR AND Readline_LIBRARY AND Ncurses_LIBRARY) + set(READLINE_FOUND TRUE) +else(Readline_INCLUDE_DIR AND Readline_LIBRARY AND Ncurses_LIBRARY) + FIND_LIBRARY(Readline_LIBRARY NAMES readline) + include(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS(Readline DEFAULT_MSG Readline_INCLUDE_DIR Readline_LIBRARY ) + MARK_AS_ADVANCED(Readline_INCLUDE_DIR Readline_LIBRARY) +endif(Readline_INCLUDE_DIR AND Readline_LIBRARY AND Ncurses_LIBRARY) + +mark_as_advanced( + Readline_ROOT_DIR + Readline_INCLUDE_DIR + Readline_LIBRARY +) diff --git a/infrastructure/cmake/build/bin_bbackupd.vcxproj.user b/infrastructure/cmake/build/bin_bbackupd.vcxproj.user new file mode 100755 index 00000000..fa1f3d34 --- /dev/null +++ b/infrastructure/cmake/build/bin_bbackupd.vcxproj.user @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerCommandArguments>testfiles\bbackupd.conf</LocalDebuggerCommandArguments> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\bbackupd</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/build/bin_bbstored.vcxproj.user b/infrastructure/cmake/build/bin_bbstored.vcxproj.user new file mode 100755 index 00000000..339cddee --- /dev/null +++ b/infrastructure/cmake/build/bin_bbstored.vcxproj.user @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerCommandArguments>testfiles/bbstored.conf</LocalDebuggerCommandArguments> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\backupstorefix</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/build/test_backupstore.vcxproj.user b/infrastructure/cmake/build/test_backupstore.vcxproj.user new file mode 100755 index 00000000..7d7b3158 --- /dev/null +++ b/infrastructure/cmake/build/test_backupstore.vcxproj.user @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\backupstore</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + <LocalDebuggerCommandArguments> + </LocalDebuggerCommandArguments> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/build/test_backupstorefix.vcxproj.user b/infrastructure/cmake/build/test_backupstorefix.vcxproj.user new file mode 100755 index 00000000..170fb496 --- /dev/null +++ b/infrastructure/cmake/build/test_backupstorefix.vcxproj.user @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\backupstorefix</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/build/test_bbackupd.vcxproj.user b/infrastructure/cmake/build/test_bbackupd.vcxproj.user new file mode 100755 index 00000000..ebf8c6a3 --- /dev/null +++ b/infrastructure/cmake/build/test_bbackupd.vcxproj.user @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerCommandArguments>-e test_basics</LocalDebuggerCommandArguments> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\bbackupd</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/build/test_common.vcxproj.user b/infrastructure/cmake/build/test_common.vcxproj.user new file mode 100755 index 00000000..e5854a80 --- /dev/null +++ b/infrastructure/cmake/build/test_common.vcxproj.user @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\common</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/build/test_httpserver.vcxproj.user b/infrastructure/cmake/build/test_httpserver.vcxproj.user new file mode 100755 index 00000000..ac1512a8 --- /dev/null +++ b/infrastructure/cmake/build/test_httpserver.vcxproj.user @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\httpserver</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/build/test_raidfile.vcxproj.user b/infrastructure/cmake/build/test_raidfile.vcxproj.user new file mode 100755 index 00000000..620aa4bb --- /dev/null +++ b/infrastructure/cmake/build/test_raidfile.vcxproj.user @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\raidfile</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/getversion.pl b/infrastructure/cmake/getversion.pl new file mode 100755 index 00000000..d7ca0333 --- /dev/null +++ b/infrastructure/cmake/getversion.pl @@ -0,0 +1,13 @@ +#!perl + +use File::Basename; +$basedir = dirname($0)."/../.."; + +-d $basedir or die "Base directory $basedir does not exist!"; +chdir $basedir or die "Failed to change to base directory: $basedir: $!"; + +require "$basedir/infrastructure/BoxPlatform.pm.in"; + +print "$BoxPlatform::product_version\n"; + +exit 0; diff --git a/infrastructure/cmake/msvc/bin_bbackupd.vcxproj.user b/infrastructure/cmake/msvc/bin_bbackupd.vcxproj.user new file mode 100755 index 00000000..fa1f3d34 --- /dev/null +++ b/infrastructure/cmake/msvc/bin_bbackupd.vcxproj.user @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerCommandArguments>testfiles\bbackupd.conf</LocalDebuggerCommandArguments> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\bbackupd</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/msvc/bin_bbstored.vcxproj.user b/infrastructure/cmake/msvc/bin_bbstored.vcxproj.user new file mode 100755 index 00000000..339cddee --- /dev/null +++ b/infrastructure/cmake/msvc/bin_bbstored.vcxproj.user @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerCommandArguments>testfiles/bbstored.conf</LocalDebuggerCommandArguments> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\backupstorefix</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/msvc/test_backupstore.vcxproj.user b/infrastructure/cmake/msvc/test_backupstore.vcxproj.user new file mode 100755 index 00000000..7d7b3158 --- /dev/null +++ b/infrastructure/cmake/msvc/test_backupstore.vcxproj.user @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\backupstore</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + <LocalDebuggerCommandArguments> + </LocalDebuggerCommandArguments> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/msvc/test_backupstorefix.vcxproj.user b/infrastructure/cmake/msvc/test_backupstorefix.vcxproj.user new file mode 100755 index 00000000..170fb496 --- /dev/null +++ b/infrastructure/cmake/msvc/test_backupstorefix.vcxproj.user @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\backupstorefix</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/msvc/test_bbackupd.vcxproj.user b/infrastructure/cmake/msvc/test_bbackupd.vcxproj.user new file mode 100755 index 00000000..ebf8c6a3 --- /dev/null +++ b/infrastructure/cmake/msvc/test_bbackupd.vcxproj.user @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerCommandArguments>-e test_basics</LocalDebuggerCommandArguments> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\bbackupd</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/msvc/test_common.vcxproj.user b/infrastructure/cmake/msvc/test_common.vcxproj.user new file mode 100755 index 00000000..e5854a80 --- /dev/null +++ b/infrastructure/cmake/msvc/test_common.vcxproj.user @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\common</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/msvc/test_httpserver.vcxproj.user b/infrastructure/cmake/msvc/test_httpserver.vcxproj.user new file mode 100755 index 00000000..ac1512a8 --- /dev/null +++ b/infrastructure/cmake/msvc/test_httpserver.vcxproj.user @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\httpserver</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/msvc/test_raidfile.vcxproj.user b/infrastructure/cmake/msvc/test_raidfile.vcxproj.user new file mode 100755 index 00000000..620aa4bb --- /dev/null +++ b/infrastructure/cmake/msvc/test_raidfile.vcxproj.user @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <LocalDebuggerWorkingDirectory>$(ProjectDir)\..\..\..\debug\test\raidfile</LocalDebuggerWorkingDirectory> + <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> + </PropertyGroup> +</Project>
\ No newline at end of file diff --git a/infrastructure/cmake/windows/CMakeLists.txt b/infrastructure/cmake/windows/CMakeLists.txt new file mode 100644 index 00000000..49a1ea4d --- /dev/null +++ b/infrastructure/cmake/windows/CMakeLists.txt @@ -0,0 +1,137 @@ +cmake_minimum_required(VERSION 2.6) + +project(BoxBackup_Windows) + +set(boxbackup_dir ${CMAKE_SOURCE_DIR}/../../..) +set_property(DIRECTORY PROPERTY EP_BASE .) +set(install_dir ${CMAKE_BINARY_DIR}/install) + +# Automate the process of downloading, building and "installing" dependencies on Windows, +# as used by AppVeyor. + +# Version of zlib to download, build, and compile Box Backup against: +set(ZLIB_VERSION 1.2.11) +# Hash of zlib-${ZLIB_VERSION}.tar.gz, to be verified after download: +set(ZLIB_HASH SHA256=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1) + +# Version of OpenSSL to download, build, and compile Box Backup against: +set(OPENSSL_VERSION 1.1.0g) +# Hash of openssl-${OPENSSL_VERSION}.tar.gz, to be verified after download: +set(OPENSSL_HASH SHA256=de4d501267da39310905cb6dc8c6121f7a2cad45a7707f76df828fe1b85073af) + +# Version of PCRE to download, build, and compile Box Backup against: +set(PCRE_VERSION 8.39) +# Hash of pcre-${PCRE_VERSION}.tar.gz, to be verified after download: +set(PCRE_HASH SHA256=ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7) + +# Version of Boost to download, unpack, and compile Box Backup against: +set(BOOST_VERSION 1.62.0) +# Hash of the Boost download file, to be verified after download: +set(BOOST_HASH SHA1=5fd97433c3f859d8cbab1eaed4156d3068ae3648) + +include(ExternalProject) + +ExternalProject_Add(zlib + URL "http://zlib.net/zlib-${ZLIB_VERSION}.tar.gz" + URL_HASH ${ZLIB_HASH} + DOWNLOAD_NO_PROGRESS 1 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} ${SUB_CMAKE_EXTRA_ARGS} + # We need to build both versions, debug and release, because cmake requires both to be + # present to generate its multi-configuration project files for Visual Studio/MSBuild. + INSTALL_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target install --config Debug + COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target install --config Release + STEP_TARGETS configure install +) + +if(WIN32) + ExternalProject_Add(openssl + DEPENDS zlib + URL "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" + URL_HASH ${OPENSSL_HASH} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND perl Configure debug-VC-WIN32 no-asm no-shared + --prefix=${install_dir} + --openssldir=etc + # Run tests before install, but don't make the main target depend on them, so that + # we don't have to run them whenever we build manually on Windows. + TEST_BEFORE_INSTALL 1 + TEST_EXCLUDE_FROM_MAIN 1 + # You would expect us to use nt.mak to compile a static library here, but mk1mf.pl uses the /MT[d] + # CRT in that case, which is incompatible with our dynamic runtime, /MD[d]. It seems that the libs + # built by ntdll.mak, which are compiled with /MD[d], are full libraries and not import libs, + # so we can link statically against them and still get a dynamic runtime. + BUILD_IN_SOURCE 1 + BUILD_COMMAND nmake /s + TEST_COMMAND nmake /s test + INSTALL_COMMAND nmake /s install + ) +elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + ExternalProject_Add(openssl + DEPENDS zlib + URL "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" + URL_HASH ${OPENSSL_HASH} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND perl Configure darwin64-x86_64-cc --prefix=${install_dir} + BUILD_IN_SOURCE 1 + ) +else() + ExternalProject_Add(openssl + DEPENDS zlib + URL "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" + URL_HASH ${OPENSSL_HASH} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND ./config --prefix=${install_dir} + BUILD_IN_SOURCE 1 + ) + +endif() + +ExternalProject_Add(pcre + # Temporarily use SVN repo until the PCRE_STATIC issue in 8.40 is fixed: + # https://vcs.pcre.org/pcre?view=revision&revision=1677 + SVN_REPOSITORY svn://vcs.exim.org/pcre/code/trunk + SVN_REVISION -r 1677 + DOWNLOAD_NO_PROGRESS 1 + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${install_dir} ${SUB_CMAKE_EXTRA_ARGS} + -DPCRE_SUPPORT_LIBREADLINE=OFF + -DPCRE_SUPPORT_LIBBZ2=OFF + # We need to build both versions, debug and release, because cmake requires both to be + # present to generate its multi-configuration project files for Visual Studio/MSBuild. + INSTALL_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target install --config Debug + COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target install --config Release +) + +string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) +ExternalProject_Add(boost + URL "http://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION}/boost_${BOOST_VERSION_UNDERSCORES}.tar.bz2" + URL_HASH ${BOOST_HASH} + # DOWNLOAD_NO_PROGRESS 1 + # Disable automatic updating (untarring) as it's slow and not necessary + UPDATE_DISCONNECTED 1 + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo "No configure step needed" + BUILD_COMMAND ${CMAKE_COMMAND} -E echo "No build step needed" + INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "No install step needed" +) + +if(BOXBACKUP_VERSION) + list(APPEND boxbackup_cmake_args "-DBOXBACKUP_VERSION=${BOXBACKUP_VERSION}") +endif() +string(REPLACE ";" " " boxbackup_cmake_args "${boxbackup_cmake_args}") + +ExternalProject_Add(boxbackup + DEPENDS zlib openssl pcre boost + SOURCE_DIR ${boxbackup_dir}/infrastructure/cmake + CMAKE_ARGS + -DZLIB_ROOT=${install_dir} + -DOPENSSL_ROOT_DIR=${install_dir} + -DPCRE_ROOT=${install_dir} + -DBOOST_ROOT=${CMAKE_BINARY_DIR}/Source/boost + -DBOX_SUPPORT_READLINE=OFF + -DCMAKE_INSTALL_PREFIX=${install_dir} + -DAPPVEYOR_MODE=1 + -DDEBUG=${DEBUG} + ${boxbackup_cmake_args} + ${SUB_CMAKE_EXTRA_ARGS} + INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "No install step needed" + STEP_TARGETS configure build +) diff --git a/config.guess b/infrastructure/config.guess index 0773d0f6..b79252d6 100755 --- a/config.guess +++ b/infrastructure/config.guess @@ -1,13 +1,12 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. +# Copyright 1992-2013 Free Software Foundation, Inc. -timestamp='2004-03-03' +timestamp='2013-06-10' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -16,24 +15,22 @@ timestamp='2004-03-03' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# along with this program; if not, see <http://www.gnu.org/licenses/>. # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - -# Originally written by Per Bothner <per@bothner.com>. -# Please send patches to <config-patches@gnu.org>. Submit a context -# diff and a properly formatted ChangeLog entry. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). +# +# Originally written by Per Bothner. # -# This script attempts to guess a canonical system name similar to -# config.sub. If it succeeds, it prints the system name on stdout, and -# exits with 0. Otherwise, it exits with 1. +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD # -# The plan is that this can be called by configure scripts if you -# don't specify an explicit build system type. +# Please send patches with a ChangeLog entry to config-patches@gnu.org. + me=`echo "$0" | sed -e 's,.*/,,'` @@ -53,8 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 -Free Software Foundation, Inc. +Copyright 1992-2013 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -66,11 +62,11 @@ Try \`$me --help' for more information." while test $# -gt 0 ; do case $1 in --time-stamp | --time* | -t ) - echo "$timestamp" ; exit 0 ;; + echo "$timestamp" ; exit ;; --version | -v ) - echo "$version" ; exit 0 ;; + echo "$version" ; exit ;; --help | --h* | -h ) - echo "$usage"; exit 0 ;; + echo "$usage"; exit ;; -- ) # Stop option processing shift; break ;; - ) # Use stdin as input. @@ -104,7 +100,7 @@ set_cc_for_build=' trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; : ${TMPDIR=/tmp} ; - { tmp=`(umask 077 && mktemp -d -q "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; @@ -123,7 +119,7 @@ case $CC_FOR_BUILD,$HOST_CC,$CC in ;; ,,*) CC_FOR_BUILD=$CC ;; ,*,*) CC_FOR_BUILD=$HOST_CC ;; -esac ;' +esac ; set_cc_for_build= ;' # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) @@ -136,12 +132,33 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown +case "${UNAME_SYSTEM}" in +Linux|GNU|GNU/*) + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + LIBC=gnu + + eval $set_cc_for_build + cat <<-EOF > $dummy.c + #include <features.h> + #if defined(__UCLIBC__) + LIBC=uclibc + #elif defined(__dietlibc__) + LIBC=dietlibc + #else + LIBC=gnu + #endif + EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` + ;; +esac + # Note: order is significant - the case branches are not exclusive. case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or - # more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*, + # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently # switched to ELF, *-*-netbsd* would select the old # object file format. This provides both forward @@ -158,6 +175,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; + sh5el) machine=sh5le-unknown ;; *) machine=${UNAME_MACHINE_ARCH}-unknown ;; esac # The Operating System including object format, if it has switched @@ -166,7 +184,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep __ELF__ >/dev/null + | grep -q __ELF__ then # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). # Return netbsd for either. FIX? @@ -176,7 +194,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in fi ;; *) - os=netbsd + os=netbsd ;; esac # The OS release @@ -196,71 +214,34 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. echo "${machine}-${os}${release}" - exit 0 ;; - amd64:OpenBSD:*:*) - echo x86_64-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - amiga:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - arc:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - cats:OpenBSD:*:*) - echo arm-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - hp300:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mac68k:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - macppc:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvme68k:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvme88k:OpenBSD:*:*) - echo m88k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvmeppc:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - pegasos:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - pmax:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - sgi:OpenBSD:*:*) - echo mipseb-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - sun3:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - wgrisc:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; + exit ;; + *:Bitrig:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} + exit ;; *:OpenBSD:*:*) - echo ${UNAME_MACHINE}-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; + UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + exit ;; *:ekkoBSD:*:*) echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} - exit 0 ;; + exit ;; + *:SolidBSD:*:*) + echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + exit ;; macppc:MirBSD:*:*) - echo powerppc-unknown-mirbsd${UNAME_RELEASE} - exit 0 ;; + echo powerpc-unknown-mirbsd${UNAME_RELEASE} + exit ;; *:MirBSD:*:*) echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} - exit 0 ;; + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` ;; *5.*) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` ;; esac # According to Compaq, /usr/sbin/psrinfo has been available on @@ -306,40 +287,46 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - exit 0 ;; - Alpha*:OpenVMS:*:*) - echo alpha-hp-vms - exit 0 ;; + # Reset EXIT trap before exiting to avoid spurious non-zero exit code. + exitcode=$? + trap '' 0 + exit $exitcode ;; Alpha\ *:Windows_NT*:*) # How do we know it's Interix rather than the generic POSIX subsystem? # Should we change UNAME_MACHINE based on the output of uname instead # of the specific Alpha model? echo alpha-pc-interix - exit 0 ;; + exit ;; 21064:Windows_NT:50:3) echo alpha-dec-winnt3.5 - exit 0 ;; + exit ;; Amiga*:UNIX_System_V:4.0:*) echo m68k-unknown-sysv4 - exit 0;; + exit ;; *:[Aa]miga[Oo][Ss]:*:*) echo ${UNAME_MACHINE}-unknown-amigaos - exit 0 ;; + exit ;; *:[Mm]orph[Oo][Ss]:*:*) echo ${UNAME_MACHINE}-unknown-morphos - exit 0 ;; + exit ;; *:OS/390:*:*) echo i370-ibm-openedition - exit 0 ;; + exit ;; + *:z/VM:*:*) + echo s390-ibm-zvmoe + exit ;; *:OS400:*:*) - echo powerpc-ibm-os400 - exit 0 ;; + echo powerpc-ibm-os400 + exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) echo arm-acorn-riscix${UNAME_RELEASE} - exit 0;; + exit ;; + arm*:riscos:*:*|arm*:RISCOS:*:*) + echo arm-unknown-riscos + exit ;; SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) echo hppa1.1-hitachi-hiuxmpp - exit 0;; + exit ;; Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. if test "`(/bin/universe) 2>/dev/null`" = att ; then @@ -347,32 +334,51 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in else echo pyramid-pyramid-bsd fi - exit 0 ;; + exit ;; NILE*:*:*:dcosx) echo pyramid-pyramid-svr4 - exit 0 ;; + exit ;; DRS?6000:unix:4.0:6*) echo sparc-icl-nx6 - exit 0 ;; - DRS?6000:UNIX_SV:4.2*:7*) + exit ;; + DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) case `/usr/bin/uname -p` in - sparc) echo sparc-icl-nx7 && exit 0 ;; + sparc) echo sparc-icl-nx7; exit ;; esac ;; + s390x:SunOS:*:*) + echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; sun4H:SunOS:5.*:*) echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; - i86pc:SunOS:5.*:*) - echo i386-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; + i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) + echo i386-pc-auroraux${UNAME_RELEASE} + exit ;; + i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) + eval $set_cc_for_build + SUN_ARCH="i386" + # If there is a compiler, see if it is configured for 64-bit objects. + # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. + # This test works for both compilers. + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + SUN_ARCH="x86_64" + fi + fi + echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; sun4*:SunOS:*:*) case "`/usr/bin/arch -k`" in Series*|S4*) @@ -381,10 +387,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in esac # Japanese Language versions have a version number like `4.1.3-JL'. echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` - exit 0 ;; + exit ;; sun3*:SunOS:*:*) echo m68k-sun-sunos${UNAME_RELEASE} - exit 0 ;; + exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 @@ -396,10 +402,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in echo sparc-sun-sunos${UNAME_RELEASE} ;; esac - exit 0 ;; + exit ;; aushp:SunOS:*:*) echo sparc-auspex-sunos${UNAME_RELEASE} - exit 0 ;; + exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not # "atarist" or "atariste" at least should have a processor @@ -409,41 +415,41 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; + exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-milan-mint${UNAME_RELEASE} + exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-hades-mint${UNAME_RELEASE} + exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-unknown-mint${UNAME_RELEASE} + exit ;; m68k:machten:*:*) echo m68k-apple-machten${UNAME_RELEASE} - exit 0 ;; + exit ;; powerpc:machten:*:*) echo powerpc-apple-machten${UNAME_RELEASE} - exit 0 ;; + exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 - exit 0 ;; + exit ;; RISC*:ULTRIX:*:*) echo mips-dec-ultrix${UNAME_RELEASE} - exit 0 ;; + exit ;; VAX*:ULTRIX*:*:*) echo vax-dec-ultrix${UNAME_RELEASE} - exit 0 ;; + exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) echo clipper-intergraph-clix${UNAME_RELEASE} - exit 0 ;; + exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c @@ -467,35 +473,36 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit (-1); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c \ - && $dummy `echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` \ - && exit 0 + $CC_FOR_BUILD -o $dummy $dummy.c && + dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`$dummy $dummyarg` && + { echo "$SYSTEM_NAME"; exit; } echo mips-mips-riscos${UNAME_RELEASE} - exit 0 ;; + exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax - exit 0 ;; + exit ;; Motorola:*:4.3:PL8-*) echo powerpc-harris-powermax - exit 0 ;; + exit ;; Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) echo powerpc-harris-powermax - exit 0 ;; + exit ;; Night_Hawk:Power_UNIX:*:*) echo powerpc-harris-powerunix - exit 0 ;; + exit ;; m88k:CX/UX:7*:*) echo m88k-harris-cxux7 - exit 0 ;; + exit ;; m88k:*:4*:R4*) echo m88k-motorola-sysv4 - exit 0 ;; + exit ;; m88k:*:3*:R3*) echo m88k-motorola-sysv3 - exit 0 ;; + exit ;; AViiON:dgux:*:*) - # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=`/usr/bin/uname -p` + # DG/UX returns AViiON for all architectures + UNAME_PROCESSOR=`/usr/bin/uname -p` if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] then if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ @@ -508,29 +515,29 @@ EOF else echo i586-dg-dgux${UNAME_RELEASE} fi - exit 0 ;; + exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) echo m88k-dolphin-sysv3 - exit 0 ;; + exit ;; M88*:*:R3*:*) # Delta 88k system running SVR3 echo m88k-motorola-sysv3 - exit 0 ;; + exit ;; XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) echo m88k-tektronix-sysv3 - exit 0 ;; + exit ;; Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) echo m68k-tektronix-bsd - exit 0 ;; + exit ;; *:IRIX*:*:*) echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` - exit 0 ;; + exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. - echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit 0 ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id + exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' i*86:AIX:*:*) echo i386-ibm-aix - exit 0 ;; + exit ;; ia64:AIX:*:*) if [ -x /usr/bin/oslevel ] ; then IBM_REV=`/usr/bin/oslevel` @@ -538,7 +545,7 @@ EOF IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} - exit 0 ;; + exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then eval $set_cc_for_build @@ -553,15 +560,19 @@ EOF exit(0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 - echo rs6000-ibm-aix3.2.5 + if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + then + echo "$SYSTEM_NAME" + else + echo rs6000-ibm-aix3.2.5 + fi elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then echo rs6000-ibm-aix3.2.4 else echo rs6000-ibm-aix3.2 fi - exit 0 ;; - *:AIX:*:[45]) + exit ;; + *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 @@ -574,28 +585,28 @@ EOF IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi echo ${IBM_ARCH}-ibm-aix${IBM_REV} - exit 0 ;; + exit ;; *:AIX:*:*) echo rs6000-ibm-aix - exit 0 ;; + exit ;; ibmrt:4.4BSD:*|romp-ibm:BSD:*) echo romp-ibm-bsd4.4 - exit 0 ;; + exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to - exit 0 ;; # report: romp-ibm BSD 4.3 + exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx - exit 0 ;; + exit ;; DPX/2?00:B.O.S.:*:*) echo m68k-bull-sysv3 - exit 0 ;; + exit ;; 9000/[34]??:4.3bsd:1.*:*) echo m68k-hp-bsd - exit 0 ;; + exit ;; hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) echo m68k-hp-bsd4.4 - exit 0 ;; + exit ;; 9000/[34678]??:HP-UX:*:*) HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` case "${UNAME_MACHINE}" in @@ -604,52 +615,52 @@ EOF 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` - sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 - 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; + sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + case "${sc_cpu_version}" in + 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 + 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 532) # CPU_PA_RISC2_0 + case "${sc_kernel_bits}" in + 32) HP_ARCH="hppa2.0n" ;; + 64) HP_ARCH="hppa2.0w" ;; '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 - esac ;; - esac + esac ;; + esac fi if [ "${HP_ARCH}" = "" ]; then eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + sed 's/^ //' << EOF >$dummy.c - #define _HPUX_SOURCE - #include <stdlib.h> - #include <unistd.h> + #define _HPUX_SOURCE + #include <stdlib.h> + #include <unistd.h> - int main () - { - #if defined(_SC_KERNEL_BITS) - long bits = sysconf(_SC_KERNEL_BITS); - #endif - long cpu = sysconf (_SC_CPU_VERSION); + int main () + { + #if defined(_SC_KERNEL_BITS) + long bits = sysconf(_SC_KERNEL_BITS); + #endif + long cpu = sysconf (_SC_CPU_VERSION); - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1"); break; - case CPU_PA_RISC2_0: - #if defined(_SC_KERNEL_BITS) - switch (bits) - { - case 64: puts ("hppa2.0w"); break; - case 32: puts ("hppa2.0n"); break; - default: puts ("hppa2.0"); break; - } break; - #else /* !defined(_SC_KERNEL_BITS) */ - puts ("hppa2.0"); break; - #endif - default: puts ("hppa1.0"); break; - } - exit (0); - } + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1"); break; + case CPU_PA_RISC2_0: + #if defined(_SC_KERNEL_BITS) + switch (bits) + { + case 64: puts ("hppa2.0w"); break; + case 32: puts ("hppa2.0n"); break; + default: puts ("hppa2.0"); break; + } break; + #else /* !defined(_SC_KERNEL_BITS) */ + puts ("hppa2.0"); break; + #endif + default: puts ("hppa1.0"); break; + } + exit (0); + } EOF (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` test -z "$HP_ARCH" && HP_ARCH=hppa @@ -657,9 +668,19 @@ EOF esac if [ ${HP_ARCH} = "hppa2.0w" ] then - # avoid double evaluation of $set_cc_for_build - test -n "$CC_FOR_BUILD" || eval $set_cc_for_build - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E -) | grep __LP64__ >/dev/null + eval $set_cc_for_build + + # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating + # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler + # generating 64-bit code. GNU and HP use different nomenclature: + # + # $ CC_FOR_BUILD=cc ./config.guess + # => hppa2.0w-hp-hpux11.23 + # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess + # => hppa64-hp-hpux11.23 + + if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + grep -q __LP64__ then HP_ARCH="hppa2.0w" else @@ -667,11 +688,11 @@ EOF fi fi echo ${HP_ARCH}-hp-hpux${HPUX_REV} - exit 0 ;; + exit ;; ia64:HP-UX:*:*) HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` echo ia64-hp-hpux${HPUX_REV} - exit 0 ;; + exit ;; 3050*:HI-UX:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c @@ -699,337 +720,345 @@ EOF exit (0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 + $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 - exit 0 ;; + exit ;; 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) echo hppa1.1-hp-bsd - exit 0 ;; + exit ;; 9000/8??:4.3bsd:*:*) echo hppa1.0-hp-bsd - exit 0 ;; + exit ;; *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) echo hppa1.0-hp-mpeix - exit 0 ;; + exit ;; hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) echo hppa1.1-hp-osf - exit 0 ;; + exit ;; hp8??:OSF1:*:*) echo hppa1.0-hp-osf - exit 0 ;; + exit ;; i*86:OSF1:*:*) if [ -x /usr/sbin/sysversion ] ; then echo ${UNAME_MACHINE}-unknown-osf1mk else echo ${UNAME_MACHINE}-unknown-osf1 fi - exit 0 ;; + exit ;; parisc*:Lites*:*:*) echo hppa1.1-hp-lites - exit 0 ;; + exit ;; C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) echo c1-convex-bsd - exit 0 ;; + exit ;; C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) if getsysinfo -f scalar_acc then echo c32-convex-bsd else echo c2-convex-bsd fi - exit 0 ;; + exit ;; C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) echo c34-convex-bsd - exit 0 ;; + exit ;; C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) echo c38-convex-bsd - exit 0 ;; + exit ;; C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) echo c4-convex-bsd - exit 0 ;; + exit ;; CRAY*Y-MP:*:*:*) echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*[A-Z]90:*:*:*) echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*TS:*:*:*) echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*T3E:*:*:*) echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*SV1:*:*:*) echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; *:UNICOS/mp:*:*) - echo nv1-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` - echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit 0 ;; + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` - echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit 0 ;; + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} - exit 0 ;; + exit ;; sparc*:BSD/OS:*:*) echo sparc-unknown-bsdi${UNAME_RELEASE} - exit 0 ;; + exit ;; *:BSD/OS:*:*) echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} - exit 0 ;; + exit ;; *:FreeBSD:*:*) - # Determine whether the default compiler uses glibc. - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include <features.h> - #if __GLIBC__ >= 2 - LIBC=gnu - #else - LIBC= - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` - # GNU/KFreeBSD systems have a "k" prefix to indicate we are using - # FreeBSD's kernel, but not the complete OS. - case ${LIBC} in gnu) kernel_only='k' ;; esac - echo ${UNAME_MACHINE}-unknown-${kernel_only}freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`${LIBC:+-$LIBC} - exit 0 ;; + UNAME_PROCESSOR=`/usr/bin/uname -p` + case ${UNAME_PROCESSOR} in + amd64) + echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + *) + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + esac + exit ;; i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin - exit 0 ;; - i*:MINGW*:*) + exit ;; + *:MINGW64*:*) + echo ${UNAME_MACHINE}-pc-mingw64 + exit ;; + *:MINGW*:*) echo ${UNAME_MACHINE}-pc-mingw32 - exit 0 ;; + exit ;; + i*:MSYS*:*) + echo ${UNAME_MACHINE}-pc-msys + exit ;; + i*:windows32*:*) + # uname -m includes "-pc" on this system. + echo ${UNAME_MACHINE}-mingw32 + exit ;; i*:PW*:*) echo ${UNAME_MACHINE}-pc-pw32 - exit 0 ;; - x86:Interix*:[34]*) - echo i586-pc-interix${UNAME_RELEASE}|sed -e 's/\..*//' - exit 0 ;; + exit ;; + *:Interix*:*) + case ${UNAME_MACHINE} in + x86) + echo i586-pc-interix${UNAME_RELEASE} + exit ;; + authenticamd | genuineintel | EM64T) + echo x86_64-unknown-interix${UNAME_RELEASE} + exit ;; + IA64) + echo ia64-unknown-interix${UNAME_RELEASE} + exit ;; + esac ;; [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) echo i${UNAME_MACHINE}-pc-mks - exit 0 ;; + exit ;; + 8664:Windows_NT:*) + echo x86_64-pc-mks + exit ;; i*:Windows_NT*:* | Pentium*:Windows_NT*:*) # How do we know it's Interix rather than the generic POSIX subsystem? # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we # UNAME_MACHINE based on the output of uname instead of i386? echo i586-pc-interix - exit 0 ;; + exit ;; i*:UWIN*:*) echo ${UNAME_MACHINE}-pc-uwin - exit 0 ;; + exit ;; + amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) + echo x86_64-unknown-cygwin + exit ;; p*:CYGWIN*:*) echo powerpcle-unknown-cygwin - exit 0 ;; + exit ;; prep*:SunOS:5.*:*) echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; *:GNU:*:*) # the GNU system - echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` - exit 0 ;; + echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + exit ;; i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix - exit 0 ;; + exit ;; + aarch64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + aarch64_be:Linux:*:*) + UNAME_MACHINE=aarch64_be + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + alpha:Linux:*:*) + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + EV5) UNAME_MACHINE=alphaev5 ;; + EV56) UNAME_MACHINE=alphaev56 ;; + PCA56) UNAME_MACHINE=alphapca56 ;; + PCA57) UNAME_MACHINE=alphapca56 ;; + EV6) UNAME_MACHINE=alphaev6 ;; + EV67) UNAME_MACHINE=alphaev67 ;; + EV68*) UNAME_MACHINE=alphaev68 ;; + esac + objdump --private-headers /bin/sh | grep -q ld.so.1 + if test "$?" = 0 ; then LIBC="gnulibc1" ; fi + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + arc:Linux:*:* | arceb:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; arm*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + eval $set_cc_for_build + if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_EABI__ + then + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + else + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi + else + echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf + fi + fi + exit ;; + avr32*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; cris:Linux:*:*) - echo cris-axis-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-axis-linux-${LIBC} + exit ;; + crisv32:Linux:*:*) + echo ${UNAME_MACHINE}-axis-linux-${LIBC} + exit ;; + frv:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + hexagon:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + i*86:Linux:*:*) + echo ${UNAME_MACHINE}-pc-linux-${LIBC} + exit ;; ia64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + m32r*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; m68*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - mips:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + mips:Linux:*:* | mips64:Linux:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c #undef CPU - #undef mips - #undef mipsel + #undef ${UNAME_MACHINE} + #undef ${UNAME_MACHINE}el #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mipsel + CPU=${UNAME_MACHINE}el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips + CPU=${UNAME_MACHINE} #else CPU= #endif #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` - test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` + test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } ;; - mips64:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #undef CPU - #undef mips64 - #undef mips64el - #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mips64el - #else - #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips64 - #else - CPU= - #endif - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` - test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 - ;; - ppc:Linux:*:*) - echo powerpc-unknown-linux-gnu - exit 0 ;; - ppc64:Linux:*:*) - echo powerpc64-unknown-linux-gnu - exit 0 ;; - alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in - EV5) UNAME_MACHINE=alphaev5 ;; - EV56) UNAME_MACHINE=alphaev56 ;; - PCA56) UNAME_MACHINE=alphapca56 ;; - PCA57) UNAME_MACHINE=alphapca56 ;; - EV6) UNAME_MACHINE=alphaev6 ;; - EV67) UNAME_MACHINE=alphaev67 ;; - EV68*) UNAME_MACHINE=alphaev68 ;; - esac - objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null - if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi - echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} - exit 0 ;; + or1k:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + or32:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + padre:Linux:*:*) + echo sparc-unknown-linux-${LIBC} + exit ;; + parisc64:Linux:*:* | hppa64:Linux:*:*) + echo hppa64-unknown-linux-${LIBC} + exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-gnu ;; - PA8*) echo hppa2.0-unknown-linux-gnu ;; - *) echo hppa-unknown-linux-gnu ;; + PA7*) echo hppa1.1-unknown-linux-${LIBC} ;; + PA8*) echo hppa2.0-unknown-linux-${LIBC} ;; + *) echo hppa-unknown-linux-${LIBC} ;; esac - exit 0 ;; - parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-gnu - exit 0 ;; + exit ;; + ppc64:Linux:*:*) + echo powerpc64-unknown-linux-${LIBC} + exit ;; + ppc:Linux:*:*) + echo powerpc-unknown-linux-${LIBC} + exit ;; + ppc64le:Linux:*:*) + echo powerpc64le-unknown-linux-${LIBC} + exit ;; + ppcle:Linux:*:*) + echo powerpcle-unknown-linux-${LIBC} + exit ;; s390:Linux:*:* | s390x:Linux:*:*) - echo ${UNAME_MACHINE}-ibm-linux - exit 0 ;; + echo ${UNAME_MACHINE}-ibm-linux-${LIBC} + exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; sh*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + tile*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + vax:Linux:*:*) + echo ${UNAME_MACHINE}-dec-linux-${LIBC} + exit ;; x86_64:Linux:*:*) - echo x86_64-unknown-linux-gnu - exit 0 ;; - i*86:Linux:*:*) - # The BFD linker knows what the default object file format is, so - # first see if it will tell us. cd to the root directory to prevent - # problems with other programs or directories called `ld' in the path. - # Set LC_ALL=C to ensure ld outputs messages in English. - ld_supported_targets=`cd /; LC_ALL=C ld --help 2>&1 \ - | sed -ne '/supported targets:/!d - s/[ ][ ]*/ /g - s/.*supported targets: *// - s/ .*// - p'` - case "$ld_supported_targets" in - elf32-i386) - TENTATIVE="${UNAME_MACHINE}-pc-linux-gnu" - ;; - a.out-i386-linux) - echo "${UNAME_MACHINE}-pc-linux-gnuaout" - exit 0 ;; - coff-i386) - echo "${UNAME_MACHINE}-pc-linux-gnucoff" - exit 0 ;; - "") - # Either a pre-BFD a.out linker (linux-gnuoldld) or - # one that does not give us useful --help. - echo "${UNAME_MACHINE}-pc-linux-gnuoldld" - exit 0 ;; - esac - # Determine whether the default compiler is a.out or elf - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include <features.h> - #ifdef __ELF__ - # ifdef __GLIBC__ - # if __GLIBC__ >= 2 - LIBC=gnu - # else - LIBC=gnulibc1 - # endif - # else - LIBC=gnulibc1 - # endif - #else - #ifdef __INTEL_COMPILER - LIBC=gnu - #else - LIBC=gnuaout - #endif - #endif - #ifdef __dietlibc__ - LIBC=dietlibc - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` - test x"${LIBC}" != x && echo "${UNAME_MACHINE}-pc-linux-${LIBC}" && exit 0 - test x"${TENTATIVE}" != x && echo "${TENTATIVE}" && exit 0 - ;; + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + xtensa*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. # earlier versions are messed up and put the nodename in both # sysname and nodename. echo i386-sequent-sysv4 - exit 0 ;; + exit ;; i*86:UNIX_SV:4.2MP:2.*) - # Unixware is an offshoot of SVR4, but it has its own version - # number series starting with 2... - # I am not positive that other SVR4 systems won't match this, + # Unixware is an offshoot of SVR4, but it has its own version + # number series starting with 2... + # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. - # Use sysv4.2uw... so that sysv4* matches it. + # Use sysv4.2uw... so that sysv4* matches it. echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} - exit 0 ;; + exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. echo ${UNAME_MACHINE}-pc-os2-emx - exit 0 ;; + exit ;; i*86:XTS-300:*:STOP) echo ${UNAME_MACHINE}-unknown-stop - exit 0 ;; + exit ;; i*86:atheos:*:*) echo ${UNAME_MACHINE}-unknown-atheos - exit 0 ;; - i*86:syllable:*:*) + exit ;; + i*86:syllable:*:*) echo ${UNAME_MACHINE}-pc-syllable - exit 0 ;; - i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*) + exit ;; + i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) echo i386-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; i*86:*DOS:*:*) echo ${UNAME_MACHINE}-pc-msdosdjgpp - exit 0 ;; + exit ;; i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then @@ -1037,15 +1066,16 @@ EOF else echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} fi - exit 0 ;; - i*86:*:5:[78]*) + exit ;; + i*86:*:5:[678]*) + # UnixWare 7.x, OpenUNIX and OpenServer 6. case `/bin/uname -X | grep "^Machine"` in *486*) UNAME_MACHINE=i486 ;; *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} - exit 0 ;; + exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name` @@ -1063,73 +1093,86 @@ EOF else echo ${UNAME_MACHINE}-pc-sysv32 fi - exit 0 ;; + exit ;; pc:*:*:*) # Left here for compatibility: - # uname -m prints for DJGPP always 'pc', but it prints nothing about - # the processor, so we play safe by assuming i386. - echo i386-pc-msdosdjgpp - exit 0 ;; + # uname -m prints for DJGPP always 'pc', but it prints nothing about + # the processor, so we play safe by assuming i586. + # Note: whatever this is, it MUST be the same as what config.sub + # prints for the "djgpp" host, or else GDB configury will decide that + # this is a cross-build. + echo i586-pc-msdosdjgpp + exit ;; Intel:Mach:3*:*) echo i386-pc-mach3 - exit 0 ;; + exit ;; paragon:*:*:*) echo i860-intel-osf1 - exit 0 ;; + exit ;; i860:*:4.*:*) # i860-SVR4 if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 fi - exit 0 ;; + exit ;; mini*:CTIX:SYS*5:*) # "miniframe" echo m68010-convergent-sysv - exit 0 ;; + exit ;; mc68k:UNIX:SYSTEM5:3.51m) echo m68k-convergent-sysv - exit 0 ;; + exit ;; M680?0:D-NIX:5.3:*) echo m68k-diab-dnix - exit 0 ;; - M68*:*:R3V[567]*:*) - test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;; - 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0) + exit ;; + M68*:*:R3V[5678]*:*) + test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; + 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) OS_REL='' test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && echo i486-ncr-sysv4.3${OS_REL} && exit 0 + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && echo i586-ncr-sysv4.3${OS_REL} && exit 0 ;; + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && echo i486-ncr-sysv4 && exit 0 ;; + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4; exit; } ;; + NCR*:*:4.2:* | MPRAS*:*:4.2:*) + OS_REL='.3' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) echo m68k-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 - exit 0 ;; + exit ;; TSUNAMI:LynxOS:2.*:*) echo sparc-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; rs6000:LynxOS:2.*:*) echo rs6000-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; - PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*) + exit ;; + PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) echo powerpc-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; SM[BE]S:UNIX_SV:*:*) echo mips-dde-sysv${UNAME_RELEASE} - exit 0 ;; + exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 - exit 0 ;; + exit ;; RM*:SINIX-*:*:*) echo mips-sni-sysv4 - exit 0 ;; + exit ;; *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` @@ -1137,68 +1180,99 @@ EOF else echo ns32k-sni-sysv fi - exit 0 ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort - # says <Richard.M.Bartel@ccMail.Census.GOV> - echo i586-unisys-sysv4 - exit 0 ;; + exit ;; + PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + # says <Richard.M.Bartel@ccMail.Census.GOV> + echo i586-unisys-sysv4 + exit ;; *:UNIX_System_V:4*:FTX*) # From Gerald Hewes <hewes@openmarket.com>. # How about differentiating between stratus architectures? -djm echo hppa1.1-stratus-sysv4 - exit 0 ;; + exit ;; *:*:*:FTX*) # From seanf@swdc.stratus.com. echo i860-stratus-sysv4 - exit 0 ;; + exit ;; + i*86:VOS:*:*) + # From Paul.Green@stratus.com. + echo ${UNAME_MACHINE}-stratus-vos + exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos - exit 0 ;; + exit ;; mc68*:A/UX:*:*) echo m68k-apple-aux${UNAME_RELEASE} - exit 0 ;; + exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 - exit 0 ;; + exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} + echo mips-nec-sysv${UNAME_RELEASE} else - echo mips-unknown-sysv${UNAME_RELEASE} + echo mips-unknown-sysv${UNAME_RELEASE} fi - exit 0 ;; + exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. echo powerpc-be-beos - exit 0 ;; + exit ;; BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. echo powerpc-apple-beos - exit 0 ;; + exit ;; BePC:BeOS:*:*) # BeOS running on Intel PC compatible. echo i586-pc-beos - exit 0 ;; + exit ;; + BePC:Haiku:*:*) # Haiku running on Intel PC compatible. + echo i586-pc-haiku + exit ;; + x86_64:Haiku:*:*) + echo x86_64-unknown-haiku + exit ;; SX-4:SUPER-UX:*:*) echo sx4-nec-superux${UNAME_RELEASE} - exit 0 ;; + exit ;; SX-5:SUPER-UX:*:*) echo sx5-nec-superux${UNAME_RELEASE} - exit 0 ;; + exit ;; SX-6:SUPER-UX:*:*) echo sx6-nec-superux${UNAME_RELEASE} - exit 0 ;; + exit ;; + SX-7:SUPER-UX:*:*) + echo sx7-nec-superux${UNAME_RELEASE} + exit ;; + SX-8:SUPER-UX:*:*) + echo sx8-nec-superux${UNAME_RELEASE} + exit ;; + SX-8R:SUPER-UX:*:*) + echo sx8r-nec-superux${UNAME_RELEASE} + exit ;; Power*:Rhapsody:*:*) echo powerpc-apple-rhapsody${UNAME_RELEASE} - exit 0 ;; + exit ;; *:Rhapsody:*:*) echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} - exit 0 ;; + exit ;; *:Darwin:*:*) - case `uname -p` in - *86) UNAME_PROCESSOR=i686 ;; - powerpc) UNAME_PROCESSOR=powerpc ;; - esac + UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown + eval $set_cc_for_build + if test "$UNAME_PROCESSOR" = unknown ; then + UNAME_PROCESSOR=powerpc + fi + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + fi echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} - exit 0 ;; + exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` if test "$UNAME_PROCESSOR" = "x86"; then @@ -1206,22 +1280,28 @@ EOF UNAME_MACHINE=pc fi echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} - exit 0 ;; + exit ;; *:QNX:*:4*) echo i386-pc-qnx - exit 0 ;; + exit ;; + NEO-?:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk${UNAME_RELEASE} + exit ;; + NSE-*:NONSTOP_KERNEL:*:*) + echo nse-tandem-nsk${UNAME_RELEASE} + exit ;; NSR-?:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk${UNAME_RELEASE} - exit 0 ;; + exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux - exit 0 ;; + exit ;; BS2000:POSIX*:*:*) echo bs2000-siemens-sysv - exit 0 ;; + exit ;; DS/*:UNIX_System_V:*:*) echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} - exit 0 ;; + exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 @@ -1232,36 +1312,55 @@ EOF UNAME_MACHINE="$cputype" fi echo ${UNAME_MACHINE}-unknown-plan9 - exit 0 ;; + exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 - exit 0 ;; + exit ;; *:TENEX:*:*) echo pdp10-unknown-tenex - exit 0 ;; + exit ;; KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) echo pdp10-dec-tops20 - exit 0 ;; + exit ;; XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) echo pdp10-xkl-tops20 - exit 0 ;; + exit ;; *:TOPS-20:*:*) echo pdp10-unknown-tops20 - exit 0 ;; + exit ;; *:ITS:*:*) echo pdp10-unknown-its - exit 0 ;; + exit ;; SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} - exit 0 ;; + echo mips-sei-seiux${UNAME_RELEASE} + exit ;; *:DragonFly:*:*) echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` - exit 0 ;; + exit ;; + *:*VMS:*:*) + UNAME_MACHINE=`(uname -p) 2>/dev/null` + case "${UNAME_MACHINE}" in + A*) echo alpha-dec-vms ; exit ;; + I*) echo ia64-dec-vms ; exit ;; + V*) echo vax-dec-vms ; exit ;; + esac ;; + *:XENIX:*:SysV) + echo i386-pc-xenix + exit ;; + i*86:skyos:*:*) + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + exit ;; + i*86:rdos:*:*) + echo ${UNAME_MACHINE}-pc-rdos + exit ;; + i*86:AROS:*:*) + echo ${UNAME_MACHINE}-pc-aros + exit ;; + x86_64:VMkernel:*:*) + echo ${UNAME_MACHINE}-unknown-esx + exit ;; esac -#echo '(No uname command or uname output not recognized.)' 1>&2 -#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2 - eval $set_cc_for_build cat >$dummy.c <<EOF #ifdef _SEQUENT_ @@ -1279,16 +1378,16 @@ main () #include <sys/param.h> printf ("m68k-sony-newsos%s\n", #ifdef NEWSOS4 - "4" + "4" #else - "" + "" #endif - ); exit (0); + ); exit (0); #endif #endif #if defined (__arm) && defined (__acorn) && defined (__unix) - printf ("arm-acorn-riscix"); exit (0); + printf ("arm-acorn-riscix\n"); exit (0); #endif #if defined (hp300) && !defined (hpux) @@ -1377,11 +1476,12 @@ main () } EOF -$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && $dummy && exit 0 +$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } # Apollos put the system type in the environment. -test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; } +test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } # Convex versions that predate uname can use getsysinfo(1) @@ -1390,22 +1490,22 @@ then case `getsysinfo -f cpu_type` in c1*) echo c1-convex-bsd - exit 0 ;; + exit ;; c2*) if getsysinfo -f scalar_acc then echo c32-convex-bsd else echo c2-convex-bsd fi - exit 0 ;; + exit ;; c34*) echo c34-convex-bsd - exit 0 ;; + exit ;; c38*) echo c38-convex-bsd - exit 0 ;; + exit ;; c4*) echo c4-convex-bsd - exit 0 ;; + exit ;; esac fi @@ -1416,7 +1516,9 @@ This script, last modified $timestamp, has failed to recognize the operating system you are using. It is advised that you download the most up to date version of the config scripts from - ftp://ftp.gnu.org/pub/gnu/config/ + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +and + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD If the version you run ($0) is already up to date, please send the following data and any information you think might be diff --git a/config.sub b/infrastructure/config.sub index 264f820a..8b612ab8 100755 --- a/config.sub +++ b/infrastructure/config.sub @@ -1,42 +1,40 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. +# Copyright 1992-2013 Free Software Foundation, Inc. -timestamp='2004-02-23' +timestamp='2013-04-24' -# This file is (in principle) common to ALL GNU software. -# The presence of a machine in this file suggests that SOME GNU software -# can handle that machine. It does not imply ALL GNU software can. -# -# This file is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - +# along with this program; if not, see <http://www.gnu.org/licenses/>. +# # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). -# Please send patches to <config-patches@gnu.org>. Submit a context -# diff and a properly formatted ChangeLog entry. + +# Please send patches with a ChangeLog entry to config-patches@gnu.org. # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. # If it is invalid, we print an error message on stderr and exit with code 1. # Otherwise, we print the canonical config type on stdout and succeed. +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD + # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases # that are meaningful with *any* GNU software. @@ -70,8 +68,7 @@ Report bugs and patches to <config-patches@gnu.org>." version="\ GNU config.sub ($timestamp) -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 -Free Software Foundation, Inc. +Copyright 1992-2013 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -83,11 +80,11 @@ Try \`$me --help' for more information." while test $# -gt 0 ; do case $1 in --time-stamp | --time* | -t ) - echo "$timestamp" ; exit 0 ;; + echo "$timestamp" ; exit ;; --version | -v ) - echo "$version" ; exit 0 ;; + echo "$version" ; exit ;; --help | --h* | -h ) - echo "$usage"; exit 0 ;; + echo "$usage"; exit ;; -- ) # Stop option processing shift; break ;; - ) # Use stdin as input. @@ -99,7 +96,7 @@ while test $# -gt 0 ; do *local*) # First pass through any local machine types. echo $1 - exit 0;; + exit ;; * ) break ;; @@ -118,11 +115,18 @@ esac # Here we must recognize all the valid KERNEL-OS combinations. maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in - nto-qnx* | linux-gnu* | linux-dietlibc | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | \ - kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | storm-chaos* | os2-emx* | rtmk-nova*) + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ + linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | \ + kopensolaris*-gnu* | \ + storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` ;; + android-linux) + os=-linux-android + basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown + ;; *) basic_machine=`echo $1 | sed 's/-[^-]*$//'` if [ $basic_machine != $1 ] @@ -145,10 +149,13 @@ case $os in -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ - -apple | -axis) + -apple | -axis | -knuth | -cray | -microblaze*) os= basic_machine=$1 ;; + -bluegene*) + os=-cnk + ;; -sim | -cisco | -oki | -wec | -winbond) os= basic_machine=$1 @@ -163,13 +170,17 @@ case $os in os=-chorusos basic_machine=$1 ;; - -chorusrdb) - os=-chorusrdb + -chorusrdb) + os=-chorusrdb basic_machine=$1 - ;; + ;; -hiux*) os=-hiuxwe2 ;; + -sco6) + os=-sco5v6 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; -sco5) os=-sco3.2v5 basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` @@ -186,6 +197,10 @@ case $os in # Don't forget version if it is 3.2v4 or newer. basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` ;; + -sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; -sco*) os=-sco3.2v2 basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` @@ -203,6 +218,12 @@ case $os in -isc*) basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` ;; + -lynx*178) + os=-lynxos178 + ;; + -lynx*5) + os=-lynxos5 + ;; -lynx*) os=-lynxos ;; @@ -227,57 +248,106 @@ case $basic_machine in # Some are omitted here because they have special meanings below. 1750a | 580 \ | a29k \ + | aarch64 | aarch64_be \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ | am33_2.0 \ - | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \ + | arc | arceb \ + | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ + | avr | avr32 \ + | be32 | be64 \ + | bfin \ | c4x | clipper \ | d10v | d30v | dlx | dsp16xx \ - | fr30 | frv \ + | epiphany \ + | fido | fr30 | frv \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ | i370 | i860 | i960 | ia64 \ | ip2k | iq2000 \ - | m32r | m68000 | m68k | m88k | mcore \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle | m68000 | m68k | m88k \ + | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ | mips | mipsbe | mipseb | mipsel | mipsle \ | mips16 \ | mips64 | mips64el \ - | mips64vr | mips64vrel \ + | mips64octeon | mips64octeonel \ | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ | mips64vr4100 | mips64vr4100el \ | mips64vr4300 | mips64vr4300el \ | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ | mipsisa32 | mipsisa32el \ | mipsisa32r2 | mipsisa32r2el \ | mipsisa64 | mipsisa64el \ | mipsisa64r2 | mipsisa64r2el \ | mipsisa64sb1 | mipsisa64sb1el \ | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ | mipstx39 | mipstx39el \ | mn10200 | mn10300 \ + | moxie \ + | mt \ | msp430 \ + | nds32 | nds32le | nds32be \ + | nios | nios2 | nios2eb | nios2el \ | ns16k | ns32k \ - | openrisc | or32 \ + | open8 \ + | or1k | or32 \ | pdp10 | pdp11 | pj | pjl \ - | powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \ + | powerpc | powerpc64 | powerpc64le | powerpcle \ | pyramid \ - | sh | sh[1234] | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \ + | rl78 | rx \ + | score \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ - | sparc | sparc64 | sparc86x | sparclet | sparclite | sparcv9 | sparcv9b \ - | strongarm \ - | tahoe | thumb | tic4x | tic80 | tron \ - | v850 | v850e \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ + | spu \ + | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ + | ubicom32 \ + | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ | we32k \ - | x86 | xscale | xstormy16 | xtensa \ - | z8k) + | x86 | xc16x | xstormy16 | xtensa \ + | z8k | z80) basic_machine=$basic_machine-unknown ;; - m6811 | m68hc11 | m6812 | m68hc12) - # Motorola 68HC11/12. + c54x) + basic_machine=tic54x-unknown + ;; + c55x) + basic_machine=tic55x-unknown + ;; + c6x) + basic_machine=tic6x-unknown + ;; + m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip) basic_machine=$basic_machine-unknown os=-none ;; m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) ;; + ms1) + basic_machine=mt-unknown + ;; + + strongarm | thumb | xscale) + basic_machine=arm-unknown + ;; + xgate) + basic_machine=$basic_machine-unknown + os=-none + ;; + xscaleeb) + basic_machine=armeb-unknown + ;; + + xscaleel) + basic_machine=armel-unknown + ;; # We use `pc' rather than `unknown' # because (1) that's what they normally are, and @@ -293,59 +363,82 @@ case $basic_machine in # Recognize the basic CPU types with company name. 580-* \ | a29k-* \ + | aarch64-* | aarch64_be-* \ | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ - | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ + | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ - | avr-* \ - | bs2000-* \ - | c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \ - | clipper-* | cydra-* \ + | avr-* | avr32-* \ + | be32-* | be64-* \ + | bfin-* | bs2000-* \ + | c[123]* | c30-* | [cjt]90-* | c4x-* \ + | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ | elxsi-* \ - | f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ + | hexagon-* \ | i*86-* | i860-* | i960-* | ia64-* \ | ip2k-* | iq2000-* \ - | m32r-* \ + | le32-* | le64-* \ + | lm32-* \ + | m32c-* | m32r-* | m32rle-* \ | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | mcore-* \ + | m88110-* | m88k-* | maxq-* | mcore-* | metag-* \ + | microblaze-* | microblazeel-* \ | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ | mips16-* \ | mips64-* | mips64el-* \ - | mips64vr-* | mips64vrel-* \ + | mips64octeon-* | mips64octeonel-* \ | mips64orion-* | mips64orionel-* \ + | mips64r5900-* | mips64r5900el-* \ + | mips64vr-* | mips64vrel-* \ | mips64vr4100-* | mips64vr4100el-* \ | mips64vr4300-* | mips64vr4300el-* \ | mips64vr5000-* | mips64vr5000el-* \ + | mips64vr5900-* | mips64vr5900el-* \ | mipsisa32-* | mipsisa32el-* \ | mipsisa32r2-* | mipsisa32r2el-* \ | mipsisa64-* | mipsisa64el-* \ | mipsisa64r2-* | mipsisa64r2el-* \ | mipsisa64sb1-* | mipsisa64sb1el-* \ | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipsr5900-* | mipsr5900el-* \ | mipstx39-* | mipstx39el-* \ + | mmix-* \ + | mt-* \ | msp430-* \ - | none-* | np1-* | nv1-* | ns16k-* | ns32k-* \ + | nds32-* | nds32le-* | nds32be-* \ + | nios-* | nios2-* | nios2eb-* | nios2el-* \ + | none-* | np1-* | ns16k-* | ns32k-* \ + | open8-* \ | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ - | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \ + | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ | pyramid-* \ - | romp-* | rs6000-* \ - | sh-* | sh[1234]-* | sh[23]e-* | sh[34]eb-* | shbe-* \ + | rl78-* | romp-* | rs6000-* | rx-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ - | sparc-* | sparc64-* | sparc86x-* | sparclet-* | sparclite-* \ - | sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \ - | tahoe-* | thumb-* \ + | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ + | sparclite-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ + | tile*-* \ | tron-* \ - | v850-* | v850e-* | vax-* \ + | ubicom32-* \ + | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ + | vax-* \ | we32k-* \ - | x86-* | x86_64-* | xps100-* | xscale-* | xstormy16-* \ - | xtensa-* \ + | x86-* | x86_64-* | xc16x-* | xps100-* \ + | xstormy16-* | xtensa*-* \ | ymp-* \ - | z8k-*) + | z8k-* | z80-*) + ;; + # Recognize the basic CPU types without company name, with glob match. + xtensa*) + basic_machine=$basic_machine-unknown ;; # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. @@ -363,7 +456,7 @@ case $basic_machine in basic_machine=a29k-amd os=-udi ;; - abacus) + abacus) basic_machine=abacus-unknown ;; adobe68k) @@ -409,6 +502,10 @@ case $basic_machine in basic_machine=m68k-apollo os=-bsd ;; + aros) + basic_machine=i386-pc + os=-aros + ;; aux) basic_machine=m68k-apple os=-aux @@ -417,10 +514,35 @@ case $basic_machine in basic_machine=ns32k-sequent os=-dynix ;; + blackfin) + basic_machine=bfin-unknown + os=-linux + ;; + blackfin-*) + basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + bluegene*) + basic_machine=powerpc-ibm + os=-cnk + ;; + c54x-*) + basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c55x-*) + basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c6x-*) + basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; c90) basic_machine=c90-cray os=-unicos ;; + cegcc) + basic_machine=arm-unknown + os=-cegcc + ;; convex-c1) basic_machine=c1-convex os=-bsd @@ -445,13 +567,20 @@ case $basic_machine in basic_machine=j90-cray os=-unicos ;; - cr16c) - basic_machine=cr16c-unknown + craynv) + basic_machine=craynv-cray + os=-unicosmp + ;; + cr16 | cr16-*) + basic_machine=cr16-unknown os=-elf ;; crds | unos) basic_machine=m68k-crds ;; + crisv32 | crisv32-* | etraxfs*) + basic_machine=crisv32-axis + ;; cris | cris-* | etrax*) basic_machine=cris-axis ;; @@ -481,6 +610,14 @@ case $basic_machine in basic_machine=m88k-motorola os=-sysv3 ;; + dicos) + basic_machine=i686-pc + os=-dicos + ;; + djgpp) + basic_machine=i586-pc + os=-msdosdjgpp + ;; dpx20 | dpx20-*) basic_machine=rs6000-bull os=-bosx @@ -592,7 +729,6 @@ case $basic_machine in i370-ibm* | ibm*) basic_machine=i370-ibm ;; -# I'm not sure what "Sysv32" means. Should this be sysv3.2? i*86v32) basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` os=-sysv32 @@ -631,6 +767,14 @@ case $basic_machine in basic_machine=m68k-isi os=-sysv ;; + m68knommu) + basic_machine=m68k-unknown + os=-linux + ;; + m68knommu-*) + basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; m88k-omron*) basic_machine=m88k-omron ;; @@ -642,10 +786,21 @@ case $basic_machine in basic_machine=ns32k-utek os=-sysv ;; + microblaze*) + basic_machine=microblaze-xilinx + ;; + mingw64) + basic_machine=x86_64-pc + os=-mingw64 + ;; mingw32) basic_machine=i386-pc os=-mingw32 ;; + mingw32ce) + basic_machine=arm-unknown + os=-mingw32ce + ;; miniframe) basic_machine=m68000-convergent ;; @@ -659,10 +814,6 @@ case $basic_machine in mips3*) basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown ;; - mmix*) - basic_machine=mmix-knuth - os=-mmixware - ;; monitor) basic_machine=m68k-rom68k os=-coff @@ -675,10 +826,21 @@ case $basic_machine in basic_machine=i386-pc os=-msdos ;; + ms1-*) + basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` + ;; + msys) + basic_machine=i386-pc + os=-msys + ;; mvs) basic_machine=i370-ibm os=-mvs ;; + nacl) + basic_machine=le32-unknown + os=-nacl + ;; ncr3000) basic_machine=i486-ncr os=-sysv4 @@ -743,9 +905,11 @@ case $basic_machine in np1) basic_machine=np1-gould ;; - nv1) - basic_machine=nv1-cray - os=-unicosmp + neo-tandem) + basic_machine=neo-tandem + ;; + nse-tandem) + basic_machine=nse-tandem ;; nsr-tandem) basic_machine=nsr-tandem @@ -754,9 +918,8 @@ case $basic_machine in basic_machine=hppa1.1-oki os=-proelf ;; - or32 | or32-*) + openrisc | openrisc-*) basic_machine=or32-unknown - os=-coff ;; os400) basic_machine=powerpc-ibm @@ -778,6 +941,14 @@ case $basic_machine in basic_machine=i860-intel os=-osf ;; + parisc) + basic_machine=hppa-unknown + os=-linux + ;; + parisc-*) + basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; pbd) basic_machine=sparc-tti ;; @@ -787,6 +958,12 @@ case $basic_machine in pc532 | pc532-*) basic_machine=ns32k-pc532 ;; + pc98) + basic_machine=i386-pc + ;; + pc98-*) + basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; pentium | p5 | k5 | k6 | nexgen | viac3) basic_machine=i586-pc ;; @@ -816,9 +993,10 @@ case $basic_machine in ;; power) basic_machine=power-ibm ;; - ppc) basic_machine=powerpc-unknown + ppc | ppcbe) basic_machine=powerpc-unknown ;; - ppc-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + ppc-* | ppcbe-*) + basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` ;; ppcle | powerpclittle | ppc-le | powerpc-little) basic_machine=powerpcle-unknown @@ -843,6 +1021,14 @@ case $basic_machine in basic_machine=i586-unknown os=-pw32 ;; + rdos | rdos64) + basic_machine=x86_64-pc + os=-rdos + ;; + rdos32) + basic_machine=i386-pc + os=-rdos + ;; rom68k) basic_machine=m68k-rom68k os=-coff @@ -869,6 +1055,10 @@ case $basic_machine in sb1el) basic_machine=mipsisa64sb1el-unknown ;; + sde) + basic_machine=mipsisa32-sde + os=-elf + ;; sei) basic_machine=mips-sei os=-seiux @@ -880,6 +1070,9 @@ case $basic_machine in basic_machine=sh-hitachi os=-hms ;; + sh5el) + basic_machine=sh5le-unknown + ;; sh64) basic_machine=sh64-unknown ;; @@ -901,6 +1094,9 @@ case $basic_machine in basic_machine=i860-stratus os=-sysv4 ;; + strongarm-* | thumb-*) + basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; sun2) basic_machine=m68000-sun ;; @@ -957,17 +1153,9 @@ case $basic_machine in basic_machine=t90-cray os=-unicos ;; - tic54x | c54x*) - basic_machine=tic54x-unknown - os=-coff - ;; - tic55x | c55x*) - basic_machine=tic55x-unknown - os=-coff - ;; - tic6x | c6x*) - basic_machine=tic6x-unknown - os=-coff + tile*) + basic_machine=$basic_machine-unknown + os=-linux-gnu ;; tx39) basic_machine=mipstx39-unknown @@ -1029,9 +1217,16 @@ case $basic_machine in basic_machine=hppa1.1-winbond os=-proelf ;; + xbox) + basic_machine=i686-pc + os=-mingw32 + ;; xps | xps100) basic_machine=xps100-honeywell ;; + xscale-* | xscalee[bl]-*) + basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` + ;; ymp) basic_machine=ymp-cray os=-unicos @@ -1040,6 +1235,10 @@ case $basic_machine in basic_machine=z8k-unknown os=-sim ;; + z80-*-coff) + basic_machine=z80-unknown + os=-sim + ;; none) basic_machine=none-none os=-none @@ -1059,6 +1258,9 @@ case $basic_machine in romp) basic_machine=romp-ibm ;; + mmix) + basic_machine=mmix-knuth + ;; rs6000) basic_machine=rs6000-ibm ;; @@ -1075,13 +1277,10 @@ case $basic_machine in we32k) basic_machine=we32k-att ;; - sh3 | sh4 | sh[34]eb | sh[1234]le | sh[23]ele) + sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) basic_machine=sh-unknown ;; - sh64) - basic_machine=sh64-unknown - ;; - sparc | sparcv9 | sparcv9b) + sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) basic_machine=sparc-sun ;; cydra) @@ -1125,9 +1324,12 @@ esac if [ x"$os" != x"" ] then case $os in - # First match some system type aliases - # that might get confused with valid system types. + # First match some system type aliases + # that might get confused with valid system types. # -solaris* is a basic system type, with this one exception. + -auroraux) + os=-auroraux + ;; -solaris1 | -solaris1.*) os=`echo $os | sed -e 's|solaris1|sunos4|'` ;; @@ -1148,26 +1350,31 @@ case $os in # Each alternative MUST END IN A *, to match a version number. # -sysv* is not here because it comes later, after sysvr4. -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ - | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\ - | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \ + | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ + | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ + | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* \ + | -aos* | -aros* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* | -openbsd* \ + | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ + | -bitrig* | -openbsd* | -solidbsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* \ - | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -linux-gnu* | -linux-uclibc* | -uxpv* | -beos* | -mpeix* | -udk* \ + | -chorusos* | -chorusrdb* | -cegcc* \ + | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ + | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ + | -linux-newlib* | -linux-musl* | -linux-uclibc* \ + | -uxpv* | -beos* | -mpeix* | -udk* \ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ - | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly*) + | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1185,7 +1392,7 @@ case $os in os=`echo $os | sed -e 's|nto|nto-qnx|'` ;; -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* | -beos* \ + | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) ;; -mac*) @@ -1206,7 +1413,7 @@ case $os in -opened*) os=-openedition ;; - -os400*) + -os400*) os=-os400 ;; -wince*) @@ -1255,7 +1462,7 @@ case $os in -sinix*) os=-sysv4 ;; - -tpf*) + -tpf*) os=-tpf ;; -triton*) @@ -1291,8 +1498,13 @@ case $os in -aros*) os=-aros ;; - -kaos*) - os=-kaos + -zvmoe) + os=-zvmoe + ;; + -dicos*) + os=-dicos + ;; + -nacl*) ;; -none) ;; @@ -1316,6 +1528,12 @@ else # system, and we'll never get to this point. case $basic_machine in + score-*) + os=-elf + ;; + spu-*) + os=-elf + ;; *-acorn) os=-riscix1.2 ;; @@ -1325,9 +1543,21 @@ case $basic_machine in arm*-semi) os=-aout ;; - c4x-* | tic4x-*) - os=-coff - ;; + c4x-* | tic4x-*) + os=-coff + ;; + hexagon-*) + os=-elf + ;; + tic54x-*) + os=-coff + ;; + tic55x-*) + os=-coff + ;; + tic6x-*) + os=-coff + ;; # This must come before the *-dec entry. pdp10-*) os=-tops20 @@ -1346,19 +1576,22 @@ case $basic_machine in ;; m68000-sun) os=-sunos3 - # This also exists in the configure program, but was not the - # default. - # os=-sunos4 ;; m68*-cisco) os=-aout ;; + mep-*) + os=-elf + ;; mips*-cisco) os=-elf ;; mips*-*) os=-elf ;; + or1k-*) + os=-elf + ;; or32-*) os=-coff ;; @@ -1371,9 +1604,15 @@ case $basic_machine in *-be) os=-beos ;; + *-haiku) + os=-haiku + ;; *-ibm) os=-aix ;; + *-knuth) + os=-mmixware + ;; *-wec) os=-proelf ;; @@ -1476,7 +1715,7 @@ case $basic_machine in -sunos*) vendor=sun ;; - -aix*) + -cnk*|-aix*) vendor=ibm ;; -beos*) @@ -1539,7 +1778,7 @@ case $basic_machine in esac echo $basic_machine$os -exit 0 +exit # Local variables: # eval: (add-hook 'write-file-hooks 'time-stamp) diff --git a/infrastructure/install-sh b/infrastructure/install-sh new file mode 100755 index 00000000..377bb868 --- /dev/null +++ b/infrastructure/install-sh @@ -0,0 +1,527 @@ +#!/bin/sh +# install - install a program, script, or datafile + +scriptversion=2011-11-20.07; # UTC + +# This originates from X11R5 (mit/util/scripts/install.sh), which was +# later released in X11R6 (xc/config/util/install.sh) with the +# following copyright and license. +# +# Copyright (C) 1994 X Consortium +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- +# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# Except as contained in this notice, the name of the X Consortium shall not +# be used in advertising or otherwise to promote the sale, use or other deal- +# ings in this Software without prior written authorization from the X Consor- +# tium. +# +# +# FSF changes to this file are in the public domain. +# +# Calling this script install-sh is preferred over install.sh, to prevent +# 'make' implicit rules from creating a file called install from it +# when there is no Makefile. +# +# This script is compatible with the BSD install script, but was written +# from scratch. + +nl=' +' +IFS=" "" $nl" + +# set DOITPROG to echo to test this script + +# Don't use :- since 4.3BSD and earlier shells don't like it. +doit=${DOITPROG-} +if test -z "$doit"; then + doit_exec=exec +else + doit_exec=$doit +fi + +# Put in absolute file names if you don't have them in your path; +# or use environment vars. + +chgrpprog=${CHGRPPROG-chgrp} +chmodprog=${CHMODPROG-chmod} +chownprog=${CHOWNPROG-chown} +cmpprog=${CMPPROG-cmp} +cpprog=${CPPROG-cp} +mkdirprog=${MKDIRPROG-mkdir} +mvprog=${MVPROG-mv} +rmprog=${RMPROG-rm} +stripprog=${STRIPPROG-strip} + +posix_glob='?' +initialize_posix_glob=' + test "$posix_glob" != "?" || { + if (set -f) 2>/dev/null; then + posix_glob= + else + posix_glob=: + fi + } +' + +posix_mkdir= + +# Desired mode of installed file. +mode=0755 + +chgrpcmd= +chmodcmd=$chmodprog +chowncmd= +mvcmd=$mvprog +rmcmd="$rmprog -f" +stripcmd= + +src= +dst= +dir_arg= +dst_arg= + +copy_on_change=false +no_target_directory= + +usage="\ +Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE + or: $0 [OPTION]... SRCFILES... DIRECTORY + or: $0 [OPTION]... -t DIRECTORY SRCFILES... + or: $0 [OPTION]... -d DIRECTORIES... + +In the 1st form, copy SRCFILE to DSTFILE. +In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. +In the 4th, create DIRECTORIES. + +Options: + --help display this help and exit. + --version display version info and exit. + + -c (ignored) + -C install only if different (preserve the last data modification time) + -d create directories instead of installing files. + -g GROUP $chgrpprog installed files to GROUP. + -m MODE $chmodprog installed files to MODE. + -o USER $chownprog installed files to USER. + -s $stripprog installed files. + -t DIRECTORY install into DIRECTORY. + -T report an error if DSTFILE is a directory. + +Environment variables override the default commands: + CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG + RMPROG STRIPPROG +" + +while test $# -ne 0; do + case $1 in + -c) ;; + + -C) copy_on_change=true;; + + -d) dir_arg=true;; + + -g) chgrpcmd="$chgrpprog $2" + shift;; + + --help) echo "$usage"; exit $?;; + + -m) mode=$2 + case $mode in + *' '* | *' '* | *' +'* | *'*'* | *'?'* | *'['*) + echo "$0: invalid mode: $mode" >&2 + exit 1;; + esac + shift;; + + -o) chowncmd="$chownprog $2" + shift;; + + -s) stripcmd=$stripprog;; + + -t) dst_arg=$2 + # Protect names problematic for 'test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac + shift;; + + -T) no_target_directory=true;; + + --version) echo "$0 $scriptversion"; exit $?;; + + --) shift + break;; + + -*) echo "$0: invalid option: $1" >&2 + exit 1;; + + *) break;; + esac + shift +done + +if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then + # When -d is used, all remaining arguments are directories to create. + # When -t is used, the destination is already specified. + # Otherwise, the last argument is the destination. Remove it from $@. + for arg + do + if test -n "$dst_arg"; then + # $@ is not empty: it contains at least $arg. + set fnord "$@" "$dst_arg" + shift # fnord + fi + shift # arg + dst_arg=$arg + # Protect names problematic for 'test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac + done +fi + +if test $# -eq 0; then + if test -z "$dir_arg"; then + echo "$0: no input file specified." >&2 + exit 1 + fi + # It's OK to call 'install-sh -d' without argument. + # This can happen when creating conditional directories. + exit 0 +fi + +if test -z "$dir_arg"; then + do_exit='(exit $ret); exit $ret' + trap "ret=129; $do_exit" 1 + trap "ret=130; $do_exit" 2 + trap "ret=141; $do_exit" 13 + trap "ret=143; $do_exit" 15 + + # Set umask so as not to create temps with too-generous modes. + # However, 'strip' requires both read and write access to temps. + case $mode in + # Optimize common cases. + *644) cp_umask=133;; + *755) cp_umask=22;; + + *[0-7]) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw='% 200' + fi + cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; + *) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw=,u+rw + fi + cp_umask=$mode$u_plus_rw;; + esac +fi + +for src +do + # Protect names problematic for 'test' and other utilities. + case $src in + -* | [=\(\)!]) src=./$src;; + esac + + if test -n "$dir_arg"; then + dst=$src + dstdir=$dst + test -d "$dstdir" + dstdir_status=$? + else + + # Waiting for this to be detected by the "$cpprog $src $dsttmp" command + # might cause directories to be created, which would be especially bad + # if $src (and thus $dsttmp) contains '*'. + if test ! -f "$src" && test ! -d "$src"; then + echo "$0: $src does not exist." >&2 + exit 1 + fi + + if test -z "$dst_arg"; then + echo "$0: no destination specified." >&2 + exit 1 + fi + dst=$dst_arg + + # If destination is a directory, append the input filename; won't work + # if double slashes aren't ignored. + if test -d "$dst"; then + if test -n "$no_target_directory"; then + echo "$0: $dst_arg: Is a directory" >&2 + exit 1 + fi + dstdir=$dst + dst=$dstdir/`basename "$src"` + dstdir_status=0 + else + # Prefer dirname, but fall back on a substitute if dirname fails. + dstdir=` + (dirname "$dst") 2>/dev/null || + expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$dst" : 'X\(//\)[^/]' \| \ + X"$dst" : 'X\(//\)$' \| \ + X"$dst" : 'X\(/\)' \| . 2>/dev/null || + echo X"$dst" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q' + ` + + test -d "$dstdir" + dstdir_status=$? + fi + fi + + obsolete_mkdir_used=false + + if test $dstdir_status != 0; then + case $posix_mkdir in + '') + # Create intermediate dirs using mode 755 as modified by the umask. + # This is like FreeBSD 'install' as of 1997-10-28. + umask=`umask` + case $stripcmd.$umask in + # Optimize common cases. + *[2367][2367]) mkdir_umask=$umask;; + .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; + + *[0-7]) + mkdir_umask=`expr $umask + 22 \ + - $umask % 100 % 40 + $umask % 20 \ + - $umask % 10 % 4 + $umask % 2 + `;; + *) mkdir_umask=$umask,go-w;; + esac + + # With -d, create the new directory with the user-specified mode. + # Otherwise, rely on $mkdir_umask. + if test -n "$dir_arg"; then + mkdir_mode=-m$mode + else + mkdir_mode= + fi + + posix_mkdir=false + case $umask in + *[123567][0-7][0-7]) + # POSIX mkdir -p sets u+wx bits regardless of umask, which + # is incompatible with FreeBSD 'install' when (umask & 300) != 0. + ;; + *) + tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ + trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0 + + if (umask $mkdir_umask && + exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1 + then + if test -z "$dir_arg" || { + # Check for POSIX incompatibilities with -m. + # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or + # other-writable bit of parent directory when it shouldn't. + # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. + ls_ld_tmpdir=`ls -ld "$tmpdir"` + case $ls_ld_tmpdir in + d????-?r-*) different_mode=700;; + d????-?--*) different_mode=755;; + *) false;; + esac && + $mkdirprog -m$different_mode -p -- "$tmpdir" && { + ls_ld_tmpdir_1=`ls -ld "$tmpdir"` + test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" + } + } + then posix_mkdir=: + fi + rmdir "$tmpdir/d" "$tmpdir" + else + # Remove any dirs left behind by ancient mkdir implementations. + rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null + fi + trap '' 0;; + esac;; + esac + + if + $posix_mkdir && ( + umask $mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" + ) + then : + else + + # The umask is ridiculous, or mkdir does not conform to POSIX, + # or it failed possibly due to a race condition. Create the + # directory the slow way, step by step, checking for races as we go. + + case $dstdir in + /*) prefix='/';; + [-=\(\)!]*) prefix='./';; + *) prefix='';; + esac + + eval "$initialize_posix_glob" + + oIFS=$IFS + IFS=/ + $posix_glob set -f + set fnord $dstdir + shift + $posix_glob set +f + IFS=$oIFS + + prefixes= + + for d + do + test X"$d" = X && continue + + prefix=$prefix$d + if test -d "$prefix"; then + prefixes= + else + if $posix_mkdir; then + (umask=$mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break + # Don't fail if two instances are running concurrently. + test -d "$prefix" || exit 1 + else + case $prefix in + *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; + *) qprefix=$prefix;; + esac + prefixes="$prefixes '$qprefix'" + fi + fi + prefix=$prefix/ + done + + if test -n "$prefixes"; then + # Don't fail if two instances are running concurrently. + (umask $mkdir_umask && + eval "\$doit_exec \$mkdirprog $prefixes") || + test -d "$dstdir" || exit 1 + obsolete_mkdir_used=true + fi + fi + fi + + if test -n "$dir_arg"; then + { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && + { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || + test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 + else + + # Make a couple of temp file names in the proper directory. + dsttmp=$dstdir/_inst.$$_ + rmtmp=$dstdir/_rm.$$_ + + # Trap to clean up those temp files at exit. + trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 + + # Copy the file name to the temp name. + (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && + + # and set any options; do chmod last to preserve setuid bits. + # + # If any of these fail, we abort the whole thing. If we want to + # ignore errors from any of these, just make sure not to ignore + # errors from the above "$doit $cpprog $src $dsttmp" command. + # + { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && + { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && + { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && + + # If -C, don't bother to copy if it wouldn't change the file. + if $copy_on_change && + old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && + new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && + + eval "$initialize_posix_glob" && + $posix_glob set -f && + set X $old && old=:$2:$4:$5:$6 && + set X $new && new=:$2:$4:$5:$6 && + $posix_glob set +f && + + test "$old" = "$new" && + $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 + then + rm -f "$dsttmp" + else + # Rename the file to the real destination. + $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || + + # The rename failed, perhaps because mv can't rename something else + # to itself, or perhaps because mv is so ancient that it does not + # support -f. + { + # Now remove or move aside any old file at destination location. + # We try this two ways since rm can't unlink itself on some + # systems and the destination file might be busy for other + # reasons. In this case, the final cleanup might fail but the new + # file should still install successfully. + { + test ! -f "$dst" || + $doit $rmcmd -f "$dst" 2>/dev/null || + { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && + { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } + } || + { echo "$0: cannot unlink or rename $dst" >&2 + (exit 1); exit 1 + } + } && + + # Now rename the file to the real destination. + $doit $mvcmd "$dsttmp" "$dst" + } + fi || exit 1 + + trap '' 0 + fi +done + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/infrastructure/m4/ax_check_compile_flag.m4 b/infrastructure/m4/ax_check_compile_flag.m4 new file mode 100644 index 00000000..c3a8d695 --- /dev/null +++ b/infrastructure/m4/ax_check_compile_flag.m4 @@ -0,0 +1,72 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 2 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/infrastructure/m4/boxbackup_tests.m4 b/infrastructure/m4/boxbackup_tests.m4 index 9e6bc2f2..59467e66 100644 --- a/infrastructure/m4/boxbackup_tests.m4 +++ b/infrastructure/m4/boxbackup_tests.m4 @@ -10,10 +10,30 @@ solaris*) ;; esac -if test "x$GXX" = "xyes"; then - # Use -Wall if we have gcc. This gives better warnings - AC_SUBST([CXXFLAGS_STRICT], ['-Wall -Wundef']) +# Enable some compiler flags if the compiler supports them. This gives better warnings +# and detects some problems early. +AX_CHECK_COMPILE_FLAG(-Wall, [cxxflags_strict="$cxxflags_strict -Wall"]) +# -Wundef would be a good idea, but Boost is full of undefined variable use, so we need +# to disable it for now so that we can concentrate on real errors: +dnl AX_CHECK_COMPILE_FLAG(-Wundef, [cxxflags_strict="$cxxflags_strict -Wundef"]) +AX_CHECK_COMPILE_FLAG(-Werror=return-type, + [cxxflags_strict="$cxxflags_strict -Werror=return-type"]) +AX_CHECK_COMPILE_FLAG(-Werror=delete-non-virtual-dtor, + [cxxflags_strict="$cxxflags_strict -Werror=delete-non-virtual-dtor"]) +AX_CHECK_COMPILE_FLAG(-Werror=undefined-bool-conversion, + [cxxflags_strict="$cxxflags_strict -Werror=undefined-bool-conversion"]) +# We should really enable -Werror=sometimes-uninitialized, but QDBM violates it: +dnl AX_CHECK_COMPILE_FLAG(-Werror=sometimes-uninitialized, +dnl [cxxflags_strict="$cxxflags_strict -Werror=sometimes-uninitialized"]) +# This error is detected by MSVC, but not usually by GCC/Clang: +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58114 +AX_CHECK_COMPILE_FLAG(-Werror=delete-incomplete, + [cxxflags_strict="$cxxflags_strict -Werror=delete-incomplete"]) +AX_CHECK_COMPILE_FLAG(-Wno-deprecated-declarations, + [cxxflags_strict="$cxxflags_strict -Wno-deprecated-declarations"]) +AC_SUBST([CXXFLAGS_STRICT], [$cxxflags_strict]) +if test "x$GXX" = "xyes"; then # Don't check for gcc -rdynamic on Solaris as it's broken, but returns 0. # On Cygwin it does nothing except cause gcc to emit a warning message. case $build_os in @@ -25,6 +45,7 @@ if test "x$GXX" = "xyes"; then # [http://readlist.com/lists/gcc.gnu.org/gcc/6/31502.html] # This is needed to get symbols in backtraces. # Note that this apparently fails on HP-UX and Solaris + save_LDFLAGS=$LDFLAGS LDFLAGS="$LDFLAGS -rdynamic" AC_MSG_CHECKING([whether gcc accepts -rdynamic]) AC_TRY_LINK([], [return 0;], @@ -33,6 +54,7 @@ if test "x$GXX" = "xyes"; then if test x"$have_rdynamic" = x"yes" ; then AC_SUBST([LDADD_RDYNAMIC], ['-rdynamic']) fi + LDFLAGS=$save_LDFLAGS ;; esac fi @@ -98,7 +120,7 @@ AX_PATH_BDB([1.x or 4.1], [ ]) # need to find libdl before trying to link openssl, apparently -AC_SEARCH_LIBS([dlsym], ["dl"]) +AC_SEARCH_LIBS([dlsym], [dl]) AC_CHECK_FUNCS([dlsym dladdr]) ## Check for Open SSL, use old versions only if explicitly requested @@ -123,21 +145,18 @@ Upgrade or read the documentation for alternatives]]) ### Checks for header files. -case $target_os in -mingw32*) ;; -winnt*) ;; -*) - AC_HEADER_DIRENT - ;; -esac - AC_HEADER_STDC AC_HEADER_SYS_WAIT -AC_CHECK_HEADERS([dlfcn.h fcntl.h getopt.h process.h pwd.h signal.h]) -AC_CHECK_HEADERS([syslog.h time.h cxxabi.h]) +AC_CHECK_HEADERS([cxxabi.h dirent.h dlfcn.h fcntl.h getopt.h netdb.h process.h pwd.h signal.h]) +AC_CHECK_HEADERS([syslog.h time.h unistd.h]) AC_CHECK_HEADERS([netinet/in.h netinet/tcp.h]) -AC_CHECK_HEADERS([sys/file.h sys/param.h sys/socket.h sys/time.h sys/types.h]) -AC_CHECK_HEADERS([sys/uio.h sys/un.h sys/wait.h sys/xattr.h]) +AC_CHECK_HEADERS([sys/file.h sys/param.h sys/poll.h sys/socket.h sys/stat.h sys/time.h]) +AC_CHECK_HEADERS([sys/types.h sys/uio.h sys/un.h sys/wait.h sys/xattr.h]) +AC_CHECK_HEADERS([sys/ucred.h],,, [ + #ifdef HAVE_SYS_PARAM_H + # include <sys/param.h> + #endif + ]) AC_CHECK_HEADERS([bsd/unistd.h]) AC_CHECK_HEADERS([sys/socket.h], [have_sys_socket_h=yes]) AC_CHECK_HEADERS([winsock2.h], [have_winsock2_h=yes]) @@ -185,10 +204,11 @@ AC_TYPE_OFF_T AC_TYPE_PID_T AC_TYPE_SIZE_T -AC_CHECK_MEMBERS([struct stat.st_flags]) -AC_CHECK_MEMBERS([struct stat.st_mtimespec]) -AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec]) -AC_CHECK_MEMBERS([struct stat.st_atimensec]) +AC_CHECK_MEMBERS([struct stat.st_flags],,, [[#include <sys/stat.h>]]) +AC_CHECK_MEMBERS([struct stat.st_atim],,, [[#include <sys/stat.h>]]) +AC_CHECK_MEMBERS([struct stat.st_atimespec],,, [[#include <sys/stat.h>]]) +AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec],,, [[#include <sys/stat.h>]]) +AC_CHECK_MEMBERS([struct stat.st_atimensec],,, [[#include <sys/stat.h>]]) AC_CHECK_MEMBERS([struct sockaddr_in.sin_len],,, [[ #include <sys/types.h> #include <netinet/in.h> @@ -197,11 +217,14 @@ AC_CHECK_MEMBERS([DIR.d_fd],,, [[#include <dirent.h>]]) AC_CHECK_MEMBERS([DIR.dd_fd],,, [[#include <dirent.h>]]) AC_CHECK_MEMBERS([struct tcp_info.tcpi_rtt],,, [[#include <netinet/tcp.h>]]) +AC_CHECK_DECLS([O_BINARY],,, [[#include <fcntl.h>]]) + +AC_CHECK_DECLS([ENOTSUP],,, [[#include <sys/errno.h>]]) AC_CHECK_DECLS([INFTIM],,, [[#include <poll.h>]]) AC_CHECK_DECLS([SO_PEERCRED],,, [[#include <sys/socket.h>]]) -AC_CHECK_DECLS([O_BINARY],,,) AC_CHECK_DECLS([SOL_TCP],,, [[#include <netinet/tcp.h>]]) AC_CHECK_DECLS([TCP_INFO],,, [[#include <netinet/tcp.h>]]) +AC_CHECK_DECLS([SYS_open, SYS_openat],,, [[#include <sys/syscall.h>]]) if test -n "$have_sys_socket_h"; then AC_CHECK_DECLS([SO_SNDBUF],,, [[#include <sys/socket.h>]]) @@ -216,6 +239,24 @@ fi # Solaris provides getpeerucred() instead of getpeereid() or SO_PEERCRED AC_CHECK_HEADERS([ucred.h]) AC_CHECK_FUNCS([getpeerucred]) +AC_CHECK_MEMBERS([struct ucred.uid, struct ucred.cr_uid],,, + [[ + #ifdef HAVE_UCRED_H + # include <ucred.h> + #endif + + #ifdef HAVE_SYS_PARAM_H + # include <sys/param.h> + #endif + + #ifdef HAVE_SYS_UCRED_H + # include <sys/ucred.h> + #endif + + #ifdef HAVE_SYS_SOCKET_H + # include <sys/socket.h> + #endif + ]]) AC_CHECK_DECLS([optreset],,, [[#include <getopt.h>]]) AC_CHECK_DECLS([dirfd],,, @@ -254,11 +295,15 @@ AC_FUNC_CLOSEDIR_VOID AC_FUNC_ERROR_AT_LINE AC_TYPE_SIGNAL AC_FUNC_STAT -AC_CHECK_FUNCS([getpeereid getpeername lchown setproctitle getpid gettimeofday waitpid ftruncate]) -AC_SEARCH_LIBS([setproctitle], ["bsd"]) +AC_CHECK_FUNCS([ftruncate getpeereid getpeername getpid gettimeofday lchown]) +AC_CHECK_FUNCS([setproctitle utimensat]) +AC_SEARCH_LIBS([setproctitle], [bsd]) # NetBSD implements kqueue too differently for us to get it fixed by 0.10 -# TODO: Remove this when NetBSD kqueue implementation is working +# TODO: Remove this when NetBSD kqueue implementation is working. The main +# thing to fix is that ServerStream needs to put a pointer into WaitForEvent, +# which wants to store it in struct kevent.udata, but on NetBSD that's an +# intptr_t instead of a void *, and it doesn't like accepting pointers. netbsd_hack=`echo $target_os | sed 's/netbsd.*/netbsd/'` if test "$netbsd_hack" != "netbsd"; then AC_CHECK_FUNCS([kqueue]) @@ -312,4 +357,10 @@ fi ;; esac - +AC_CHECK_PROGS(default_debugger, [lldb gdb]) +AC_ARG_WITH([debugger], + [AS_HELP_STRING([--with-debugger=<gdb|lldb|...>], + [use this debugger in t-gdb scripts to debug tests @<:@default=lldb if present, otherwise gdb@:>@])], + [], + [with_debugger=$default_debugger]) +AC_SUBST([with_debugger]) diff --git a/infrastructure/makebuildenv.pl.in b/infrastructure/makebuildenv.pl.in index 005f2d25..d5ac9f2f 100755 --- a/infrastructure/makebuildenv.pl.in +++ b/infrastructure/makebuildenv.pl.in @@ -14,21 +14,21 @@ $|=1; print "Box build environment setup.\n"; -my @implicit_deps = ('lib/common'); +# Modules that everything else depends on, without being explicitly specified. +# Deprecated because we can't specify dependencies between them in the usual way, +# and it's totally unnecessary anyway: +my @implicit_deps; # work out platform variables use lib 'infrastructure'; use BoxPlatform; -print "Building on '$build_os'.\n\n"; +print "Building on '$build_os $build_os_ver' using ". + ($bsd_make ? "BSD" : "GNU")." $make_command.\n\n"; # keep copy of command line args my $makebuildenv_args = join(' ',@ARGV); -# do command line arguments -my $compile_line_extra = $platform_compile_line_extra; -my $link_line_extra = $platform_link_line_extra; - # make sure local files directory exists unless(-d 'local') { @@ -39,9 +39,6 @@ unless(-d 'local') # flags about the environment my %env_flags; -$module_dependency{"lib/common"} = ["lib/win32"]; -push @implicit_deps, "lib/win32"; - # print "Flag: $_\n" for(keys %env_flags); # seed autogen code @@ -211,7 +208,7 @@ for(@modules_files) if(m/\AOMIT:(.+)/) { - if($1 eq $build_os or $1 eq $target_os) + if($1 eq $build_os or $1 eq $ac_target_os) { $modules_omitted = 1; $modules_omitting = 1; @@ -376,17 +373,159 @@ for my $mod (@modules, @implicit_deps) closedir DIR; } -# Then write a makefile for each module +my $default_cflags = '@CFLAGS@'; +my $default_cxxflags = '@CXXFLAGS@'; +$default_cflags =~ s/ -O2//g; +$default_cxxflags =~ s/ -O2//g; +my $debug_base_dir = 'debug'; +my $release_base_dir = 'release'; +my $debugger = '@with_debugger@'; + +my $release_flags = "-O2"; +if ($target_windows) +{ + $release_flags = "-O0 -g"; +} + +# Then write a master Makefile, and a mini-Makefile for each module print "done\n\nGenerating Makefiles...\n"; -my %module_resources_win32; +my $makefile_ifdef_prefix = $bsd_make ? "." : ""; +my $autoconf_cppflags = '@CPPFLAGS@'; +my $autoconf_cxxflags = '@CXXFLAGS_STRICT@'; +my $autoconf_ldflags = '@LDFLAGS@'; + +open MASTER_MAKEFILE, ">Makefile" or die "Makefile: $!"; +print MASTER_MAKEFILE <<__E; +# +# AUTOMATICALLY GENERATED FILE +# do not edit! +# +# + +# "parcels" is defined near the end of this Makefile. +default: parcels + +CC = @CC@ +CXX = @CXX@ +AR = @AR@ +RANLIB = @RANLIB@ +PERL = @PERL@ +WINDRES = @WINDRES@ + +# Work around a mistake in QDBM (using <angled> includes for a file not in the +# system path) by adding it to the include path with -I. +DEFAULT_CFLAGS = $autoconf_cppflags $default_cflags $autoconf_cxxflags \\ + $extra_platform_defines $platform_compile_line_extra \\ + -DBOX_VERSION="\\"$product_version\\"" -Iqdbm +DEFAULT_CXXFLAGS = $autoconf_cppflags $default_cxxflags $autoconf_cxxflags \\ + $extra_platform_defines $platform_compile_line_extra \\ + -DBOX_VERSION="\\"$product_version\\"" +LDFLAGS += $autoconf_ldflags \@LDADD_RDYNAMIC@ $platform_link_line_extra + +RELEASE_CFLAGS = \$(DEFAULT_CFLAGS) -DBOX_RELEASE_BUILD $release_flags +RELEASE_CXXFLAGS = \$(DEFAULT_CXXFLAGS) -DBOX_RELEASE_BUILD $release_flags +RELEASE_OUTBASE = $release_base_dir +# http://gcc.gnu.org/onlinedocs/libstdc++/manual/debug_mode_using.html#debug_mode.using.mode +DEBUG_CFLAGS = \$(DEFAULT_CFLAGS) -g -O0 -D_GLIBCXX_DEBUG +DEBUG_CXXFLAGS = \$(DEFAULT_CXXFLAGS) -g -O0 -D_GLIBCXX_DEBUG +DEBUG_OUTBASE = $debug_base_dir + +__E + +if ($bsd_make) +{ + print MASTER_MAKEFILE <<__E; +.ifdef V +HIDE = +_CC = \$(CC) +_CXX = \$(CXX) +_LINK = \$(CXX) +_WINDRES = \$(WINDRES) +_AR = \$(AR) +_RANLIB = \$(RANLIB) +_PERL = \$(PERL) +.else +HIDE = @ +_CC = @ echo " [CC] " \$(*F) && \$(CC) +_CXX = @ echo " [CXX] " \$(*F) && \$(CXX) +_LINK = @ echo " [LINK] " \$(*F) && \$(CXX) +_WINDRES = @ echo " [WINDRES]" \$(*F) && \$(WINDRES) +_AR = @ echo " [AR] " \$(*F) && \$(AR) +_RANLIB = @ echo " [RANLIB] " \$(*F) && \$(RANLIB) +_PERL = @ echo " [PERL] " \$(*F) && \$(PERL) >/dev/null +.endif + +__E +} +else +{ + print MASTER_MAKEFILE <<__E; +HIDE = \$(if \$(V),,@) +_CC = \$(if \$(V),\$(CC), @ echo " [CC] \$<" && \$(CC)) +_CXX = \$(if \$(V),\$(CXX), @ echo " [CXX] \$<" && \$(CXX)) +_LINK = \$(if \$(V),\$(CXX), @ echo " [LINK] \$@" && \$(CXX)) +_WINDRES = \$(if \$(V),\$(WINDRES), @ echo " [WINDRES] \$<" && \$(WINDRES)) +_AR = \$(if \$(V),\$(AR), @ echo " [AR] \$@" && \$(AR)) +_RANLIB = \$(if \$(V),\$(RANLIB), @ echo " [RANLIB] \$@" && \$(RANLIB)) +_PERL = \$(if \$(V),\$(PERL), @ echo " [PERL] \$@" && \$(PERL) >/dev/null) +__E +} + +my %module_resources_win32; +my @debug_build_targets; +my @release_build_targets; +my @all_clean_targets; +my @all_realclean_targets; +my %mod_type_name; +my %mod_end_targets; +my %library_targets; + +# First, loop over all modules and quickly categorise them and generate the target filenames. +# We need this to write correct dependency info into the makefile when modules are listed +# out of order in modules.txt, which isn't strictly necessary but may look tidier. for my $mod (@implicit_deps, @modules) { print $mod,"\n"; my ($type,$name) = split /\//,$mod; - + if (not $name) + { + # External modules such as qdbm have no "type" in the directory path. + $name = $mod; + $type = 'lib'; + } + + # is target a library? + my $target_is_library = ($type ne 'bin' && $type ne 'test'); + + # make target name + my $end_target_file = $name; + + if ($target_is_library) + { + $end_target_file .= '.a'; + } + else + { + $end_target_file .= $platform_exe_ext; + } + + $end_target_file = '_test'.$platform_exe_ext if $type eq 'test'; + + $mod_type_name{$mod} = [$type, $name]; + $mod_end_targets{$mod} = $end_target_file; + $library_targets{$mod} = $target_is_library; +} + +# Now loop over them again, generating makefile instructions etc. +for my $mod (@implicit_deps, @modules) +{ + my ($type,$name) = @{$mod_type_name{$mod}}; + my $end_target_file = $mod_end_targets{$mod}; + my $target_is_library = $library_targets{$mod}; + # add additional files for tests if($type eq 'test') { @@ -464,7 +603,7 @@ __E close FL; } - print TESTFILE "$runcmd\n"; + print TESTFILE "$runcmd\nexit_status=\$?\n"; if (-d "$module/testfiles") { @@ -473,14 +612,24 @@ kill_daemons __E } + print TESTFILE "exit \$exit_status\n"; close TESTFILE; } - writetestfile("$mod/_t", "GLIBCXX_FORCE_NEW=1 ". - './test' . $platform_exe_ext . ' "$@"', $mod); - writetestfile("$mod/_t-gdb", "GLIBCXX_FORCE_NEW=1 ". - 'gdb ./test' . $platform_exe_ext . ' "$@"', $mod); - + writetestfile("$mod/t", "GLIBCXX_FORCE_NEW=1 ". + './_test' . $platform_exe_ext . ' "$@"', $mod); + + if($debugger) + { + writetestfile("$mod/t-gdb", "GLIBCXX_FORCE_NEW=1 ". + $debugger . ' ./_test' . $platform_exe_ext . ' "$@"', $mod); + } + else + { + writetestfile("$mod/t-gdb", + "echo 'No debugger was detected by configure script'\n". + "exit 2"); + } } my @all_deps_for_module; @@ -514,123 +663,70 @@ __E } } } - - - # make include path - my $include_paths = join(' ',map {'-I../../'.$_} @all_deps_for_module); - - # is target a library? - my $target_is_library = ($type ne 'bin' && $type ne 'test'); - # make target name - my $end_target = $name; + # get the list of library things to add -- in order of dependency + # so things link properly + my @lib_files; + my @dep_targets; + my @include_search_dirs; - if ($target_is_library) - { - $end_target .= '.a'; - } - else + foreach my $dep (reverse @all_deps_for_module) { - $end_target .= $platform_exe_ext; + my $dep_target = $mod_end_targets{$dep}; + die "No output file found for $dep" unless $dep_target; + $dep_target = "$dep/$dep_target"; + push @dep_targets, $dep_target; + + if ($dep =~ m|^lib\/(.+)$|) + { + push @lib_files, $dep_target; + push @include_search_dirs, $dep; + } + elsif ($dep =~ m|^([^/]+)$|) + { + push @lib_files, $dep_target; + push @include_search_dirs, $dep; + } } - $end_target = 'test'.$platform_exe_ext if $type eq 'test'; + # make include path + my $cpp_include_paths = join(' ',map {"-I$_"} @include_search_dirs); + print MASTER_MAKEFILE "${type}_${name}_includes = $cpp_include_paths\n"; # adjust for outdir - $end_target = '$(OUTDIR)/' . $end_target; + my $debug_end_target = "\$(DEBUG_OUTBASE)/$mod/$end_target_file"; + my $release_end_target = "\$(RELEASE_OUTBASE)/$mod/$end_target_file"; + push @debug_build_targets, $debug_end_target; + push @release_build_targets, $release_end_target; # start the makefile - my $mk_name_extra = ($bsd_make)?'':'X'; - open MAKE,">$mod/Makefile".$mk_name_extra or die "Can't open Makefile for $mod\n"; - my $debug_link_extra = ($target_is_library)?'':'../../debug/lib/debug/debug.a'; - - my $default_cxxflags = '@CXXFLAGS@'; - $default_cxxflags =~ s/ -O2//g; - - my $release_flags = "-O2"; - if ($target_windows) - { - $release_flags = "-O0 -g"; - } - - print MAKE <<__E; + open MINI_MODULE_MAKEFILE,">$mod/Makefile" or die "Can't open Makefile for $mod\n"; + print MINI_MODULE_MAKEFILE <<__E; # # AUTOMATICALLY GENERATED FILE # do not edit! # -# -CXX = @CXX@ -AR = @AR@ -RANLIB = @RANLIB@ -PERL = @PERL@ -WINDRES = @WINDRES@ - -DEFAULT_CXXFLAGS = @CPPFLAGS@ $default_cxxflags @CXXFLAGS_STRICT@ \\ - $include_paths $extra_platform_defines \\ - -DBOX_VERSION="\\"$product_version\\"" -LDFLAGS += @LDFLAGS@ @LDADD_RDYNAMIC@ - -.ifdef RELEASE -CXXFLAGS += -DBOX_RELEASE_BUILD $release_flags \$(DEFAULT_CXXFLAGS) -OUTBASE = ../../release -OUTDIR = ../../release/$mod -DEPENDMAKEFLAGS = -D RELEASE -VARIENT = RELEASE -.else -CXXFLAGS += -g \$(DEFAULT_CXXFLAGS) -OUTBASE = ../../debug -OUTDIR = ../../debug/$mod -DEPENDMAKEFLAGS = -VARIENT = DEBUG -.endif +${makefile_ifdef_prefix}ifdef RELEASE +TARGET = $release_base_dir/$mod/$end_target_file +${makefile_ifdef_prefix}else +TARGET = $debug_base_dir/$mod/$end_target_file +${makefile_ifdef_prefix}endif -__E +.PHONY: default +default: + \$(MAKE) -C ../.. \$(TARGET) - if ($bsd_make) - { - print MAKE <<__E; -.ifdef V -HIDE = -_CXX = \$(CXX) -_LINK = \$(CXX) -_WINDRES = \$(WINDRES) -_AR = \$(AR) -_RANLIB = \$(RANLIB) -_PERL = \$(PERL) -.else -HIDE = @ -_CXX = @ echo " [CXX] " \$(*F) && \$(CXX) -_LINK = @ echo " [LINK] " \$(*F) && \$(CXX) -_WINDRES = @ echo " [WINDRES]" \$(*F) && \$(WINDRES) -_AR = @ echo " [AR] " \$(*F) && \$(AR) -_RANLIB = @ echo " [RANLIB] " \$(*F) && \$(RANLIB) -_PERL = @ echo " [PERL] " \$(*F) && \$(PERL) >/dev/null -.endif - -__E - } - else - { - print MAKE <<__E; -HIDE = \$(if \$(V),,@) -_CXX = \$(if \$(V),\$(CXX), @ echo " [CXX] \$<" && \$(CXX)) -_LINK = \$(if \$(V),\$(CXX), @ echo " [LINK] \$@" && \$(CXX)) -_WINDRES = \$(if \$(V),\$(WINDRES), @ echo " [WINDRES] \$<" && \$(WINDRES)) -_AR = \$(if \$(V),\$(AR), @ echo " [AR] \$@" && \$(AR)) -_RANLIB = \$(if \$(V),\$(RANLIB), @ echo " [RANLIB] \$@" && \$(RANLIB)) -_PERL = \$(if \$(V),\$(PERL), @ echo " [PERL] \$@" && \$(PERL) >/dev/null) +.PHONY: clean +clean: + \$(MAKE) -C ../.. clean_${type}_${name} +.PHONY: realclean +realclean: + \$(MAKE) -C ../.. realclean_${type}_${name} __E - } + close MINI_MODULE_MAKEFILE; - # if there is a Makefile.pre, include it now - if(-e "$mod/Makefile.pre") - { - print MAKE ".include <Makefile.pre>\n\n"; - } - - # read directory - opendir DIR,$mod; + opendir DIR, $mod; my @items = readdir DIR; closedir DIR; @@ -643,10 +739,10 @@ __E { # Read items my $d = "$mod/$di"; - opendir DIR,$d; + opendir DIR, $d; my @i = readdir DIR; closedir DIR; - for(@i) + for (@i) { next if m/\A\./; push @autogen_items,"$di/$_" @@ -675,294 +771,587 @@ __E } } - # ready for the rest of the details... - my $make; - - # then... do the cpp files... - my @obj_base; - for my $file (@items) + # write the recipes for debug and release builds of each file + foreach my $var_prefix ('DEBUG', 'RELEASE') { - my $is_cpp = $file =~ m/\A(.+)\.cpp\Z/i; - my $is_rc = $file =~ m/\A(.+)\.rc\Z/i; - my $base = $1; + my $make; - if ($target_windows) - { - next if not $is_cpp and not $is_rc; - } - else + # then... do the cpp files... + my @obj_base; + for my $file (@items) { - next if not $is_cpp; - } + my $is_c = $file =~ m/\A(.+)\.c\Z/i; + my $is_cpp = $file =~ m/\A(.+)\.cpp\Z/i; + my $is_rc = $file =~ m/\A(.+)\.rc\Z/i; + my $base = $1; - next if $file =~ /\A\._/; # Temp Mac OS Resource hack + # Don't try to compile .rc files except on Windows: + next if not $is_c and not $is_cpp and not ($is_rc and $target_windows); + next if $file =~ /\A\._/; # Temp Mac OS Resource hack - # store for later - push @obj_base,$base; - - # get the file... - open FL,"$mod/$file"; - my $f; - read FL,$f,-s "$mod/$file"; - close FL; + # store for later + push @obj_base, $base; - my %dep; + # get the file... + open FL, "$mod/$file"; + my $f; + read FL, $f, -s "$mod/$file"; + close FL; + + my %dep; - while($f =~ m/\#include\s+"([^"]+?)"/g) - { - insert_dep($1, \%dep) if exists $hfiles{$1}; + while($f =~ m/\#include\s+"([^"]+?)"/g) + { + insert_dep($1, \%dep) if exists $hfiles{$1}; + } + + # output filename + my $out_name = "\$(${var_prefix}_OUTBASE)/$mod/$base.o"; + + # write the line for this cpp file + my @dep_paths = map + { + ($hfiles{$_} eq $mod) + ? "$mod/$_" + : $hfiles{$_}."/$_" + } + keys %dep; + + $make .= "$out_name: $mod/$file @dep_paths\n"; + + if ($is_c) + { + $make .= "\t\$(_CC) \$(${var_prefix}_CFLAGS) ". + "\$(${type}_${name}_includes) -DBOX_MODULE=\"\\\"$mod\\\"\" " . + "-c $mod/$file -o $out_name\n\n"; + } + if ($is_cpp) + { + $make .= "\t\$(_CXX) \$(${var_prefix}_CXXFLAGS) ". + "\$(${type}_${name}_includes) -DBOX_MODULE=\"\\\"$mod\\\"\" " . + "-c $mod/$file -o $out_name\n\n"; + } + elsif ($is_rc) + { + $make .= "\t\$(_WINDRES) $file $out_name\n\n"; + my $res_list = $module_resources_win32{$mod}; + $res_list ||= []; + push @$res_list, $base.'.o'; + $module_resources_win32{$mod} = $res_list; + } } + + # need to see if the extra makefile fragments require extra object files + # or include any more makefiles + my @objs = @obj_base; + my @makefile_includes; - # output filename - my $out_name = '$(OUTDIR)/'.$base.'.o'; + additional_objects_from_make_fragment("$mod/Makefile.extra", \@objs, \@makefile_includes); + additional_objects_from_make_fragment("$mod/Makefile.extra.$build_os", \@objs, \@makefile_includes); + + my $prefixed_end_target = "\$(${var_prefix}_OUTBASE)/$mod/$end_target_file"; + my $o_file_list = join(' ',map {"\$(${var_prefix}_OUTBASE)/$mod/$_.o"} sort @objs); + my @prefixed_lib_files = map {"\$(${var_prefix}_OUTBASE)/$_"} @lib_files; + my @prefixed_dep_targets = map {"\$(${var_prefix}_OUTBASE)/$_"} @dep_targets; + + print MASTER_MAKEFILE "$prefixed_end_target: $o_file_list"; + print MASTER_MAKEFILE " @prefixed_dep_targets" unless $target_is_library; + print MASTER_MAKEFILE "\n"; - # write the line for this cpp file - my @dep_paths = map - { - ($hfiles{$_} eq $mod) - ? $_ - : '../../'.$hfiles{$_}."/$_" + if ($target_windows) + { + foreach my $dep (@all_deps_for_module) + { + my $res_list = $module_resources_win32{$dep}; + next unless $res_list; + $o_file_list .= ' '.join(' ', + map {"\$(${var_prefix}_OUTBASE)/$dep/$_"} @$res_list); + } } - keys %dep; - $make .= $out_name.': '.join(' ',$file,@dep_paths)."\n"; - - if ($is_cpp) + # stuff to make the final target... + if($target_is_library) { - $make .= "\t\$(_CXX) \$(CXXFLAGS) $compile_line_extra ". - "-DBOX_MODULE=\"\\\"$mod\\\"\" " . - "-c $file -o $out_name\n\n"; + # make a library archive... + print MASTER_MAKEFILE "\t\$(HIDE) (rm -f $prefixed_end_target)\n"; + print MASTER_MAKEFILE "\t\$(_AR) cq $prefixed_end_target $o_file_list\n"; + print MASTER_MAKEFILE "\t\$(_RANLIB) $prefixed_end_target\n"; } - elsif ($is_rc) + else { - $make .= "\t\$(_WINDRES) $file $out_name\n\n"; - my $res_list = $module_resources_win32{$mod}; - $res_list ||= []; - push @$res_list, $base.'.o'; - $module_resources_win32{$mod} = $res_list; + # work out library options + # need to be... least used first, in absolute order they appear in the modules.txt file + my @libops; + + sub libops_fill + { + my ($module, $libops_ref) = @_; + + my $library_link_opts = $module_library_link_opts{$module}; + if ($library_link_opts) + { + push @$libops_ref, @$library_link_opts; + } + + my $deps = $module_dependency{$module}; + foreach my $dep (@$deps) + { + libops_fill($dep, $libops_ref); + } + } + + libops_fill($mod,\@libops); + + my $lo = ''; + my %ldone; + for(@libops) + { + next if exists $ldone{$_}; + $lo .= ' '.$_; + $ldone{$_} = 1; + } + + # link line... + print MASTER_MAKEFILE "\t\$(_LINK) \$(LDFLAGS) " . + "-o $prefixed_end_target $o_file_list " . + "@prefixed_lib_files $lo $platform_lib_files\n"; } - } - - my $has_deps = ($#{$module_dependency{$mod}} >= 0); -# ----- # always has dependencies with debug library - $has_deps = 1; - $has_deps = 0 if $target_is_library; - # Dependency stuff - my $deps_makeinfo; - if($has_deps) - { - if($bsd_make) + # tests need to copy the test file over + if($type eq 'test') { - $deps_makeinfo = <<'__E'; -.BEGIN:: -.ifndef NODEPS -. if $(.TARGETS) == "" + print MASTER_MAKEFILE <<__E; + \$(HIDE) cp $mod/t \$(${var_prefix}_OUTBASE)/$mod/t + \$(HIDE) chmod u+x \$(${var_prefix}_OUTBASE)/$mod/t + \$(HIDE) cp $mod/t-gdb \$(${var_prefix}_OUTBASE)/$mod/t-gdb + \$(HIDE) chmod u+x \$(${var_prefix}_OUTBASE)/$mod/t-gdb __E } - else + + print MASTER_MAKEFILE $make,"\n"; + + for(@makefile_includes) { - # gnu make - $deps_makeinfo = <<'__E'; -.PHONY: dep_modules -dep_modules: -ifndef NODEPS -ifeq ($(strip $(.TARGETS)),) -__E + print MASTER_MAKEFILE "\n.include <$_>\n"; } +} + + print MASTER_MAKEFILE <<__E; +clean_${type}_${name}: + rm -rf \$(DEBUG_OUTBASE)/$mod/* + rm -rf \$(RELEASE_OUTBASE)/$mod/* + +realclean_${type}_${name}: clean_${type}_${name} + rm -f $mod/t $mod/t-gdb $mod/Makefile + find $mod -name 'autogen_*' -type f -exec rm -f {} \\; +__E + push @all_clean_targets, "clean_${type}_${name}"; + push @all_realclean_targets, "realclean_${type}_${name}"; + + my $includes = ""; + + if(-e "$mod/Makefile.extra") + { + $includes .= ".include <$mod/Makefile.extra>\n\n"; + } + if(-e "$mod/Makefile.extra.$build_os") + { + $includes .= ".include <$mod/Makefile.extra.$build_os>\n\n"; + } + + if(!$bsd_make) + { + # need to post process this into a GNU makefile + $includes =~ s/\A\.\s*(ifdef|else|endif|ifndef)/$1/; + $includes =~ s/\A\.\s*include\s+<(.+?)>/include $1/; + $includes =~ s/-D\s+(\w+)/$1=1/g; + } + + print MASTER_MAKEFILE $includes; +} + +my @parcels; +my %parcel_contents; + +sub starts_with ($$) +{ + my ($string,$expected) = @_; + return substr($string, 0, length $expected) eq $expected; +} + +sub os_matches ($) +{ + my ($prefix_string) = @_; + my @prefixes = split m'\,', $prefix_string; + foreach my $prefix (@prefixes) + { + return 1 if starts_with($build_os, $prefix); + return 1 if starts_with($ac_target_os, $prefix); + return 1 if starts_with("$ac_target_cpu-$ac_target_os", + $prefix); + return 1 if starts_with($ac_target, $prefix); + } + return 0; +} + +my $copy_command = "cp -p"; + +open PARCELS,"parcels.txt" or die "Can't open parcels file"; +{ + my $cur_parcel = ''; + while(<PARCELS>) + { + chomp; s/#.+\Z//; s/\s+\Z//; s/\s+/ /g; + next unless m/\S/; - # run make for things we require - for my $dep (@all_deps_for_module) + # omit bits on some platforms? + next if m/\AEND-OMIT/; + if(m/\AOMIT:(.+)/) { - my $dep_target = ""; - if ($dep =~ m|^lib/(.*)|) + if (os_matches($1)) { - $dep_target = "\$(OUTBASE)/$dep/$1.a"; + while(<PARCELS>) + { + last if m/\AEND-OMIT/; + } } - elsif ($dep =~ m|^.*/(.*)|) + next; + } + + if (m'\AONLY:(.+)') + { + if (not os_matches($1)) { - $dep_target = "\$(OUTBASE)/$dep/$1$platform_exe_ext"; + while (<PARCELS>) + { + last if m'\AEND-ONLY'; + } } - else + next; + } + next if (m'\AEND-ONLY'); + + if (m'\AEXCEPT:(.+)') + { + if (os_matches($1)) { - $dep_target = "lib$dep.a"; + while (<PARCELS>) + { + last if m'\AEND-EXCEPT'; + } } - - $deps_makeinfo .= <<EOF; - \$(HIDE) ( \\ - cd ../../$dep; \\ - \$(MAKE) $sub_make_options -q \$(DEPENDMAKEFLAGS) -D NODEPS $dep_target \\ - || \$(MAKE) $sub_make_options \$(DEPENDMAKEFLAGS) -D NODEPS $dep_target \\ - ) -EOF + next; } - - $deps_makeinfo .= ".\tendif\n.endif\n\n"; - } - print MAKE $deps_makeinfo if $bsd_make; - - # get the list of library things to add -- in order of dependency - # so things link properly - my @lib_files; - foreach my $dep (reverse @all_deps_for_module) - { - if ($dep =~ m|^lib\/(.+)$|) + next if (m'\AEND-EXCEPT'); + + # new parcel, or a new parcel definition? + if(m/\A\s+(.+)\Z/) { - push @lib_files, "\$(OUTBASE)/$dep/$1.a"; + push @{$parcel_contents{$cur_parcel}},$1 } - elsif ($dep =~ m|^([^/]+)$|) + else { - push @lib_files, "../../$dep/lib$1.a"; + $cur_parcel = $_; + push @parcels,$_; } } +} +close PARCELS; - # need to see if the extra makefile fragments require extra object files - # or include any more makefiles - my @objs = @obj_base; - my @makefile_includes; - - additional_objects_from_make_fragment("$mod/Makefile.extra", \@objs, \@makefile_includes); - additional_objects_from_make_fragment("$mod/Makefile.extra.$build_os", \@objs, \@makefile_includes); +# create parcels directory +mkdir "parcels",0755; +mkdir "parcels/scripts",0755; + +print MASTER_MAKEFILE "parcels:\t",join(' ',map {"build-".$_} @parcels),"\n\n"; + +my $runtest_script = $target_windows ? './infrastructure/mingw/runtest.sh' + : './runtest.pl'; + +print MASTER_MAKEFILE <<__END_OF_FRAGMENT; +.PHONY: test +test: debug_tests release_tests - my $o_file_list = join(' ',map {'$(OUTDIR)/'.$_.'.o'} sort @objs); +.PHONY: debug_tests +debug_tests: + $runtest_script ALL debug - if ($has_deps and not $bsd_make) +.PHONY: release_tests +release_tests: + $runtest_script ALL release + +.PHONY: docs +docs: + cd docs; \$(MAKE) + +__END_OF_FRAGMENT + +my $release_flag = BoxPlatform::make_flag('RELEASE'); +my @parcel_targets; + +for my $parcel (@parcels) +{ + my $version = BoxPlatform::parcel_root($parcel); + my $make_target = BoxPlatform::parcel_target($parcel); + my $dir = BoxPlatform::parcel_dir($parcel); + my @parcel_deps; + + # Need to use BSD install on Solaris + my $install_bin = $build_os eq 'SunOS' ? '/usr/ucb/install' : 'install'; + + unless ($target_windows) { - print MAKE ".PHONY: all\n" . - "all: dep_modules $end_target\n\n"; + open SCRIPT,">parcels/scripts/install-$parcel" or die + "Can't open installer script for $parcel for writing"; + print SCRIPT "#!/bin/sh\n\n"; } - print MAKE $end_target,': ',$o_file_list; - print MAKE " @lib_files" unless $target_is_library; - print MAKE "\n"; - - if ($target_windows) + for(@{$parcel_contents{$parcel}}) { - foreach my $dep (@all_deps_for_module) + my @args = split /\s+/; + + my ($type,$name,$dest) = @args; + my $optional = 0; + my $install = 1; + + if ($type eq 'optional') { - my $res_list = $module_resources_win32{$dep}; - next unless $res_list; - $o_file_list .= ' '.join(' ', - map {'$(OUTBASE)/'.$dep."/$_"} @$res_list); + $optional = 1; + shift @args; + ($type,$name,$dest) = @args; } - } - # stuff to make the final target... - if($target_is_library) - { - # make a library archive... - print MAKE "\t\$(HIDE) (echo -n > $end_target; rm $end_target)\n"; - print MAKE "\t\$(_AR) cq $end_target $o_file_list\n"; - print MAKE "\t\$(_RANLIB) $end_target\n"; - } - else - { - # work out library options - # need to be... least used first, in absolute order they appear in the modules.txt file - my @libops; - - sub libops_fill + if ($type eq 'noinstall') { - my ($module, $libops_ref) = @_; - - my $library_link_opts = $module_library_link_opts{$module}; - if ($library_link_opts) + $install = 0; + shift @args; + ($type,$name,$dest) = @args; + } + + if($type eq 'bin') + { + my $exeext = $platform_exe_ext; + print MASTER_MAKEFILE <<EOF; +$dir/$name$exeext: release/bin/$name/$name$exeext + mkdir -p $dir + $copy_command release/bin/$name/$name$exeext $dir + +EOF + push @parcel_deps, "$dir/$name$exeext"; + } + elsif ($type eq 'script') + { + # Replace any variables ($ac_target etc.) with their + # values. + $name =~ s|(\$[^/ ]+)|$1|eeg; + my $fullpath = $name; + my $filename = $name; + # remove path from script name + $filename =~ s{.*/}{}; + + print MASTER_MAKEFILE <<EOF; +$dir/$filename: $fullpath + mkdir -p $dir +EOF + + if ($optional) { - push @$libops_ref, @$library_link_opts; + print MASTER_MAKEFILE "\ttest -r $fullpath " . + "&& $copy_command $fullpath $dir || true\n"; } - - my $deps = $module_dependency{$module}; - foreach my $dep (@$deps) + else { - libops_fill($dep, $libops_ref); + print MASTER_MAKEFILE "\t$copy_command $fullpath $dir\n"; } + + print MASTER_MAKEFILE "\n"; + + push @parcel_deps, "$dir/$filename"; } - - libops_fill($mod,\@libops); - - my $lo = ''; - my %ldone; - for(@libops) + elsif($type eq 'man') { - next if exists $ldone{$_}; - $lo .= ' '.$_; - $ldone{$_} = 1; + print MASTER_MAKEFILE <<EOF; +$dir/${name}.gz: docs/man/${name}.gz + mkdir -p $dir + $copy_command docs/man/${name}.gz $dir + +EOF + # Releases have the docs pre-made, but users + # may want to rebuild them for some reason. + my $docbook_source = "docs/docbook/${name}"; + $docbook_source =~ s/\.[58]$/.xml/; + print MASTER_MAKEFILE <<EOF; +docs/man/${name}.gz: $docbook_source docs/docbook/bb-man.xsl + cd docs; \$(MAKE) man/${name}.gz + +EOF + push @parcel_deps, "$dir/${name}.gz"; } - - # link line... - print MAKE "\t\$(_LINK) \$(LDFLAGS) $link_line_extra " . - "-o $end_target $o_file_list " . - "@lib_files $lo $platform_lib_files\n"; - } + elsif($type eq 'html') + { + print MASTER_MAKEFILE <<EOF; +$dir/docs/${name}.html: docs/htmlguide/man-html/${name}.html + mkdir -p $dir/docs + $copy_command docs/htmlguide/man-html/${name}.html $dir/docs - # tests need to copy the test file over - if($type eq 'test') - { - print MAKE "\tcp _t \$(OUTDIR)/t\n\tchmod u+x \$(OUTDIR)/t\n"; - print MAKE "\tcp _t-gdb \$(OUTDIR)/t-gdb\n\tchmod u+x \$(OUTDIR)/t-gdb\n"; +EOF + # Releases have the docs pre-made, but users + # may want to rebuild them for some reason. + my $docbook_source = "docs/docbook/${name}.xml"; + print MASTER_MAKEFILE <<EOF; +docs/htmlguide/man-html/${name}.html: $docbook_source docs/docbook/bb-nochunk-book.xsl + cd docs; \$(MAKE) htmlguide/man-html/${name}.html + +EOF + push @parcel_deps, "$dir/docs/${name}.html"; + } + elsif ($type eq 'subdir') + { + push @parcel_deps, "build_${type}_${name}"; + } } - # dependency line? - print MAKE "\n"; + print MASTER_MAKEFILE <<EOF; +build-$parcel: $make_target - # module dependencies for GNU make? - print MAKE $deps_makeinfo if !$bsd_make; - - # print the rest of the file - print MAKE $make,"\n"; - - # and a clean target - print MAKE <<EOF; -clean: - -rm -rf \$(OUTDIR)/* -. ifndef SUBCLEAN +$make_target: @parcel_deps + test -d $dir || mkdir $dir EOF - for my $dep (@all_deps_for_module) - { - print MAKE "\t\$(HIDE) (cd ../../$dep; \$(MAKE) \$(DEPENDMAKEFLAGS) -D SUBCLEAN clean)\n"; - } - print MAKE ".\tendif\n"; + + push @parcel_targets, "build-$parcel"; - # include any extra stuff - print MAKE "\n\n"; - if(-e "$mod/Makefile.extra") - { - print MAKE ".include <Makefile.extra>\n\n"; - } - if(-e "$mod/Makefile.extra.$build_os") + for(@{$parcel_contents{$parcel}}) { - print MAKE ".include <Makefile.extra.$build_os>\n\n"; + my @args = split /\s+/; + + my ($type,$name,$dest) = @args; + + my $optional = 0; + my $install = 1; + + if ($type eq 'optional') + { + $optional = 1; + shift @args; + ($type,$name,$dest) = @args; + } + + if ($type eq 'noinstall') + { + $install = 0; + shift @args; + ($type,$name,$dest) = @args; + } + + if ($type eq 'script') + { + # remove path from script name + $name =~ s{.*/}{}; + } + + if ($type eq 'html') + { + $dest = "share/doc/@PACKAGE_TARNAME@"; + $name = "docs/$name.html"; + } + + if ($type eq 'man') + { + $name =~ /([0-9])$/; + $dest = "man/man$1"; + $name =~ s/$/\.gz/; + } + + if ($install and not $target_windows and not $type eq "subdir") + { + my $local_install_dir = $install_into_dir; + if (defined $dest) + { + if ($dest =~ m,^/,) + { + # Don't add $prefix if $dest is a literal path + $local_install_dir = $dest; + } + else + { + $local_install_dir = "@prefix@/$dest"; + } + } + print SCRIPT "mkdir -p " . + "\${DESTDIR}$local_install_dir/\n"; + print SCRIPT "$install_bin $name " . + "\${DESTDIR}$local_install_dir\n"; + } } - for(@makefile_includes) - { - print MAKE ".include <$_>\n\n"; + + unless ($target_windows) + { + close SCRIPT; + chmod 0755,"parcels/scripts/install-$parcel"; } - # and finally a target for rebuilding the build system - print MAKE "\nbuildsystem:\n\t(cd ../..; perl ./infrastructure/makebuildenv.pl $makebuildenv_args)\n\n"; - - close MAKE; + my $root = BoxPlatform::parcel_root($parcel); - if(!$bsd_make) + unless ($target_windows) { - # need to post process this into a GNU makefile - open MAKE,">$mod/Makefile"; - open MAKEB,"$mod/MakefileX"; + print MASTER_MAKEFILE "\tcp parcels/scripts/install-$parcel $dir\n"; + } - while(<MAKEB>) - { - s/\A\.\s*(ifdef|else|endif|ifndef)/$1/; - s/\A\.\s*include\s+<(.+?)>/include $1/; - s/-D\s+(\w+)/$1=1/g; - print MAKE; - } + print MASTER_MAKEFILE "\t(cd parcels; tar cf - $root | gzip -9 - > $root.tgz )\n"; + + print MASTER_MAKEFILE "\n"; - close MAKEB; - close MAKE; - unlink "$mod/MakefileX"; + unless ($target_windows) + { + print MASTER_MAKEFILE "install-$parcel:\n"; + print MASTER_MAKEFILE "\t(cd $dir; ./install-$parcel)\n\n"; } } +print MASTER_MAKEFILE <<EOF; + +debug_build: @debug_build_targets +release_build: @release_build_targets +parcels: @parcel_targets + +# well-known targets that users might expect to be able to build: +install: + cat local/install.msg + +clean: @all_clean_targets + cd docs; \$(MAKE) clean + +realclean: clean @all_realclean_targets + find release debug -type f -exec rm -f {} \\; + +# and finally a target for rebuilding the build system: +buildsystem: + perl ./infrastructure/makebuildenv.pl $makebuildenv_args +EOF + +for my $parcel (@parcels) +{ + # need to use -f to avoid error if they don't exist (already cleaned) + print MASTER_MAKEFILE "\trm -rf ", BoxPlatform::parcel_dir($parcel), "\n"; + print MASTER_MAKEFILE "\trm -f ", BoxPlatform::parcel_target($parcel), "\n"; +} + +open INSTALLMSG,">local/install.msg" or die "Can't open install message file for writing"; +print INSTALLMSG <<__E; + +Parcels need to be installed separately, and as root. Type one of the following: + +__E + +for(@parcels) +{ + print INSTALLMSG " $make_command install-".$_."\n"; +} +print INSTALLMSG "\n"; + +close INSTALLMSG; + +close MASTER_MAKEFILE; + print "\nType 'cd <module_dir>; $make_command' to build a module\n\n"; if($modules_omitted) diff --git a/infrastructure/makeparcels.pl.in b/infrastructure/makeparcels.pl.in deleted file mode 100755 index 5afdf730..00000000 --- a/infrastructure/makeparcels.pl.in +++ /dev/null @@ -1,405 +0,0 @@ -#!@PERL@ - -use strict; -use lib 'infrastructure'; -use BoxPlatform; - -my @parcels; -my %parcel_contents; - -sub starts_with ($$) -{ - my ($string,$expected) = @_; - return substr($string, 0, length $expected) eq $expected; -} - -sub os_matches ($) -{ - my ($prefix_string) = @_; - my @prefixes = split m'\,', $prefix_string; - foreach my $prefix (@prefixes) - { - return 1 if starts_with($build_os, $prefix); - return 1 if starts_with($target_os, $prefix); - } - return 0; -} - -my $copy_command = "cp -p"; - -if ($build_os eq 'CYGWIN') -{ - $copy_command = "cp -pu"; # faster -} - -open PARCELS,"parcels.txt" or die "Can't open parcels file"; -{ - my $cur_parcel = ''; - while(<PARCELS>) - { - chomp; s/#.+\Z//; s/\s+\Z//; s/\s+/ /g; - next unless m/\S/; - - # omit bits on some platforms? - next if m/\AEND-OMIT/; - if(m/\AOMIT:(.+)/) - { - if (os_matches($1)) - { - while(<PARCELS>) - { - last if m/\AEND-OMIT/; - } - } - next; - } - - if (m'\AONLY:(.+)') - { - if (not os_matches($1)) - { - while (<PARCELS>) - { - last if m'\AEND-ONLY'; - } - } - next; - } - next if (m'\AEND-ONLY'); - - if (m'\AEXCEPT:(.+)') - { - if (os_matches($1)) - { - while (<PARCELS>) - { - last if m'\AEND-EXCEPT'; - } - } - next; - } - next if (m'\AEND-EXCEPT'); - - # new parcel, or a new parcel definition? - if(m/\A\s+(.+)\Z/) - { - push @{$parcel_contents{$cur_parcel}},$1 - } - else - { - $cur_parcel = $_; - push @parcels,$_; - } - } -} -close PARCELS; - -# create parcels directory -mkdir "parcels",0755; -mkdir "parcels/scripts",0755; - -# write master makefile - -open MAKE,">Makefile" or die "Can't open master Makefile for writing"; - -print MAKE <<__E; -# -# AUTOMATICALLY GENERATED FILE -# do not edit! -# -# - -MAKE = $make_command - -__E - -print MAKE "all:\t",join(' ',map {"build-".$_} @parcels),"\n\n"; - -print MAKE <<__END_OF_FRAGMENT; -test: release/common/test - -release/common/test: - ./runtest.pl ALL release - -.PHONY: docs -docs: - cd docs; \$(MAKE) - -__END_OF_FRAGMENT - -my $release_flag = BoxPlatform::make_flag('RELEASE'); -my @clean_deps; - -for my $parcel (@parcels) -{ - my $version = BoxPlatform::parcel_root($parcel); - my $target = BoxPlatform::parcel_target($parcel); - my $dir = BoxPlatform::parcel_dir($parcel); - my @parcel_deps; - - # Need to use BSD install on Solaris - my $install_bin = $build_os eq 'SunOS' ? '/usr/ucb/install' : 'install'; - - unless ($target_windows) - { - open SCRIPT,">parcels/scripts/install-$parcel" or die - "Can't open installer script for $parcel for writing"; - print SCRIPT "#!/bin/sh\n\n"; - } - - for(@{$parcel_contents{$parcel}}) - { - my @args = split /\s+/; - - my ($type,$name,$dest) = @args; - my $optional = 0; - my $install = 1; - - if ($type eq 'optional') - { - $optional = 1; - shift @args; - ($type,$name,$dest) = @args; - } - - if ($type eq 'noinstall') - { - $install = 0; - shift @args; - ($type,$name,$dest) = @args; - } - - if($type eq 'bin') - { - my $exeext = $platform_exe_ext; - print MAKE <<EOF; -$dir/$name$exeext: release/bin/$name/$name$exeext - mkdir -p $dir - $copy_command release/bin/$name/$name$exeext $dir - -.PHONY: release/bin/$name/$name$exeext -release/bin/$name/$name$exeext: - (cd bin/$name; \$(MAKE) $release_flag) - -EOF - push @parcel_deps, "$dir/$name$exeext"; - } - elsif ($type eq 'script') - { - my $fullpath = $name; - my $filename = $name; - # remove path from script name - $filename =~ s{.*/}{}; - - print MAKE <<EOF; -$dir/$filename: $fullpath - mkdir -p $dir -EOF - - if ($optional) - { - print MAKE "\ttest -r $fullpath " . - "&& $copy_command $fullpath $dir || true\n"; - } - else - { - print MAKE "\t$copy_command $fullpath $dir\n"; - } - - print MAKE "\n"; - - push @parcel_deps, "$dir/$filename"; - } - elsif($type eq 'man') - { - print MAKE <<EOF; -$dir/${name}.gz: docs/man/${name}.gz - mkdir -p $dir - $copy_command docs/man/${name}.gz $dir - -EOF - # Releases have the docs pre-made, but users - # may want to rebuild them for some reason. - my $docbook_source = "docs/docbook/${name}"; - $docbook_source =~ s/\.[58]$/.xml/; - print MAKE <<EOF; -docs/man/${name}.gz: $docbook_source docs/docbook/bb-man.xsl - cd docs; \$(MAKE) man/${name}.gz - -EOF - push @parcel_deps, "$dir/${name}.gz"; - } - elsif($type eq 'html') - { - print MAKE <<EOF; -$dir/docs/${name}.html: docs/htmlguide/man-html/${name}.html - mkdir -p $dir/docs - $copy_command docs/htmlguide/man-html/${name}.html $dir/docs - -EOF - # Releases have the docs pre-made, but users - # may want to rebuild them for some reason. - my $docbook_source = "docs/docbook/${name}.xml"; - print MAKE <<EOF; -docs/htmlguide/man-html/${name}.html: $docbook_source docs/docbook/bb-nochunk-book.xsl - cd docs; \$(MAKE) htmlguide/man-html/${name}.html - -EOF - push @parcel_deps, "$dir/docs/${name}.html"; - } - elsif ($type eq 'subdir') - { - shift @args; - my $subdir = shift @args; - print MAKE <<EOF; -.PHONY: $name-build $name-clean - -$name-build: - cd $subdir; \$(MAKE) @args - -$name-clean: - cd $name; \$(MAKE) clean -EOF - push @parcel_deps, "$name-build"; - push @clean_deps, "$name-clean"; - } - } - - print MAKE <<EOF; -build-$parcel: $target - -$target: @parcel_deps - test -d $dir || mkdir $dir -EOF - - for(@{$parcel_contents{$parcel}}) - { - my @args = split /\s+/; - - my ($type,$name,$dest) = @args; - - my $optional = 0; - my $install = 1; - - if ($type eq 'optional') - { - $optional = 1; - shift @args; - ($type,$name,$dest) = @args; - } - - if ($type eq 'noinstall') - { - $install = 0; - shift @args; - ($type,$name,$dest) = @args; - } - - if ($type eq 'script') - { - # remove path from script name - $name =~ s{.*/}{}; - } - - if ($type eq 'html') - { - $dest = "share/doc/@PACKAGE_TARNAME@"; - $name = "docs/$name.html"; - } - - if ($type eq 'man') - { - $name =~ /([0-9])$/; - $dest = "man/man$1"; - $name =~ s/$/\.gz/; - } - - if ($install and not $target_windows and not $type eq "subdir") - { - my $local_install_dir = $install_into_dir; - if (defined $dest) - { - if ($dest =~ m,^/,) - { - # Don't add $prefix if $dest is a literal path - $local_install_dir = $dest; - } - else - { - $local_install_dir = "@prefix@/$dest"; - } - } - print SCRIPT "mkdir -p " . - "\${DESTDIR}$local_install_dir/\n"; - print SCRIPT "$install_bin $name " . - "\${DESTDIR}$local_install_dir\n"; - } - } - - unless ($target_windows) - { - close SCRIPT; - chmod 0755,"parcels/scripts/install-$parcel"; - } - - my $root = BoxPlatform::parcel_root($parcel); - - unless ($target_windows) - { - print MAKE "\tcp parcels/scripts/install-$parcel $dir\n"; - } - - print MAKE "\t(cd parcels; tar cf - $root | gzip -9 - > $root.tgz )\n"; - - print MAKE "\n"; - - unless ($target_windows) - { - print MAKE "install-$parcel:\n"; - print MAKE "\t(cd $dir; ./install-$parcel)\n\n"; - } -} - -print MAKE <<EOF; -install: - cat local/install.msg - -clean: @clean_deps - cd docs; \$(MAKE) clean -EOF - -if ($build_os eq 'CYGWIN') -{ - print MAKE "\tfind release debug -type f | xargs -r rm\n"; - print MAKE "\tfind . -name 'autogen_*' -type f | xargs -r rm\n"; -} -else -{ - print MAKE "\tfind release debug -type f -exec rm -f {} \\;\n"; - print MAKE "\tfind . -name 'autogen_*' -type f -exec rm -f {} \\;\n"; -} - -for my $parcel (@parcels) -{ - # need to use -f to avoid error if they don't exist (already cleaned) - print MAKE "\trm -rf ", BoxPlatform::parcel_dir($parcel), "\n"; - print MAKE "\trm -f ", BoxPlatform::parcel_target($parcel), "\n"; -} - -close MAKE; - -open INSTALLMSG,">local/install.msg" or die "Can't open install message file for writing"; -print INSTALLMSG <<__E; - -Parcels need to be installed separately, and as root. Type one of the following: - -__E - -for(@parcels) -{ - print INSTALLMSG " $make_command install-".$_."\n"; -} -print INSTALLMSG "\n"; - -close INSTALLMSG; - diff --git a/infrastructure/mingw/configure.sh b/infrastructure/mingw/configure.sh deleted file mode 100755 index 46a61637..00000000 --- a/infrastructure/mingw/configure.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh - -DEP_PATH=/usr/i686-pc-mingw32 - -if [ ! -r "$DEP_PATH/lib/libssl.a" ]; then - echo "Error: install OpenSSL as instructed by" \ - "docs/backup/win32_build_on_cygwin_using_mingw.txt" >&2 - exit 2 -fi - -if [ ! -r "$DEP_PATH/lib/libpcreposix.a" \ - -o ! -r "$DEP_PATH/lib/libpcre.a" \ - -o ! -r "$DEP_PATH/include/pcreposix.h" ]; then - echo "Error: install PCRE as instructed by" \ - "docs/backup/win32_build_on_cygwin_using_mingw.txt" >&2 - exit 2 -fi - -LIBZ_PATH="${DEP_PATH}/sys-root/mingw/lib" - -if [ ! -r "$LIBZ_PATH/libz.dll.a" ]; then - echo "Error: upgrade your Cygwin mingw-zlib-devel package" >&2 - exit 2 -fi - -if [ ! -x "configure" ]; then - if ! ./bootstrap; then - echo "Error: bootstrap failed, aborting." >&2 - exit 1 - fi -fi - -if ! ./configure "$@" --target=i686-pc-mingw32 \ - CFLAGS="-mno-cygwin -mthreads" \ - CPPFLAGS="-mno-cygwin" \ - CXXFLAGS="-mno-cygwin -mthreads" \ - LDFLAGS="-Wl,-Bstatic -mno-cygwin -mthreads -L${DEP_PATH}/lib -L${LIBZ_PATH}" -then - echo "Error: configure failed, aborting." >&2 - exit 1 -fi - -exit 0 diff --git a/infrastructure/msvc/2003/bbackupctl.vcproj b/infrastructure/msvc/2003/bbackupctl.vcproj deleted file mode 100644 index 02f7482e..00000000 --- a/infrastructure/msvc/2003/bbackupctl.vcproj +++ /dev/null @@ -1,159 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="7.10"
- Name="bbackupctl"
- ProjectGUID="{9FD51412-E945-4457-A17A-CA3C505CF431}"
- Keyword="Win32Proj">
- <Platforms>
- <Platform
- Name="Win32"/>
- </Platforms>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="1"
- CharacterSet="2">
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\..\db-4.2.52.NC\build_win32";"$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;BOX_RELEASE_BUILD "
- MinimalRebuild="TRUE"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="4"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib"
- OutputFile="$(OutDir)/bbackupctl.exe"
- LinkIncremental="2"
- GenerateDebugInformation="TRUE"
- ProgramDatabaseFile="$(OutDir)/bbackupctl.pdb"
- SubSystem="1"
- TargetMachine="1"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCWebDeploymentTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="1"
- CharacterSet="2"
- WholeProgramOptimization="TRUE">
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="TRUE"
- OptimizeForProcessor="1"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING"
- RuntimeLibrary="0"
- BufferSecurityCheck="FALSE"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="3"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib"
- OutputFile="$(OutDir)/bbackupctl.exe"
- LinkIncremental="1"
- IgnoreDefaultLibraryNames=""
- GenerateDebugInformation="TRUE"
- SubSystem="1"
- OptimizeReferences="2"
- EnableCOMDATFolding="2"
- OptimizeForWindows98="1"
- TargetMachine="1"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCWebDeploymentTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}">
- <Filter
- Name="bin"
- Filter="">
- <Filter
- Name="bbackupctl"
- Filter="">
- <File
- RelativePath="..\..\..\bin\bbackupctl\bbackupctl.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\win32\WinNamedPipeStream.cpp">
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}">
- <File
- RelativePath="..\..\..\lib\win32\WinNamedPipeStream.h">
- </File>
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
- <File
- RelativePath="..\..\..\lib\win32\messages.rc">
- </File>
- </Filter>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2003/bbackupd.vcproj b/infrastructure/msvc/2003/bbackupd.vcproj deleted file mode 100644 index f34db0cc..00000000 --- a/infrastructure/msvc/2003/bbackupd.vcproj +++ /dev/null @@ -1,219 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="7.10"
- Name="bbackupd"
- ProjectGUID="{22D325FB-9131-4BD6-B390-968F0491D687}"
- Keyword="Win32Proj">
- <Platforms>
- <Platform
- Name="Win32"/>
- </Platforms>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="1"
- CharacterSet="2">
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(SolutionDir)..\..\..\..\db-4.2.52.NC\build_win32";"$(SolutionDir)..\..\..\..\boost_1_31_0";"$(SolutionDir)..\..\..\..\openssl\include";"$(SolutionDir)..\..\..\..\zlib\include";"$(SolutionDir)..\..\..\lib\backupclient";"$(SolutionDir)..\..\..\lib\server";"$(SolutionDir)..\..\..\lib\crypto";"$(SolutionDir)..\..\..\lib\compress";"$(SolutionDir)..\..\..\lib\win32";"$(SolutionDir)..\..\..\lib\common\""
- PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;BOX_RELEASE_BUILD "
- MinimalRebuild="TRUE"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="4"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib"
- OutputFile="$(OutDir)/bbackupd.exe"
- LinkIncremental="2"
- IgnoreAllDefaultLibraries="FALSE"
- GenerateDebugInformation="TRUE"
- ProgramDatabaseFile="$(OutDir)/bbackupd.pdb"
- SubSystem="1"
- TargetMachine="1"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCWebDeploymentTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="1"
- CharacterSet="2"
- WholeProgramOptimization="TRUE">
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="TRUE"
- OptimizeForProcessor="1"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0""
- PreprocessorDefinitions="WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;BOOST_REGEX_NO_LIB"
- RuntimeLibrary="0"
- BufferSecurityCheck="FALSE"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="3"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib"
- OutputFile="$(OutDir)/bbackupd.exe"
- LinkIncremental="1"
- IgnoreDefaultLibraryNames=""
- GenerateDebugInformation="TRUE"
- SubSystem="1"
- OptimizeReferences="2"
- EnableCOMDATFolding="2"
- OptimizeForWindows98="1"
- TargetMachine="1"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCWebDeploymentTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}">
- <Filter
- Name="bin"
- Filter="">
- <Filter
- Name="bbackupd"
- Filter="">
- <File
- RelativePath="..\..\..\bin\bbackupd\autogen_ClientException.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientContext.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientDeleteList.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientDirectoryRecord.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientInodeToIDMap.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupDaemon.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\bbackupd.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\Win32BackupService.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\Win32ServiceFunctions.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\win32\WinNamedPipeStream.cpp">
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}">
- <Filter
- Name="bin"
- Filter="">
- <Filter
- Name="bbackupd"
- Filter="">
- <File
- RelativePath="..\..\..\bin\bbackupd\autogen_ClientException.h">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientContext.h">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientDeleteList.h">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientDirectoryRecord.h">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientInodeToIDMap.h">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupDaemon.h">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\Win32BackupService.h">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\Win32ServiceFunctions.h">
- </File>
- <File
- RelativePath="..\..\..\lib\win32\WinNamedPipeStream.h">
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
- <File
- RelativePath="..\..\..\lib\win32\messages.rc">
- </File>
- </Filter>
- <File
- RelativePath="..\..\..\ReadMe.txt">
- </File>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2003/boxbackup.sln b/infrastructure/msvc/2003/boxbackup.sln deleted file mode 100644 index d9a28041..00000000 --- a/infrastructure/msvc/2003/boxbackup.sln +++ /dev/null @@ -1,57 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 8.00
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "boxquery", "boxquery.vcproj", "{FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}"
- ProjectSection(ProjectDependencies) = postProject
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "common", "common.vcproj", "{A089CEE6-EBF0-4232-A0C0-74850A8127A6}"
- ProjectSection(ProjectDependencies) = postProject
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbackupd", "bbackupd.vcproj", "{22D325FB-9131-4BD6-B390-968F0491D687}"
- ProjectSection(ProjectDependencies) = postProject
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "win32test", "win32test.vcproj", "{28C29E72-76A2-4D0C-B35B-12D446733D2E}"
- ProjectSection(ProjectDependencies) = postProject
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbackupctl", "bbackupctl.vcproj", "{9FD51412-E945-4457-A17A-CA3C505CF431}"
- ProjectSection(ProjectDependencies) = postProject
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
- EndProjectSection
-EndProject
-Global
- GlobalSection(SolutionConfiguration) = preSolution
- Debug = Debug
- Release = Release
- EndGlobalSection
- GlobalSection(ProjectConfiguration) = postSolution
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Debug.ActiveCfg = Debug|Win32
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Debug.Build.0 = Debug|Win32
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Release.ActiveCfg = Release|Win32
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Release.Build.0 = Release|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Debug.ActiveCfg = Debug|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Debug.Build.0 = Debug|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Release.ActiveCfg = Release|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Release.Build.0 = Release|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Debug.ActiveCfg = Debug|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Debug.Build.0 = Debug|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Release.ActiveCfg = Release|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Release.Build.0 = Release|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Debug.ActiveCfg = Debug|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Debug.Build.0 = Debug|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Release.ActiveCfg = Release|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Release.Build.0 = Release|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Debug.ActiveCfg = Debug|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Debug.Build.0 = Debug|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Release.ActiveCfg = Release|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Release.Build.0 = Release|Win32
- EndGlobalSection
- GlobalSection(ExtensibilityGlobals) = postSolution
- EndGlobalSection
- GlobalSection(ExtensibilityAddIns) = postSolution
- EndGlobalSection
-EndGlobal
diff --git a/infrastructure/msvc/2003/boxquery.vcproj b/infrastructure/msvc/2003/boxquery.vcproj deleted file mode 100644 index 6ac09024..00000000 --- a/infrastructure/msvc/2003/boxquery.vcproj +++ /dev/null @@ -1,174 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="7.10"
- Name="boxquery"
- ProjectGUID="{FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}"
- RootNamespace="boxquery"
- Keyword="Win32Proj">
- <Platforms>
- <Platform
- Name="Win32"/>
- </Platforms>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="1"
- CharacterSet="2">
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0""
- PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;BOX_RELEASE_BUILD "
- MinimalRebuild="TRUE"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="4"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib"
- OutputFile="$(OutDir)/bbackupquery.exe"
- LinkIncremental="2"
- GenerateDebugInformation="TRUE"
- ProgramDatabaseFile="$(OutDir)/boxquery.pdb"
- SubSystem="1"
- TargetMachine="1"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCWebDeploymentTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="1"
- CharacterSet="2"
- WholeProgramOptimization="TRUE">
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="TRUE"
- OptimizeForProcessor="1"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0""
- PreprocessorDefinitions="WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;BOOST_REGEX_NO_LIB"
- RuntimeLibrary="0"
- BufferSecurityCheck="FALSE"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="3"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib"
- OutputFile="$(OutDir)/bbackupquery.exe"
- LinkIncremental="1"
- IgnoreDefaultLibraryNames=""
- GenerateDebugInformation="FALSE"
- SubSystem="1"
- OptimizeReferences="2"
- EnableCOMDATFolding="2"
- OptimizeForWindows98="1"
- TargetMachine="1"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCWebDeploymentTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}">
- <Filter
- Name="bin"
- Filter="">
- <Filter
- Name="backupquery"
- Filter="">
- <File
- RelativePath="..\..\..\bin\bbackupquery\autogen_Documentation.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupquery\BackupQueries.cpp">
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupquery\bbackupquery.cpp">
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}">
- <Filter
- Name="bin"
- Filter="">
- <Filter
- Name="backupquery"
- Filter="">
- <File
- RelativePath="..\..\..\bin\bbackupquery\BackupQueries.h">
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
- <File
- RelativePath="..\..\..\lib\win32\messages.rc">
- </File>
- </Filter>
- <File
- RelativePath="..\..\..\ReadMe.txt">
- </File>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2003/common.vcproj b/infrastructure/msvc/2003/common.vcproj deleted file mode 100644 index fb18b76a..00000000 --- a/infrastructure/msvc/2003/common.vcproj +++ /dev/null @@ -1,672 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="7.10"
- Name="common"
- ProjectGUID="{A089CEE6-EBF0-4232-A0C0-74850A8127A6}"
- Keyword="Win32Proj">
- <Platforms>
- <Platform
- Name="Win32"/>
- </Platforms>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="4"
- CharacterSet="2">
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0\""
- PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;BOX_RELEASE_BUILD "
- MinimalRebuild="TRUE"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="4"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLibrarianTool"
- OutputFile="$(OutDir)/common.lib"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="4"
- CharacterSet="2"
- WholeProgramOptimization="TRUE">
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="TRUE"
- OptimizeForProcessor="1"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\";"$(SolutionDir)..\..\..\..\boost_1_31_0\""
- PreprocessorDefinitions="BOOST_REGEX_NO_LIB;WIN32;BOX_RELEASE_BUILD;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING"
- RuntimeLibrary="0"
- BufferSecurityCheck="FALSE"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="3"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLibrarianTool"
- OutputFile="$(OutDir)/common.lib"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}">
- <Filter
- Name="lib"
- Filter="">
- <Filter
- Name="compress"
- Filter="">
- <File
- RelativePath="..\..\..\lib\compress\autogen_CompressException.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\compress\CompressStream.cpp">
- </File>
- </Filter>
- <Filter
- Name="common"
- Filter="">
- <File
- RelativePath="..\..\..\lib\common\autogen_CommonException.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\autogen_ConversionException.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxException.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTime.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTimeToText.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\CollectInBufferStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Configuration.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\ConversionString.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\DebugAssertFailed.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\DebugMemLeakFinder.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\DebugPrintf.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\EventWatchFilesystemObject.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\ExcludeList.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\FdGetLine.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\FileStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\IOStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\IOStreamGetLine.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Logging.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemBlockStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\PartialReadStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\PathUtils.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\ReadGatherStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\ReadLoggingStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\StreamableMemBlock.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\UnixUser.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Utils.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\common\WaitForEvent.cpp">
- </File>
- </Filter>
- <Filter
- Name="backupclient"
- Filter="">
- <File
- RelativePath="..\..\..\lib\backupclient\autogen_BackupProtocolClient.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\autogen_BackupStoreException.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientCryptoKeys.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientFileAttributes.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientMakeExcludeList.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientRestore.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupDaemonConfigVerify.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreDirectory.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFile.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCmbDiff.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCmbIdx.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCombine.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCryptVar.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileDiff.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileEncodeStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFilename.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFilenameClear.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileRevDiff.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreObjectDump.cpp">
- </File>
- </Filter>
- <Filter
- Name="crypto"
- Filter="">
- <File
- RelativePath="..\..\..\lib\crypto\autogen_CipherException.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherAES.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherBlowfish.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherContext.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherDescription.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\MD5Digest.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\Random.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\RollingChecksum.cpp">
- </File>
- </Filter>
- <Filter
- Name="win32"
- Filter="">
- <File
- RelativePath="..\..\..\lib\win32\emu.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\win32\getopt_long.cxx">
- </File>
- </Filter>
- <Filter
- Name="server"
- Filter="">
- <File
- RelativePath="..\..\..\lib\server\autogen_ConnectionException.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\autogen_ServerException.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\Daemon.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\LocalProcessStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\Protocol.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolObject.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolUncertainStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\Socket.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketStream.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketStreamTLS.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\SSLLib.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\TLSContext.cpp">
- </File>
- <File
- RelativePath="..\..\..\lib\server\WinNamedPipeStream.cpp">
- </File>
-
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}">
- <Filter
- Name="lib"
- Filter="">
- <Filter
- Name="compress"
- Filter="">
- <File
- RelativePath="..\..\..\lib\compress\autogen_CompressException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\compress\Compress.h">
- </File>
- <File
- RelativePath="..\..\..\lib\compress\CompressException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\compress\CompressStream.h">
- </File>
- </Filter>
- <Filter
- Name="common"
- Filter="">
- <File
- RelativePath="..\..\..\lib\common\autogen_CommonException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\autogen_ConversionException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BannerText.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BeginStructPackForWire.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Box.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxPlatform.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxPortsAndFiles.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTime.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTimeToText.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTimeToUnix.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\CollectInBufferStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\CommonException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Configuration.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Conversion.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\EndStructPackForWire.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\EventWatchFilesystemObject.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\ExcludeList.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\FdGetLine.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\FileModificationTime.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\FileStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Guards.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\IOStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\IOStreamGetLine.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\LinuxWorkaround.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\LocalProcessStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Logging.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\MainHelper.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemBlockStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemLeakFinder.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemLeakFindOff.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemLeakFindOn.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\NamedLock.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\PartialReadStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\PathUtils.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\ReadGatherStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\StreamableMemBlock.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\TemporaryDirectory.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Test.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Timer.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\UnixUser.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\Utils.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\WaitForEvent.h">
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxVersion.h">
- </File>
- </Filter>
- <Filter
- Name="backupclient"
- Filter="">
- <File
- RelativePath="..\..\..\lib\backupclient\autogen_BackupProtocolClient.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\autogen_BackupStoreException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientCryptoKeys.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientFileAttributes.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientMakeExcludeList.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientRestore.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupDaemonConfigVerify.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreConstants.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreDirectory.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFile.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCryptVar.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileEncodeStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFilename.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFilenameClear.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileWire.h">
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreObjectMagic.h">
- </File>
- </Filter>
- <Filter
- Name="crypto"
- Filter="">
- <File
- RelativePath="..\..\..\lib\crypto\autogen_CipherException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherAES.h">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherBlowfish.h">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherContext.h">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherDescription.h">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\MD5Digest.h">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\Random.h">
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\RollingChecksum.h">
- </File>
- </Filter>
- <Filter
- Name="win32"
- Filter="">
- <File
- RelativePath="..\..\..\lib\win32\emu.h">
- </File>
- <File
- RelativePath="..\..\..\lib\win32\getopt.h">
- </File>
- <File
- RelativePath="..\..\..\lib\win32\WinNamedPipeStream.h">
- </File>
- </Filter>
- <Filter
- Name="server"
- Filter="">
- <File
- RelativePath="..\..\..\lib\server\autogen_ConnectionException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\autogen_ServerException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\Daemon.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\Protocol.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolObject.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolUncertainStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolWire.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\ServerException.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\ServerStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\ServerTLS.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\Socket.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketListen.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketStream.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketStreamTLS.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\SSLLib.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\TLSContext.h">
- </File>
- <File
- RelativePath="..\..\..\lib\server\WinNamedPipeStream.h">
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
- </Filter>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2003/win32test.vcproj b/infrastructure/msvc/2003/win32test.vcproj deleted file mode 100644 index 2ef7164e..00000000 --- a/infrastructure/msvc/2003/win32test.vcproj +++ /dev/null @@ -1,148 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="7.10"
- Name="win32test"
- ProjectGUID="{28C29E72-76A2-4D0C-B35B-12D446733D2E}"
- Keyword="Win32Proj">
- <Platforms>
- <Platform
- Name="Win32"/>
- </Platforms>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="1"
- CharacterSet="2">
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\bin\bbackupd";"$(ProjectDir)..\..\..\..\db-4.2.52.NC\build_win32";"$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING"
- MinimalRebuild="TRUE"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="4"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib"
- OutputFile="$(OutDir)/win32test.exe"
- LinkIncremental="2"
- GenerateDebugInformation="TRUE"
- ProgramDatabaseFile="$(OutDir)/win32test.pdb"
- SubSystem="1"
- TargetMachine="1"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCWebDeploymentTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="1"
- CharacterSet="2"
- WholeProgramOptimization="TRUE">
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="TRUE"
- OptimizeForProcessor="1"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\bin\bbackupd";"$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING"
- RuntimeLibrary="0"
- BufferSecurityCheck="FALSE"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="TRUE"
- DebugInformationFormat="3"/>
- <Tool
- Name="VCCustomBuildTool"/>
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib $(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib"
- OutputFile="$(OutDir)/win32test.exe"
- LinkIncremental="1"
- IgnoreDefaultLibraryNames=""
- GenerateDebugInformation="TRUE"
- SubSystem="1"
- OptimizeReferences="2"
- EnableCOMDATFolding="2"
- OptimizeForWindows98="1"
- TargetMachine="1"/>
- <Tool
- Name="VCMIDLTool"/>
- <Tool
- Name="VCPostBuildEventTool"/>
- <Tool
- Name="VCPreBuildEventTool"/>
- <Tool
- Name="VCPreLinkEventTool"/>
- <Tool
- Name="VCResourceCompilerTool"/>
- <Tool
- Name="VCWebServiceProxyGeneratorTool"/>
- <Tool
- Name="VCXMLDataGeneratorTool"/>
- <Tool
- Name="VCWebDeploymentTool"/>
- <Tool
- Name="VCManagedWrapperGeneratorTool"/>
- <Tool
- Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}">
- <File
- RelativePath="..\..\..\lib\win32\emu.cpp">
- </File>
- <File
- RelativePath="..\..\..\test\win32\testlibwin32.cpp">
- </File>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}">
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
- </Filter>
- <File
- RelativePath="..\..\..\ReadMe.txt">
- </File>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2005/bbackupctl.vcproj b/infrastructure/msvc/2005/bbackupctl.vcproj deleted file mode 100644 index 216a284b..00000000 --- a/infrastructure/msvc/2005/bbackupctl.vcproj +++ /dev/null @@ -1,222 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="8.00"
- Name="bbackupctl"
- ProjectGUID="{9FD51412-E945-4457-A17A-CA3C505CF431}"
- RootNamespace="bbackupctl"
- Keyword="Win32Proj"
- >
- <Platforms>
- <Platform
- Name="Win32"
- />
- </Platforms>
- <ToolFiles>
- </ToolFiles>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="1"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
- MinimalRebuild="true"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="4"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib Advapi32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\pcre.lib"
- OutputFile="$(OutDir)/bbackupctl.exe"
- LinkIncremental="2"
- GenerateDebugInformation="true"
- ProgramDatabaseFile="$(OutDir)/bbackupctl.pdb"
- SubSystem="1"
- TargetMachine="1"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCWebDeploymentTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="1"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- WholeProgramOptimization="1"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
- RuntimeLibrary="0"
- BufferSecurityCheck="false"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="3"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib"
- OutputFile="$(OutDir)/bbackupctl.exe"
- LinkIncremental="1"
- IgnoreDefaultLibraryNames=""
- GenerateDebugInformation="true"
- SubSystem="1"
- OptimizeReferences="2"
- EnableCOMDATFolding="2"
- OptimizeForWindows98="1"
- TargetMachine="1"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCWebDeploymentTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
- >
- <Filter
- Name="bin"
- >
- <Filter
- Name="bbackupctl"
- >
- <File
- RelativePath="..\..\..\bin\bbackupctl\bbackupctl.cpp"
- >
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
- >
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
- >
- <File
- RelativePath="..\..\..\lib\win32\messages.rc"
- >
- </File>
- </Filter>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2005/bbackupd.vcproj b/infrastructure/msvc/2005/bbackupd.vcproj deleted file mode 100644 index ac8eb86a..00000000 --- a/infrastructure/msvc/2005/bbackupd.vcproj +++ /dev/null @@ -1,299 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="8.00"
- Name="bbackupd"
- ProjectGUID="{22D325FB-9131-4BD6-B390-968F0491D687}"
- RootNamespace="bbackupd"
- Keyword="Win32Proj"
- >
- <Platforms>
- <Platform
- Name="Win32"
- />
- </Platforms>
- <ToolFiles>
- </ToolFiles>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="1"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(SolutionDir)..\..\..\lib\backupclient";"$(SolutionDir)..\..\..\lib\common";"$(SolutionDir)..\..\..\lib\compress";"$(SolutionDir)..\..\..\lib\crypto";"$(SolutionDir)..\..\..\lib\server";"$(SolutionDir)..\..\..\lib\win32";"$(SolutionDir)..\..\..\..\openssl\inc32";"$(SolutionDir)..\..\..\..\zlib\include""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
- MinimalRebuild="true"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="4"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib Advapi32.lib User32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\pcre.lib"
- OutputFile="$(OutDir)/bbackupd.exe"
- LinkIncremental="2"
- IgnoreAllDefaultLibraries="false"
- GenerateDebugInformation="true"
- ProgramDatabaseFile="$(OutDir)/bbackupd.pdb"
- SubSystem="1"
- TargetMachine="1"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCWebDeploymentTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="1"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- WholeProgramOptimization="1"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(SolutionDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
- RuntimeLibrary="0"
- BufferSecurityCheck="false"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="3"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib"
- OutputFile="$(OutDir)/bbackupd.exe"
- LinkIncremental="1"
- IgnoreDefaultLibraryNames=""
- GenerateDebugInformation="true"
- SubSystem="1"
- OptimizeReferences="2"
- EnableCOMDATFolding="2"
- OptimizeForWindows98="1"
- TargetMachine="1"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCWebDeploymentTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
- >
- <Filter
- Name="bin"
- >
- <Filter
- Name="bbackupd"
- >
- <File
- RelativePath="..\..\..\bin\bbackupd\autogen_ClientException.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientContext.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientDeleteList.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientDirectoryRecord.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientInodeToIDMap.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupDaemon.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\bbackupd.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\Win32BackupService.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\Win32ServiceFunctions.cpp"
- >
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
- >
- <Filter
- Name="bin"
- >
- <Filter
- Name="bbackupd"
- >
- <File
- RelativePath="..\..\..\bin\bbackupd\autogen_ClientException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientContext.h"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientDeleteList.h"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientDirectoryRecord.h"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupClientInodeToIDMap.h"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\BackupDaemon.h"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\Win32BackupService.h"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupd\Win32ServiceFunctions.h"
- >
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
- >
- <File
- RelativePath="..\..\..\lib\win32\messages.rc"
- >
- </File>
- </Filter>
- <File
- RelativePath="..\..\..\ReadMe.txt"
- >
- </File>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2005/boxbackup.sln b/infrastructure/msvc/2005/boxbackup.sln deleted file mode 100644 index 833066e9..00000000 --- a/infrastructure/msvc/2005/boxbackup.sln +++ /dev/null @@ -1,55 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 9.00
-# Visual C++ Express 2005
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "boxquery", "boxquery.vcproj", "{FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}"
- ProjectSection(ProjectDependencies) = postProject
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "common", "common.vcproj", "{A089CEE6-EBF0-4232-A0C0-74850A8127A6}"
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbackupd", "bbackupd.vcproj", "{22D325FB-9131-4BD6-B390-968F0491D687}"
- ProjectSection(ProjectDependencies) = postProject
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "win32test", "win32test.vcproj", "{28C29E72-76A2-4D0C-B35B-12D446733D2E}"
- ProjectSection(ProjectDependencies) = postProject
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbackupctl", "bbackupctl.vcproj", "{9FD51412-E945-4457-A17A-CA3C505CF431}"
- ProjectSection(ProjectDependencies) = postProject
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6} = {A089CEE6-EBF0-4232-A0C0-74850A8127A6}
- EndProjectSection
-EndProject
-Global
- GlobalSection(SolutionConfigurationPlatforms) = preSolution
- Debug|Win32 = Debug|Win32
- Release|Win32 = Release|Win32
- EndGlobalSection
- GlobalSection(ProjectConfigurationPlatforms) = postSolution
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Debug|Win32.ActiveCfg = Debug|Win32
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Debug|Win32.Build.0 = Debug|Win32
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Release|Win32.ActiveCfg = Release|Win32
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Release|Win32.Build.0 = Release|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Debug|Win32.ActiveCfg = Debug|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Debug|Win32.Build.0 = Debug|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Release|Win32.ActiveCfg = Release|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Release|Win32.Build.0 = Release|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Debug|Win32.ActiveCfg = Debug|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Debug|Win32.Build.0 = Debug|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Release|Win32.ActiveCfg = Release|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Release|Win32.Build.0 = Release|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Debug|Win32.ActiveCfg = Debug|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Debug|Win32.Build.0 = Debug|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Release|Win32.ActiveCfg = Release|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Release|Win32.Build.0 = Release|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Debug|Win32.ActiveCfg = Debug|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Debug|Win32.Build.0 = Debug|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Release|Win32.ActiveCfg = Release|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Release|Win32.Build.0 = Release|Win32
- EndGlobalSection
- GlobalSection(SolutionProperties) = preSolution
- HideSolutionNode = FALSE
- EndGlobalSection
-EndGlobal
diff --git a/infrastructure/msvc/2005/boxbackup.suo b/infrastructure/msvc/2005/boxbackup.suo Binary files differdeleted file mode 100644 index 534f337c..00000000 --- a/infrastructure/msvc/2005/boxbackup.suo +++ /dev/null diff --git a/infrastructure/msvc/2005/boxquery.vcproj b/infrastructure/msvc/2005/boxquery.vcproj deleted file mode 100644 index 776c0ac9..00000000 --- a/infrastructure/msvc/2005/boxquery.vcproj +++ /dev/null @@ -1,246 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="8.00"
- Name="boxquery"
- ProjectGUID="{FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}"
- RootNamespace="boxquery"
- Keyword="Win32Proj"
- >
- <Platforms>
- <Platform
- Name="Win32"
- />
- </Platforms>
- <ToolFiles>
- </ToolFiles>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="1"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\pcre";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
- MinimalRebuild="true"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="4"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib Advapi32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\pcre.lib"
- OutputFile="$(OutDir)/bbackupquery.exe"
- LinkIncremental="2"
- GenerateDebugInformation="true"
- ProgramDatabaseFile="$(OutDir)/boxquery.pdb"
- SubSystem="1"
- TargetMachine="1"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCWebDeploymentTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="1"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- WholeProgramOptimization="1"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\..\openssl\include";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\lib\common\""
- PreprocessorDefinitions="WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;PCRE_STATIC"
- RuntimeLibrary="0"
- BufferSecurityCheck="false"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="3"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib"
- OutputFile="$(OutDir)/bbackupquery.exe"
- LinkIncremental="1"
- IgnoreDefaultLibraryNames=""
- GenerateDebugInformation="false"
- SubSystem="1"
- OptimizeReferences="2"
- EnableCOMDATFolding="2"
- OptimizeForWindows98="1"
- TargetMachine="1"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCWebDeploymentTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
- >
- <Filter
- Name="bin"
- >
- <Filter
- Name="backupquery"
- >
- <File
- RelativePath="..\..\..\bin\bbackupquery\autogen_Documentation.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupquery\BackupQueries.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\bin\bbackupquery\bbackupquery.cpp"
- >
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
- >
- <Filter
- Name="bin"
- >
- <Filter
- Name="backupquery"
- >
- <File
- RelativePath="..\..\..\bin\bbackupquery\BackupQueries.h"
- >
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
- >
- <File
- RelativePath="..\..\..\lib\win32\messages.rc"
- >
- </File>
- </Filter>
- <File
- RelativePath="..\..\..\ReadMe.txt"
- >
- </File>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2005/common.vcproj b/infrastructure/msvc/2005/common.vcproj deleted file mode 100644 index 256bce06..00000000 --- a/infrastructure/msvc/2005/common.vcproj +++ /dev/null @@ -1,896 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="8.00"
- Name="common"
- ProjectGUID="{A089CEE6-EBF0-4232-A0C0-74850A8127A6}"
- RootNamespace="common"
- Keyword="Win32Proj"
- >
- <Platforms>
- <Platform
- Name="Win32"
- />
- </Platforms>
- <ToolFiles>
- </ToolFiles>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="4"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- >
- <Tool
- Name="VCPreBuildEventTool"
- Description="Determining Version Number"
- CommandLine="perl $(InputDir)..\getversion.pl"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\..\pcre";$(NOINHERIT)"
- PreprocessorDefinitions="WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
- MinimalRebuild="true"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- WarnAsError="false"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="4"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLibrarianTool"
- OutputFile="$(OutDir)/common.lib"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="4"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- WholeProgramOptimization="1"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common\";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include";"$(ProjectDir)..\..\..\..\pcre\pcre-6.7\""
- PreprocessorDefinitions="WIN32;BOX_RELEASE_BUILD;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
- RuntimeLibrary="0"
- BufferSecurityCheck="false"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="3"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLibrarianTool"
- OutputFile="$(OutDir)/common.lib"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
- >
- <Filter
- Name="lib"
- >
- <Filter
- Name="compress"
- >
- <File
- RelativePath="..\..\..\lib\compress\autogen_CompressException.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\compress\CompressStream.cpp"
- >
- </File>
- </Filter>
- <Filter
- Name="common"
- >
- <File
- RelativePath="..\..\..\lib\common\autogen_CommonException.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\autogen_ConversionException.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxException.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTime.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTimeToText.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\CollectInBufferStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Configuration.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\ConversionString.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\DebugAssertFailed.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\DebugMemLeakFinder.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\DebugPrintf.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\EventWatchFilesystemObject.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\ExcludeList.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\FdGetLine.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\FileStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\IOStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\IOStreamGetLine.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Logging.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemBlockStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\PartialReadStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\PathUtils.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\ReadGatherStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\ReadLoggingStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\StreamableMemBlock.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Timer.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\UnixUser.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Utils.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\WaitForEvent.cpp"
- >
- </File>
- </Filter>
- <Filter
- Name="backupclient"
- >
- <File
- RelativePath="..\..\..\lib\backupclient\autogen_BackupProtocolClient.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\autogen_BackupStoreException.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientCryptoKeys.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientFileAttributes.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientMakeExcludeList.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientRestore.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupDaemonConfigVerify.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreDirectory.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFile.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCmbDiff.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCmbIdx.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCombine.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCryptVar.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileDiff.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileEncodeStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFilename.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFilenameClear.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileRevDiff.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreObjectDump.cpp"
- >
- </File>
- </Filter>
- <Filter
- Name="crypto"
- >
- <File
- RelativePath="..\..\..\lib\crypto\autogen_CipherException.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherAES.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherBlowfish.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherContext.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherDescription.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\MD5Digest.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\Random.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\RollingChecksum.cpp"
- >
- </File>
- </Filter>
- <Filter
- Name="win32"
- >
- <File
- RelativePath="..\..\..\lib\win32\emu.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\win32\getopt_long.cxx"
- >
- </File>
- </Filter>
- <Filter
- Name="server"
- >
- <File
- RelativePath="..\..\..\lib\server\autogen_ConnectionException.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\autogen_ServerException.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\Daemon.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\LocalProcessStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\Protocol.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolObject.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolUncertainStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\Socket.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketStream.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketStreamTLS.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\SSLLib.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\TLSContext.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\WinNamedPipeStream.cpp"
- >
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
- >
- <Filter
- Name="lib"
- >
- <Filter
- Name="compress"
- >
- <File
- RelativePath="..\..\..\lib\compress\autogen_CompressException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\compress\Compress.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\compress\CompressException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\compress\CompressStream.h"
- >
- </File>
- </Filter>
- <Filter
- Name="common"
- >
- <File
- RelativePath="..\..\..\lib\common\autogen_CommonException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\autogen_ConversionException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BannerText.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BeginStructPackForWire.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Box.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxConfig-MSVC.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxPlatform.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxPortsAndFiles.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTime.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTimeToText.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxTimeToUnix.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\BoxVersion.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\CollectInBufferStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\CommonException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Configuration.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Conversion.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\EndStructPackForWire.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\EventWatchFilesystemObject.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\ExcludeList.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\FdGetLine.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\FileModificationTime.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\FileStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Guards.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\IOStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\IOStreamGetLine.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\LocalProcessStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Logging.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\MainHelper.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemBlockStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemLeakFinder.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemLeakFindOff.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\MemLeakFindOn.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\NamedLock.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\PartialReadStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\PathUtils.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\ReadGatherStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\ReadLoggingStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\StreamableMemBlock.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\TemporaryDirectory.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Test.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Timer.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\UnixUser.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\Utils.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\common\WaitForEvent.h"
- >
- </File>
- </Filter>
- <Filter
- Name="backupclient"
- >
- <File
- RelativePath="..\..\..\lib\backupclient\autogen_BackupProtocolClient.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\autogen_BackupStoreException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientCryptoKeys.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientFileAttributes.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientMakeExcludeList.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupClientRestore.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupDaemonConfigVerify.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreConstants.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreDirectory.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFile.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileCryptVar.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileEncodeStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFilename.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFilenameClear.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreFileWire.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\backupclient\BackupStoreObjectMagic.h"
- >
- </File>
- </Filter>
- <Filter
- Name="crypto"
- >
- <File
- RelativePath="..\..\..\lib\crypto\autogen_CipherException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherAES.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherBlowfish.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherContext.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherDescription.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\CipherException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\MD5Digest.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\Random.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\crypto\RollingChecksum.h"
- >
- </File>
- </Filter>
- <Filter
- Name="win32"
- >
- <File
- RelativePath="..\..\..\lib\win32\emu.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\win32\getopt.h"
- >
- </File>
- </Filter>
- <Filter
- Name="server"
- >
- <File
- RelativePath="..\..\..\lib\server\autogen_ConnectionException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\autogen_ServerException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\Daemon.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\Protocol.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolObject.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolUncertainStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\ProtocolWire.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\ServerException.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\ServerStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\ServerTLS.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\Socket.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketListen.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketStream.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\SocketStreamTLS.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\SSLLib.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\TLSContext.h"
- >
- </File>
- <File
- RelativePath="..\..\..\lib\server\WinNamedPipeStream.h"
- >
- </File>
- </Filter>
- </Filter>
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
- >
- </Filter>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2005/win32test.vcproj b/infrastructure/msvc/2005/win32test.vcproj deleted file mode 100644 index 0f97c302..00000000 --- a/infrastructure/msvc/2005/win32test.vcproj +++ /dev/null @@ -1,218 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
- ProjectType="Visual C++"
- Version="8.00"
- Name="win32test"
- ProjectGUID="{28C29E72-76A2-4D0C-B35B-12D446733D2E}"
- RootNamespace="win32test"
- Keyword="Win32Proj"
- >
- <Platforms>
- <Platform
- Name="Win32"
- />
- </Platforms>
- <ToolFiles>
- </ToolFiles>
- <Configurations>
- <Configuration
- Name="Debug|Win32"
- OutputDirectory="..\..\..\Debug"
- IntermediateDirectory="..\..\..\Debug"
- ConfigurationType="1"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\bin\bbackupd";"$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include""
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
- MinimalRebuild="true"
- BasicRuntimeChecks="3"
- RuntimeLibrary="1"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="4"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib Advapi32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Debug\common.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\debug\pcre.lib"
- OutputFile="$(OutDir)/win32test.exe"
- LinkIncremental="2"
- GenerateDebugInformation="true"
- ProgramDatabaseFile="$(OutDir)/win32test.pdb"
- SubSystem="1"
- TargetMachine="1"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCWebDeploymentTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- <Configuration
- Name="Release|Win32"
- OutputDirectory="..\..\..\Release"
- IntermediateDirectory="..\..\..\Release"
- ConfigurationType="1"
- InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
- CharacterSet="2"
- WholeProgramOptimization="1"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- />
- <Tool
- Name="VCCLCompilerTool"
- EnableFiberSafeOptimizations="true"
- AdditionalIncludeDirectories=""$(ProjectDir)..\..\..\bin\bbackupd";"$(ProjectDir)..\..\..\lib\backupclient";"$(ProjectDir)..\..\..\lib\common\";"$(ProjectDir)..\..\..\lib\compress";"$(ProjectDir)..\..\..\lib\crypto";"$(ProjectDir)..\..\..\lib\server";"$(ProjectDir)..\..\..\lib\win32";"$(ProjectDir)..\..\..\..\openssl\inc32";"$(ProjectDir)..\..\..\..\zlib\include""
- PreprocessorDefinitions="WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC"
- RuntimeLibrary="0"
- BufferSecurityCheck="false"
- UsePrecompiledHeader="0"
- WarningLevel="3"
- Detect64BitPortabilityProblems="true"
- DebugInformationFormat="3"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="Ws2_32.lib $(ProjectDir)..\..\..\..\zlib\lib\zdll.lib $(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib $(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib $(ProjectDir)..\..\..\Release\common.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib $(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib"
- OutputFile="$(OutDir)/win32test.exe"
- LinkIncremental="1"
- IgnoreDefaultLibraryNames=""
- GenerateDebugInformation="true"
- SubSystem="1"
- OptimizeReferences="2"
- EnableCOMDATFolding="2"
- OptimizeForWindows98="1"
- TargetMachine="1"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCWebDeploymentTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- </Configurations>
- <References>
- </References>
- <Files>
- <Filter
- Name="Source Files"
- Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
- UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
- >
- <File
- RelativePath="..\..\..\lib\win32\emu.cpp"
- >
- </File>
- <File
- RelativePath="..\..\..\test\win32\testlibwin32.cpp"
- >
- </File>
- </Filter>
- <Filter
- Name="Header Files"
- Filter="h;hpp;hxx;hm;inl;inc;xsd"
- UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
- >
- </Filter>
- <Filter
- Name="Resource Files"
- Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
- UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
- >
- </Filter>
- <File
- RelativePath="..\..\..\ReadMe.txt"
- >
- </File>
- </Files>
- <Globals>
- </Globals>
-</VisualStudioProject>
diff --git a/infrastructure/msvc/2010/bbackupctl.vcxproj b/infrastructure/msvc/2010/bbackupctl.vcxproj deleted file mode 100644 index 5751b4d8..00000000 --- a/infrastructure/msvc/2010/bbackupctl.vcxproj +++ /dev/null @@ -1,109 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{9FD51412-E945-4457-A17A-CA3C505CF431}</ProjectGuid>
- <RootNamespace>bbackupctl</RootNamespace>
- <Keyword>Win32Proj</Keyword>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup>
- <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</IntDir>
- <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <Optimization>Disabled</Optimization>
- <AdditionalIncludeDirectories>$(ProjectDir)..\..\..\lib\backupclient;$(ProjectDir)..\..\..\lib\common;$(ProjectDir)..\..\..\lib\compress;$(ProjectDir)..\..\..\lib\crypto;$(ProjectDir)..\..\..\lib\server;$(ProjectDir)..\..\..\lib\win32;$(ProjectDir)..\..\..\..\openssl\include;$(ProjectDir)..\..\..\..\zlib\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;QDBM_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <MinimalRebuild>true</MinimalRebuild>
- <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <ProgramDatabaseFile>$(OutDir)bbackupctl.pdb</ProgramDatabaseFile>
- <SubSystem>Console</SubSystem>
- <TargetMachine>MachineX86</TargetMachine>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
- <AdditionalIncludeDirectories>$(ProjectDir)..\..\..\lib\backupclient;$(ProjectDir)..\..\..\lib\server;$(ProjectDir)..\..\..\lib\crypto;$(ProjectDir)..\..\..\..\openssl\inc32;$(ProjectDir)..\..\..\lib\compress;$(ProjectDir)..\..\..\..\zlib\include;$(ProjectDir)..\..\..\lib\win32;$(ProjectDir)..\..\..\lib\common\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
- <BufferSecurityCheck>false</BufferSecurityCheck>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
- </ClCompile>
- <Link>
- <AdditionalDependencies>Ws2_32.lib;$(ProjectDir)..\..\..\..\zlib\lib\zdll.lib;$(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib;$(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib;$(ProjectDir)..\..\..\Release\common.lib;$(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib;$(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <OutputFile>$(OutDir)bbackupctl.exe</OutputFile>
- <IgnoreSpecificDefaultLibraries>%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <SubSystem>Console</SubSystem>
- <OptimizeReferences>true</OptimizeReferences>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <TargetMachine>MachineX86</TargetMachine>
- </Link>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\bin\bbackupctl\bbackupctl.cpp" />
- </ItemGroup>
- <ItemGroup>
- <ResourceCompile Include="..\..\..\lib\win32\messages.rc" />
- </ItemGroup>
- <ItemGroup>
- <ProjectReference Include="common.vcxproj">
- <Project>{a089cee6-ebf0-4232-a0c0-74850a8127a6}</Project>
- <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
- </ProjectReference>
- <ProjectReference Include="libbackupclient.vcxproj">
- <Project>{32604097-c934-4711-b1ad-206336640e70}</Project>
- </ProjectReference>
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/2010/bbackupd.vcxproj b/infrastructure/msvc/2010/bbackupd.vcxproj deleted file mode 100644 index a98bd852..00000000 --- a/infrastructure/msvc/2010/bbackupd.vcxproj +++ /dev/null @@ -1,139 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{22D325FB-9131-4BD6-B390-968F0491D687}</ProjectGuid>
- <RootNamespace>bbackupd</RootNamespace>
- <Keyword>Win32Proj</Keyword>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- <UseOfMfc>false</UseOfMfc>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup>
- <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</IntDir>
- <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <Optimization>Disabled</Optimization>
- <AdditionalIncludeDirectories>$(SolutionDir)..\..\..\lib\backupstore;$(SolutionDir)..\..\..\lib\backupclient;$(SolutionDir)..\..\..\lib\common;$(SolutionDir)..\..\..\lib\compress;$(SolutionDir)..\..\..\lib\crypto;$(SolutionDir)..\..\..\lib\server;$(SolutionDir)..\..\..\lib\win32;$(SolutionDir)..\..\..\qdbm;$(SolutionDir)..\..\..\..\openssl\include;$(SolutionDir)..\..\..\..\zlib\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>ENABLE_VSS;WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;QDBM_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <MinimalRebuild>true</MinimalRebuild>
- <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
- </ClCompile>
- <Link>
- <AdditionalDependencies>VssApi.lib;$(OutDir)\libbackupclient.lib;$(OutDir)\libbackupstore.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <ProgramDatabaseFile>$(OutDir)bbackupd.pdb</ProgramDatabaseFile>
- <SubSystem>Console</SubSystem>
- <TargetMachine>MachineX86</TargetMachine>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
- <AdditionalIncludeDirectories>$(ProjectDir)..\..\..\lib\backupclient;$(ProjectDir)..\..\..\lib\server;$(ProjectDir)..\..\..\lib\crypto;$(SolutionDir)..\..\..\..\openssl\inc32;$(ProjectDir)..\..\..\lib\compress;$(ProjectDir)..\..\..\..\zlib\include;$(ProjectDir)..\..\..\lib\win32;$(ProjectDir)..\..\..\lib\common\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
- <BufferSecurityCheck>false</BufferSecurityCheck>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
- </ClCompile>
- <Link>
- <AdditionalDependencies>Ws2_32.lib;$(ProjectDir)..\..\..\..\zlib\lib\zdll.lib;$(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib;$(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib;$(ProjectDir)..\..\..\Release\common.lib;$(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib;$(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <OutputFile>$(OutDir)bbackupd.exe</OutputFile>
- <IgnoreSpecificDefaultLibraries>%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <SubSystem>Console</SubSystem>
- <OptimizeReferences>true</OptimizeReferences>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <TargetMachine>MachineX86</TargetMachine>
- </Link>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\bin\bbackupd\autogen_ClientException.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupd\BackupClientContext.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupd\BackupClientDeleteList.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupd\BackupClientDirectoryRecord.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupd\BackupClientInodeToIDMap.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupd\BackupDaemon.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupd\bbackupd.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupd\Win32BackupService.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupd\Win32ServiceFunctions.cpp" />
- </ItemGroup>
- <ItemGroup>
- <ClInclude Include="..\..\..\bin\bbackupd\autogen_ClientException.h" />
- <ClInclude Include="..\..\..\bin\bbackupd\BackupClientContext.h" />
- <ClInclude Include="..\..\..\bin\bbackupd\BackupClientDeleteList.h" />
- <ClInclude Include="..\..\..\bin\bbackupd\BackupClientDirectoryRecord.h" />
- <ClInclude Include="..\..\..\bin\bbackupd\BackupClientInodeToIDMap.h" />
- <ClInclude Include="..\..\..\bin\bbackupd\BackupDaemon.h" />
- <ClInclude Include="..\..\..\bin\bbackupd\Win32BackupService.h" />
- <ClInclude Include="..\..\..\bin\bbackupd\Win32ServiceFunctions.h" />
- </ItemGroup>
- <ItemGroup>
- <ResourceCompile Include="..\..\..\lib\win32\messages.rc" />
- </ItemGroup>
- <ItemGroup>
- <ProjectReference Include="common.vcxproj">
- <Project>{a089cee6-ebf0-4232-a0c0-74850a8127a6}</Project>
- <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
- <Private>true</Private>
- <CopyLocalSatelliteAssemblies>false</CopyLocalSatelliteAssemblies>
- <LinkLibraryDependencies>true</LinkLibraryDependencies>
- <UseLibraryDependencyInputs>false</UseLibraryDependencyInputs>
- </ProjectReference>
- <ProjectReference Include="qdbm.vcxproj">
- <Project>{72af22a7-b339-4fdf-b6ae-ca6522d4bb8d}</Project>
- <Private>true</Private>
- <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
- <CopyLocalSatelliteAssemblies>false</CopyLocalSatelliteAssemblies>
- <LinkLibraryDependencies>true</LinkLibraryDependencies>
- <UseLibraryDependencyInputs>false</UseLibraryDependencyInputs>
- </ProjectReference>
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/2010/bbstoreaccounts.vcxproj b/infrastructure/msvc/2010/bbstoreaccounts.vcxproj deleted file mode 100644 index abaf06db..00000000 --- a/infrastructure/msvc/2010/bbstoreaccounts.vcxproj +++ /dev/null @@ -1,84 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{C8A1509C-F91B-4140-BD51-B87FF24FB95F}</ProjectGuid>
- <RootNamespace>bbstoreaccounts</RootNamespace>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>true</UseDebugLibraries>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>false</UseDebugLibraries>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <OutDir>$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <IntDir>$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>Disabled</Optimization>
- <AdditionalIncludeDirectories>$(SolutionDir)..\..\..\lib\backupclient;$(SolutionDir)..\..\..\lib\backupstore;$(SolutionDir)..\..\..\lib\raidfile;$(SolutionDir)..\..\..\lib\common;$(SolutionDir)..\..\..\lib\compress;$(SolutionDir)..\..\..\lib\crypto;$(SolutionDir)..\..\..\lib\server;$(SolutionDir)..\..\..\lib\win32;$(SolutionDir)..\..\..\qdbm;$(SolutionDir)..\..\..\..\openssl\include;$(SolutionDir)..\..\..\..\zlib\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;QDBM_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- </Link>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\bin\bbstoreaccounts\bbstoreaccounts.cpp" />
- </ItemGroup>
- <ItemGroup>
- <ProjectReference Include="common.vcxproj">
- <Project>{a089cee6-ebf0-4232-a0c0-74850a8127a6}</Project>
- </ProjectReference>
- <ProjectReference Include="libbackupstore.vcxproj">
- <Project>{97d89aef-2be4-4e34-8703-03ba67bf4494}</Project>
- </ProjectReference>
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/2010/bbstored.vcxproj b/infrastructure/msvc/2010/bbstored.vcxproj deleted file mode 100644 index 7d09f75d..00000000 --- a/infrastructure/msvc/2010/bbstored.vcxproj +++ /dev/null @@ -1,93 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{D8404314-73DD-4270-8205-BE677F8FDAC7}</ProjectGuid>
- <RootNamespace>bbstored</RootNamespace>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>true</UseDebugLibraries>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>false</UseDebugLibraries>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <OutDir>$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <IntDir>$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;QDBM_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- <AdditionalIncludeDirectories>$(SolutionDir)..\..\..\lib\backupclient;$(SolutionDir)..\..\..\lib\backupstore;$(SolutionDir)..\..\..\lib\raidfile;$(SolutionDir)..\..\..\lib\common;$(SolutionDir)..\..\..\lib\compress;$(SolutionDir)..\..\..\lib\crypto;$(SolutionDir)..\..\..\lib\server;$(SolutionDir)..\..\..\lib\win32;$(SolutionDir)..\..\..\qdbm;$(SolutionDir)..\..\..\..\openssl\include;$(SolutionDir)..\..\..\..\zlib\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- </Link>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ProjectReference Include="common.vcxproj">
- <Project>{a089cee6-ebf0-4232-a0c0-74850a8127a6}</Project>
- </ProjectReference>
- <ProjectReference Include="libbackupstore.vcxproj">
- <Project>{97d89aef-2be4-4e34-8703-03ba67bf4494}</Project>
- </ProjectReference>
- </ItemGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\bin\bbstored\BackupStoreDaemon.cpp" />
- <ClCompile Include="..\..\..\bin\bbstored\bbstored.cpp" />
- <ClCompile Include="..\..\..\bin\bbstored\BBStoreDHousekeeping.cpp" />
- </ItemGroup>
- <ItemGroup>
- <ClInclude Include="..\..\..\bin\bbstored\BackupStoreDaemon.h" />
- </ItemGroup>
- <ItemGroup>
- <None Include="..\..\..\bin\bbstored\bbstored-certs.in" />
- <None Include="..\..\..\bin\bbstored\bbstored-config.in" />
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/2010/boxbackup.sln b/infrastructure/msvc/2010/boxbackup.sln deleted file mode 100644 index 384ff58c..00000000 --- a/infrastructure/msvc/2010/boxbackup.sln +++ /dev/null @@ -1,89 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 11.00
-# Visual C++ Express 2010
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbackupquery", "boxquery.vcxproj", "{FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}"
- ProjectSection(ProjectDependencies) = postProject
- {32604097-C934-4711-B1AD-206336640E70} = {32604097-C934-4711-B1AD-206336640E70}
- {97D89AEF-2BE4-4E34-8703-03BA67BF4494} = {97D89AEF-2BE4-4E34-8703-03BA67BF4494}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "common", "common.vcxproj", "{A089CEE6-EBF0-4232-A0C0-74850A8127A6}"
- ProjectSection(ProjectDependencies) = postProject
- {72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D} = {72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbackupd", "bbackupd.vcxproj", "{22D325FB-9131-4BD6-B390-968F0491D687}"
- ProjectSection(ProjectDependencies) = postProject
- {32604097-C934-4711-B1AD-206336640E70} = {32604097-C934-4711-B1AD-206336640E70}
- {72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D} = {72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D}
- {97D89AEF-2BE4-4E34-8703-03BA67BF4494} = {97D89AEF-2BE4-4E34-8703-03BA67BF4494}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "win32test", "win32test.vcxproj", "{28C29E72-76A2-4D0C-B35B-12D446733D2E}"
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbackupctl", "bbackupctl.vcxproj", "{9FD51412-E945-4457-A17A-CA3C505CF431}"
- ProjectSection(ProjectDependencies) = postProject
- {32604097-C934-4711-B1AD-206336640E70} = {32604097-C934-4711-B1AD-206336640E70}
- {97D89AEF-2BE4-4E34-8703-03BA67BF4494} = {97D89AEF-2BE4-4E34-8703-03BA67BF4494}
- EndProjectSection
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "qdbm", "qdbm.vcxproj", "{72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D}"
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbstored", "bbstored.vcxproj", "{D8404314-73DD-4270-8205-BE677F8FDAC7}"
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbackupstore", "libbackupstore.vcxproj", "{97D89AEF-2BE4-4E34-8703-03BA67BF4494}"
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bbstoreaccounts", "bbstoreaccounts.vcxproj", "{C8A1509C-F91B-4140-BD51-B87FF24FB95F}"
-EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbackupclient", "libbackupclient.vcxproj", "{32604097-C934-4711-B1AD-206336640E70}"
-EndProject
-Global
- GlobalSection(SolutionConfigurationPlatforms) = preSolution
- Debug|Win32 = Debug|Win32
- Release|Win32 = Release|Win32
- EndGlobalSection
- GlobalSection(ProjectConfigurationPlatforms) = postSolution
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Debug|Win32.ActiveCfg = Debug|Win32
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Debug|Win32.Build.0 = Debug|Win32
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Release|Win32.ActiveCfg = Release|Win32
- {FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}.Release|Win32.Build.0 = Release|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Debug|Win32.ActiveCfg = Debug|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Debug|Win32.Build.0 = Debug|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Release|Win32.ActiveCfg = Release|Win32
- {A089CEE6-EBF0-4232-A0C0-74850A8127A6}.Release|Win32.Build.0 = Release|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Debug|Win32.ActiveCfg = Debug|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Debug|Win32.Build.0 = Debug|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Release|Win32.ActiveCfg = Release|Win32
- {22D325FB-9131-4BD6-B390-968F0491D687}.Release|Win32.Build.0 = Release|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Debug|Win32.ActiveCfg = Debug|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Debug|Win32.Build.0 = Debug|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Release|Win32.ActiveCfg = Release|Win32
- {28C29E72-76A2-4D0C-B35B-12D446733D2E}.Release|Win32.Build.0 = Release|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Debug|Win32.ActiveCfg = Debug|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Debug|Win32.Build.0 = Debug|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Release|Win32.ActiveCfg = Release|Win32
- {9FD51412-E945-4457-A17A-CA3C505CF431}.Release|Win32.Build.0 = Release|Win32
- {72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D}.Debug|Win32.ActiveCfg = Debug|Win32
- {72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D}.Debug|Win32.Build.0 = Debug|Win32
- {72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D}.Release|Win32.ActiveCfg = Release|Win32
- {72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D}.Release|Win32.Build.0 = Release|Win32
- {D8404314-73DD-4270-8205-BE677F8FDAC7}.Debug|Win32.ActiveCfg = Debug|Win32
- {D8404314-73DD-4270-8205-BE677F8FDAC7}.Debug|Win32.Build.0 = Debug|Win32
- {D8404314-73DD-4270-8205-BE677F8FDAC7}.Release|Win32.ActiveCfg = Release|Win32
- {D8404314-73DD-4270-8205-BE677F8FDAC7}.Release|Win32.Build.0 = Release|Win32
- {97D89AEF-2BE4-4E34-8703-03BA67BF4494}.Debug|Win32.ActiveCfg = Debug|Win32
- {97D89AEF-2BE4-4E34-8703-03BA67BF4494}.Debug|Win32.Build.0 = Debug|Win32
- {97D89AEF-2BE4-4E34-8703-03BA67BF4494}.Release|Win32.ActiveCfg = Release|Win32
- {97D89AEF-2BE4-4E34-8703-03BA67BF4494}.Release|Win32.Build.0 = Release|Win32
- {C8A1509C-F91B-4140-BD51-B87FF24FB95F}.Debug|Win32.ActiveCfg = Debug|Win32
- {C8A1509C-F91B-4140-BD51-B87FF24FB95F}.Debug|Win32.Build.0 = Debug|Win32
- {C8A1509C-F91B-4140-BD51-B87FF24FB95F}.Release|Win32.ActiveCfg = Release|Win32
- {C8A1509C-F91B-4140-BD51-B87FF24FB95F}.Release|Win32.Build.0 = Release|Win32
- {32604097-C934-4711-B1AD-206336640E70}.Debug|Win32.ActiveCfg = Debug|Win32
- {32604097-C934-4711-B1AD-206336640E70}.Debug|Win32.Build.0 = Debug|Win32
- {32604097-C934-4711-B1AD-206336640E70}.Release|Win32.ActiveCfg = Release|Win32
- {32604097-C934-4711-B1AD-206336640E70}.Release|Win32.Build.0 = Release|Win32
- EndGlobalSection
- GlobalSection(SolutionProperties) = preSolution
- HideSolutionNode = FALSE
- EndGlobalSection
-EndGlobal
diff --git a/infrastructure/msvc/2010/boxquery.vcxproj b/infrastructure/msvc/2010/boxquery.vcxproj deleted file mode 100644 index c419c4b7..00000000 --- a/infrastructure/msvc/2010/boxquery.vcxproj +++ /dev/null @@ -1,123 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{FE9EC666-4B3A-4370-B3D4-DEBD4A21F36E}</ProjectGuid>
- <RootNamespace>boxquery</RootNamespace>
- <Keyword>Win32Proj</Keyword>
- <ProjectName>bbackupquery</ProjectName>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup>
- <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</IntDir>
- <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <Optimization>Disabled</Optimization>
- <AdditionalIncludeDirectories>$(ProjectDir)..\..\..\lib\backupstore;$(ProjectDir)..\..\..\lib\backupclient;$(ProjectDir)..\..\..\lib\common;$(ProjectDir)..\..\..\lib\compress;$(ProjectDir)..\..\..\lib\crypto;$(ProjectDir)..\..\..\lib\server;$(ProjectDir)..\..\..\lib\win32;$(ProjectDir)..\..\..\..\pcre;$(ProjectDir)..\..\..\..\openssl\include;$(ProjectDir)..\..\..\..\zlib\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;QDBM_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <MinimalRebuild>true</MinimalRebuild>
- <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <ProgramDatabaseFile>$(OutDir)boxquery.pdb</ProgramDatabaseFile>
- <SubSystem>Console</SubSystem>
- <TargetMachine>MachineX86</TargetMachine>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
- <AdditionalIncludeDirectories>$(ProjectDir)..\..\..\lib\backupclient;$(ProjectDir)..\..\..\lib\server;$(ProjectDir)..\..\..\lib\crypto;$(ProjectDir)..\..\..\..\openssl\include;$(ProjectDir)..\..\..\lib\compress;$(ProjectDir)..\..\..\..\zlib\include;$(ProjectDir)..\..\..\lib\win32;$(ProjectDir)..\..\..\lib\common\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
- <BufferSecurityCheck>false</BufferSecurityCheck>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
- </ClCompile>
- <Link>
- <AdditionalDependencies>Ws2_32.lib;$(ProjectDir)..\..\..\..\zlib\lib\zdll.lib;$(ProjectDir)..\..\..\Release\common.lib;$(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib;$(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib;$(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib;$(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <OutputFile>$(OutDir)bbackupquery.exe</OutputFile>
- <IgnoreSpecificDefaultLibraries>%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>
- <GenerateDebugInformation>false</GenerateDebugInformation>
- <SubSystem>Console</SubSystem>
- <OptimizeReferences>true</OptimizeReferences>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <TargetMachine>MachineX86</TargetMachine>
- </Link>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\bin\bbackupquery\autogen_Documentation.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupquery\BackupQueries.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupquery\bbackupquery.cpp" />
- <ClCompile Include="..\..\..\bin\bbackupquery\CommandCompletion.cpp" />
- </ItemGroup>
- <ItemGroup>
- <ClInclude Include="..\..\..\bin\bbackupquery\BackupQueries.h" />
- <ClInclude Include="..\..\..\bin\bbackupquery\BoxBackupCompareParams.h" />
- </ItemGroup>
- <ItemGroup>
- <ResourceCompile Include="..\..\..\lib\win32\messages.rc" />
- </ItemGroup>
- <ItemGroup>
- <None Include="..\..\..\ReadMe.txt" />
- </ItemGroup>
- <ItemGroup>
- <ProjectReference Include="common.vcxproj">
- <Project>{a089cee6-ebf0-4232-a0c0-74850a8127a6}</Project>
- <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
- </ProjectReference>
- <ProjectReference Include="libbackupclient.vcxproj">
- <Project>{32604097-c934-4711-b1ad-206336640e70}</Project>
- </ProjectReference>
- <ProjectReference Include="libbackupstore.vcxproj">
- <Project>{97d89aef-2be4-4e34-8703-03ba67bf4494}</Project>
- </ProjectReference>
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/2010/common.vcxproj b/infrastructure/msvc/2010/common.vcxproj deleted file mode 100644 index a644891a..00000000 --- a/infrastructure/msvc/2010/common.vcxproj +++ /dev/null @@ -1,251 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{A089CEE6-EBF0-4232-A0C0-74850A8127A6}</ProjectGuid>
- <RootNamespace>common</RootNamespace>
- <Keyword>Win32Proj</Keyword>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>StaticLibrary</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>StaticLibrary</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup>
- <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</IntDir>
- <IncludePath Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(IncludePath)</IncludePath>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <PreBuildEvent />
- <ClCompile>
- <Optimization>Disabled</Optimization>
- <AdditionalIncludeDirectories>$(ProjectDir)..\..\..\qdbm;$(ProjectDir)..\..\..\lib\common;$(ProjectDir)..\..\..\lib\compress;$(ProjectDir)..\..\..\lib\crypto;$(ProjectDir)..\..\..\lib\server;$(ProjectDir)..\..\..\lib\win32;$(ProjectDir)..\..\..\..\openssl\include;$(ProjectDir)..\..\..\..\zlib\include;$(ProjectDir)..\..\..\..\pcre</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;QDBM_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <MinimalRebuild>true</MinimalRebuild>
- <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <TreatWarningAsError>false</TreatWarningAsError>
- <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
- </ClCompile>
- <Lib />
- <CustomBuildStep />
- <CustomBuildStep />
- <CustomBuildStep />
- <Lib>
- <AdditionalDependencies>Ws2_32.lib;$(ProjectDir)..\..\..\..\pcre\build\vc2010\debug\pcre.lib;$(ProjectDir)..\..\..\..\pcre\build\vc2010\debug\pcreposix.lib;$(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib;$(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib;$(ProjectDir)..\..\..\..\zlib\lib\zdll.lib;%(AdditionalDependencies)</AdditionalDependencies>
- </Lib>
- <PreBuildEvent>
- <Command>perl $(ProjectDir)..\getversion.pl</Command>
- </PreBuildEvent>
- <PreBuildEvent>
- <Message>Determining Version Number</Message>
- </PreBuildEvent>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
- <AdditionalIncludeDirectories>$(ProjectDir)..\..\..\lib\common\;$(ProjectDir)..\..\..\lib\compress;$(ProjectDir)..\..\..\lib\crypto;$(ProjectDir)..\..\..\lib\server;$(ProjectDir)..\..\..\lib\win32;$(ProjectDir)..\..\..\..\openssl\inc32;$(ProjectDir)..\..\..\..\zlib\include;$(ProjectDir)..\..\..\..\pcre\pcre-6.7\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;BOX_RELEASE_BUILD;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
- <BufferSecurityCheck>false</BufferSecurityCheck>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
- </ClCompile>
- <Lib>
- <OutputFile>$(OutDir)common.lib</OutputFile>
- </Lib>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\lib\common\BufferedStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\BufferedWriteStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\FileModificationTime.cpp" />
- <ClCompile Include="..\..\..\lib\common\GetLine.cpp" />
- <ClCompile Include="..\..\..\lib\common\InvisibleTempFileStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\NamedLock.cpp" />
- <ClCompile Include="..\..\..\lib\common\RateLimitingStream.cpp" />
- <ClCompile Include="..\..\..\lib\compress\autogen_CompressException.cpp" />
- <ClCompile Include="..\..\..\lib\compress\CompressStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\autogen_CommonException.cpp" />
- <ClCompile Include="..\..\..\lib\common\autogen_ConversionException.cpp" />
- <ClCompile Include="..\..\..\lib\common\BoxException.cpp" />
- <ClCompile Include="..\..\..\lib\common\BoxTime.cpp" />
- <ClCompile Include="..\..\..\lib\common\BoxTimeToText.cpp" />
- <ClCompile Include="..\..\..\lib\common\CollectInBufferStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\Configuration.cpp" />
- <ClCompile Include="..\..\..\lib\common\ConversionString.cpp" />
- <ClCompile Include="..\..\..\lib\common\DebugAssertFailed.cpp" />
- <ClCompile Include="..\..\..\lib\common\DebugMemLeakFinder.cpp" />
- <ClCompile Include="..\..\..\lib\common\DebugPrintf.cpp" />
- <ClCompile Include="..\..\..\lib\common\EventWatchFilesystemObject.cpp" />
- <ClCompile Include="..\..\..\lib\common\ExcludeList.cpp" />
- <ClCompile Include="..\..\..\lib\common\FdGetLine.cpp" />
- <ClCompile Include="..\..\..\lib\common\FileStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\IOStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\IOStreamGetLine.cpp" />
- <ClCompile Include="..\..\..\lib\common\Logging.cpp" />
- <ClCompile Include="..\..\..\lib\common\MemBlockStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\PartialReadStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\PathUtils.cpp" />
- <ClCompile Include="..\..\..\lib\common\ReadGatherStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\ReadLoggingStream.cpp" />
- <ClCompile Include="..\..\..\lib\common\StreamableMemBlock.cpp" />
- <ClCompile Include="..\..\..\lib\common\Timer.cpp" />
- <ClCompile Include="..\..\..\lib\common\UnixUser.cpp" />
- <ClCompile Include="..\..\..\lib\common\Utils.cpp" />
- <ClCompile Include="..\..\..\lib\common\WaitForEvent.cpp" />
- <ClCompile Include="..\..\..\lib\crypto\autogen_CipherException.cpp" />
- <ClCompile Include="..\..\..\lib\crypto\CipherAES.cpp" />
- <ClCompile Include="..\..\..\lib\crypto\CipherBlowfish.cpp" />
- <ClCompile Include="..\..\..\lib\crypto\CipherContext.cpp" />
- <ClCompile Include="..\..\..\lib\crypto\CipherDescription.cpp" />
- <ClCompile Include="..\..\..\lib\crypto\CryptoUtils.cpp" />
- <ClCompile Include="..\..\..\lib\crypto\MD5Digest.cpp" />
- <ClCompile Include="..\..\..\lib\crypto\Random.cpp" />
- <ClCompile Include="..\..\..\lib\crypto\RollingChecksum.cpp" />
- <ClCompile Include="..\..\..\lib\server\Message.cpp" />
- <ClCompile Include="..\..\..\lib\server\TcpNice.cpp" />
- <ClCompile Include="..\..\..\lib\win32\emu.cpp" />
- <ClCompile Include="..\..\..\lib\win32\getopt_long.cpp" />
- <ClCompile Include="..\..\..\lib\server\autogen_ConnectionException.cpp" />
- <ClCompile Include="..\..\..\lib\server\autogen_ServerException.cpp" />
- <ClCompile Include="..\..\..\lib\server\Daemon.cpp" />
- <ClCompile Include="..\..\..\lib\server\LocalProcessStream.cpp" />
- <ClCompile Include="..\..\..\lib\server\Protocol.cpp" />
- <ClCompile Include="..\..\..\lib\server\ProtocolUncertainStream.cpp" />
- <ClCompile Include="..\..\..\lib\server\Socket.cpp" />
- <ClCompile Include="..\..\..\lib\server\SocketStream.cpp" />
- <ClCompile Include="..\..\..\lib\server\SocketStreamTLS.cpp" />
- <ClCompile Include="..\..\..\lib\server\SSLLib.cpp" />
- <ClCompile Include="..\..\..\lib\server\TLSContext.cpp" />
- <ClCompile Include="..\..\..\lib\server\WinNamedPipeStream.cpp" />
- </ItemGroup>
- <ItemGroup>
- <ClInclude Include="..\..\..\lib\common\BufferedStream.h" />
- <ClInclude Include="..\..\..\lib\common\BufferedWriteStream.h" />
- <ClInclude Include="..\..\..\lib\common\GetLine.h" />
- <ClInclude Include="..\..\..\lib\common\InvisibleTempFileStream.h" />
- <ClInclude Include="..\..\..\lib\common\RateLimitingStream.h" />
- <ClInclude Include="..\..\..\lib\compress\autogen_CompressException.h" />
- <ClInclude Include="..\..\..\lib\compress\Compress.h" />
- <ClInclude Include="..\..\..\lib\compress\CompressException.h" />
- <ClInclude Include="..\..\..\lib\compress\CompressStream.h" />
- <ClInclude Include="..\..\..\lib\common\autogen_CommonException.h" />
- <ClInclude Include="..\..\..\lib\common\autogen_ConversionException.h" />
- <ClInclude Include="..\..\..\lib\common\BannerText.h" />
- <ClInclude Include="..\..\..\lib\common\BeginStructPackForWire.h" />
- <ClInclude Include="..\..\..\lib\common\Box.h" />
- <ClInclude Include="..\..\..\lib\common\BoxConfig-MSVC.h" />
- <ClInclude Include="..\..\..\lib\common\BoxException.h" />
- <ClInclude Include="..\..\..\lib\common\BoxPlatform.h" />
- <ClInclude Include="..\..\..\lib\common\BoxPortsAndFiles.h" />
- <ClInclude Include="..\..\..\lib\common\BoxTime.h" />
- <ClInclude Include="..\..\..\lib\common\BoxTimeToText.h" />
- <ClInclude Include="..\..\..\lib\common\BoxTimeToUnix.h" />
- <ClInclude Include="..\..\..\lib\common\BoxVersion.h" />
- <ClInclude Include="..\..\..\lib\common\CollectInBufferStream.h" />
- <ClInclude Include="..\..\..\lib\common\CommonException.h" />
- <ClInclude Include="..\..\..\lib\common\Configuration.h" />
- <ClInclude Include="..\..\..\lib\common\Conversion.h" />
- <ClInclude Include="..\..\..\lib\common\EndStructPackForWire.h" />
- <ClInclude Include="..\..\..\lib\common\EventWatchFilesystemObject.h" />
- <ClInclude Include="..\..\..\lib\common\ExcludeList.h" />
- <ClInclude Include="..\..\..\lib\common\FdGetLine.h" />
- <ClInclude Include="..\..\..\lib\common\FileModificationTime.h" />
- <ClInclude Include="..\..\..\lib\common\FileStream.h" />
- <ClInclude Include="..\..\..\lib\common\Guards.h" />
- <ClInclude Include="..\..\..\lib\common\IOStream.h" />
- <ClInclude Include="..\..\..\lib\common\IOStreamGetLine.h" />
- <ClInclude Include="..\..\..\lib\crypto\CryptoUtils.h" />
- <ClInclude Include="..\..\..\lib\server\LocalProcessStream.h" />
- <ClInclude Include="..\..\..\lib\common\Logging.h" />
- <ClInclude Include="..\..\..\lib\common\MainHelper.h" />
- <ClInclude Include="..\..\..\lib\common\MemBlockStream.h" />
- <ClInclude Include="..\..\..\lib\common\MemLeakFinder.h" />
- <ClInclude Include="..\..\..\lib\common\MemLeakFindOff.h" />
- <ClInclude Include="..\..\..\lib\common\MemLeakFindOn.h" />
- <ClInclude Include="..\..\..\lib\common\NamedLock.h" />
- <ClInclude Include="..\..\..\lib\common\PartialReadStream.h" />
- <ClInclude Include="..\..\..\lib\common\PathUtils.h" />
- <ClInclude Include="..\..\..\lib\common\ReadGatherStream.h" />
- <ClInclude Include="..\..\..\lib\common\ReadLoggingStream.h" />
- <ClInclude Include="..\..\..\lib\common\StreamableMemBlock.h" />
- <ClInclude Include="..\..\..\lib\common\TemporaryDirectory.h" />
- <ClInclude Include="..\..\..\lib\common\Test.h" />
- <ClInclude Include="..\..\..\lib\common\Timer.h" />
- <ClInclude Include="..\..\..\lib\common\UnixUser.h" />
- <ClInclude Include="..\..\..\lib\common\Utils.h" />
- <ClInclude Include="..\..\..\lib\common\WaitForEvent.h" />
- <ClInclude Include="..\..\..\lib\crypto\autogen_CipherException.h" />
- <ClInclude Include="..\..\..\lib\crypto\CipherAES.h" />
- <ClInclude Include="..\..\..\lib\crypto\CipherBlowfish.h" />
- <ClInclude Include="..\..\..\lib\crypto\CipherContext.h" />
- <ClInclude Include="..\..\..\lib\crypto\CipherDescription.h" />
- <ClInclude Include="..\..\..\lib\crypto\CipherException.h" />
- <ClInclude Include="..\..\..\lib\crypto\MD5Digest.h" />
- <ClInclude Include="..\..\..\lib\crypto\Random.h" />
- <ClInclude Include="..\..\..\lib\crypto\RollingChecksum.h" />
- <ClInclude Include="..\..\..\lib\server\Message.h" />
- <ClInclude Include="..\..\..\lib\server\TcpNice.h" />
- <ClInclude Include="..\..\..\lib\win32\emu.h" />
- <ClInclude Include="..\..\..\lib\win32\getopt.h" />
- <ClInclude Include="..\..\..\lib\server\autogen_ConnectionException.h" />
- <ClInclude Include="..\..\..\lib\server\autogen_ServerException.h" />
- <ClInclude Include="..\..\..\lib\server\Daemon.h" />
- <ClInclude Include="..\..\..\lib\server\Protocol.h" />
- <ClInclude Include="..\..\..\lib\server\ProtocolUncertainStream.h" />
- <ClInclude Include="..\..\..\lib\server\ProtocolWire.h" />
- <ClInclude Include="..\..\..\lib\server\ServerException.h" />
- <ClInclude Include="..\..\..\lib\server\ServerStream.h" />
- <ClInclude Include="..\..\..\lib\server\ServerTLS.h" />
- <ClInclude Include="..\..\..\lib\server\Socket.h" />
- <ClInclude Include="..\..\..\lib\server\SocketListen.h" />
- <ClInclude Include="..\..\..\lib\server\SocketStream.h" />
- <ClInclude Include="..\..\..\lib\server\SocketStreamTLS.h" />
- <ClInclude Include="..\..\..\lib\server\SSLLib.h" />
- <ClInclude Include="..\..\..\lib\server\TLSContext.h" />
- <ClInclude Include="..\..\..\lib\server\WinNamedPipeStream.h" />
- </ItemGroup>
- <ItemGroup>
- <ProjectReference Include="qdbm.vcxproj">
- <Project>{72af22a7-b339-4fdf-b6ae-ca6522d4bb8d}</Project>
- </ProjectReference>
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/2010/libbackupclient.vcxproj b/infrastructure/msvc/2010/libbackupclient.vcxproj deleted file mode 100644 index a38b3cf5..00000000 --- a/infrastructure/msvc/2010/libbackupclient.vcxproj +++ /dev/null @@ -1,94 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\lib\backupclient\BackupClientCryptoKeys.cpp" />
- <ClCompile Include="..\..\..\lib\backupclient\BackupClientMakeExcludeList.cpp" />
- <ClCompile Include="..\..\..\lib\backupclient\BackupClientRestore.cpp" />
- <ClCompile Include="..\..\..\lib\backupclient\BackupDaemonConfigVerify.cpp" />
- <ClCompile Include="..\..\..\lib\backupclient\BackupStoreObjectDump.cpp" />
- </ItemGroup>
- <ItemGroup>
- <ClInclude Include="..\..\..\lib\backupclient\BackupClientCryptoKeys.h" />
- <ClInclude Include="..\..\..\lib\backupclient\BackupClientMakeExcludeList.h" />
- <ClInclude Include="..\..\..\lib\backupclient\BackupClientRestore.h" />
- <ClInclude Include="..\..\..\lib\backupclient\BackupDaemonConfigVerify.h" />
- </ItemGroup>
- <ItemGroup>
- <ProjectReference Include="libbackupstore.vcxproj">
- <Project>{97d89aef-2be4-4e34-8703-03ba67bf4494}</Project>
- </ProjectReference>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{32604097-C934-4711-B1AD-206336640E70}</ProjectGuid>
- <RootNamespace>libbackupstore</RootNamespace>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>StaticLibrary</ConfigurationType>
- <UseDebugLibraries>true</UseDebugLibraries>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>false</UseDebugLibraries>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <OutDir>$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <TargetExt>.lib</TargetExt>
- <IntDir>$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>Disabled</Optimization>
- <AdditionalIncludeDirectories>$(SolutionDir)..\..\..\lib\backupclient;$(SolutionDir)..\..\..\lib\backupstore;$(SolutionDir)..\..\..\lib\raidfile;$(SolutionDir)..\..\..\lib\common;$(SolutionDir)..\..\..\lib\compress;$(SolutionDir)..\..\..\lib\crypto;$(SolutionDir)..\..\..\lib\server;$(SolutionDir)..\..\..\lib\win32;$(SolutionDir)..\..\..\qdbm;$(SolutionDir)..\..\..\..\openssl\include;$(SolutionDir)..\..\..\..\zlib\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;QDBM_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <AdditionalDependencies>Ws2_32.lib;Advapi32.lib;User32.lib;$(ProjectDir)..\..\..\..\zlib\lib\zdll.lib;$(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib;$(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib;$(ProjectDir)..\..\..\$(Configuration)\common.lib;%(AdditionalDependencies)</AdditionalDependencies>
- </Link>
- <Lib />
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- </Link>
- </ItemDefinitionGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/2010/libbackupstore.vcxproj b/infrastructure/msvc/2010/libbackupstore.vcxproj deleted file mode 100644 index 7f83d04a..00000000 --- a/infrastructure/msvc/2010/libbackupstore.vcxproj +++ /dev/null @@ -1,151 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{97D89AEF-2BE4-4E34-8703-03BA67BF4494}</ProjectGuid>
- <RootNamespace>libbackupstore</RootNamespace>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>StaticLibrary</ConfigurationType>
- <UseDebugLibraries>true</UseDebugLibraries>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>false</UseDebugLibraries>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <OutDir>$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <TargetExt>.lib</TargetExt>
- <IntDir>$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>Disabled</Optimization>
- <AdditionalIncludeDirectories>$(SolutionDir)..\..\..\lib\backupclient;$(SolutionDir)..\..\..\lib\backupstore;$(SolutionDir)..\..\..\lib\raidfile;$(SolutionDir)..\..\..\lib\common;$(SolutionDir)..\..\..\lib\compress;$(SolutionDir)..\..\..\lib\crypto;$(SolutionDir)..\..\..\lib\server;$(SolutionDir)..\..\..\lib\win32;$(SolutionDir)..\..\..\qdbm;$(SolutionDir)..\..\..\..\openssl\include;$(SolutionDir)..\..\..\..\zlib\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;QDBM_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <AdditionalDependencies>Ws2_32.lib;Advapi32.lib;User32.lib;$(ProjectDir)..\..\..\..\zlib\lib\zdll.lib;$(ProjectDir)..\..\..\..\openssl\lib\libeay32.lib;$(ProjectDir)..\..\..\..\openssl\lib\ssleay32.lib;$(ProjectDir)..\..\..\$(Configuration)\common.lib;%(AdditionalDependencies)</AdditionalDependencies>
- </Link>
- <Lib />
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- </Link>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\lib\backupstore\autogen_BackupProtocol.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\autogen_BackupStoreException.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupClientFileAttributes.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupCommands.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreAccountDatabase.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreAccounts.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreCheck.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreCheck2.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreCheckData.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreConfigVerify.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreContext.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreDirectory.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFile.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFileCmbDiff.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFileCmbIdx.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFileCombine.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFileCryptVar.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFileDiff.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFileEncodeStream.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFilename.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFilenameClear.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreFileRevDiff.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreInfo.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\BackupStoreRefCountDatabase.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\HousekeepStoreAccount.cpp" />
- <ClCompile Include="..\..\..\lib\backupstore\StoreStructure.cpp" />
- <ClCompile Include="..\..\..\lib\raidfile\autogen_RaidFileException.cpp" />
- <ClCompile Include="..\..\..\lib\raidfile\RaidFileController.cpp" />
- <ClCompile Include="..\..\..\lib\raidfile\RaidFileRead.cpp" />
- <ClCompile Include="..\..\..\lib\raidfile\RaidFileUtil.cpp" />
- <ClCompile Include="..\..\..\lib\raidfile\RaidFileWrite.cpp" />
- </ItemGroup>
- <ItemGroup>
- <ClInclude Include="..\..\..\lib\backupstore\autogen_BackupProtocol.h" />
- <ClInclude Include="..\..\..\lib\backupstore\autogen_BackupStoreException.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupClientFileAttributes.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupConstants.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreAccountDatabase.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreAccounts.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreCheck.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreConfigVerify.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreConstants.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreContext.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreDirectory.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreException.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreFile.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreFileCryptVar.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreFileEncodeStream.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreFilename.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreFilenameClear.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreFileWire.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreInfo.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreObjectMagic.h" />
- <ClInclude Include="..\..\..\lib\backupstore\BackupStoreRefCountDatabase.h" />
- <ClInclude Include="..\..\..\lib\backupstore\HousekeepStoreAccount.h" />
- <ClInclude Include="..\..\..\lib\backupstore\RunStatusProvider.h" />
- <ClInclude Include="..\..\..\lib\backupstore\StoreStructure.h" />
- <ClInclude Include="..\..\..\lib\raidfile\autogen_RaidFileException.h" />
- <ClInclude Include="..\..\..\lib\raidfile\RaidFileController.h" />
- <ClInclude Include="..\..\..\lib\raidfile\RaidFileException.h" />
- <ClInclude Include="..\..\..\lib\raidfile\RaidFileRead.h" />
- <ClInclude Include="..\..\..\lib\raidfile\RaidFileUtil.h" />
- <ClInclude Include="..\..\..\lib\raidfile\RaidFileWrite.h" />
- </ItemGroup>
- <ItemGroup>
- <None Include="..\..\..\lib\backupstore\backupprotocol.txt" />
- <None Include="..\..\..\lib\backupstore\BackupStoreException.txt" />
- <None Include="..\..\..\lib\backupstore\Makefile.extra" />
- </ItemGroup>
- <ItemGroup>
- <ProjectReference Include="common.vcxproj">
- <Project>{a089cee6-ebf0-4232-a0c0-74850a8127a6}</Project>
- </ProjectReference>
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/2010/qdbm.vcxproj b/infrastructure/msvc/2010/qdbm.vcxproj deleted file mode 100644 index 77bf8205..00000000 --- a/infrastructure/msvc/2010/qdbm.vcxproj +++ /dev/null @@ -1,81 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{72AF22A7-B339-4FDF-B6AE-CA6522D4BB8D}</ProjectGuid>
- <RootNamespace>qdbm</RootNamespace>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>StaticLibrary</ConfigurationType>
- <UseDebugLibraries>true</UseDebugLibraries>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <UseDebugLibraries>false</UseDebugLibraries>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <TargetExt>.lib</TargetExt>
- <OutDir>$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- <IntDir>$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>Disabled</Optimization>
- <CompileAsManaged>false</CompileAsManaged>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- <PreprocessorDefinitions>QDBM_STATIC;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- </Link>
- <Lib />
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <WarningLevel>Level3</WarningLevel>
- <Optimization>MaxSpeed</Optimization>
- <FunctionLevelLinking>true</FunctionLevelLinking>
- <IntrinsicFunctions>true</IntrinsicFunctions>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <OptimizeReferences>true</OptimizeReferences>
- </Link>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ClInclude Include="..\..\..\qdbm\depot.h" />
- <ClInclude Include="..\..\..\qdbm\myconf.h" />
- </ItemGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\qdbm\depot.c" />
- <ClCompile Include="..\..\..\qdbm\myconf.c" />
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/2010/win32test.vcxproj b/infrastructure/msvc/2010/win32test.vcxproj deleted file mode 100644 index 0c61adb3..00000000 --- a/infrastructure/msvc/2010/win32test.vcxproj +++ /dev/null @@ -1,108 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <ItemGroup Label="ProjectConfigurations">
- <ProjectConfiguration Include="Debug|Win32">
- <Configuration>Debug</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- <ProjectConfiguration Include="Release|Win32">
- <Configuration>Release</Configuration>
- <Platform>Win32</Platform>
- </ProjectConfiguration>
- </ItemGroup>
- <PropertyGroup Label="Globals">
- <ProjectGuid>{28C29E72-76A2-4D0C-B35B-12D446733D2E}</ProjectGuid>
- <RootNamespace>win32test</RootNamespace>
- <Keyword>Win32Proj</Keyword>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- <WholeProgramOptimization>true</WholeProgramOptimization>
- </PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
- <ConfigurationType>Application</ConfigurationType>
- <CharacterSet>MultiByte</CharacterSet>
- </PropertyGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
- <ImportGroup Label="ExtensionSettings">
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
- <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
- <Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
- </ImportGroup>
- <PropertyGroup Label="UserMacros" />
- <PropertyGroup>
- <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir)..\..\..\$(Configuration)\$(ProjectName)\</IntDir>
- <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</OutDir>
- <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\..\Release\</IntDir>
- <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
- </PropertyGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
- <ClCompile>
- <Optimization>Disabled</Optimization>
- <AdditionalIncludeDirectories>$(ProjectDir)..\..\..\bin\bbackupd;$(ProjectDir)..\..\..\lib\backupstore;$(ProjectDir)..\..\..\lib\backupclient;$(ProjectDir)..\..\..\lib\common;$(ProjectDir)..\..\..\lib\compress;$(ProjectDir)..\..\..\lib\crypto;$(ProjectDir)..\..\..\lib\server;$(ProjectDir)..\..\..\lib\win32;$(ProjectDir)..\..\..\..\openssl\include;$(ProjectDir)..\..\..\..\zlib\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;QDBM_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <MinimalRebuild>true</MinimalRebuild>
- <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
- <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
- </ClCompile>
- <Link>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <ProgramDatabaseFile>$(OutDir)win32test.pdb</ProgramDatabaseFile>
- <SubSystem>Console</SubSystem>
- <TargetMachine>MachineX86</TargetMachine>
- <ShowProgress>NotSet</ShowProgress>
- </Link>
- </ItemDefinitionGroup>
- <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
- <ClCompile>
- <EnableFiberSafeOptimizations>true</EnableFiberSafeOptimizations>
- <AdditionalIncludeDirectories>$(ProjectDir)..\..\..\bin\bbackupd;$(ProjectDir)..\..\..\lib\backupclient;$(ProjectDir)..\..\..\lib\common\;$(ProjectDir)..\..\..\lib\compress;$(ProjectDir)..\..\..\lib\crypto;$(ProjectDir)..\..\..\lib\server;$(ProjectDir)..\..\..\lib\win32;$(ProjectDir)..\..\..\..\openssl\inc32;$(ProjectDir)..\..\..\..\zlib\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <PreprocessorDefinitions>WIN32;BOX_RELEASE_BUILD;_CONSOLE;PLATFORM_DISABLE_MEM_LEAK_TESTING;_CRT_SECURE_NO_DEPRECATE;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
- <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
- <BufferSecurityCheck>false</BufferSecurityCheck>
- <PrecompiledHeader>
- </PrecompiledHeader>
- <WarningLevel>Level3</WarningLevel>
- <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
- </ClCompile>
- <Link>
- <AdditionalDependencies>Ws2_32.lib;$(ProjectDir)..\..\..\..\zlib\lib\zdll.lib;$(ProjectDir)..\..\..\..\openssl\out32dll\libeay32.lib;$(ProjectDir)..\..\..\..\openssl\out32dll\ssleay32.lib;$(ProjectDir)..\..\..\Release\common.lib;$(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcreposix.lib;$(ProjectDir)..\..\..\..\pcre\bin\release\lib_pcre.lib;%(AdditionalDependencies)</AdditionalDependencies>
- <OutputFile>$(OutDir)win32test.exe</OutputFile>
- <IgnoreSpecificDefaultLibraries>%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>
- <GenerateDebugInformation>true</GenerateDebugInformation>
- <SubSystem>Console</SubSystem>
- <OptimizeReferences>true</OptimizeReferences>
- <EnableCOMDATFolding>true</EnableCOMDATFolding>
- <TargetMachine>MachineX86</TargetMachine>
- </Link>
- </ItemDefinitionGroup>
- <ItemGroup>
- <ClCompile Include="..\..\..\lib\win32\emu.cpp" />
- <ClCompile Include="..\..\..\test\win32\testlibwin32.cpp" />
- </ItemGroup>
- <ItemGroup>
- <None Include="..\..\..\ReadMe.txt" />
- </ItemGroup>
- <ItemGroup>
- <ProjectReference Include="common.vcxproj">
- <Project>{a089cee6-ebf0-4232-a0c0-74850a8127a6}</Project>
- <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
- </ProjectReference>
- </ItemGroup>
- <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
- <ImportGroup Label="ExtensionTargets">
- </ImportGroup>
-</Project>
\ No newline at end of file diff --git a/infrastructure/msvc/fake-config.sub.pl b/infrastructure/msvc/fake-config.sub.pl deleted file mode 100644 index fa9b6839..00000000 --- a/infrastructure/msvc/fake-config.sub.pl +++ /dev/null @@ -1,18 +0,0 @@ -#!perl
-
-use strict;
-use warnings;
-use Cwd;
-
-require "infrastructure\\BoxPlatform.pm.in";
-my $wd = getcwd();
-my $dummy = $BoxPlatform::product_version;
-
-while(<>)
-{
- s|\@build_dir@|$wd|;
- s|\@client_parcel_dir@|$wd/Debug|;
- s|\@box_version@|$BoxPlatform::product_version|;
- m|[^@](@[^@]+@)| and die "Unknown variable: $1";
- print;
-}
\ No newline at end of file diff --git a/infrastructure/msvc/getversion.pl b/infrastructure/msvc/getversion.pl deleted file mode 100644 index 0cf9cbcb..00000000 --- a/infrastructure/msvc/getversion.pl +++ /dev/null @@ -1,34 +0,0 @@ -#!perl - -$basedir = $0; -$basedir =~ s/\\[^\\]*$//; -$basedir =~ s/\\[^\\]*$//; -$basedir =~ s/\\[^\\]*$//; -$basedir =~ s/\\[^\\]*$//; -$basedir =~ s/\\[^\\]*$//; --d $basedir or die "$basedir: $!"; -chdir $basedir or die "$basedir: $!"; - -require "$basedir\\infrastructure\\BoxPlatform.pm.in"; - -my $verfile = "$basedir/lib/common/BoxVersion.h"; -my $newver = "#define BOX_VERSION \"$BoxPlatform::product_version\"\n"; -my $oldver = ""; - -if (-r $verfile) -{ - open VERSIONFILE, "< $verfile" or die "$verfile: $!"; - $oldver = <VERSIONFILE>; - close VERSIONFILE; - -} - -if ($newver ne $oldver) -{ - open VERSIONFILE, "> $verfile" or die "BoxVersion.h: $!"; - print VERSIONFILE $newver; - close VERSIONFILE; -} - -print $BoxPlatform::product_version; -exit 0; diff --git a/infrastructure/travis-build.sh b/infrastructure/travis-build.sh new file mode 100755 index 00000000..74b58a3d --- /dev/null +++ b/infrastructure/travis-build.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +set -e +set -x + +if [ "$TRAVIS_OS_NAME" = "osx" ]; then + brew update + # Travis appears to have Boost and OpenSSL installed already: + # brew install boost ccache openssl + ls /usr/local /usr/local/opt /usr/local/opt/openssl + brew install ccache +fi + +ccache -s + +if [ "$BUILD" = 'cmake' ]; then + if [ -z "$TEST_TARGET" ]; then + echo "TEST_TARGET must be set to 'release' or 'debug' for CMake builds" + exit 2 + fi + + if [ "$TRAVIS_OS_NAME" = "osx" ]; then + EXTRA_ARGS="-DOPENSSL_ROOT_DIR=/usr/local/opt/openssl -DBOOST_ROOT=/usr/local/opt/boost" + fi + + cd `dirname $0` + mkdir -p cmake/build + cd cmake/build + cmake --version + cmake -DCMAKE_BUILD_TYPE:STRING=$TEST_TARGET $EXTRA_ARGS "$@" .. + make + + [ "$TEST" = "n" ] || ctest -C $TEST_TARGET -V +else + if [ "$TRAVIS_OS_NAME" = "osx" ]; then + EXTRA_ARGS="--with-ssl-lib=/usr/local/opt/openssl/lib --with-ssl-headers=/usr/local/opt/openssl/include --with-boost=/usr/local/opt/boost" + fi + + cd `dirname $0`/.. + ./bootstrap + ./configure CC="ccache $CC" CXX="ccache $CXX" $EXTRA_ARGS "$@" + grep CXX config.status + make V=1 $EXTRA_MAKE_ARGS + + [ "$TEST" = "n" ] || ./runtest.pl ALL $TEST_TARGET + + if [ "$TEST_TARGET" = "release" ]; then + make + make parcels + fi +fi + +ccache -s diff --git a/lib/backupclient/BackupClientRestore.cpp b/lib/backupclient/BackupClientRestore.cpp index db72c4bd..d3300604 100644 --- a/lib/backupclient/BackupClientRestore.cpp +++ b/lib/backupclient/BackupClientRestore.cpp @@ -204,13 +204,13 @@ typedef struct // -------------------------------------------------------------------------- // // Function -// Name: BackupClientRestoreDir(BackupProtocolClient &, +// Name: BackupClientRestoreDir(BackupProtocolCallable &, // int64_t, const char *, bool) // Purpose: Restore a directory // Created: 23/11/03 // // -------------------------------------------------------------------------- -static int BackupClientRestoreDir(BackupProtocolClient &rConnection, +static int BackupClientRestoreDir(BackupProtocolCallable &rConnection, int64_t DirectoryID, const std::string &rRemoteDirectoryName, const std::string &rLocalDirectoryName, RestoreParams &Params, RestoreResumeInfo &rLevel) @@ -224,7 +224,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, DIRECTORY_SEPARATOR_ASCHAR + rLevel.mNextLevelLocalName); BackupClientRestoreDir(rConnection, rLevel.mNextLevelID, - rRemoteDirectoryName + '/' + + rRemoteDirectoryName + '/' + rLevel.mNextLevelLocalName, localDirname, Params, *rLevel.mpNextLevel); @@ -232,7 +232,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, rLevel.mRestoredObjects.insert(rLevel.mNextLevelID); // Remove the level for the recursed directory - rLevel.RemoveLevel(); + rLevel.RemoveLevel(); } // Create the local directory, if not already done. @@ -299,7 +299,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, } std::string parentDirectoryName(rLocalDirectoryName); - if(parentDirectoryName[parentDirectoryName.size() - 1] == + if(parentDirectoryName[parentDirectoryName.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR) { parentDirectoryName.resize(parentDirectoryName.size() - 1); @@ -309,7 +309,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, if(lastSlash == std::string::npos) { - // might be a forward slash separator, + // might be a forward slash separator, // especially in the unit tests! lastSlash = parentDirectoryName.rfind('/'); } @@ -813,7 +813,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, // -------------------------------------------------------------------------- // // Function -// Name: BackupClientRestore(BackupProtocolClient &, int64_t, +// Name: BackupClientRestore(BackupProtocolCallable &, int64_t, // const char *, bool, bool, bool, bool, bool) // Purpose: Restore a directory on the server to a local // directory on the disc. The local directory must not @@ -840,7 +840,7 @@ static int BackupClientRestoreDir(BackupProtocolClient &rConnection, // Created: 23/11/03 // // -------------------------------------------------------------------------- -int BackupClientRestore(BackupProtocolClient &rConnection, +int BackupClientRestore(BackupProtocolCallable &rConnection, int64_t DirectoryID, const std::string& RemoteDirectoryName, const std::string& LocalDirectoryName, bool PrintDots, bool RestoreDeleted, bool UndeleteAfterRestoreDeleted, bool Resume, @@ -889,7 +889,7 @@ int BackupClientRestore(BackupProtocolClient &rConnection, } // Restore the directory - int result = BackupClientRestoreDir(rConnection, DirectoryID, + int result = BackupClientRestoreDir(rConnection, DirectoryID, RemoteDirectoryName, LocalDirectoryName, params, params.mResumeInfo); if (result != Restore_Complete) diff --git a/lib/backupclient/BackupClientRestore.h b/lib/backupclient/BackupClientRestore.h index 77f09c2e..cdbedea7 100644 --- a/lib/backupclient/BackupClientRestore.h +++ b/lib/backupclient/BackupClientRestore.h @@ -7,10 +7,10 @@ // // -------------------------------------------------------------------------- -#ifndef BACKUPSCLIENTRESTORE_H -#define BACKUPSCLIENTRESTORE__H +#ifndef BACKUPCLIENTRESTORE_H +#define BACKUPCLIENTRESTORE_H -class BackupProtocolClient; +class BackupProtocolCallable; enum { @@ -22,7 +22,7 @@ enum Restore_CompleteWithErrors, }; -int BackupClientRestore(BackupProtocolClient &rConnection, +int BackupClientRestore(BackupProtocolCallable &rConnection, int64_t DirectoryID, const std::string& RemoteDirectoryName, const std::string& LocalDirectoryName, @@ -32,5 +32,5 @@ int BackupClientRestore(BackupProtocolClient &rConnection, bool Resume, bool ContinueAfterErrors); -#endif // BACKUPSCLIENTRESTORE__H +#endif // BACKUPCLIENTRESTORE_H diff --git a/lib/backupclient/BackupDaemonConfigVerify.cpp b/lib/backupclient/BackupDaemonConfigVerify.cpp index a3b95335..865ee413 100644 --- a/lib/backupclient/BackupDaemonConfigVerify.cpp +++ b/lib/backupclient/BackupDaemonConfigVerify.cpp @@ -11,11 +11,12 @@ #include "BackupDaemonConfigVerify.h" #include "Daemon.h" #include "BoxPortsAndFiles.h" +#include "BackupConstants.h" #include "MemLeakFindOn.h" -static const ConfigurationVerifyKey backuplocationkeys[] = +static const ConfigurationVerifyKey backuplocationkeys[] = { ConfigurationVerifyKey("ExcludeFile", ConfigTest_MultiValueAllowed), ConfigurationVerifyKey("ExcludeFilesRegex", ConfigTest_MultiValueAllowed), @@ -28,7 +29,7 @@ static const ConfigurationVerifyKey backuplocationkeys[] = ConfigurationVerifyKey("Path", ConfigTest_Exists | ConfigTest_LastEntry) }; -static const ConfigurationVerify backuplocations[] = +static const ConfigurationVerify backuplocations[] = { { "*", @@ -39,12 +40,22 @@ static const ConfigurationVerify backuplocations[] = } }; -static const ConfigurationVerifyKey verifyserverkeys[] = +static const ConfigurationVerifyKey verifyserverkeys[] = { DAEMON_VERIFY_SERVER_KEYS }; -static const ConfigurationVerify verifyserver[] = +static const ConfigurationVerifyKey verifys3keys[] = +{ + // These values are only required for Amazon S3-compatible stores + ConfigurationVerifyKey("HostName", ConfigTest_Exists), + ConfigurationVerifyKey("Port", ConfigTest_Exists | ConfigTest_IsInt, 80), + ConfigurationVerifyKey("BasePath", ConfigTest_Exists), + ConfigurationVerifyKey("AccessKey", ConfigTest_Exists), + ConfigurationVerifyKey("SecretKey", ConfigTest_Exists | ConfigTest_LastEntry) +}; + +static const ConfigurationVerify verifyserver[] = { { "Server", @@ -54,6 +65,13 @@ static const ConfigurationVerify verifyserver[] = 0 }, { + "S3Store", + 0, + verifys3keys, + 0, + 0 + }, + { "BackupLocations", backuplocations, 0, @@ -62,12 +80,12 @@ static const ConfigurationVerify verifyserver[] = } }; -static const ConfigurationVerifyKey verifyrootkeys[] = +static const ConfigurationVerifyKey verifyrootkeys[] = { - ConfigurationVerifyKey("AccountNumber", - ConfigTest_Exists | ConfigTest_IsUint32), ConfigurationVerifyKey("UpdateStoreInterval", ConfigTest_Exists | ConfigTest_IsInt), + ConfigurationVerifyKey("BackupErrorDelay", + ConfigTest_IsInt, BACKUP_ERROR_RETRY_SECONDS), ConfigurationVerifyKey("MinimumFileAge", ConfigTest_Exists | ConfigTest_IsInt), ConfigurationVerifyKey("MaxUploadWait", @@ -89,9 +107,6 @@ static const ConfigurationVerifyKey verifyrootkeys[] = ConfigTest_Exists | ConfigTest_IsInt), ConfigurationVerifyKey("DiffingUploadSizeThreshold", ConfigTest_Exists | ConfigTest_IsInt), - ConfigurationVerifyKey("StoreHostname", ConfigTest_Exists), - ConfigurationVerifyKey("StorePort", ConfigTest_IsInt, - BOX_PORT_BBSTORED), ConfigurationVerifyKey("ExtendedLogging", ConfigTest_IsBool, false), // extended log to syslog ConfigurationVerifyKey("ExtendedLogFile", 0), @@ -102,6 +117,8 @@ static const ConfigurationVerifyKey verifyrootkeys[] = // enable logging to a file ConfigurationVerifyKey("LogFileLevel", 0), // set the level of verbosity of file logging + ConfigurationVerifyKey("LogFileOverwrite", ConfigTest_IsBool, false), + // overwrite the log file on each backup ConfigurationVerifyKey("CommandSocket", 0), // not compulsory to have this ConfigurationVerifyKey("KeepAliveTime", ConfigTest_IsInt), @@ -120,13 +137,18 @@ static const ConfigurationVerifyKey verifyrootkeys[] = ConfigurationVerifyKey("TcpNice", ConfigTest_IsBool, false), // optional enable of tcp nice/background mode - ConfigurationVerifyKey("CertificateFile", ConfigTest_Exists), - ConfigurationVerifyKey("PrivateKeyFile", ConfigTest_Exists), - ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_Exists), ConfigurationVerifyKey("KeysFile", ConfigTest_Exists), - ConfigurationVerifyKey("DataDirectory", - ConfigTest_Exists | ConfigTest_LastEntry), + ConfigurationVerifyKey("DataDirectory", ConfigTest_Exists), + // These values are only required for bbstored stores: + ConfigurationVerifyKey("StoreHostname", 0), + ConfigurationVerifyKey("StorePort", ConfigTest_IsInt, + BOX_PORT_BBSTORED), + ConfigurationVerifyKey("AccountNumber", + ConfigTest_IsUint32), + ConfigurationVerifyKey("CertificateFile", 0), + ConfigurationVerifyKey("PrivateKeyFile", 0), + ConfigurationVerifyKey("TrustedCAsFile", ConfigTest_LastEntry), }; const ConfigurationVerify BackupDaemonConfigVerify = diff --git a/bin/bbackupd/ClientException.txt b/lib/backupclient/ClientException.txt index 04f88620..04f88620 100644 --- a/bin/bbackupd/ClientException.txt +++ b/lib/backupclient/ClientException.txt diff --git a/bin/bbackupd/Makefile.extra b/lib/backupclient/Makefile.extra index 25ceb1e7..25ceb1e7 100644 --- a/bin/bbackupd/Makefile.extra +++ b/lib/backupclient/Makefile.extra diff --git a/lib/backupstore/BackgroundTask.h b/lib/backupstore/BackgroundTask.h new file mode 100644 index 00000000..bae9162f --- /dev/null +++ b/lib/backupstore/BackgroundTask.h @@ -0,0 +1,39 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackgroundTask.h +// Purpose: Declares the BackgroundTask interface. +// Created: 2014/04/07 +// +// -------------------------------------------------------------------------- + +#ifndef BACKGROUNDTASK__H +#define BACKGROUNDTASK__H + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackgroundTask +// Purpose: Provides a RunBackgroundTask() method which allows +// background tasks such as polling the command socket +// to happen while a file is being uploaded. If it +// returns false, the current task should be aborted. +// Created: 2014/04/07 +// +// -------------------------------------------------------------------------- +class BackgroundTask +{ + public: + enum State { + Unknown = 0, + Scanning_Dirs, + Searching_Blocks, + Uploading_Full, + Uploading_Patch, + }; + virtual ~BackgroundTask() { } + virtual bool RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum) = 0; +}; + +#endif // BACKGROUNDTASK__H diff --git a/lib/backupstore/BackupAccountControl.cpp b/lib/backupstore/BackupAccountControl.cpp new file mode 100644 index 00000000..331ef841 --- /dev/null +++ b/lib/backupstore/BackupAccountControl.cpp @@ -0,0 +1,267 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupAccountControl.cpp +// Purpose: Client-side account management for Amazon S3 stores +// Created: 2015/06/27 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <climits> +#include <iostream> + +#include "autogen_CommonException.h" +#include "autogen_BackupStoreException.h" +#include "BackupAccountControl.h" +#include "BackupStoreConstants.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreInfo.h" +#include "Configuration.h" +#include "HTTPResponse.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +void BackupAccountControl::CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit) +{ + if(SoftLimit > HardLimit) + { + BOX_FATAL("Soft limit must be less than the hard limit."); + exit(1); + } + if(SoftLimit > ((HardLimit * MAX_SOFT_LIMIT_SIZE) / 100)) + { + BOX_WARNING("We recommend setting the soft limit below " << + MAX_SOFT_LIMIT_SIZE << "% of the hard limit, or " << + HumanReadableSize((HardLimit * MAX_SOFT_LIMIT_SIZE) + / 100) << " in this case."); + } +} + +int64_t BackupAccountControl::SizeStringToBlocks(const char *string, int blockSize) +{ + // Get number + char *endptr = (char*)string; + int64_t number = strtol(string, &endptr, 0); + if(endptr == string || number == LONG_MIN || number == LONG_MAX) + { + BOX_FATAL("'" << string << "' is not a valid number."); + exit(1); + } + + // Check units + switch(*endptr) + { + case 'M': + case 'm': + // Units: Mb + return (number * 1024*1024) / blockSize; + break; + + case 'G': + case 'g': + // Units: Gb + return (number * 1024*1024*1024) / blockSize; + break; + + case 'B': + case 'b': + // Units: Blocks + // Easy! Just return the number specified. + return number; + break; + + default: + BOX_FATAL(string << " has an invalid units specifier " + "(use B for blocks, M for MB, G for GB, eg 2GB)"); + exit(1); + break; + } +} + +std::string BackupAccountControl::BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize) +{ + return FormatUsageBar(Blocks, Blocks * BlockSize, MaxBlocks * BlockSize, + mMachineReadableOutput); +} + +int BackupAccountControl::PrintAccountInfo(const BackupStoreInfo& info, + int BlockSize) +{ + // Then print out lots of info + std::cout << FormatUsageLineStart("Account ID", mMachineReadableOutput) << + BOX_FORMAT_ACCOUNT(info.GetAccountID()) << std::endl; + std::cout << FormatUsageLineStart("Account Name", mMachineReadableOutput) << + info.GetAccountName() << std::endl; + std::cout << FormatUsageLineStart("Last object ID", mMachineReadableOutput) << + BOX_FORMAT_OBJECTID(info.GetLastObjectIDUsed()) << std::endl; + std::cout << FormatUsageLineStart("Used", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksUsed(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Current files", + mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInCurrentFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Old files", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInOldFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Deleted files", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInDeletedFiles(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksInDirectories(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Soft limit", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksSoftLimit(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Hard limit", mMachineReadableOutput) << + BlockSizeToString(info.GetBlocksHardLimit(), + info.GetBlocksHardLimit(), BlockSize) << std::endl; + std::cout << FormatUsageLineStart("Client store marker", mMachineReadableOutput) << + info.GetClientStoreMarker() << std::endl; + std::cout << FormatUsageLineStart("Current Files", mMachineReadableOutput) << + info.GetNumCurrentFiles() << std::endl; + std::cout << FormatUsageLineStart("Old Files", mMachineReadableOutput) << + info.GetNumOldFiles() << std::endl; + std::cout << FormatUsageLineStart("Deleted Files", mMachineReadableOutput) << + info.GetNumDeletedFiles() << std::endl; + std::cout << FormatUsageLineStart("Directories", mMachineReadableOutput) << + info.GetNumDirectories() << std::endl; + std::cout << FormatUsageLineStart("Enabled", mMachineReadableOutput) << + (info.IsAccountEnabled() ? "yes" : "no") << std::endl; + + return 0; +} + +S3BackupAccountControl::S3BackupAccountControl(const Configuration& config, + bool machineReadableOutput) +: BackupAccountControl(config, machineReadableOutput) +{ + if(!mConfig.SubConfigurationExists("S3Store")) + { + THROW_EXCEPTION_MESSAGE(CommonException, + InvalidConfiguration, + "The S3Store configuration subsection is required " + "when S3Store mode is enabled"); + } + const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); + + mBasePath = s3config.GetKeyValue("BasePath"); + if(mBasePath.size() == 0) + { + mBasePath = "/"; + } + else + { + if(mBasePath[0] != '/' || mBasePath[mBasePath.size() - 1] != '/') + { + THROW_EXCEPTION_MESSAGE(CommonException, + InvalidConfiguration, + "If S3Store.BasePath is not empty then it must start and " + "end with a slash, e.g. '/subdir/', but it currently does not."); + } + } + + mapS3Client.reset(new S3Client( + s3config.GetKeyValue("HostName"), + s3config.GetKeyValueInt("Port"), + s3config.GetKeyValue("AccessKey"), + s3config.GetKeyValue("SecretKey"))); + + mapFileSystem.reset(new S3BackupFileSystem(mConfig, mBasePath, *mapS3Client)); +} + +std::string S3BackupAccountControl::GetFullURL(const std::string ObjectPath) const +{ + const Configuration s3config = mConfig.GetSubConfiguration("S3Store"); + return std::string("http://") + s3config.GetKeyValue("HostName") + ":" + + s3config.GetKeyValue("Port") + GetFullPath(ObjectPath); +} + +int S3BackupAccountControl::CreateAccount(const std::string& name, int32_t SoftLimit, + int32_t HardLimit) +{ + // Try getting the info file. If we get a 200 response then it already + // exists, and we should bail out. If we get a 404 then it's safe to + // continue. Otherwise something else is wrong and we should bail out. + std::string info_url = GetFullURL(S3_INFO_FILE_NAME); + + HTTPResponse response = GetObject(S3_INFO_FILE_NAME); + if(response.GetResponseCode() == HTTPResponse::Code_OK) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, AccountAlreadyExists, + "The BackupStoreInfo file already exists at this URL: " << + info_url); + } + + if(response.GetResponseCode() != HTTPResponse::Code_NotFound) + { + mapS3Client->CheckResponse(response, std::string("Failed to check for an " + "existing BackupStoreInfo file at this URL: ") + info_url); + } + + BackupStoreInfo info(0, // fake AccountID for S3 stores + info_url, // FileName, + SoftLimit, HardLimit); + info.SetAccountName(name); + + // And an empty directory + BackupStoreDirectory rootDir(BACKUPSTORE_ROOT_DIRECTORY_ID, BACKUPSTORE_ROOT_DIRECTORY_ID); + int64_t rootDirSize = mapFileSystem->PutDirectory(rootDir); + + // Update the store info to reflect the size of the root directory + info.ChangeBlocksUsed(rootDirSize); + info.ChangeBlocksInDirectories(rootDirSize); + info.AdjustNumDirectories(1); + int64_t id = info.AllocateObjectID(); + ASSERT(id == BACKUPSTORE_ROOT_DIRECTORY_ID); + + CollectInBufferStream out; + info.Save(out); + out.SetForReading(); + + response = PutObject(S3_INFO_FILE_NAME, out); + mapS3Client->CheckResponse(response, std::string("Failed to upload the new BackupStoreInfo " + "file to this URL: ") + info_url); + + // Now get the file again, to check that it really worked. + response = GetObject(S3_INFO_FILE_NAME); + mapS3Client->CheckResponse(response, std::string("Failed to download the new BackupStoreInfo " + "file that we just created: ") + info_url); + + return 0; +} + +std::string S3BackupFileSystem::GetDirectoryURI(int64_t ObjectID) +{ + std::ostringstream out; + out << mBasePath << "dirs/" << BOX_FORMAT_OBJECTID(ObjectID) << ".dir"; + return out.str(); +} + +std::auto_ptr<HTTPResponse> S3BackupFileSystem::GetDirectory(BackupStoreDirectory& rDir) +{ + std::string uri = GetDirectoryURI(rDir.GetObjectID()); + HTTPResponse response = mrClient.GetObject(uri); + mrClient.CheckResponse(response, + std::string("Failed to download directory: ") + uri); + return std::auto_ptr<HTTPResponse>(new HTTPResponse(response)); +} + +int S3BackupFileSystem::PutDirectory(BackupStoreDirectory& rDir) +{ + CollectInBufferStream out; + rDir.WriteToStream(out); + out.SetForReading(); + + std::string uri = GetDirectoryURI(rDir.GetObjectID()); + HTTPResponse response = mrClient.PutObject(uri, out); + mrClient.CheckResponse(response, + std::string("Failed to upload directory: ") + uri); + + int blocks = (out.GetSize() + S3_NOTIONAL_BLOCK_SIZE - 1) / S3_NOTIONAL_BLOCK_SIZE; + return blocks; +} + diff --git a/lib/backupstore/BackupAccountControl.h b/lib/backupstore/BackupAccountControl.h new file mode 100644 index 00000000..00118ec2 --- /dev/null +++ b/lib/backupstore/BackupAccountControl.h @@ -0,0 +1,91 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupAccountControl.h +// Purpose: Client-side account management for Amazon S3 stores +// Created: 2015/06/27 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPACCOUNTCONTROL__H +#define BACKUPACCOUNTCONTROL__H + +#include <string> + +#include "BackupStoreAccountDatabase.h" +#include "HTTPResponse.h" +#include "S3Client.h" + +class BackupStoreDirectory; +class BackupStoreInfo; +class Configuration; + +class BackupAccountControl +{ +protected: + const Configuration& mConfig; + bool mMachineReadableOutput; + +public: + BackupAccountControl(const Configuration& config, + bool machineReadableOutput = false) + : mConfig(config), + mMachineReadableOutput(machineReadableOutput) + { } + void CheckSoftHardLimits(int64_t SoftLimit, int64_t HardLimit); + int64_t SizeStringToBlocks(const char *string, int BlockSize); + std::string BlockSizeToString(int64_t Blocks, int64_t MaxBlocks, int BlockSize); + int PrintAccountInfo(const BackupStoreInfo& info, int BlockSize); +}; + +class S3BackupFileSystem +{ +private: + std::string mBasePath; + S3Client& mrClient; +public: + S3BackupFileSystem(const Configuration& config, const std::string& BasePath, + S3Client& rClient) + : mBasePath(BasePath), + mrClient(rClient) + { } + std::string GetDirectoryURI(int64_t ObjectID); + std::auto_ptr<HTTPResponse> GetDirectory(BackupStoreDirectory& rDir); + int PutDirectory(BackupStoreDirectory& rDir); +}; + +class S3BackupAccountControl : public BackupAccountControl +{ +private: + std::string mBasePath; + std::auto_ptr<S3Client> mapS3Client; + std::auto_ptr<S3BackupFileSystem> mapFileSystem; +public: + S3BackupAccountControl(const Configuration& config, + bool machineReadableOutput = false); + std::string GetFullPath(const std::string ObjectPath) const + { + return mBasePath + ObjectPath; + } + std::string GetFullURL(const std::string ObjectPath) const; + int CreateAccount(const std::string& name, int32_t SoftLimit, int32_t HardLimit); + int GetBlockSize() { return 4096; } + HTTPResponse GetObject(const std::string& name) + { + return mapS3Client->GetObject(GetFullPath(name)); + } + HTTPResponse PutObject(const std::string& name, IOStream& rStreamToSend, + const char* pContentType = NULL) + { + return mapS3Client->PutObject(GetFullPath(name), rStreamToSend, + pContentType); + } +}; + +// max size of soft limit as percent of hard limit +#define MAX_SOFT_LIMIT_SIZE 97 +#define S3_INFO_FILE_NAME "boxbackup.info" +#define S3_NOTIONAL_BLOCK_SIZE 1048576 + +#endif // BACKUPACCOUNTCONTROL__H + diff --git a/lib/backupstore/BackupClientFileAttributes.cpp b/lib/backupstore/BackupClientFileAttributes.cpp index d76432ba..37140301 100644 --- a/lib/backupstore/BackupClientFileAttributes.cpp +++ b/lib/backupstore/BackupClientFileAttributes.cpp @@ -55,20 +55,20 @@ BEGIN_STRUCTURE_PACKING_FOR_WIRE typedef struct { int32_t AttributeType; - u_int32_t UID; - u_int32_t GID; - u_int64_t ModificationTime; - u_int64_t AttrModificationTime; - u_int32_t UserDefinedFlags; - u_int32_t FileGenerationNumber; - u_int16_t Mode; + uint32_t UID; + uint32_t GID; + uint64_t ModificationTime; + uint64_t AttrModificationTime; + uint32_t UserDefinedFlags; + uint32_t FileGenerationNumber; + uint16_t Mode; // Symbolic link filename may follow // Extended attribute (xattr) information may follow, format is: - // u_int32_t Size of extended attribute block (excluding this word) + // uint32_t Size of extended attribute block (excluding this word) // For each of NumberOfAttributes (sorted by AttributeName): - // u_int16_t AttributeNameLength + // uint16_t AttributeNameLength // char AttributeName[AttributeNameLength] - // u_int32_t AttributeValueLength + // uint32_t AttributeValueLength // unsigned char AttributeValue[AttributeValueLength] // AttributeName is 0 terminated, AttributeValue is not (and may be binary data) } attr_StreamFormat; @@ -117,7 +117,7 @@ namespace BackupClientFileAttributes::BackupClientFileAttributes() : mpClearAttributes(0) { - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); + ASSERT(sizeof(uint64_t) == sizeof(box_time_t)); } // -------------------------------------------------------------------------- @@ -131,7 +131,7 @@ BackupClientFileAttributes::BackupClientFileAttributes() BackupClientFileAttributes::BackupClientFileAttributes(const EMU_STRUCT_STAT &st) : mpClearAttributes(0) { - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); + ASSERT(sizeof(uint64_t) == sizeof(box_time_t)); StreamableMemBlock *pnewAttr = new StreamableMemBlock; FillAttributes(*pnewAttr, (const char *)NULL, st, true); @@ -411,7 +411,7 @@ void BackupClientFileAttributes::ReadAttributes(const std::string& Filename, // __time64_t winTime = BoxTimeToSeconds( // pnewAttr->ModificationTime); - u_int64_t modTime = box_ntoh64(pattr->ModificationTime); + uint64_t modTime = box_ntoh64(pattr->ModificationTime); box_time_t modSecs = BoxTimeToSeconds(modTime); __time64_t winTime = modSecs; @@ -545,7 +545,7 @@ void BackupClientFileAttributes::FillAttributesLink( void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBlock, const std::string& Filename) { -#ifdef HAVE_SYS_XATTR_H +#if defined HAVE_LLISTXATTR && defined HAVE_LGETXATTR int listBufferSize = 10000; char* list = new char[listBufferSize]; @@ -582,30 +582,30 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc // Leave space for attr block size later int xattrBlockSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); + xattrSize += sizeof(uint32_t); // Loop for each attribute for(std::vector<std::string>::const_iterator attrKeyI = attrKeys.begin(); attrKeyI!=attrKeys.end(); ++attrKeyI) { std::string attrKey(*attrKeyI); - if(xattrSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t)>static_cast<unsigned int>(xattrBufferSize)) + if(xattrSize+sizeof(uint16_t)+attrKey.size()+1+sizeof(uint32_t)>static_cast<unsigned int>(xattrBufferSize)) { - xattrBufferSize = (xattrBufferSize+sizeof(u_int16_t)+attrKey.size()+1+sizeof(u_int32_t))*2; + xattrBufferSize = (xattrBufferSize+sizeof(uint16_t)+attrKey.size()+1+sizeof(uint32_t))*2; outputBlock.ResizeBlock(xattrBufferSize); buffer = static_cast<unsigned char*>(outputBlock.GetBuffer()); } // Store length and text for attibute name - u_int16_t keyLength = htons(attrKey.size()+1); - std::memcpy(buffer+xattrSize, &keyLength, sizeof(u_int16_t)); - xattrSize += sizeof(u_int16_t); + uint16_t keyLength = htons(attrKey.size()+1); + std::memcpy(buffer+xattrSize, &keyLength, sizeof(uint16_t)); + xattrSize += sizeof(uint16_t); std::memcpy(buffer+xattrSize, attrKey.c_str(), attrKey.size()+1); xattrSize += attrKey.size()+1; // Leave space for value size int valueSizeOffset = xattrSize; - xattrSize += sizeof(u_int32_t); + xattrSize += sizeof(uint32_t); // Find size of attribute (must call with buffer and length 0 on some platforms, // as -1 is returned if the data doesn't fit.) @@ -642,21 +642,30 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc xattrSize += valueSize; // Fill in value size - u_int32_t valueLength = htonl(valueSize); - std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(u_int32_t)); + uint32_t valueLength = htonl(valueSize); + std::memcpy(buffer+valueSizeOffset, &valueLength, sizeof(uint32_t)); } // Fill in attribute block size - u_int32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(u_int32_t)); - std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(u_int32_t)); + uint32_t xattrBlockLength = htonl(xattrSize-xattrBlockSizeOffset-sizeof(uint32_t)); + std::memcpy(buffer+xattrBlockSizeOffset, &xattrBlockLength, sizeof(uint32_t)); outputBlock.ResizeBlock(xattrSize); } else if(listSize<0) { - if(errno == EOPNOTSUPP || errno == EACCES) + if(errno == EOPNOTSUPP || errno == EACCES +#if HAVE_DECL_ENOTSUP + // NetBSD uses ENOTSUP instead + // https://mail-index.netbsd.org/tech-kern/2011/12/13/msg012185.html + || errno == ENOTSUP +#endif + ) { - // fail silently + // Not supported by OS, or not on this filesystem + BOX_TRACE(BOX_SYS_ERRNO_MESSAGE(errno, + BOX_FILE_MESSAGE(Filename, "Failed to " + "list extended attributes"))); } else if(errno == ERANGE) { @@ -664,12 +673,17 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc "attributes of '" << Filename << "': " "buffer too small, not backed up"); } + else if(errno == ENOENT) + { + BOX_ERROR("Failed to list extended " + "attributes of '" << Filename << "': " + "file no longer exists"); + } else { - BOX_LOG_SYS_ERROR("Failed to list extended " - "attributes of '" << Filename << "', " - "not backed up"); - THROW_EXCEPTION(CommonException, OSFileError); + THROW_SYS_FILE_ERROR("Failed to list extended " + "attributes for unknown reason", Filename, + CommonException, OSFileError); } } } @@ -679,7 +693,7 @@ void BackupClientFileAttributes::FillExtendedAttr(StreamableMemBlock &outputBloc throw; } delete[] list; -#endif +#endif // defined HAVE_LLISTXATTR && defined HAVE_LGETXATTR } // -------------------------------------------------------------------------- @@ -839,7 +853,7 @@ void BackupClientFileAttributes::WriteAttributes(const std::string& Filename, #endif } - if(static_cast<int>(xattrOffset+sizeof(u_int32_t))<=mpClearAttributes->GetSize()) + if(static_cast<int>(xattrOffset+sizeof(uint32_t))<=mpClearAttributes->GetSize()) { WriteExtendedAttr(Filename, xattrOffset); } @@ -978,13 +992,13 @@ void BackupClientFileAttributes::EnsureClearAvailable() const // -------------------------------------------------------------------------- void BackupClientFileAttributes::WriteExtendedAttr(const std::string& Filename, int xattrOffset) const { -#ifdef HAVE_SYS_XATTR_H +#if defined HAVE_LSETXATTR const char* buffer = static_cast<char*>(mpClearAttributes->GetBuffer()); - u_int32_t xattrBlockLength = 0; - std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(u_int32_t)); + uint32_t xattrBlockLength = 0; + std::memcpy(&xattrBlockLength, buffer+xattrOffset, sizeof(uint32_t)); int xattrBlockSize = ntohl(xattrBlockLength); - xattrOffset += sizeof(u_int32_t); + xattrOffset += sizeof(uint32_t); int xattrEnd = xattrOffset+xattrBlockSize; if(xattrEnd>mpClearAttributes->GetSize()) @@ -995,18 +1009,18 @@ void BackupClientFileAttributes::WriteExtendedAttr(const std::string& Filename, while(xattrOffset<xattrEnd) { - u_int16_t keyLength = 0; - std::memcpy(&keyLength, buffer+xattrOffset, sizeof(u_int16_t)); + uint16_t keyLength = 0; + std::memcpy(&keyLength, buffer+xattrOffset, sizeof(uint16_t)); int keySize = ntohs(keyLength); - xattrOffset += sizeof(u_int16_t); + xattrOffset += sizeof(uint16_t); const char* key = buffer+xattrOffset; xattrOffset += keySize; - u_int32_t valueLength = 0; - std::memcpy(&valueLength, buffer+xattrOffset, sizeof(u_int32_t)); + uint32_t valueLength = 0; + std::memcpy(&valueLength, buffer+xattrOffset, sizeof(uint32_t)); int valueSize = ntohl(valueLength); - xattrOffset += sizeof(u_int32_t); + xattrOffset += sizeof(uint32_t); // FIXME: Warn on EOPNOTSUPP if(::lsetxattr(Filename.c_str(), key, buffer+xattrOffset, diff --git a/lib/backupstore/BackupCommands.cpp b/lib/backupstore/BackupCommands.cpp index 318ce55a..22ef0215 100644 --- a/lib/backupstore/BackupCommands.cpp +++ b/lib/backupstore/BackupCommands.cpp @@ -52,11 +52,66 @@ return PROTOCOL_ERROR(Err_SessionReadOnly); \ } + // -------------------------------------------------------------------------- // // Function -// Name: BackupProtocolVersion::DoCommand(Protocol &, BackupStoreContext &) -// Purpose: Return the current version, or an error if the requested version isn't allowed +// Name: BackupProtocolMessage::HandleException(BoxException& e) +// Purpose: Return an error message appropriate to the passed-in +// exception, or rethrow it. +// Created: 2014/09/14 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupProtocolMessage> BackupProtocolReplyable::HandleException(BoxException& e) const +{ + if(e.GetType() == RaidFileException::ExceptionType && + e.GetSubType() == RaidFileException::RaidFileDoesntExist) + { + return PROTOCOL_ERROR(Err_DoesNotExist); + } + else if (e.GetType() == BackupStoreException::ExceptionType) + { + if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify) + { + return PROTOCOL_ERROR(Err_FileDoesNotVerify); + } + else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit) + { + return PROTOCOL_ERROR(Err_StorageLimitExceeded); + } + else if(e.GetSubType() == BackupStoreException::MultiplyReferencedObject) + { + return PROTOCOL_ERROR(Err_MultiplyReferencedObject); + } + else if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory) + { + return PROTOCOL_ERROR(Err_DoesNotExistInDirectory); + } + else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory) + { + return PROTOCOL_ERROR(Err_TargetNameExists); + } + else if(e.GetSubType() == BackupStoreException::ObjectDoesNotExist) + { + return PROTOCOL_ERROR(Err_DoesNotExist); + } + else if(e.GetSubType() == BackupStoreException::PatchChainInfoBadInDirectory) + { + return PROTOCOL_ERROR(Err_PatchConsistencyError); + } + } + + throw; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolVersion::DoCommand(Protocol &, +// BackupStoreContext &) +// Purpose: Return the current version, or an error if the +// requested version isn't allowed // Created: 2003/08/20 // // -------------------------------------------------------------------------- @@ -93,7 +148,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc // and that the client actually has an account on this machine if(mClientID != rContext.GetClientID()) { - BOX_WARNING("Failed login from client ID " << + BOX_WARNING("Failed login from client ID " << BOX_FORMAT_ACCOUNT(mClientID) << ": " "wrong certificate for this account"); return PROTOCOL_ERROR(Err_BadLogin); @@ -101,7 +156,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc if(!rContext.GetClientHasAccount()) { - BOX_WARNING("Failed login from client ID " << + BOX_WARNING("Failed login from client ID " << BOX_FORMAT_ACCOUNT(mClientID) << ": " "no such account on this server"); return PROTOCOL_ERROR(Err_BadLogin); @@ -117,17 +172,17 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc BOX_FORMAT_ACCOUNT(mClientID)); return PROTOCOL_ERROR(Err_CannotLockStoreForWriting); } - + // Debug: check we got the lock ASSERT(!rContext.SessionIsReadOnly()); } - + // Load the store info rContext.LoadStoreInfo(); if(!rContext.GetBackupStoreInfo().IsAccountEnabled()) { - BOX_WARNING("Refused login from disabled client ID " << + BOX_WARNING("Refused login from disabled client ID " << BOX_FORMAT_ACCOUNT(mClientID)); return PROTOCOL_ERROR(Err_DisabledAccount); } @@ -137,9 +192,9 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc // Mark the next phase rContext.SetPhase(BackupStoreContext::Phase_Commands); - + // Log login - BOX_NOTICE("Login from Client ID " << + BOX_NOTICE("Login from Client ID " << BOX_FORMAT_ACCOUNT(mClientID) << " " "(name=" << rContext.GetAccountName() << "): " << (((mFlags & Flags_ReadOnly) != Flags_ReadOnly) @@ -164,14 +219,15 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolLogin::DoCommand(BackupProtoc // -------------------------------------------------------------------------- std::auto_ptr<BackupProtocolMessage> BackupProtocolFinished::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const { - BOX_NOTICE("Session finished for Client ID " << + // can be called in any phase + + BOX_NOTICE("Session finished for Client ID " << BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << " " "(name=" << rContext.GetAccountName() << ")"); // Let the context know about it rContext.ReceivedFinishCommand(); - // can be called in any phase return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolFinished); } @@ -191,26 +247,15 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolListDirectory::DoCommand(Back // Store the listing to a stream std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream); - try - { - // Ask the context for a directory - const BackupStoreDirectory &rdir( - rContext.GetDirectory(mObjectID)); - rdir.WriteToStream(*stream, mFlagsMustBeSet, - mFlagsNotToBeSet, mSendAttributes, - false /* never send dependency info to the client */); - } - catch (RaidFileException &e) - { - if (e.GetSubType() == RaidFileException::RaidFileDoesntExist) - { - return PROTOCOL_ERROR(Err_DoesNotExist); - } - throw; - } + // Ask the context for a directory + const BackupStoreDirectory &rdir( + rContext.GetDirectory(mObjectID)); + rdir.WriteToStream(*stream, mFlagsMustBeSet, + mFlagsNotToBeSet, mSendAttributes, + false /* never send dependency info to the client */); stream->SetForReading(); - + // Get the protocol to send the stream rProtocol.SendStreamAfterCommand(static_cast< std::auto_ptr<IOStream> > (stream)); @@ -226,7 +271,9 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolListDirectory::DoCommand(Back // Created: 2003/09/02 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION @@ -237,7 +284,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand(BackupPr { return hookResult; } - + // Check that the diff from file actually exists, if it's specified if(mDiffFromFileID != 0) { @@ -247,35 +294,13 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolStoreFile::DoCommand(BackupPr return PROTOCOL_ERROR(Err_DiffFromFileDoesNotExist); } } - - // A stream follows, which contains the file - std::auto_ptr<IOStream> filestream(rProtocol.ReceiveStream()); - + // Ask the context to store it - int64_t id = 0; - try - { - id = rContext.AddFile(*filestream, mDirectoryObjectID, - mModificationTime, mAttributesHash, mDiffFromFileID, - mFilename, - true /* mark files with same name as old versions */); - } - catch(BackupStoreException &e) - { - if(e.GetSubType() == BackupStoreException::AddedFileDoesNotVerify) - { - return PROTOCOL_ERROR(Err_FileDoesNotVerify); - } - else if(e.GetSubType() == BackupStoreException::AddedFileExceedsStorageLimit) - { - return PROTOCOL_ERROR(Err_StorageLimitExceeded); - } - else - { - throw; - } - } - + int64_t id = rContext.AddFile(rDataStream, mDirectoryObjectID, + mModificationTime, mAttributesHash, mDiffFromFileID, + mFilename, + true /* mark files with same name as old versions */); + // Tell the caller what the file ID was return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(id)); } @@ -298,7 +323,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObject::DoCommand(BackupPr // Check the object exists if(!rContext.ObjectExists(mObjectID)) { - return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(NoObject)); + return PROTOCOL_ERROR(Err_DoesNotExist); } // Open the object @@ -315,7 +340,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObject::DoCommand(BackupPr // // Function // Name: BackupProtocolGetFile::DoCommand(Protocol &, BackupStoreContext &) -// Purpose: Command to get an file object from the server -- may have to do a bit of +// Purpose: Command to get an file object from the server -- may have to do a bit of // work to get the object. // Created: 2003/09/03 // @@ -357,13 +382,13 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt en = rdir.FindEntryByID(id); if(en == 0) { - BOX_ERROR("Object " << + BOX_ERROR("Object " << BOX_FORMAT_OBJECTID(mObjectID) << - " in dir " << + " in dir " << BOX_FORMAT_OBJECTID(mInDirectory) << " for account " << BOX_FORMAT_ACCOUNT(rContext.GetClientID()) << - " references object " << + " references object " << BOX_FORMAT_OBJECTID(id) << " which does not exist in dir"); return PROTOCOL_ERROR(Err_PatchConsistencyError); @@ -371,89 +396,73 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt id = en->GetDependsNewer(); } while(en != 0 && id != 0); - + // OK! The last entry in the chain is the full file, the others are patches back from it. // Open the last one, which is the current from file std::auto_ptr<IOStream> from(rContext.OpenObject(patchChain[patchChain.size() - 1])); - + // Then, for each patch in the chain, do a combine for(int p = ((int)patchChain.size()) - 2; p >= 0; --p) { // ID of patch int64_t patchID = patchChain[p]; - + // Open it a couple of times std::auto_ptr<IOStream> diff(rContext.OpenObject(patchID)); std::auto_ptr<IOStream> diff2(rContext.OpenObject(patchID)); - + // Choose a temporary filename for the result of the combination std::ostringstream fs; - fs << rContext.GetStoreRoot() << ".recombinetemp." << p; - std::string tempFn = + fs << rContext.GetAccountRoot() << ".recombinetemp." << p; + std::string tempFn = RaidFileController::DiscSetPathToFileSystemPath( rContext.GetStoreDiscSet(), fs.str(), p + 16); - + // Open the temporary file - std::auto_ptr<IOStream> combined; - try - { - { - // Write nastily to allow this to work with gcc 2.x - std::auto_ptr<IOStream> t( - new InvisibleTempFileStream( - tempFn.c_str(), - O_RDWR | O_CREAT | - O_EXCL | O_BINARY | - O_TRUNC)); - combined = t; - } - } - catch(...) - { - // Make sure it goes - ::unlink(tempFn.c_str()); - throw; - } - + std::auto_ptr<IOStream> combined( + new InvisibleTempFileStream( + tempFn, O_RDWR | O_CREAT | O_EXCL | + O_BINARY | O_TRUNC)); + // Do the combining BackupStoreFile::CombineFile(*diff, *diff2, *from, *combined); - + // Move to the beginning of the combined file combined->Seek(0, IOStream::SeekType_Absolute); - + // Then shuffle round for the next go if (from.get()) from->Close(); from = combined; } - + // Now, from contains a nice file to send to the client. Reorder it { // Write nastily to allow this to work with gcc 2.x std::auto_ptr<IOStream> t(BackupStoreFile::ReorderFileToStreamOrder(from.get(), true /* take ownership */)); stream = t; } - + // Release from file to avoid double deletion from.release(); } else { // Simple case: file already exists on disc ready to go - + // Open the object std::auto_ptr<IOStream> object(rContext.OpenObject(mObjectID)); BufferedStream buf(*object); - + // Verify it if(!BackupStoreFile::VerifyEncodedFileFormat(buf)) { return PROTOCOL_ERROR(Err_FileDoesNotVerify); } - + // Reset stream -- seek to beginning object->Seek(0, IOStream::SeekType_Absolute); - + // Reorder the stream/file into stream order { // Write nastily to allow this to work with gcc 2.x @@ -461,15 +470,15 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt stream = t; } - // Object will be deleted when the stream is deleted, - // so can release the object auto_ptr here to avoid + // Object will be deleted when the stream is deleted, + // so can release the object auto_ptr here to avoid // premature deletion object.release(); } // Stream the reordered stream to the peer rProtocol.SendStreamAfterCommand(stream); - + // Tell the caller what the file was return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(mObjectID)); } @@ -483,18 +492,38 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetFile::DoCommand(BackupProt // Created: 2003/09/04 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const +{ + return BackupProtocolCreateDirectory2(mContainingDirectoryID, + mAttributesModTime, 0 /* ModificationTime */, + mDirectoryName).DoCommand(rProtocol, rContext, rDataStream); +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolCreateDirectory2::DoCommand(Protocol &, BackupStoreContext &) +// Purpose: Create directory command, with a specific +// modification time. +// Created: 2014/02/11 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory2::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - - // Get the stream containing the attributes - std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); - // Collect the attributes -- do this now so no matter what the outcome, + + // Collect the attributes -- do this now so no matter what the outcome, // the data has been absorbed. StreamableMemBlock attr; - attr.Set(*attrstream, rProtocol.GetTimeout()); - + attr.Set(rDataStream, rProtocol.GetTimeout()); + // Check to see if the hard limit has been exceeded if(rContext.HardLimitExceeded()) { @@ -503,8 +532,10 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand(Ba } bool alreadyExists = false; - int64_t id = rContext.AddDirectory(mContainingDirectoryID, mDirectoryName, attr, mAttributesModTime, alreadyExists); - + int64_t id = rContext.AddDirectory(mContainingDirectoryID, + mDirectoryName, attr, mAttributesModTime, mModificationTime, + alreadyExists); + if(alreadyExists) { return PROTOCOL_ERROR(Err_DirectoryAlreadyExists); @@ -524,17 +555,17 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolCreateDirectory::DoCommand(Ba // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolChangeDirAttributes::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> BackupProtocolChangeDirAttributes::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - // Get the stream containing the attributes - std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); - // Collect the attributes -- do this now so no matter what the outcome, + // Collect the attributes -- do this now so no matter what the outcome, // the data has been absorbed. StreamableMemBlock attr; - attr.Set(*attrstream, rProtocol.GetTimeout()); + attr.Set(rDataStream, rProtocol.GetTimeout()); // Get the context to do it's magic rContext.ChangeDirAttributes(mObjectID, attr, mAttributesModTime); @@ -552,17 +583,18 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolChangeDirAttributes::DoComman // Created: 2003/09/06 // // -------------------------------------------------------------------------- -std::auto_ptr<BackupProtocolMessage> BackupProtocolSetReplacementFileAttributes::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +std::auto_ptr<BackupProtocolMessage> +BackupProtocolSetReplacementFileAttributes::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext, + IOStream& rDataStream) const { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - // Get the stream containing the attributes - std::auto_ptr<IOStream> attrstream(rProtocol.ReceiveStream()); - // Collect the attributes -- do this now so no matter what the outcome, + // Collect the attributes -- do this now so no matter what the outcome, // the data has been absorbed. StreamableMemBlock attr; - attr.Set(*attrstream, rProtocol.GetTimeout()); + attr.Set(rDataStream, rProtocol.GetTimeout()); // Get the context to do it's magic int64_t objectID = 0; @@ -577,7 +609,6 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolSetReplacementFileAttributes: } - // -------------------------------------------------------------------------- // // Function @@ -644,19 +675,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolDeleteDirectory::DoCommand(Ba } // Context handles this - try - { - rContext.DeleteDirectory(mObjectID); - } - catch (BackupStoreException &e) - { - if(e.GetSubType() == BackupStoreException::MultiplyReferencedObject) - { - return PROTOCOL_ERROR(Err_MultiplyReferencedObject); - } - - throw; - } + rContext.DeleteDirectory(mObjectID); // return the object ID return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(mObjectID)); @@ -722,29 +741,11 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolMoveObject::DoCommand(BackupP { CHECK_PHASE(Phase_Commands) CHECK_WRITEABLE_SESSION - + // Let context do this, but modify error reporting on exceptions... - try - { - rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory, - mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName, - (mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject); - } - catch(BackupStoreException &e) - { - if(e.GetSubType() == BackupStoreException::CouldNotFindEntryInDirectory) - { - return PROTOCOL_ERROR(Err_DoesNotExist); - } - else if(e.GetSubType() == BackupStoreException::NameAlreadyExistsInDirectory) - { - return PROTOCOL_ERROR(Err_TargetNameExists); - } - else - { - throw; - } - } + rContext.MoveObject(mObjectID, mMoveFromDirectory, mMoveToDirectory, + mNewFilename, (mFlags & Flags_MoveAllWithSameName) == Flags_MoveAllWithSameName, + (mFlags & Flags_AllowMoveOverDeletedObject) == Flags_AllowMoveOverDeletedObject); // Return the object ID return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolSuccess(mObjectID)); @@ -762,21 +763,21 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolMoveObject::DoCommand(BackupP std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const { CHECK_PHASE(Phase_Commands) - + // Create a stream for the list of filenames std::auto_ptr<CollectInBufferStream> stream(new CollectInBufferStream); // Object and directory IDs int64_t objectID = mObjectID; int64_t dirID = mContainingDirectoryID; - + // Data to return in the reply int32_t numNameElements = 0; int16_t objectFlags = 0; int64_t modTime = 0; uint64_t attrModHash = 0; bool haveModTimes = false; - + do { // Check the directory really exists @@ -786,7 +787,28 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back } // Load up the directory - const BackupStoreDirectory &rdir(rContext.GetDirectory(dirID)); + const BackupStoreDirectory *pDir; + + try + { + pDir = &rContext.GetDirectory(dirID); + } + catch(BackupStoreException &e) + { + if(e.GetSubType() == BackupStoreException::ObjectDoesNotExist) + { + // If this can't be found, then there is a problem... + // tell the caller it can't be found. + return std::auto_ptr<BackupProtocolMessage>( + new BackupProtocolObjectName( + BackupProtocolObjectName::NumNameElements_ObjectDoesntExist, + 0, 0, 0)); + } + + throw; + } + + const BackupStoreDirectory& rdir(*pDir); // Find the element in this directory and store it's name if(objectID != ObjectID_DirectoryOnly) @@ -799,13 +821,13 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back // Abort! return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolObjectName(BackupProtocolObjectName::NumNameElements_ObjectDoesntExist, 0, 0, 0)); } - + // Store flags? if(objectFlags == 0) { objectFlags = en->GetFlags(); } - + // Store modification times? if(!haveModTimes) { @@ -813,14 +835,14 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back attrModHash = en->GetAttributesHash(); haveModTimes = true; } - + // Store the name in the stream en->GetName().WriteToStream(*stream); - + // Count of name elements ++numNameElements; } - + // Setup for next time round objectID = dirID; dirID = rdir.GetContainerID(); @@ -831,7 +853,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetObjectName::DoCommand(Back if(numNameElements > 0) { // Get the stream ready to go - stream->SetForReading(); + stream->SetForReading(); // Tell the protocol to send the stream rProtocol.SendStreamAfterCommand(static_cast< std::auto_ptr<IOStream> >(stream)); } @@ -856,10 +878,10 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByID::DoCommand( // Open the file std::auto_ptr<IOStream> stream(rContext.OpenObject(mObjectID)); - + // Move the file pointer to the block index BackupStoreFile::MoveStreamPositionToBlockIndex(*stream); - + // Return the stream to the client rProtocol.SendStreamAfterCommand(stream); @@ -882,7 +904,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByName::DoComman // Get the directory const BackupStoreDirectory &dir(rContext.GetDirectory(mInDirectory)); - + // Find the latest object ID within it which has the same name int64_t objectID = 0; BackupStoreDirectory::Iterator i(dir); @@ -898,7 +920,7 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByName::DoComman } } } - + // Found anything? if(objectID == 0) { @@ -908,10 +930,10 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetBlockIndexByName::DoComman // Open the file std::auto_ptr<IOStream> stream(rContext.OpenObject(objectID)); - + // Move the file pointer to the block index BackupStoreFile::MoveStreamPositionToBlockIndex(*stream); - + // Return the stream to the client rProtocol.SendStreamAfterCommand(stream); @@ -934,11 +956,11 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetAccountUsage::DoCommand(Ba // Get store info from context const BackupStoreInfo &rinfo(rContext.GetBackupStoreInfo()); - + // Find block size RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(rinfo.GetDiscSetNumber())); - + // Return info return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolAccountUsage( rinfo.GetBlocksUsed(), @@ -968,3 +990,48 @@ std::auto_ptr<BackupProtocolMessage> BackupProtocolGetIsAlive::DoCommand(BackupP // return std::auto_ptr<BackupProtocolMessage>(new BackupProtocolIsAlive()); } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupProtocolGetAccountUsage2::DoCommand(BackupProtocolReplyable &, BackupStoreContext &) +// Purpose: Return the amount of disc space used +// Created: 26/12/13 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupProtocolMessage> BackupProtocolGetAccountUsage2::DoCommand( + BackupProtocolReplyable &rProtocol, BackupStoreContext &rContext) const +{ + CHECK_PHASE(Phase_Commands) + + // Get store info from context + const BackupStoreInfo &info(rContext.GetBackupStoreInfo()); + + // Find block size + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet &rdiscSet(rcontroller.GetDiscSet(info.GetDiscSetNumber())); + + // Return info + BackupProtocolAccountUsage2* usage = new BackupProtocolAccountUsage2(); + std::auto_ptr<BackupProtocolMessage> reply(usage); + #define COPY(name) usage->Set ## name (info.Get ## name ()) + COPY(AccountName); + usage->SetAccountEnabled(info.IsAccountEnabled()); + COPY(ClientStoreMarker); + usage->SetBlockSize(rdiscSet.GetBlockSize()); + COPY(LastObjectIDUsed); + COPY(BlocksUsed); + COPY(BlocksInCurrentFiles); + COPY(BlocksInOldFiles); + COPY(BlocksInDeletedFiles); + COPY(BlocksInDirectories); + COPY(BlocksSoftLimit); + COPY(BlocksHardLimit); + COPY(NumCurrentFiles); + COPY(NumOldFiles); + COPY(NumDeletedFiles); + COPY(NumDirectories); + #undef COPY + + return reply; +} diff --git a/lib/backupstore/BackupConstants.h b/lib/backupstore/BackupConstants.h index 19d06a15..195dc621 100644 --- a/lib/backupstore/BackupConstants.h +++ b/lib/backupstore/BackupConstants.h @@ -13,6 +13,9 @@ // 15 minutes to timeout (milliseconds) #define BACKUP_STORE_TIMEOUT (15*60*1000) +// Time to wait for retry after a backup error +#define BACKUP_ERROR_RETRY_SECONDS 100 + // Should the store daemon convert files to Raid immediately? #define BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY true diff --git a/lib/backupstore/BackupProtocol.h b/lib/backupstore/BackupProtocol.h new file mode 100644 index 00000000..d9070c73 --- /dev/null +++ b/lib/backupstore/BackupProtocol.h @@ -0,0 +1,71 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupProtocol.h +// Purpose: A thin wrapper around autogen_BackupProtocol.h +// Created: 2014/01/05 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPPROTOCOL__H +#define BACKUPPROTOCOL__H + +#include <autogen_BackupProtocol.h> +#include <BackupStoreConstants.h> +#include <BackupStoreContext.h> + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupProtocolLocal2 +// Purpose: BackupProtocolLocal with a few more IQ points +// Created: 2014/09/20 +// +// -------------------------------------------------------------------------- +class BackupProtocolLocal2 : public BackupProtocolLocal +{ +private: + BackupStoreContext mContext; + int32_t mAccountNumber; + bool mReadOnly; + +protected: + BackupStoreContext& GetContext() { return mContext; } + +public: + BackupProtocolLocal2(int32_t AccountNumber, + const std::string& ConnectionDetails, + const std::string& AccountRootDir, int DiscSetNumber, + bool ReadOnly) + // This is rather ugly: the BackupProtocolLocal constructor must not + // touch the Context, because it's not initialised yet! + : BackupProtocolLocal(mContext), + mContext(AccountNumber, (HousekeepingInterface *)NULL, + ConnectionDetails), + mAccountNumber(AccountNumber), + mReadOnly(ReadOnly) + { + mContext.SetClientHasAccount(AccountRootDir, DiscSetNumber); + QueryVersion(BACKUP_STORE_SERVER_VERSION); + QueryLogin(AccountNumber, + ReadOnly ? BackupProtocolLogin::Flags_ReadOnly : 0); + } + virtual ~BackupProtocolLocal2() { } + + std::auto_ptr<BackupProtocolFinished> Query(const BackupProtocolFinished &rQuery) + { + std::auto_ptr<BackupProtocolFinished> finished = + BackupProtocolLocal::Query(rQuery); + mContext.ReleaseWriteLock(); + return finished; + } + + void Reopen() + { + QueryVersion(BACKUP_STORE_SERVER_VERSION); + QueryLogin(mAccountNumber, + mReadOnly ? BackupProtocolLogin::Flags_ReadOnly : 0); + } +}; + +#endif // BACKUPPROTOCOL__H diff --git a/lib/backupstore/backupprotocol.txt b/lib/backupstore/BackupProtocol.txt index aa987e70..5921d009 100644 --- a/lib/backupstore/backupprotocol.txt +++ b/lib/backupstore/BackupProtocol.txt @@ -7,6 +7,7 @@ IdentString Box-Backup:v=C ServerContextClass BackupStoreContext BackupStoreContext.h AddType Filename BackupStoreFilenameClear BackupStoreFilenameClear.h +AddType String std::string ImplementLog Server syslog ImplementLog Client syslog @@ -76,8 +77,7 @@ SetClientStoreMarker 6 Command(Success) GetObject 10 Command(Success) int64 ObjectID - CONSTANT NoObject 0 - # reply has stream following, if ObjectID != NoObject + # reply has stream following (if successful) MoveObject 11 Command(Success) @@ -123,6 +123,14 @@ CreateDirectory 20 Command(Success) StreamWithCommand # stream following containing attributes +CreateDirectory2 46 Command(Success) StreamWithCommand + int64 ContainingDirectoryID + int64 AttributesModTime + int64 ModificationTime + Filename DirectoryName + # stream following containing attributes + + ListDirectory 21 Command(Success) int64 ObjectID int16 FlagsMustBeSet @@ -151,6 +159,7 @@ ChangeDirAttributes 22 Command(Success) StreamWithCommand DeleteDirectory 23 Command(Success) int64 ObjectID + UndeleteDirectory 24 Command(Success) int64 ObjectID # may not have exactly the desired effect if files within in have been deleted before the directory was deleted. @@ -233,3 +242,25 @@ GetIsAlive 42 Command(IsAlive) IsAlive 43 Reply # no data members +GetAccountUsage2 44 Command(AccountUsage2) + # no data members + +AccountUsage2 45 Reply + String AccountName + bool AccountEnabled + int64 ClientStoreMarker + int32 BlockSize + int64 LastObjectIDUsed + int64 BlocksUsed + int64 BlocksInCurrentFiles + int64 BlocksInOldFiles + int64 BlocksInDeletedFiles + int64 BlocksInDirectories + int64 BlocksSoftLimit + int64 BlocksHardLimit + int64 NumCurrentFiles + int64 NumOldFiles + int64 NumDeletedFiles + int64 NumDirectories + +# 46 is CreateDirectory2 diff --git a/lib/backupstore/BackupStoreAccountDatabase.cpp b/lib/backupstore/BackupStoreAccountDatabase.cpp index 201491a3..c5f012fc 100644 --- a/lib/backupstore/BackupStoreAccountDatabase.cpp +++ b/lib/backupstore/BackupStoreAccountDatabase.cpp @@ -247,7 +247,8 @@ void BackupStoreAccountDatabase::Write() { // Write out the entry char line[256]; // more than enough for a couple of integers in string form - int s = ::sprintf(line, "%x:%d\n", i->second.GetID(), i->second.GetDiscSet()); + int s = ::snprintf(line, sizeof(line), "%x:%d\n", + i->second.GetID(), i->second.GetDiscSet()); if(::write(file, line, s) != s) { THROW_EXCEPTION(CommonException, OSFileError) diff --git a/lib/backupstore/BackupStoreAccounts.cpp b/lib/backupstore/BackupStoreAccounts.cpp index 18500fc1..7955b3c4 100644 --- a/lib/backupstore/BackupStoreAccounts.cpp +++ b/lib/backupstore/BackupStoreAccounts.cpp @@ -9,19 +9,29 @@ #include "Box.h" -#include <stdio.h> +#include <algorithm> +#include <climits> +#include <cstdio> +#include <cstring> +#include <iostream> #include "BackupStoreAccounts.h" #include "BackupStoreAccountDatabase.h" +#include "BackupStoreCheck.h" +#include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreInfo.h" #include "BackupStoreRefCountDatabase.h" #include "BoxPortsAndFiles.h" +#include "HousekeepStoreAccount.h" +#include "NamedLock.h" +#include "RaidFileController.h" #include "RaidFileWrite.h" #include "StoreStructure.h" #include "UnixUser.h" +#include "Utils.h" #include "MemLeakFindOn.h" @@ -110,10 +120,7 @@ void BackupStoreAccounts::Create(int32_t ID, int DiscSet, int64_t SizeSoftLimit, info->Save(); // Create the refcount database - BackupStoreRefCountDatabase::CreateNew(Entry); - std::auto_ptr<BackupStoreRefCountDatabase> refcount( - BackupStoreRefCountDatabase::Load(Entry, false)); - refcount->AddReference(BACKUPSTORE_ROOT_DIRECTORY_ID); + BackupStoreRefCountDatabase::Create(Entry)->Commit(); } // As the original user... @@ -151,9 +158,9 @@ void BackupStoreAccounts::GetAccountRoot(int32_t ID, std::string &rRootDirOut, i std::string BackupStoreAccounts::MakeAccountRootDir(int32_t ID, int DiscSet) { char accid[64]; // big enough! - ::sprintf(accid, "%08x" DIRECTORY_SEPARATOR, ID); - return std::string(std::string(BOX_RAIDFILE_ROOT_BBSTORED - DIRECTORY_SEPARATOR) + accid); + ::snprintf(accid, sizeof(accid) - 1, "%08x" DIRECTORY_SEPARATOR, ID); + return std::string(BOX_RAIDFILE_ROOT_BBSTORED DIRECTORY_SEPARATOR) + + accid; } @@ -198,6 +205,391 @@ void BackupStoreAccounts::LockAccount(int32_t ID, NamedLock& rNamedLock) { THROW_EXCEPTION_MESSAGE(BackupStoreException, CouldNotLockStoreAccount, "Failed to get exclusive " - "lock on account " << ID); + "lock on account " << BOX_FORMAT_ACCOUNT(ID)); } } + +int BackupStoreAccountsControl::BlockSizeOfDiscSet(int discSetNum) +{ + // Get controller, check disc set number + RaidFileController &controller(RaidFileController::GetController()); + if(discSetNum < 0 || discSetNum >= controller.GetNumDiscSets()) + { + BOX_FATAL("Disc set " << discSetNum << " does not exist."); + exit(1); + } + + // Return block size + return controller.GetDiscSet(discSetNum).GetBlockSize(); +} + +int BackupStoreAccountsControl::SetLimit(int32_t ID, const char *SoftLimitStr, + const char *HardLimitStr) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change limits."); + return 1; + } + + // Load the info + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, rootDir, + discSetNum, false /* Read/Write */)); + + // Change the limits + int blocksize = BlockSizeOfDiscSet(discSetNum); + int64_t softlimit = SizeStringToBlocks(SoftLimitStr, blocksize); + int64_t hardlimit = SizeStringToBlocks(HardLimitStr, blocksize); + CheckSoftHardLimits(softlimit, hardlimit); + info->ChangeLimits(softlimit, hardlimit); + + // Save + info->Save(); + + BOX_NOTICE("Limits on account " << BOX_FORMAT_ACCOUNT(ID) << + " changed to " << softlimit << " soft, " << + hardlimit << " hard."); + + return 0; +} + +int BackupStoreAccountsControl::SetAccountName(int32_t ID, const std::string& rNewAccountName) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change name."); + return 1; + } + + // Load the info + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, false /* Read/Write */)); + + info->SetAccountName(rNewAccountName); + + // Save + info->Save(); + + BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << + " name changed to " << rNewAccountName); + + return 0; +} + +int BackupStoreAccountsControl::PrintAccountInfo(int32_t ID) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + + if(!OpenAccount(ID, rootDir, discSetNum, user, + NULL /* no write lock needed for this read-only operation */)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to display info."); + return 1; + } + + // Load it in + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, true /* ReadOnly */)); + + return BackupAccountControl::PrintAccountInfo(*info, + BlockSizeOfDiscSet(discSetNum)); +} + +int BackupStoreAccountsControl::SetAccountEnabled(int32_t ID, bool enabled) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " to change enabled flag."); + return 1; + } + + // Load it in + std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(ID, + rootDir, discSetNum, false /* ReadOnly */)); + info->SetAccountEnabled(enabled); + info->Save(); + return 0; +} + +int BackupStoreAccountsControl::DeleteAccount(int32_t ID, bool AskForConfirmation) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + // Obtain a write lock, as the daemon user + if(!OpenAccount(ID, rootDir, discSetNum, user, &writeLock)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for deletion."); + return 1; + } + + // Check user really wants to do this + if(AskForConfirmation) + { + BOX_WARNING("Really delete account " << + BOX_FORMAT_ACCOUNT(ID) << "? (type 'yes' to confirm)"); + char response[256]; + if(::fgets(response, sizeof(response), stdin) == 0 || ::strcmp(response, "yes\n") != 0) + { + BOX_NOTICE("Deletion cancelled."); + return 0; + } + } + + // Back to original user, but write lock is maintained + user.reset(); + + std::auto_ptr<BackupStoreAccountDatabase> db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Delete from account database + db->DeleteEntry(ID); + + // Write back to disc + db->Write(); + + // Remove the store files... + + // First, become the user specified in the config file + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Become the right user + if(!username.empty()) + { + // Username specified, change... + user.reset(new UnixUser(username)); + user->ChangeProcessUser(true /* temporary */); + // Change will be undone when user goes out of scope + } + + // Secondly, work out which directories need wiping + std::vector<std::string> toDelete; + RaidFileController &rcontroller(RaidFileController::GetController()); + RaidFileDiscSet discSet(rcontroller.GetDiscSet(discSetNum)); + for(RaidFileDiscSet::const_iterator i(discSet.begin()); i != discSet.end(); ++i) + { + if(std::find(toDelete.begin(), toDelete.end(), *i) == toDelete.end()) + { + toDelete.push_back((*i) + DIRECTORY_SEPARATOR + rootDir); + } + } + + // NamedLock will throw an exception if it can't delete the lockfile, + // which it can't if it doesn't exist. Now that we've deleted the account, + // nobody can open it anyway, so it's safe to unlock. + writeLock.ReleaseLock(); + + int retcode = 0; + + // Thirdly, delete the directories... + for(std::vector<std::string>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d) + { + BOX_NOTICE("Deleting store directory " << (*d) << "..."); + // Just use the rm command to delete the files +#ifdef WIN32 + std::string cmd("rmdir /s/q "); + std::string dir = *d; + + // rmdir doesn't understand forward slashes, so replace them all. + for(std::string::iterator i = dir.begin(); i != dir.end(); i++) + { + if(*i == '/') + { + *i = '\\'; + } + } + cmd += dir; +#else + std::string cmd("rm -rf "); + cmd += *d; +#endif + // Run command + if(::system(cmd.c_str()) != 0) + { + BOX_ERROR("Failed to delete files in " << (*d) << + ", delete them manually."); + retcode = 1; + } + } + + // Success! + return retcode; +} + +bool BackupStoreAccountsControl::OpenAccount(int32_t ID, std::string &rRootDirOut, + int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock) +{ + // Load in the account database + std::auto_ptr<BackupStoreAccountDatabase> db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Exists? + if(!db->EntryExists(ID)) + { + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " does not exist."); + return false; + } + + // Get info from the database + BackupStoreAccounts acc(*db); + acc.GetAccountRoot(ID, rRootDirOut, rDiscSetOut); + + // Get the user under which the daemon runs + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Become the right user + if(!username.empty()) + { + // Username specified, change... + apUser.reset(new UnixUser(username)); + apUser->ChangeProcessUser(true /* temporary */); + // Change will be undone when apUser goes out of scope + // in the caller. + } + + if(pLock) + { + acc.LockAccount(ID, *pLock); + } + + return true; +} + +int BackupStoreAccountsControl::CheckAccount(int32_t ID, bool FixErrors, bool Quiet, + bool ReturnNumErrorsFound) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + NamedLock writeLock; + + if(!OpenAccount(ID, rootDir, discSetNum, user, + FixErrors ? &writeLock : NULL)) // don't need a write lock if not making changes + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for checking."); + return 1; + } + + // Check it + BackupStoreCheck check(rootDir, discSetNum, ID, FixErrors, Quiet); + check.Check(); + + if(ReturnNumErrorsFound) + { + return check.GetNumErrorsFound(); + } + else + { + return check.ErrorsFound() ? 1 : 0; + } +} + +int BackupStoreAccountsControl::CreateAccount(int32_t ID, int32_t DiscNumber, + int32_t SoftLimit, int32_t HardLimit) +{ + // Load in the account database + std::auto_ptr<BackupStoreAccountDatabase> db( + BackupStoreAccountDatabase::Read( + mConfig.GetKeyValue("AccountDatabase"))); + + // Already exists? + if(db->EntryExists(ID)) + { + BOX_ERROR("Account " << BOX_FORMAT_ACCOUNT(ID) << + " already exists."); + return 1; + } + + // Get the user under which the daemon runs + std::string username; + { + const Configuration &rserverConfig(mConfig.GetSubConfiguration("Server")); + if(rserverConfig.KeyExists("User")) + { + username = rserverConfig.GetKeyValue("User"); + } + } + + // Create it. + BackupStoreAccounts acc(*db); + acc.Create(ID, DiscNumber, SoftLimit, HardLimit, username); + + BOX_NOTICE("Account " << BOX_FORMAT_ACCOUNT(ID) << " created."); + + return 0; +} + +int BackupStoreAccountsControl::HousekeepAccountNow(int32_t ID) +{ + std::string rootDir; + int discSetNum; + std::auto_ptr<UnixUser> user; // used to reset uid when we return + + if(!OpenAccount(ID, rootDir, discSetNum, user, + NULL /* housekeeping locks the account itself */)) + { + BOX_ERROR("Failed to open account " << BOX_FORMAT_ACCOUNT(ID) + << " for housekeeping."); + return 1; + } + + HousekeepStoreAccount housekeeping(ID, rootDir, discSetNum, NULL); + bool success = housekeeping.DoHousekeeping(); + + if(!success) + { + BOX_ERROR("Failed to lock account " << BOX_FORMAT_ACCOUNT(ID) + << " for housekeeping: perhaps a client is " + "still connected?"); + return 1; + } + else + { + BOX_TRACE("Finished housekeeping on account " << + BOX_FORMAT_ACCOUNT(ID)); + return 0; + } +} + diff --git a/lib/backupstore/BackupStoreAccounts.h b/lib/backupstore/BackupStoreAccounts.h index 3163f15c..bcc3cf1c 100644 --- a/lib/backupstore/BackupStoreAccounts.h +++ b/lib/backupstore/BackupStoreAccounts.h @@ -13,6 +13,7 @@ #include <string> #include "BackupStoreAccountDatabase.h" +#include "BackupAccountControl.h" #include "NamedLock.h" // -------------------------------------------------------------------------- @@ -51,5 +52,34 @@ private: BackupStoreAccountDatabase &mrDatabase; }; +class Configuration; +class UnixUser; + +class BackupStoreAccountsControl : public BackupAccountControl +{ +public: + BackupStoreAccountsControl(const Configuration& config, + bool machineReadableOutput = false) + : BackupAccountControl(config, machineReadableOutput) + { } + int BlockSizeOfDiscSet(int discSetNum); + bool OpenAccount(int32_t ID, std::string &rRootDirOut, + int &rDiscSetOut, std::auto_ptr<UnixUser> apUser, NamedLock* pLock); + int SetLimit(int32_t ID, const char *SoftLimitStr, + const char *HardLimitStr); + int SetAccountName(int32_t ID, const std::string& rNewAccountName); + int PrintAccountInfo(int32_t ID); + int SetAccountEnabled(int32_t ID, bool enabled); + int DeleteAccount(int32_t ID, bool AskForConfirmation); + int CheckAccount(int32_t ID, bool FixErrors, bool Quiet, + bool ReturnNumErrorsFound = false); + int CreateAccount(int32_t ID, int32_t DiscNumber, int32_t SoftLimit, + int32_t HardLimit); + int HousekeepAccountNow(int32_t ID); +}; + +// max size of soft limit as percent of hard limit +#define MAX_SOFT_LIMIT_SIZE 97 + #endif // BACKUPSTOREACCOUNTS__H diff --git a/lib/backupstore/BackupStoreCheck.cpp b/lib/backupstore/BackupStoreCheck.cpp index f2302337..b53ebf6d 100644 --- a/lib/backupstore/BackupStoreCheck.cpp +++ b/lib/backupstore/BackupStoreCheck.cpp @@ -17,11 +17,13 @@ #endif #include "autogen_BackupStoreException.h" +#include "BackupStoreAccountDatabase.h" #include "BackupStoreCheck.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreFile.h" #include "BackupStoreObjectMagic.h" +#include "BackupStoreRefCountDatabase.h" #include "RaidFileController.h" #include "RaidFileException.h" #include "RaidFileRead.h" @@ -58,7 +60,7 @@ BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNum mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), mBlocksInDirectories(0), - mNumFiles(0), + mNumCurrentFiles(0), mNumOldFiles(0), mNumDeletedFiles(0), mNumDirectories(0) @@ -78,6 +80,24 @@ BackupStoreCheck::~BackupStoreCheck() { // Clean up FreeInfo(); + + // Avoid an exception if we forget to discard mapNewRefs + if (mapNewRefs.get()) + { + // Discard() can throw exception, but destructors aren't supposed to do that, so + // just catch and log them. + try + { + mapNewRefs->Discard(); + } + catch(BoxException &e) + { + BOX_ERROR("Error while destroying BackupStoreCheck: discarding " + "the refcount database threw an exception: " << e.what()); + } + + mapNewRefs.reset(); + } } @@ -92,15 +112,21 @@ BackupStoreCheck::~BackupStoreCheck() // -------------------------------------------------------------------------- void BackupStoreCheck::Check() { - std::string writeLockFilename; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); - ASSERT(FileExists(writeLockFilename)); + if(mFixErrors) + { + std::string writeLockFilename; + StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename); + ASSERT(FileExists(writeLockFilename)); + } if(!mQuiet && mFixErrors) { - BOX_NOTICE("Will fix errors encountered during checking."); + BOX_INFO("Will fix errors encountered during checking."); } + BackupStoreAccountDatabase::Entry account(mAccountID, mDiscSetNumber); + mapNewRefs = BackupStoreRefCountDatabase::Create(account); + // Phase 1, check objects if(!mQuiet) { @@ -109,14 +135,14 @@ void BackupStoreCheck::Check() BOX_INFO("Phase 1, check objects..."); } CheckObjects(); - + // Phase 2, check directories if(!mQuiet) { BOX_INFO("Phase 2, check directories..."); } CheckDirectories(); - + // Phase 3, check root if(!mQuiet) { @@ -138,16 +164,38 @@ void BackupStoreCheck::Check() } FixDirsWithWrongContainerID(); FixDirsWithLostDirs(); - + // Phase 6, regenerate store info if(!mQuiet) { BOX_INFO("Phase 6, regenerate store info..."); } WriteNewStoreInfo(); - -// DUMP_OBJECT_INFO - + + try + { + std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs = + BackupStoreRefCountDatabase::Load(account, false); + mNumberErrorsFound += mapNewRefs->ReportChangesTo(*apOldRefs); + } + catch(BoxException &e) + { + BOX_WARNING("Reference count database was missing or " + "corrupted, cannot check it for errors."); + mNumberErrorsFound++; + } + + // force file to be saved and closed before releasing the lock below + if(mFixErrors) + { + mapNewRefs->Commit(); + } + else + { + mapNewRefs->Discard(); + } + mapNewRefs.reset(); + if(mNumberErrorsFound > 0) { BOX_WARNING("Finished checking store account ID " << @@ -250,16 +298,18 @@ void BackupStoreCheck::CheckObjects() { // Make sure the starting root dir doesn't end with '/'. std::string start(mStoreRoot); - if(start.size() > 0 && start[start.size() - 1] == '/') + if(start.size() > 0 && ( + start[start.size() - 1] == '/' || + start[start.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR)) { start.resize(start.size() - 1); } - - maxDir = CheckObjectsScanDir(0, 1, mStoreRoot); + + maxDir = CheckObjectsScanDir(0, 1, start); BOX_TRACE("Max dir starting ID is " << BOX_FORMAT_OBJECTID(maxDir)); } - + // Then go through and scan all the objects within those directories for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH)) { @@ -287,24 +337,25 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const // If any of the directories is missing, create it. RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); - + if(!rdiscSet.IsNonRaidSet()) { unsigned int numDiscs = rdiscSet.size(); - + for(unsigned int l = 0; l < numDiscs; ++l) { // build name std::string dn(rdiscSet[l] + DIRECTORY_SEPARATOR + rDirName); - struct stat st; + EMU_STRUCT_STAT st; - if(stat(dn.c_str(), &st) != 0 && errno == ENOENT) + if(EMU_STAT(dn.c_str(), &st) != 0 && + errno == ENOENT) { if(mkdir(dn.c_str(), 0755) != 0) { THROW_SYS_FILE_ERROR("Failed to " "create missing RaidFile " - "directory", dn, + "directory", dn, RaidFileException, OSError); } } @@ -334,7 +385,7 @@ int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const else { BOX_ERROR("Spurious or invalid directory " << - rDirName << DIRECTORY_SEPARATOR << + rDirName << DIRECTORY_SEPARATOR << (*i) << " found, " << (mFixErrors?"deleting":"delete manually")); ++mNumberErrorsFound; @@ -361,11 +412,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) std::string dirName; StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */); // Check expectations - ASSERT(dirName.size() > 4 && + ASSERT(dirName.size() > 4 && dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR); // Remove the filename from it dirName.resize(dirName.size() - 4); // four chars for "/o00" - + // Check directory exists if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName)) { @@ -377,14 +428,14 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) std::vector<std::string> files; RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName, RaidFileRead::DirReadType_FilesOnly, files); - + // Array of things present bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)]; for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l) { idsPresent[l] = false; } - + // Parse each entry, building up a list of object IDs which are present in the dir. // This is done so that whatever order is retured from the directory, objects are scanned // in order. @@ -405,7 +456,8 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) fileOK = false; } // info and refcount databases are OK in the root directory - else if(*i == "info" || *i == "refcount.db") + else if(*i == "info" || *i == "refcount.db" || + *i == "refcount.rdb" || *i == "refcount.rdbX") { fileOK = true; } @@ -413,11 +465,11 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) { fileOK = false; } - + if(!fileOK) { // Unexpected or bad file, delete it - BOX_ERROR("Spurious file " << dirName << + BOX_ERROR("Spurious file " << dirName << DIRECTORY_SEPARATOR << (*i) << " found" << (mFixErrors?", deleting":"")); ++mNumberErrorsFound; @@ -428,7 +480,7 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) } } } - + // Check all the objects found in this directory for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i) { @@ -436,7 +488,8 @@ void BackupStoreCheck::CheckObjectsDir(int64_t StartID) { // Check the object is OK, and add entry char leaf[8]; - ::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i); + ::snprintf(leaf, sizeof(leaf), + DIRECTORY_SEPARATOR "o%02x", i); if(!CheckAndAddObject(StartID | i, dirName + leaf)) { // File was bad, delete it @@ -480,7 +533,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, std::auto_ptr<RaidFileRead> file( RaidFileRead::Open(mDiscSetNumber, rFilename)); size = file->GetDiscUsageInBlocks(); - + // Read in first four bytes -- don't have to worry about // retrying if not all bytes read as is RaidFile uint32_t signature; @@ -491,7 +544,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, } // Seek back to beginning file->Seek(0, IOStream::SeekType_Absolute); - + // Then... check depending on the type switch(ntohl(signature)) { @@ -519,7 +572,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, // Error caught, not a good file then, let it be deleted return false; } - + // Got a container ID? (ie check was successful) if(containerID == -1) { @@ -538,13 +591,13 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, // If it looks like a good object, and it's non-RAID, and // this is a RAID set, then convert it to RAID. - + RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(mDiscSetNumber)); if(!rdiscSet.IsNonRaidSet()) { // See if the file exists - RaidFileUtil::ExistType existance = + RaidFileUtil::ExistType existance = RaidFileUtil::RaidFileExists(rdiscSet, rFilename); if(existance == RaidFileUtil::NonRaid) { @@ -565,7 +618,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, if(mFixErrors) { std::auto_ptr<RaidFileRead> read( - RaidFileRead::Open(mDiscSetNumber, + RaidFileRead::Open(mDiscSetNumber, rFilename)); RaidFileWrite write(mDiscSetNumber, rFilename); write.Open(true /* overwrite */); @@ -575,7 +628,7 @@ bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID, } } } - + // Report success return true; } @@ -636,7 +689,7 @@ int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream) // Wrong object ID return -1; } - + // Return container ID return dir.GetContainerID(); } @@ -669,7 +722,7 @@ void BackupStoreCheck::CheckDirectories() { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; - + for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); @@ -703,9 +756,9 @@ void BackupStoreCheck::CheckDirectories() BOX_FORMAT_OBJECTID(pblock->mID[e]) << " was OK after fixing"); } - + if(isModified && mFixErrors) - { + { BOX_WARNING("Writing modified directory to disk: " << BOX_FORMAT_OBJECTID(pblock->mID[e])); RaidFileWrite fixed(mDiscSetNumber, filename); @@ -714,64 +767,7 @@ void BackupStoreCheck::CheckDirectories() fixed.Commit(true /* convert to raid representation now */); } - // Count valid entries - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next()) != 0) - { - int32_t iIndex; - IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); - - ASSERT(piBlock != 0 || - mDirsWhichContainLostDirs.find(en->GetObjectID()) - != mDirsWhichContainLostDirs.end()); - if (piBlock) - { - // Normally it would exist and this - // check would not be necessary, but - // we might have missing directories - // that we will recreate later. - // cf mDirsWhichContainLostDirs. - uint8_t iflags = GetFlags(piBlock, iIndex); - SetFlags(piBlock, iIndex, iflags | Flags_IsContained); - } - - if(en->IsDir()) - { - mNumDirectories++; - } - else if(!en->IsFile()) - { - BOX_TRACE("Not counting object " << - BOX_FORMAT_OBJECTID(en->GetObjectID()) << - " with flags " << en->GetFlags()); - } - else // it's a good file, add to sizes - if(en->IsOld() && en->IsDeleted()) - { - BOX_WARNING("File " << - BOX_FORMAT_OBJECTID(en->GetObjectID()) << - " is both old and deleted, " - "this should not happen!"); - } - else if(en->IsOld()) - { - mNumFiles++; - mNumOldFiles++; - mBlocksInOldFiles += en->GetSizeInBlocks(); - } - else if(en->IsDeleted()) - { - mNumFiles++; - mNumDeletedFiles++; - mBlocksInDeletedFiles += en->GetSizeInBlocks(); - } - else - { - mNumFiles++; - mBlocksInCurrentFiles += en->GetSizeInBlocks(); - } - } + CountDirectoryEntries(dir); } } } @@ -798,8 +794,8 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) ++mNumberErrorsFound; isModified = true; } - - // Go through, and check that everything in that directory exists and is valid + + // Go through, and check that every entry exists and is valid BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) @@ -824,14 +820,14 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) { // Just remove the entry badEntry = true; - BOX_ERROR("Directory ID " << + BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(dir.GetObjectID()) << - " references object " << + " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist."); ++mNumberErrorsFound; } - + // Is this entry worth keeping? if(badEntry) { @@ -846,16 +842,16 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) { BOX_ERROR("Removing directory entry " << BOX_FORMAT_OBJECTID(*d) << " from " - "directory " << + "directory " << BOX_FORMAT_OBJECTID(dir.GetObjectID())); ++mNumberErrorsFound; dir.DeleteEntry(*d); } - + // Mark as modified restart = true; isModified = true; - + // Errors found } } @@ -863,6 +859,84 @@ bool BackupStoreCheck::CheckDirectory(BackupStoreDirectory& dir) return isModified; } +// Count valid remaining entries and the number of blocks in them. +void BackupStoreCheck::CountDirectoryEntries(BackupStoreDirectory& dir) +{ + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + int32_t iIndex; + IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex); + bool badEntry = false; + bool wasAlreadyContained = false; + + ASSERT(piBlock != 0 || + mDirsWhichContainLostDirs.find(en->GetObjectID()) + != mDirsWhichContainLostDirs.end()); + + if (piBlock) + { + // Normally it would exist and this + // check would not be necessary, but + // we might have missing directories + // that we will recreate later. + // cf mDirsWhichContainLostDirs. + uint8_t iflags = GetFlags(piBlock, iIndex); + wasAlreadyContained = (iflags & Flags_IsContained); + SetFlags(piBlock, iIndex, iflags | Flags_IsContained); + } + + if(wasAlreadyContained) + { + // don't double-count objects that are + // contained by another directory as well. + } + else if(en->IsDir()) + { + mNumDirectories++; + } + else if(!en->IsFile()) + { + BOX_TRACE("Not counting object " << + BOX_FORMAT_OBJECTID(en->GetObjectID()) << + " with flags " << en->GetFlags()); + } + else // it's a file + { + // Add to sizes? + // If piBlock was zero, then wasAlreadyContained + // might be uninitialized; but we only process + // files here, and if a file's piBlock was zero + // then badEntry would be set above, so we + // wouldn't be here. + ASSERT(!badEntry) + + // It can be both old and deleted. + // If neither, then it's current. + if(en->IsDeleted()) + { + mNumDeletedFiles++; + mBlocksInDeletedFiles += en->GetSizeInBlocks(); + } + + if(en->IsOld()) + { + mNumOldFiles++; + mBlocksInOldFiles += en->GetSizeInBlocks(); + } + + if(!en->IsDeleted() && !en->IsOld()) + { + mNumCurrentFiles++; + mBlocksInCurrentFiles += en->GetSizeInBlocks(); + } + } + + mapNewRefs->AddReference(en->GetObjectID()); + } +} + bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified) { @@ -871,8 +945,7 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, ASSERT(piBlock != 0); uint8_t iflags = GetFlags(piBlock, IndexInDirBlock); - bool badEntry = false; - + // Is the type the same? if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir()) { @@ -882,73 +955,67 @@ bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which has a different type than expected."); - badEntry = true; ++mNumberErrorsFound; + return false; // remove this entry } + // Check that the entry is not already contained. - else if(iflags & Flags_IsContained) + if(iflags & Flags_IsContained) { BOX_ERROR("Directory ID " << BOX_FORMAT_OBJECTID(DirectoryID) << " references object " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << " which is already contained."); - badEntry = true; ++mNumberErrorsFound; + return false; // remove this entry } - else + + // Not already contained by another directory. + // Don't set the flag until later, after we finish repairing + // the directory and removing all bad entries. + + // Check that the container ID of the object is correct + if(piBlock->mContainer[IndexInDirBlock] != DirectoryID) { - // Not already contained by another directory. - // Don't set the flag until later, after we finish repairing - // the directory and removing all bad entries. - - // Check that the container ID of the object is correct - if(piBlock->mContainer[IndexInDirBlock] != DirectoryID) + // Needs fixing... + if(iflags & Flags_IsDir) { - // Needs fixing... - if(iflags & Flags_IsDir) - { - // Add to will fix later list - BOX_ERROR("Directory ID " << - BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) - << " has wrong container ID."); - mDirsWithWrongContainerID.push_back(rEntry.GetObjectID()); - ++mNumberErrorsFound; - } - else - { - // This is OK for files, they might move - BOX_NOTICE("File ID " << - BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) - << " has different container ID, " - "probably moved"); - } - - // Fix entry for now - piBlock->mContainer[IndexInDirBlock] = DirectoryID; + // Add to will fix later list + BOX_ERROR("Directory ID " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) + << " has wrong container ID."); + mDirsWithWrongContainerID.push_back(rEntry.GetObjectID()); + ++mNumberErrorsFound; + } + else + { + // This is OK for files, they might move + BOX_INFO("File ID " << + BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) + << " has different container ID, " + "probably moved"); } + + // Fix entry for now + piBlock->mContainer[IndexInDirBlock] = DirectoryID; } - - // Check the object size, if it's OK and a file - if(!badEntry && !rEntry.IsDir()) + + // Check the object size + if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock]) { - if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock]) - { - // Wrong size, correct it. - rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]); + // Wrong size, correct it. + BOX_ERROR("Directory " << BOX_FORMAT_OBJECTID(DirectoryID) << + " entry for " << BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) << + " has wrong size " << rEntry.GetSizeInBlocks() << + ", should be " << piBlock->mObjectSizeInBlocks[IndexInDirBlock]); - // Mark as changed - rIsModified = true; - ++mNumberErrorsFound; + rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]); - // Tell user - BOX_ERROR("Directory ID " << - BOX_FORMAT_OBJECTID(DirectoryID) << - " has wrong size for object " << - BOX_FORMAT_OBJECTID(rEntry.GetObjectID())); - } + // Mark as changed + rIsModified = true; + ++mNumberErrorsFound; } - return !badEntry; + return true; // don't delete this entry } - diff --git a/lib/backupstore/BackupStoreCheck.h b/lib/backupstore/BackupStoreCheck.h index 178a873a..5353c968 100644 --- a/lib/backupstore/BackupStoreCheck.h +++ b/lib/backupstore/BackupStoreCheck.h @@ -20,6 +20,7 @@ class IOStream; class BackupStoreFilename; +class BackupStoreRefCountDatabase; /* @@ -130,8 +131,9 @@ private: bool CheckDirectory(BackupStoreDirectory& dir); bool CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry, int64_t DirectoryID, bool& rIsModified); + void CountDirectoryEntries(BackupStoreDirectory& dir); int64_t CheckFile(int64_t ObjectID, IOStream &rStream); - int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); + int64_t CheckDirInitial(int64_t ObjectID, IOStream &rStream); // Fixing functions bool TryToRecreateDirectory(int64_t MissingDirectoryID); @@ -195,6 +197,9 @@ private: // Set of extra directories added std::set<BackupStoreCheck_ID_t> mDirsAdded; + + // The refcount database, being reconstructed as the check/fix progresses + std::auto_ptr<BackupStoreRefCountDatabase> mapNewRefs; // Misc stuff int32_t mLostDirNameSerial; @@ -206,7 +211,7 @@ private: int64_t mBlocksInOldFiles; int64_t mBlocksInDeletedFiles; int64_t mBlocksInDirectories; - int64_t mNumFiles; + int64_t mNumCurrentFiles; int64_t mNumOldFiles; int64_t mNumDeletedFiles; int64_t mNumDirectories; diff --git a/lib/backupstore/BackupStoreCheck2.cpp b/lib/backupstore/BackupStoreCheck2.cpp index 90e21e7f..13831a09 100644 --- a/lib/backupstore/BackupStoreCheck2.cpp +++ b/lib/backupstore/BackupStoreCheck2.cpp @@ -20,6 +20,7 @@ #include "BackupStoreFileWire.h" #include "BackupStoreInfo.h" #include "BackupStoreObjectMagic.h" +#include "BackupStoreRefCountDatabase.h" #include "MemBlockStream.h" #include "RaidFileRead.h" #include "RaidFileWrite.h" @@ -40,7 +41,7 @@ void BackupStoreCheck::CheckRoot() { int32_t index = 0; IDBlock *pblock = LookupID(BACKUPSTORE_ROOT_DIRECTORY_ID, index); - + if(pblock != 0) { // Found it. Which is lucky. Mark it as contained. @@ -49,9 +50,9 @@ void BackupStoreCheck::CheckRoot() else { BOX_WARNING("Root directory doesn't exist"); - + ++mNumberErrorsFound; - + if(mFixErrors) { // Create a new root directory @@ -78,7 +79,7 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain } BackupStoreDirectory dir(DirectoryID, ContainingDirID); - + // Serialise to disc std::string filename; StoreStructure::MakeObjectFilename(DirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); @@ -87,10 +88,10 @@ void BackupStoreCheck::CreateBlankDirectory(int64_t DirectoryID, int64_t Contain dir.WriteToStream(obj); int64_t size = obj.GetDiscUsageInBlocks(); obj.Commit(true /* convert to raid now */); - + // Record the fact we've done this mDirsAdded.insert(DirectoryID); - + // Add to sizes mBlocksUsed += size; mBlocksInDirectories += size; @@ -131,15 +132,16 @@ void BackupStoreCheck::CheckUnattachedObjects() { IDBlock *pblock = i->second; int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE; - + for(int e = 0; e < bentries; ++e) { uint8_t flags = GetFlags(pblock, e); if((flags & Flags_IsContained) == 0) { // Unattached object... + int64_t ObjectID = pblock->mID[e]; BOX_ERROR("Object " << - BOX_FORMAT_OBJECTID(pblock->mID[e]) << + BOX_FORMAT_OBJECTID(ObjectID) << " is unattached."); ++mNumberErrorsFound; @@ -149,6 +151,8 @@ void BackupStoreCheck::CheckUnattachedObjects() if((flags & Flags_IsDir) == Flags_IsDir) { // Directory. Just put into lost and found. + // (It doesn't contain its filename, so we + // can't recreate the entry in the parent) putIntoDirectoryID = GetLostAndFoundDirID(); } else @@ -157,7 +161,9 @@ void BackupStoreCheck::CheckUnattachedObjects() { int64_t diffFromObjectID = 0; std::string filename; - StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* don't attempt to make sure the dir exists */); + StoreStructure::MakeObjectFilename(ObjectID, + mStoreRoot, mDiscSetNumber, filename, + false /* don't attempt to make sure the dir exists */); // The easiest way to do this is to verify it again. Not such a bad penalty, because // this really shouldn't be done very often. @@ -170,20 +176,22 @@ void BackupStoreCheck::CheckUnattachedObjects() // Just delete it to be safe. if(diffFromObjectID != 0) { - BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " is unattached, and is a patch. Deleting, cannot reliably recover."); - + BOX_WARNING("Object " << BOX_FORMAT_OBJECTID(ObjectID) << " is unattached, and is a patch. Deleting, cannot reliably recover."); + // Delete this object instead if(mFixErrors) { RaidFileWrite del(mDiscSetNumber, filename); del.Delete(); } - + + mBlocksUsed -= pblock->mObjectSizeInBlocks[e]; + // Move on to next item continue; } } - + // Files contain their original filename, so perhaps the orginal directory still exists, // or we can infer the existance of a directory? // Look for a matching entry in the mDirsWhichContainLostDirs map. @@ -249,9 +257,10 @@ void BackupStoreCheck::CheckUnattachedObjects() } // Add it to the directory - pFixer->InsertObject(pblock->mID[e], + pFixer->InsertObject(ObjectID, ((flags & Flags_IsDir) == Flags_IsDir), lostDirNameSerial); + mapNewRefs->AddReference(ObjectID); } } } @@ -284,7 +293,7 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) // Not a missing directory, can't recreate. return false; } - + // Can recreate this! Wooo! if(!mFixErrors) { @@ -297,12 +306,12 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) BOX_WARNING("Recreating missing directory " << BOX_FORMAT_OBJECTID(MissingDirectoryID)); - + // Create a blank directory BackupStoreDirectory dir(MissingDirectoryID, missing->second /* containing dir ID */); // Note that this directory already contains a directory entry pointing to // this dir, so it doesn't have to be added. - + // Serialise to disc std::string filename; StoreStructure::MakeObjectFilename(MissingDirectoryID, mStoreRoot, mDiscSetNumber, filename, true /* make sure the dir exists */); @@ -310,10 +319,10 @@ bool BackupStoreCheck::TryToRecreateDirectory(int64_t MissingDirectoryID) root.Open(false /* don't allow overwriting */); dir.WriteToStream(root); root.Commit(true /* convert to raid now */); - + // Record the fact we've done this mDirsAdded.insert(MissingDirectoryID); - + // Remove the entry from the map, so this doesn't happen again mDirsWhichContainLostDirs.erase(missing); @@ -328,7 +337,7 @@ BackupStoreDirectoryFixer::BackupStoreDirectoryFixer(std::string storeRoot, // Generate filename StoreStructure::MakeObjectFilename(ID, mStoreRoot, mDiscSetNumber, mFilename, false /* don't make sure the dir exists */); - + // Read it in std::auto_ptr<RaidFileRead> file( RaidFileRead::Open(mDiscSetNumber, mFilename)); @@ -347,7 +356,7 @@ void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, { // Directory -- simply generate a name for it. char name[32]; - ::sprintf(name, "dir%08x", lostDirNameSerial); + ::snprintf(name, sizeof(name), "dir%08x", lostDirNameSerial); objectStoreFilename.SetAsClearFilename(name); } else @@ -370,7 +379,7 @@ void BackupStoreDirectoryFixer::InsertObject(int64_t ObjectID, bool IsDirectory, (ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 -#endif +#endif )) { // This should never happen, everything has been @@ -393,7 +402,7 @@ BackupStoreDirectoryFixer::~BackupStoreDirectoryFixer() { // Fix any flags which have been broken, which there's a good chance of doing mDirectory.CheckAndFix(); - + // Write it out RaidFileWrite root(mDiscSetNumber, mFilename); root.Open(true /* allow overwriting */); @@ -438,7 +447,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() while(true) { char name[32]; - ::sprintf(name, "lost+found%d", n++); + ::snprintf(name, sizeof(name), "lost+found%d", n++); lostAndFound.SetAsClearFilename(name); if(!dir.NameInUse(lostAndFound)) { @@ -453,7 +462,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() // Create a blank directory CreateBlankDirectory(id, BACKUPSTORE_ROOT_DIRECTORY_ID); - + // Add an entry for it dir.AddEntry(lostAndFound, 0, id, 0, BackupStoreDirectory::Entry::Flags_Dir, 0); @@ -462,7 +471,7 @@ int64_t BackupStoreCheck::GetLostAndFoundDirID() root.Open(true /* allow overwriting */); dir.WriteToStream(root); root.Commit(true /* convert to raid now */); - + // Store mLostAndFoundDirectoryID = id; @@ -494,7 +503,7 @@ void BackupStoreCheck::FixDirsWithWrongContainerID() int32_t index = 0; IDBlock *pblock = LookupID(*i, index); if(pblock == 0) continue; - + // Load in BackupStoreDirectory dir; std::string filename; @@ -506,7 +515,7 @@ void BackupStoreCheck::FixDirsWithWrongContainerID() // Adjust container ID dir.SetContainerID(pblock->mContainer[index]); - + // Write it out RaidFileWrite root(mDiscSetNumber, filename); root.Open(true /* allow overwriting */); @@ -539,7 +548,7 @@ void BackupStoreCheck::FixDirsWithLostDirs() int32_t index = 0; IDBlock *pblock = LookupID(i->second, index); if(pblock == 0) continue; - + // Load in BackupStoreDirectory dir; std::string filename; @@ -551,10 +560,10 @@ void BackupStoreCheck::FixDirsWithLostDirs() // Delete the dodgy entry dir.DeleteEntry(i->first); - + // Fix it up dir.CheckAndFix(); - + // Write it out RaidFileWrite root(mDiscSetNumber, filename); root.Open(true /* allow overwriting */); @@ -587,49 +596,42 @@ void BackupStoreCheck::WriteNewStoreInfo() ++mNumberErrorsFound; } - BOX_NOTICE("Total files: " << mNumFiles << " (of which " + BOX_INFO("Current files: " << mNumCurrentFiles << ", " "old files: " << mNumOldFiles << ", " - "deleted files: " << mNumDeletedFiles << "), " + "deleted files: " << mNumDeletedFiles << ", " "directories: " << mNumDirectories); - // Minimum soft and hard limits + // Minimum soft and hard limits to ensure that nothing gets deleted + // by housekeeping. int64_t minSoft = ((mBlocksUsed * 11) / 10) + 1024; int64_t minHard = ((minSoft * 11) / 10) + 1024; - // Need to do anything? - if(pOldInfo.get() != 0 && - mNumberErrorsFound == 0 && - pOldInfo->GetAccountID() == mAccountID) - { - // Leave the store info as it is, no need to alter it because nothing really changed, - // and the only essential thing was that the account ID was correct, which is was. - return; - } - - // NOTE: We will always build a new store info, so the client store marker gets changed. + int64_t softLimit = pOldInfo.get() ? pOldInfo->GetBlocksSoftLimit() : minSoft; + int64_t hardLimit = pOldInfo.get() ? pOldInfo->GetBlocksHardLimit() : minHard; - // Work out the new limits - int64_t softLimit = minSoft; - int64_t hardLimit = minHard; - if(pOldInfo.get() != 0 && pOldInfo->GetBlocksSoftLimit() > minSoft) + if(mNumberErrorsFound && pOldInfo.get()) { - softLimit = pOldInfo->GetBlocksSoftLimit(); - } - else - { - BOX_WARNING("Soft limit for account changed to ensure " - "housekeeping doesn't delete files on next run."); - } - if(pOldInfo.get() != 0 && pOldInfo->GetBlocksHardLimit() > minHard) - { - hardLimit = pOldInfo->GetBlocksHardLimit(); - } - else - { - BOX_WARNING("Hard limit for account changed to ensure " - "housekeeping doesn't delete files on next run."); + if(pOldInfo->GetBlocksSoftLimit() > minSoft) + { + softLimit = pOldInfo->GetBlocksSoftLimit(); + } + else + { + BOX_WARNING("Soft limit for account changed to ensure " + "housekeeping doesn't delete files on next run."); + } + + if(pOldInfo->GetBlocksHardLimit() > minHard) + { + hardLimit = pOldInfo->GetBlocksHardLimit(); + } + else + { + BOX_WARNING("Hard limit for account changed to ensure " + "housekeeping doesn't delete files on next run."); + } } - + // Object ID int64_t lastObjID = mLastIDInInfo; if(mLostAndFoundDirectoryID != 0) @@ -662,11 +664,24 @@ void BackupStoreCheck::WriteNewStoreInfo() hardLimit, (pOldInfo.get() ? pOldInfo->IsAccountEnabled() : true), *extra_data)); - info->AdjustNumFiles(mNumFiles); + info->AdjustNumCurrentFiles(mNumCurrentFiles); info->AdjustNumOldFiles(mNumOldFiles); info->AdjustNumDeletedFiles(mNumDeletedFiles); info->AdjustNumDirectories(mNumDirectories); + // If there are any errors (apart from wrong block counts), then we + // should reset the ClientStoreMarker to zero, which + // CreateForRegeneration does. But if there are no major errors, then + // we should maintain the old ClientStoreMarker, to avoid invalidating + // the client's directory cache. + if (pOldInfo.get() && !mNumberErrorsFound) + { + BOX_INFO("No major errors found, preserving old " + "ClientStoreMarker: " << + pOldInfo->GetClientStoreMarker()); + info->SetClientStoreMarker(pOldInfo->GetClientStoreMarker()); + } + if(pOldInfo.get()) { mNumberErrorsFound += info->ReportChangesTo(*pOldInfo); @@ -676,7 +691,7 @@ void BackupStoreCheck::WriteNewStoreInfo() if(mFixErrors) { info->Save(); - BOX_NOTICE("New store info file written successfully."); + BOX_INFO("New store info file written successfully."); } } @@ -695,7 +710,7 @@ void BackupStoreCheck::WriteNewStoreInfo() bool BackupStoreDirectory::CheckAndFix() { bool changed = false; - + // Check that if a file depends on a new version, that version is in this directory bool restart; @@ -718,11 +733,11 @@ bool BackupStoreDirectory::CheckAndFix() "on newer version " << FMT_OID(dependsNewer) << " which doesn't exist"); - + // Remove delete *i; mEntries.erase(i); - + // Mark as changed changed = true; @@ -751,7 +766,7 @@ bool BackupStoreDirectory::CheckAndFix() } } while(restart); - + // Check that if a file has a dependency marked, it exists, and remove it if it doesn't { std::vector<Entry*>::iterator i(mEntries.begin()); @@ -768,7 +783,7 @@ bool BackupStoreDirectory::CheckAndFix() "info cleared"); (*i)->SetDependsOlder(0); - + // Mark as changed changed = true; } @@ -780,7 +795,7 @@ bool BackupStoreDirectory::CheckAndFix() { // Reset change marker ch = false; - + // Search backwards -- so see newer versions first std::vector<Entry*>::iterator i(mEntries.end()); if(i == mEntries.begin()) @@ -806,10 +821,8 @@ bool BackupStoreDirectory::CheckAndFix() } else { - bool isDir = (((*i)->GetFlags() & Entry::Flags_Dir) == Entry::Flags_Dir); - // Check mutually exclusive flags - if(isDir && (((*i)->GetFlags() & Entry::Flags_File) == Entry::Flags_File)) + if((*i)->IsDir() && (*i)->IsFile()) { // Bad! Unset the file flag BOX_TRACE("Entry " << FMT_i << @@ -863,29 +876,29 @@ bool BackupStoreDirectory::CheckAndFix() } } } - + if(removeEntry) { // Mark something as changed, in loop ch = true; - + // Mark something as globally changed changed = true; - + // erase the thing from the list Entry *pentry = (*i); mEntries.erase(i); // And delete the entry object delete pentry; - + // Stop going around this loop, as the iterator is now invalid break; } } while(i != mEntries.begin()); } while(ch != false); - + return changed; } diff --git a/lib/backupstore/BackupStoreContext.cpp b/lib/backupstore/BackupStoreContext.cpp index 2c98b1d7..1a782df4 100644 --- a/lib/backupstore/BackupStoreContext.cpp +++ b/lib/backupstore/BackupStoreContext.cpp @@ -30,13 +30,14 @@ #include "MemLeakFindOn.h" -// Maximum number of directories to keep in the cache -// When the cache is bigger than this, everything gets -// deleted. +// Maximum number of directories to keep in the cache When the cache is bigger +// than this, everything gets deleted. In tests, we set the cache size to zero +// to ensure that it's always flushed, which is very inefficient but helps to +// catch programming errors (use of freed data). #ifdef BOX_RELEASE_BUILD #define MAX_CACHE_SIZE 32 #else - #define MAX_CACHE_SIZE 2 + #define MAX_CACHE_SIZE 0 #endif // Allow the housekeeping process 4 seconds to release an account @@ -54,19 +55,22 @@ // // -------------------------------------------------------------------------- BackupStoreContext::BackupStoreContext(int32_t ClientID, - HousekeepingInterface &rDaemon, const std::string& rConnectionDetails) - : mConnectionDetails(rConnectionDetails), - mClientID(ClientID), - mrDaemon(rDaemon), - mProtocolPhase(Phase_START), - mClientHasAccount(false), - mStoreDiscSet(-1), - mReadOnly(true), - mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), - mpTestHook(NULL) + HousekeepingInterface* pHousekeeping, const std::string& rConnectionDetails) +: mConnectionDetails(rConnectionDetails), + mClientID(ClientID), + mpHousekeeping(pHousekeeping), + mProtocolPhase(Phase_START), + mClientHasAccount(false), + mStoreDiscSet(-1), + mReadOnly(true), + mSaveStoreInfoDelay(STORE_INFO_SAVE_DELAY), + mpTestHook(NULL) +// If you change the initialisers, be sure to update +// BackupStoreContext::ReceivedFinishCommand as well! { } + // -------------------------------------------------------------------------- // // Function @@ -77,11 +81,19 @@ BackupStoreContext::BackupStoreContext(int32_t ClientID, // -------------------------------------------------------------------------- BackupStoreContext::~BackupStoreContext() { + ClearDirectoryCache(); +} + + +void BackupStoreContext::ClearDirectoryCache() +{ // Delete the objects in the cache - for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); + i != mDirectoryCache.end(); ++i) { delete (i->second); } + mDirectoryCache.clear(); } @@ -103,6 +115,7 @@ void BackupStoreContext::CleanUp() } } + // -------------------------------------------------------------------------- // // Function @@ -118,6 +131,20 @@ void BackupStoreContext::ReceivedFinishCommand() // Save the store info, not delayed SaveStoreInfo(false); } + + // Just in case someone wants to reuse a local protocol object, + // put the context back to its initial state. + mProtocolPhase = BackupStoreContext::Phase_Version; + + // Avoid the need to check version again, by not resetting + // mClientHasAccount, mAccountRootDir or mStoreDiscSet + + mReadOnly = true; + mSaveStoreInfoDelay = STORE_INFO_SAVE_DELAY; + mpTestHook = NULL; + mapStoreInfo.reset(); + mapRefCount.reset(); + ClearDirectoryCache(); } @@ -133,19 +160,19 @@ bool BackupStoreContext::AttemptToGetWriteLock() { // Make the filename of the write lock file std::string writeLockFile; - StoreStructure::MakeWriteLockFilename(mStoreRoot, mStoreDiscSet, writeLockFile); + StoreStructure::MakeWriteLockFilename(mAccountRootDir, mStoreDiscSet, writeLockFile); // Request the lock bool gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - - if(!gotLock) + + if(!gotLock && mpHousekeeping) { // The housekeeping process might have the thing open -- ask it to stop char msg[256]; - int msgLen = sprintf(msg, "r%x\n", mClientID); + int msgLen = snprintf(msg, sizeof(msg), "r%x\n", mClientID); // Send message - mrDaemon.SendMessageToHousekeepingProcess(msg, msgLen); - + mpHousekeeping->SendMessageToHousekeepingProcess(msg, msgLen); + // Then try again a few times int tries = MAX_WAIT_FOR_HOUSEKEEPING_TO_RELEASE_ACCOUNT; do @@ -153,16 +180,16 @@ bool BackupStoreContext::AttemptToGetWriteLock() ::sleep(1 /* second */); --tries; gotLock = mWriteLock.TryAndGetLock(writeLockFile.c_str(), 0600 /* restrictive file permissions */); - + } while(!gotLock && tries > 0); } - + if(gotLock) { // Got the lock, mark as not read only mReadOnly = false; } - + return gotLock; } @@ -181,16 +208,16 @@ void BackupStoreContext::LoadStoreInfo() { THROW_EXCEPTION(BackupStoreException, StoreInfoAlreadyLoaded) } - + // Load it up! - std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mStoreRoot, mStoreDiscSet, mReadOnly)); - + std::auto_ptr<BackupStoreInfo> i(BackupStoreInfo::Load(mClientID, mAccountRootDir, mStoreDiscSet, mReadOnly)); + // Check it if(i->GetAccountID() != mClientID) { THROW_EXCEPTION(BackupStoreException, StoreInfoForWrongAccount) } - + // Keep the pointer to it mapStoreInfo = i; @@ -203,12 +230,11 @@ void BackupStoreContext::LoadStoreInfo() } catch(BoxException &e) { - BOX_WARNING("Reference count database is missing or corrupted, " - "creating a new one, expect housekeeping to find and " - "fix problems with reference counts later."); - - BackupStoreRefCountDatabase::CreateForRegeneration(account); - mapRefCount = BackupStoreRefCountDatabase::Load(account, false); + THROW_EXCEPTION_MESSAGE(BackupStoreException, + CorruptReferenceCountDatabase, "Reference count " + "database is missing or corrupted, cannot safely open " + "account. Housekeeping will fix this automatically " + "when it next runs."); } } @@ -227,6 +253,7 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -242,7 +269,7 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) } } - // Want to save now + // Want to save now mapStoreInfo->Save(); // Set count for next delay @@ -263,83 +290,111 @@ void BackupStoreContext::SaveStoreInfo(bool AllowDelay) void BackupStoreContext::MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists) { // Delegate to utility function - StoreStructure::MakeObjectFilename(ObjectID, mStoreRoot, mStoreDiscSet, rOutput, EnsureDirectoryExists); + StoreStructure::MakeObjectFilename(ObjectID, mAccountRootDir, mStoreDiscSet, rOutput, EnsureDirectoryExists); } // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::GetDirectoryInternal(int64_t) -// Purpose: Return a reference to a directory. Valid only until the -// next time a function which affects directories is called. -// Mainly this funciton, and creation of files. -// Private version of this, which returns non-const directories. +// Name: BackupStoreContext::GetDirectoryInternal(int64_t, +// bool) +// Purpose: Return a reference to a directory. Valid only until +// the next time a function which affects directories +// is called. Mainly this function, and creation of +// files. Private version of this, which returns +// non-const directories. Unless called with +// AllowFlushCache == false, the cache may be flushed, +// invalidating all directory references that you may +// be holding, so beware. // Created: 2003/09/02 // // -------------------------------------------------------------------------- -BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) +BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID, + bool AllowFlushCache) { // Get the filename std::string filename; MakeObjectFilename(ObjectID, filename); - + int64_t oldRevID = 0, newRevID = 0; + // Already in cache? std::map<int64_t, BackupStoreDirectory*>::iterator item(mDirectoryCache.find(ObjectID)); - if(item != mDirectoryCache.end()) - { - // Check the revision ID of the file -- does it need refreshing? - int64_t revID = 0; - if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &revID)) - { - THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) - } - - if(revID == item->second->GetRevisionID()) + if(item != mDirectoryCache.end()) { +#ifndef BOX_RELEASE_BUILD // it might be in the cache, but invalidated + // in which case, delete it instead of returning it. + if(!item->second->IsInvalidated()) +#else + if(true) +#endif { - // Looks good... return the cached object - BOX_TRACE("Returning object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " from cache, modtime = " << revID); - return *(item->second); + oldRevID = item->second->GetRevisionID(); + + // Check the revision ID of the file -- does it need refreshing? + if(!RaidFileRead::FileExists(mStoreDiscSet, filename, &newRevID)) + { + THROW_EXCEPTION(BackupStoreException, DirectoryHasBeenDeleted) + } + + if(newRevID == oldRevID) + { + // Looks good... return the cached object + BOX_TRACE("Returning object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " from cache, modtime = " << newRevID) + return *(item->second); + } } - - BOX_TRACE("Refreshing object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " in cache, modtime changed from " << - item->second->GetRevisionID() << " to " << revID); // Delete this cached object delete item->second; mDirectoryCache.erase(item); } - + // Need to load it up - + // First check to see if the cache is too big - if(mDirectoryCache.size() > MAX_CACHE_SIZE) + if(mDirectoryCache.size() > MAX_CACHE_SIZE && AllowFlushCache) { - // Very simple. Just delete everything! - for(std::map<int64_t, BackupStoreDirectory*>::iterator i(mDirectoryCache.begin()); i != mDirectoryCache.end(); ++i) + // Very simple. Just delete everything! But in debug builds, + // leave the entries in the cache and invalidate them instead, + // so that any attempt to access them will cause an assertion + // failure that helps to track down the error. +#ifdef BOX_RELEASE_BUILD + ClearDirectoryCache(); +#else + for(std::map<int64_t, BackupStoreDirectory*>::iterator + i = mDirectoryCache.begin(); + i != mDirectoryCache.end(); i++) { - delete (i->second); + i->second->Invalidate(); } - mDirectoryCache.clear(); +#endif } // Get a RaidFileRead to read it - int64_t revID = 0; - std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename, &revID)); - ASSERT(revID != 0); - - // New directory object - std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory); - + std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, + filename, &newRevID)); + + ASSERT(newRevID != 0); + + if (oldRevID == 0) + { + BOX_TRACE("Loading object " << BOX_FORMAT_OBJECTID(ObjectID) << + " with modtime " << newRevID); + } + else + { + BOX_TRACE("Refreshing object " << BOX_FORMAT_OBJECTID(ObjectID) << + " in cache, modtime changed from " << oldRevID << + " to " << newRevID); + } + // Read it from the stream, then set it's revision ID BufferedStream buf(*objectFile); - dir->ReadFromStream(buf, IOStream::TimeOutInfinite); - dir->SetRevisionID(revID); - + std::auto_ptr<BackupStoreDirectory> dir(new BackupStoreDirectory(buf)); + dir->SetRevisionID(newRevID); + // Make sure the size of the directory is available for writing the dir back int64_t dirSize = objectFile->GetDiscUsageInBlocks(); ASSERT(dirSize > 0); @@ -348,7 +403,7 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) // Store in cache BackupStoreDirectory *pdir = dir.release(); try - { + { mDirectoryCache[ObjectID] = pdir; } catch(...) @@ -356,11 +411,12 @@ BackupStoreDirectory &BackupStoreContext::GetDirectoryInternal(int64_t ObjectID) delete pdir; throw; } - + // Return it return *pdir; } + // -------------------------------------------------------------------------- // // Function @@ -381,12 +437,12 @@ int64_t BackupStoreContext::AllocateObjectID() // to try for finding an unused ID. // (Sizes used in the store info are fixed by the housekeeping process) int retryLimit = (STORE_INFO_SAVE_DELAY * 2); - + while(retryLimit > 0) { // Attempt to allocate an ID from the store int64_t id = mapStoreInfo->AllocateObjectID(); - + // Generate filename std::string filename; MakeObjectFilename(id, filename); @@ -396,17 +452,17 @@ int64_t BackupStoreContext::AllocateObjectID() // Success! return id; } - + // Decrement retry count, and try again --retryLimit; - + // Mark that the store info should be saved as soon as possible mSaveStoreInfoDelay = 0; - + BOX_WARNING("When allocating object ID, found that " << BOX_FORMAT_OBJECTID(id) << " is already in use"); } - + THROW_EXCEPTION(BackupStoreException, CouldNotFindUnusedIDDuringAllocation) } @@ -431,11 +487,12 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - + // This is going to be a bit complex to make sure it copes OK // with things going wrong. // The only thing which isn't safe is incrementing the object ID @@ -444,13 +501,13 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // be corrected the next time the account has a housekeeping run, // and the object ID allocation code is tolerant of missed IDs. // (the info is written lazily, so these are necessary) - + // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - + // Allocate the next ID int64_t id = AllocateObjectID(); - + // Stream the file to disc std::string fn; MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); @@ -458,12 +515,13 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, RaidFileWrite *ppreviousVerStoreFile = 0; bool reversedDiffIsCompletelyDifferent = false; int64_t oldVersionNewBlocksUsed = 0; + BackupStoreInfo::Adjustment adjustment = {}; + try { RaidFileWrite storeFile(mStoreDiscSet, fn); storeFile.Open(false /* no overwriting */); - // size adjustment from use of patch in old file int64_t spaceSavedByConversionToPatch = 0; // Diff or full file? @@ -482,12 +540,12 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, { THROW_EXCEPTION(BackupStoreException, DiffFromIDNotFoundInDirectory) } - + // Diff file, needs to be recreated. // Choose a temporary filename. std::string tempFn(RaidFileController::DiscSetPathToFileSystemPath(mStoreDiscSet, fn + ".difftemp", 1 /* NOT the same disc as the write file, to avoid using lots of space on the same disc unnecessarily */)); - + try { // Open it twice @@ -506,13 +564,13 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, THROW_EXCEPTION(CommonException, OSFileError); } #endif - + // Stream the incoming diff to this temporary file if(!rFile.CopyStreamTo(diff, BACKUP_STORE_TIMEOUT)) { THROW_EXCEPTION(BackupStoreException, ReadFileFromStreamTimedOut) } - + // Verify the diff diff.Seek(0, IOStream::SeekType_Absolute); if(!BackupStoreFile::VerifyEncodedFileFormat(diff)) @@ -526,7 +584,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Filename of the old version std::string oldVersionFilename; MakeObjectFilename(DiffFromFileID, oldVersionFilename, false /* no need to make sure the directory it's in exists */); - + // Reassemble that diff -- open previous file, and combine the patch and file std::auto_ptr<RaidFileRead> from(RaidFileRead::Open(mStoreDiscSet, oldVersionFilename)); BackupStoreFile::CombineFile(diff, diff2, *from, storeFile); @@ -539,15 +597,26 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, diff.Seek(0, IOStream::SeekType_Absolute); BackupStoreFile::ReverseDiffFile(diff, *from, *from2, *ppreviousVerStoreFile, DiffFromFileID, &reversedDiffIsCompletelyDifferent); - + // Store disc space used oldVersionNewBlocksUsed = ppreviousVerStoreFile->GetDiscUsageInBlocks(); - + // And make a space adjustment for the size calculation spaceSavedByConversionToPatch = from->GetDiscUsageInBlocks() - oldVersionNewBlocksUsed; + adjustment.mBlocksUsed -= spaceSavedByConversionToPatch; + // The code below will change the patch from a + // Current file to an Old file, so we need to + // account for it as a Current file here. + adjustment.mBlocksInCurrentFiles -= + spaceSavedByConversionToPatch; + + // Don't adjust anything else here. We'll do it + // when we update the directory just below, + // which also accounts for non-diff replacements. + // Everything cleans up here... } catch(...) @@ -557,14 +626,17 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, throw; } } - + // Get the blocks used newObjectBlocksUsed = storeFile.GetDiscUsageInBlocks(); - + adjustment.mBlocksUsed += newObjectBlocksUsed; + adjustment.mBlocksInCurrentFiles += newObjectBlocksUsed; + adjustment.mNumCurrentFiles++; + // Exceeds the hard limit? - int64_t newBlocksUsed = mapStoreInfo->GetBlocksUsed() + - newObjectBlocksUsed - spaceSavedByConversionToPatch; - if(newBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) + int64_t newTotalBlocksUsed = mapStoreInfo->GetBlocksUsed() + + adjustment.mBlocksUsed; + if(newTotalBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) { THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) // The store file will be deleted automatically by the RaidFile object @@ -581,7 +653,7 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, delete ppreviousVerStoreFile; ppreviousVerStoreFile = 0; } - + throw; } @@ -596,21 +668,34 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Error! Delete the file RaidFileWrite del(mStoreDiscSet, fn); del.Delete(); - + // Exception THROW_EXCEPTION(BackupStoreException, AddedFileDoesNotVerify) } - } - + } + // Modify the directory -- first make all files with the same name // marked as an old version - int64_t blocksInOldFiles = 0; try { + // Adjust the entry for the object that we replaced with a + // patch, above. + BackupStoreDirectory::Entry *poldEntry = NULL; + + if(DiffFromFileID != 0) + { + // Get old version entry + poldEntry = dir.FindEntryByID(DiffFromFileID); + ASSERT(poldEntry != 0); + + // Adjust size of old entry + int64_t oldSize = poldEntry->GetSizeInBlocks(); + poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); + } + if(MarkFileWithSameNameAsOldVersions) { BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *e = 0; while((e = i.Next()) != 0) { @@ -626,43 +711,30 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, e->AddFlags(BackupStoreDirectory::Entry::Flags_OldVersion); // Can safely do this, because we know we won't be here if it's already // an old version - blocksInOldFiles += e->GetSizeInBlocks(); + adjustment.mBlocksInOldFiles += e->GetSizeInBlocks(); + adjustment.mBlocksInCurrentFiles -= e->GetSizeInBlocks(); + adjustment.mNumOldFiles++; + adjustment.mNumCurrentFiles--; } } } } - + // Then the new entry BackupStoreDirectory::Entry *pnewEntry = dir.AddEntry(rFilename, - ModificationTime, id, newObjectBlocksUsed, - BackupStoreDirectory::Entry::Flags_File, - AttributesHash); + ModificationTime, id, newObjectBlocksUsed, + BackupStoreDirectory::Entry::Flags_File, + AttributesHash); - // Adjust for the patch back stuff? - if(DiffFromFileID != 0) + // Adjust dependency info of file? + if(DiffFromFileID && poldEntry && !reversedDiffIsCompletelyDifferent) { - // Get old version entry - BackupStoreDirectory::Entry *poldEntry = dir.FindEntryByID(DiffFromFileID); - ASSERT(poldEntry != 0); - - // Adjust dependency info of file? - if(!reversedDiffIsCompletelyDifferent) - { - poldEntry->SetDependsNewer(id); - pnewEntry->SetDependsOlder(DiffFromFileID); - } - - // Adjust size of old entry - int64_t oldSize = poldEntry->GetSizeInBlocks(); - poldEntry->SetSizeInBlocks(oldVersionNewBlocksUsed); - - // And adjust blocks used count, for later adjustment - newObjectBlocksUsed += (oldVersionNewBlocksUsed - oldSize); - blocksInOldFiles += (oldVersionNewBlocksUsed - oldSize); + poldEntry->SetDependsNewer(id); + pnewEntry->SetDependsOlder(DiffFromFileID); } // Write the directory back to disc - SaveDirectory(dir, InDirectory); + SaveDirectory(dir); // Commit the old version's new patched version, now that the directory safely reflects // the state of the files on disc. @@ -678,47 +750,42 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // Back out on adding that file RaidFileWrite del(mStoreDiscSet, fn); del.Delete(); - + // Remove this entry from the cache RemoveDirectoryFromCache(InDirectory); - + // Delete any previous version store file if(ppreviousVerStoreFile != 0) { delete ppreviousVerStoreFile; ppreviousVerStoreFile = 0; } - + // Don't worry about the incremented number in the store info throw; } - + // Check logic ASSERT(ppreviousVerStoreFile == 0); - + // Modify the store info + mapStoreInfo->AdjustNumCurrentFiles(adjustment.mNumCurrentFiles); + mapStoreInfo->AdjustNumOldFiles(adjustment.mNumOldFiles); + mapStoreInfo->AdjustNumDeletedFiles(adjustment.mNumDeletedFiles); + mapStoreInfo->AdjustNumDirectories(adjustment.mNumDirectories); + mapStoreInfo->ChangeBlocksUsed(adjustment.mBlocksUsed); + mapStoreInfo->ChangeBlocksInCurrentFiles(adjustment.mBlocksInCurrentFiles); + mapStoreInfo->ChangeBlocksInOldFiles(adjustment.mBlocksInOldFiles); + mapStoreInfo->ChangeBlocksInDeletedFiles(adjustment.mBlocksInDeletedFiles); + mapStoreInfo->ChangeBlocksInDirectories(adjustment.mBlocksInDirectories); - if(DiffFromFileID == 0) - { - mapStoreInfo->AdjustNumFiles(1); - } - else - { - mapStoreInfo->AdjustNumOldFiles(1); - } - - mapStoreInfo->ChangeBlocksUsed(newObjectBlocksUsed); - mapStoreInfo->ChangeBlocksInCurrentFiles(newObjectBlocksUsed - - blocksInOldFiles); - mapStoreInfo->ChangeBlocksInOldFiles(blocksInOldFiles); - // Increment reference count on the new directory to one mapRefCount->AddReference(id); - + // Save the store info -- can cope if this exceptions because infomation // will be rebuilt by housekeeping, and ID allocation can recover. SaveStoreInfo(false); - + // Return the ID to the caller return id; } @@ -728,8 +795,11 @@ int64_t BackupStoreContext::AddFile(IOStream &rFile, int64_t InDirectory, // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::DeleteFile(const BackupStoreFilename &, int64_t, int64_t &) -// Purpose: Deletes a file, returning true if the file existed. Object ID returned too, set to zero if not found. +// Name: BackupStoreContext::DeleteFile( +// const BackupStoreFilename &, int64_t, int64_t &) +// Purpose: Deletes a file by name, returning true if the file +// existed. Object ID returned too, set to zero if not +// found. // Created: 2003/10/21 // // -------------------------------------------------------------------------- @@ -754,9 +824,6 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ bool madeChanges = false; rObjectIDOut = 0; // not found - // Count of deleted blocks - int64_t blocksDel = 0; - try { // Iterate through directory, only looking at files which haven't been deleted @@ -769,14 +836,28 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ if(e->GetName() == rFilename) { // Check that it's definately not already deleted - ASSERT((e->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) == 0); + ASSERT(!e->IsDeleted()); // Set deleted flag e->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); // Mark as made a change madeChanges = true; - // Can safely do this, because we know we won't be here if it's already - // an old version - blocksDel += e->GetSizeInBlocks(); + + int64_t blocks = e->GetSizeInBlocks(); + mapStoreInfo->AdjustNumDeletedFiles(1); + mapStoreInfo->ChangeBlocksInDeletedFiles(blocks); + + // We're marking all old versions as deleted. + // This is how a file can be old and deleted + // at the same time. So we don't subtract from + // number or size of old files. But if it was + // a current file, then it's not any more, so + // we do need to adjust the current counts. + if(!e->IsOld()) + { + mapStoreInfo->AdjustNumCurrentFiles(-1); + mapStoreInfo->ChangeBlocksInCurrentFiles(-blocks); + } + // Is this the last version? if((e->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) == 0) { @@ -786,19 +867,12 @@ bool BackupStoreContext::DeleteFile(const BackupStoreFilename &rFilename, int64_ } } } - + // Save changes? if(madeChanges) { // Save the directory back - SaveDirectory(dir, InDirectory); - - // Modify the store info, and write - // It definitely wasn't an old or deleted version - mapStoreInfo->AdjustNumFiles(-1); - mapStoreInfo->AdjustNumDeletedFiles(1); - mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); - + SaveDirectory(dir); SaveStoreInfo(false); } } @@ -871,16 +945,16 @@ bool BackupStoreContext::UndeleteFile(int64_t ObjectID, int64_t InDirectory) } } } - + // Save changes? if(madeChanges) { // Save the directory back - SaveDirectory(dir, InDirectory); - + SaveDirectory(dir); + // Modify the store info, and write mapStoreInfo->ChangeBlocksInDeletedFiles(blocksDel); - + // Maybe postponed save of store info SaveStoreInfo(); } @@ -919,27 +993,27 @@ void BackupStoreContext::RemoveDirectoryFromCache(int64_t ObjectID) // -------------------------------------------------------------------------- // // Function -// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &, int64_t) +// Name: BackupStoreContext::SaveDirectory(BackupStoreDirectory &) // Purpose: Save directory back to disc, update time in cache // Created: 2003/09/04 // // -------------------------------------------------------------------------- -void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID) +void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir) { if(mapStoreInfo.get() == 0) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - if(rDir.GetObjectID() != ObjectID) - { - THROW_EXCEPTION(BackupStoreException, Internal) - } + + int64_t ObjectID = rDir.GetObjectID(); try { // Write to disc, adjust size in store info std::string dirfn; MakeObjectFilename(ObjectID, dirfn); + int64_t old_dir_size = rDir.GetUserInfo1_SizeInBlocks(); + { RaidFileWrite writeDir(mStoreDiscSet, dirfn); writeDir.Open(true /* allow overwriting */); @@ -953,7 +1027,7 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - + // Make sure the size of the directory is available for writing the dir back ASSERT(dirSize > 0); int64_t sizeAdjustment = dirSize - rDir.GetUserInfo1_SizeInBlocks(); @@ -962,6 +1036,7 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // Update size stored in directory rDir.SetUserInfo1_SizeInBlocks(dirSize); } + // Refresh revision ID in cache { int64_t revid = 0; @@ -969,8 +1044,41 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec { THROW_EXCEPTION(BackupStoreException, Internal) } + + BOX_TRACE("Saved directory " << + BOX_FORMAT_OBJECTID(ObjectID) << + ", modtime = " << revid); + rDir.SetRevisionID(revid); } + + // Update the directory entry in the grandparent, to ensure + // that it reflects the current size of the parent directory. + int64_t new_dir_size = rDir.GetUserInfo1_SizeInBlocks(); + if(new_dir_size != old_dir_size && + ObjectID != BACKUPSTORE_ROOT_DIRECTORY_ID) + { + int64_t ContainerID = rDir.GetContainerID(); + BackupStoreDirectory& parent( + GetDirectoryInternal(ContainerID)); + // rDir is now invalid + BackupStoreDirectory::Entry* en = + parent.FindEntryByID(ObjectID); + if(!en) + { + BOX_ERROR("Missing entry for directory " << + BOX_FORMAT_OBJECTID(ObjectID) << + " in directory " << + BOX_FORMAT_OBJECTID(ContainerID) << + " while trying to update dir size in parent"); + } + else + { + ASSERT(en->GetSizeInBlocks() == old_dir_size); + en->SetSizeInBlocks(new_dir_size); + SaveDirectory(parent); + } + } } catch(...) { @@ -991,20 +1099,26 @@ void BackupStoreContext::SaveDirectory(BackupStoreDirectory &rDir, int64_t Objec // Created: 2003/09/04 // // -------------------------------------------------------------------------- -int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists) +int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, + const BackupStoreFilename &rFilename, + const StreamableMemBlock &Attributes, + int64_t AttributesModTime, + int64_t ModificationTime, + bool &rAlreadyExists) { if(mapStoreInfo.get() == 0) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - + // Flags as not already existing rAlreadyExists = false; - + // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); @@ -1030,19 +1144,31 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF // Create an empty directory with the given attributes on disc std::string fn; MakeObjectFilename(id, fn, true /* make sure the directory it's in exists */); + int64_t dirSize; + { BackupStoreDirectory emptyDir(id, InDirectory); // add the atttribues emptyDir.SetAttributes(Attributes, AttributesModTime); - + // Write... RaidFileWrite dirFile(mStoreDiscSet, fn); dirFile.Open(false /* no overwriting */); emptyDir.WriteToStream(dirFile); // Get disc usage, before it's commited - int64_t dirSize = dirFile.GetDiscUsageInBlocks(); + dirSize = dirFile.GetDiscUsageInBlocks(); + + // Exceeds the hard limit? + int64_t newTotalBlocksUsed = mapStoreInfo->GetBlocksUsed() + + dirSize; + if(newTotalBlocksUsed > mapStoreInfo->GetBlocksHardLimit()) + { + THROW_EXCEPTION(BackupStoreException, AddedFileExceedsStorageLimit) + // The file will be deleted automatically by the RaidFile object + } + // Commit the file - dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + dirFile.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); // Make sure the size of the directory is added to the usage counts in the info ASSERT(dirSize > 0); @@ -1050,12 +1176,14 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF mapStoreInfo->ChangeBlocksInDirectories(dirSize); // Not added to cache, so don't set the size in the directory } - + // Then add it into the parent directory try { - dir.AddEntry(rFilename, 0 /* modification time */, id, 0 /* blocks used */, BackupStoreDirectory::Entry::Flags_Dir, 0 /* attributes mod time */); - SaveDirectory(dir, InDirectory); + dir.AddEntry(rFilename, ModificationTime, id, dirSize, + BackupStoreDirectory::Entry::Flags_Dir, + 0 /* attributes hash */); + SaveDirectory(dir); // Increment reference count on the new directory to one mapRefCount->AddReference(id); @@ -1065,12 +1193,12 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF // Back out on adding that directory RaidFileWrite del(mStoreDiscSet, fn); del.Delete(); - + // Remove this entry from the cache RemoveDirectoryFromCache(InDirectory); - + // Don't worry about the incremented number in the store info - throw; + throw; } // Save the store info (may not be postponed) @@ -1081,6 +1209,7 @@ int64_t BackupStoreContext::AddDirectory(int64_t InDirectory, const BackupStoreF return id; } + // -------------------------------------------------------------------------- // // Function @@ -1096,6 +1225,7 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } + if(mReadOnly) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) @@ -1103,9 +1233,6 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) // Containing directory int64_t InDirectory = 0; - - // Count of blocks deleted - int64_t blocksDeleted = 0; try { @@ -1113,18 +1240,18 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { // In block, because dir may not be valid after the delete directory call BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - + // Store the directory it's in for later InDirectory = dir.GetContainerID(); // Depth first delete of contents - DeleteDirectoryRecurse(ObjectID, blocksDeleted, Undelete); + DeleteDirectoryRecurse(ObjectID, Undelete); } - + // Remove the entry from the directory it's in ASSERT(InDirectory != 0); BackupStoreDirectory &parentDir(GetDirectoryInternal(InDirectory)); - + BackupStoreDirectory::Iterator i(parentDir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), @@ -1141,18 +1268,16 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) { en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); } - + // Save it - SaveDirectory(parentDir, InDirectory); - + SaveDirectory(parentDir); + // Done break; } } - + // Update blocks deleted count - mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete?(0 - blocksDeleted):(blocksDeleted)); - mapStoreInfo->AdjustNumDirectories(-1); SaveStoreInfo(false); } catch(...) @@ -1162,6 +1287,7 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) } } + // -------------------------------------------------------------------------- // // Function @@ -1170,18 +1296,18 @@ void BackupStoreContext::DeleteDirectory(int64_t ObjectID, bool Undelete) // Created: 2003/10/21 // // -------------------------------------------------------------------------- -void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete) +void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete) { try { // Does things carefully to avoid using a directory in the cache after recursive call // because it may have been deleted. - + // Do sub directories { // Get the directory... BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - + // Then scan it for directories std::vector<int64_t> subDirs; BackupStoreDirectory::Iterator i(dir); @@ -1204,30 +1330,46 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBloc subDirs.push_back(en->GetObjectID()); } } - + // Done with the directory for now. Recurse to sub directories for(std::vector<int64_t>::const_iterator i = subDirs.begin(); i != subDirs.end(); ++i) { - DeleteDirectoryRecurse((*i), rBlocksDeletedOut, Undelete); + DeleteDirectoryRecurse(*i, Undelete); } } - + // Then, delete the files. Will need to load the directory again because it might have // been removed from the cache. { // Get the directory... BackupStoreDirectory &dir(GetDirectoryInternal(ObjectID)); - + // Changes made? bool changesMade = false; - - // Run through files + + // Run through files BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(Undelete?(BackupStoreDirectory::Entry::Flags_Deleted):(BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING), Undelete?(0):(BackupStoreDirectory::Entry::Flags_Deleted))) != 0) // Ignore deleted directories (or not deleted if Undelete) { + // Keep count of the deleted blocks + if(en->IsFile()) + { + int64_t size = en->GetSizeInBlocks(); + ASSERT(en->IsDeleted() == Undelete); + // Don't adjust counters for old files, + // because it can be both old and deleted. + if(!en->IsOld()) + { + mapStoreInfo->ChangeBlocksInCurrentFiles(Undelete ? size : -size); + mapStoreInfo->AdjustNumCurrentFiles(Undelete ? 1 : -1); + } + mapStoreInfo->ChangeBlocksInDeletedFiles(Undelete ? -size : size); + mapStoreInfo->AdjustNumDeletedFiles(Undelete ? -1 : 1); + } + // Add/remove the deleted flags if(Undelete) { @@ -1237,21 +1379,15 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBloc { en->AddFlags(BackupStoreDirectory::Entry::Flags_Deleted); } - - // Keep count of the deleted blocks - if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) - { - rBlocksDeletedOut += en->GetSizeInBlocks(); - } - + // Did something changesMade = true; } - + // Save the directory if(changesMade) { - SaveDirectory(dir, ObjectID); + SaveDirectory(dir); } } } @@ -1263,7 +1399,6 @@ void BackupStoreContext::DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBloc } - // -------------------------------------------------------------------------- // // Function @@ -1284,15 +1419,15 @@ void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const Streamable } try - { + { // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(Directory)); - + // Set attributes dir.SetAttributes(Attributes, AttributesModTime); - + // Save back - SaveDirectory(dir, Directory); + SaveDirectory(dir); } catch(...) { @@ -1301,6 +1436,7 @@ void BackupStoreContext::ChangeDirAttributes(int64_t Directory, const Streamable } } + // -------------------------------------------------------------------------- // // Function @@ -1324,7 +1460,7 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena { // Get the directory we want to modify BackupStoreDirectory &dir(GetDirectoryInternal(InDirectory)); - + // Find the file entry BackupStoreDirectory::Entry *en = 0; // Iterate through current versions of files, only @@ -1338,10 +1474,10 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena { // Set attributes en->SetAttributes(Attributes, AttributesHash); - + // Tell caller the object ID rObjectIDOut = en->GetObjectID(); - + // Done break; } @@ -1351,16 +1487,16 @@ bool BackupStoreContext::ChangeFileAttributes(const BackupStoreFilename &rFilena // Didn't find it return false; } - + // Save back - SaveDirectory(dir, InDirectory); + SaveDirectory(dir); } catch(...) { RemoveDirectoryFromCache(InDirectory); throw; } - + // Changed, everything OK return true; } @@ -1380,7 +1516,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + // Note that we need to allow object IDs a little bit greater than the last one in the store info, // because the store info may not have got saved in an error condition. Max greater ID is // STORE_INFO_SAVE_DELAY in this case, *2 to be safe. @@ -1389,7 +1525,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) // Obviously bad object ID return false; } - + // Test to see if it exists on the disc std::string filename; MakeObjectFilename(ObjectID, filename); @@ -1398,7 +1534,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) // RaidFile reports no file there return false; } - + // Do we need to be more specific? if(MustBe != ObjectExists_Anything) { @@ -1406,7 +1542,7 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) std::auto_ptr<RaidFileRead> objectFile(RaidFileRead::Open(mStoreDiscSet, filename)); // Read the first integer - u_int32_t magic; + uint32_t magic; if(!objectFile->ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in how many read if failure */)) { // Failed to get any bytes, must have failed @@ -1422,17 +1558,17 @@ bool BackupStoreContext::ObjectExists(int64_t ObjectID, int MustBe) #endif // Right one? - u_int32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; - + uint32_t requiredMagic = (MustBe == ObjectExists_File)?OBJECTMAGIC_FILE_MAGIC_VALUE_V1:OBJECTMAGIC_DIR_MAGIC_VALUE; + // Check if(ntohl(magic) != requiredMagic) { return false; } - + // File is implicitly closed } - + return true; } @@ -1451,7 +1587,7 @@ std::auto_ptr<IOStream> BackupStoreContext::OpenObject(int64_t ObjectID) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + // Attempt to open the file std::string fn; MakeObjectFilename(ObjectID, fn); @@ -1473,7 +1609,7 @@ int64_t BackupStoreContext::GetClientStoreMarker() { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + return mapStoreInfo->GetClientStoreMarker(); } @@ -1536,7 +1672,7 @@ void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) { THROW_EXCEPTION(BackupStoreException, ContextIsReadOnly) } - + mapStoreInfo->SetClientStoreMarker(ClientStoreMarker); SaveStoreInfo(false /* don't delay saving this */); } @@ -1550,7 +1686,9 @@ void BackupStoreContext::SetClientStoreMarker(int64_t ClientStoreMarker) // Created: 12/11/03 // // -------------------------------------------------------------------------- -void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) +void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, + int64_t MoveToDirectory, const BackupStoreFilename &rNewFilename, + bool MoveAllWithSameName, bool AllowMoveOverDeletedObject) { if(mReadOnly) { @@ -1561,7 +1699,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, int64_t targetSearchExcludeFlags = (AllowMoveOverDeletedObject) ?(BackupStoreDirectory::Entry::Flags_Deleted) :(BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); - + // Special case if the directories are the same... if(MoveFromDirectory == MoveToDirectory) { @@ -1569,16 +1707,16 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, { // Get the first directory BackupStoreDirectory &dir(GetDirectoryInternal(MoveFromDirectory)); - + // Find the file entry BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); - + // Error if not found if(en == 0) { THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) } - + // Check the new name doens't already exist (optionally ignoring deleted files) { BackupStoreDirectory::Iterator i(dir); @@ -1591,7 +1729,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } } } - + // Need to get all the entries with the same name? if(MoveAllWithSameName) { @@ -1612,45 +1750,46 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, // Just copy this one en->SetName(rNewFilename); } - + // Save the directory back - SaveDirectory(dir, MoveFromDirectory); + SaveDirectory(dir); } catch(...) { RemoveDirectoryFromCache(MoveToDirectory); // either will do, as they're the same throw; } - + return; } - // Got to be careful how this is written, as we can't guarentte that if we have two - // directories open, the first won't be deleted as the second is opened. (cache) + // Got to be careful how this is written, as we can't guarantee that + // if we have two directories open, the first won't be deleted as the + // second is opened. (cache) // List of entries to move std::vector<BackupStoreDirectory::Entry *> moving; - + // list of directory IDs which need to have containing dir id changed std::vector<int64_t> dirsToChangeContainingID; try { // First of all, get copies of the entries to move to the to directory. - + { // Get the first directory BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - + // Find the file entry BackupStoreDirectory::Entry *en = from.FindEntryByID(ObjectID); - + // Error if not found if(en == 0) { THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) } - + // Need to get all the entries with the same name? if(MoveAllWithSameName) { @@ -1663,7 +1802,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, { // Copy moving.push_back(new BackupStoreDirectory::Entry(*c)); - + // Check for containing directory correction if(c->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(c->GetObjectID()); } @@ -1679,13 +1818,13 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) dirsToChangeContainingID.push_back(en->GetObjectID()); } } - + // Secondly, insert them into the to directory, and save it - + { // To directory BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - + // Check the new name doens't already exist { BackupStoreDirectory::Iterator i(to); @@ -1698,7 +1837,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } } } - + // Copy the entries into it, changing the name as we go for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) { @@ -1706,9 +1845,9 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, en->SetName(rNewFilename); to.AddEntry(*en); // adds copy } - + // Save back - SaveDirectory(to, MoveToDirectory); + SaveDirectory(to); } // Thirdly... remove them from the first directory -- but if it fails, attempt to delete them from the to directory @@ -1716,57 +1855,57 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, { // Get directory BackupStoreDirectory &from(GetDirectoryInternal(MoveFromDirectory)); - + // Delete each one for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) { from.DeleteEntry((*i)->GetObjectID()); } - + // Save back - SaveDirectory(from, MoveFromDirectory); + SaveDirectory(from); } catch(...) { // UNDO modification to To directory - + // Get directory BackupStoreDirectory &to(GetDirectoryInternal(MoveToDirectory)); - + // Delete each one for(std::vector<BackupStoreDirectory::Entry *>::iterator i(moving.begin()); i != moving.end(); ++i) { to.DeleteEntry((*i)->GetObjectID()); } - + // Save back - SaveDirectory(to, MoveToDirectory); + SaveDirectory(to); // Throw the error throw; } - + // Finally... for all the directories we moved, modify their containing directory ID for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) { // Load the directory BackupStoreDirectory &change(GetDirectoryInternal(*i)); - + // Modify containing dir ID change.SetContainerID(MoveToDirectory); - + // Save it back - SaveDirectory(change, *i); + SaveDirectory(change); } } catch(...) { - // Make sure directories aren't in the cache, as they may have been modified + // Make sure directories aren't in the cache, as they may have been modified RemoveDirectoryFromCache(MoveToDirectory); RemoveDirectoryFromCache(MoveFromDirectory); for(std::vector<int64_t>::iterator i(dirsToChangeContainingID.begin()); i != dirsToChangeContainingID.end(); ++i) { - RemoveDirectoryFromCache(*i); + RemoveDirectoryFromCache(*i); } while(!moving.empty()) @@ -1775,7 +1914,7 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, moving.pop_back(); } throw; - } + } // Clean up while(!moving.empty()) @@ -1786,7 +1925,6 @@ void BackupStoreContext::MoveObject(int64_t ObjectID, int64_t MoveFromDirectory, } - // -------------------------------------------------------------------------- // // Function @@ -1801,8 +1939,7 @@ const BackupStoreInfo &BackupStoreContext::GetBackupStoreInfo() const { THROW_EXCEPTION(BackupStoreException, StoreInfoNotLoaded) } - + return *(mapStoreInfo.get()); } - diff --git a/lib/backupstore/BackupStoreContext.h b/lib/backupstore/BackupStoreContext.h index c33e7d50..48448360 100644 --- a/lib/backupstore/BackupStoreContext.h +++ b/lib/backupstore/BackupStoreContext.h @@ -45,7 +45,8 @@ class HousekeepingInterface class BackupStoreContext { public: - BackupStoreContext(int32_t ClientID, HousekeepingInterface &rDaemon, + BackupStoreContext(int32_t ClientID, + HousekeepingInterface* mpHousekeeping, const std::string& rConnectionDetails); ~BackupStoreContext(); private: @@ -64,7 +65,7 @@ public: Phase_Login = 1, Phase_Commands = 2 }; - + int GetPhase() const {return mProtocolPhase;} std::string GetPhaseName() const { @@ -80,14 +81,23 @@ public: } } void SetPhase(int NewPhase) {mProtocolPhase = NewPhase;} - + // Read only locking bool SessionIsReadOnly() {return mReadOnly;} bool AttemptToGetWriteLock(); - void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mStoreRoot = rStoreRoot; mStoreDiscSet = StoreDiscSet;} + // Not really an API, but useful for BackupProtocolLocal2. + void ReleaseWriteLock() + { + if(mWriteLock.GotLock()) + { + mWriteLock.ReleaseLock(); + } + } + + void SetClientHasAccount(const std::string &rStoreRoot, int StoreDiscSet) {mClientHasAccount = true; mAccountRootDir = rStoreRoot; mStoreDiscSet = StoreDiscSet;} bool GetClientHasAccount() const {return mClientHasAccount;} - const std::string &GetStoreRoot() const {return mStoreRoot;} + const std::string &GetAccountRoot() const {return mAccountRootDir;} int GetStoreDiscSet() const {return mStoreDiscSet;} // Store info @@ -106,7 +116,7 @@ public: // Client marker int64_t GetClientStoreMarker(); void SetClientStoreMarker(int64_t ClientStoreMarker); - + // Usage information void GetStoreDiscUsageInfo(int64_t &rBlocksUsed, int64_t &rBlocksSoftLimit, int64_t &rBlocksHardLimit); bool HardLimitExceeded(); @@ -125,13 +135,24 @@ public: const BackupStoreDirectory &GetDirectory(int64_t ObjectID) { // External callers aren't allowed to change it -- this function - // merely turns the the returned directory const. + // merely turns the returned directory const. return GetDirectoryInternal(ObjectID); } - + // Manipulating files/directories - int64_t AddFile(IOStream &rFile, int64_t InDirectory, int64_t ModificationTime, int64_t AttributesHash, int64_t DiffFromFileID, const BackupStoreFilename &rFilename, bool MarkFileWithSameNameAsOldVersions); - int64_t AddDirectory(int64_t InDirectory, const BackupStoreFilename &rFilename, const StreamableMemBlock &Attributes, int64_t AttributesModTime, bool &rAlreadyExists); + int64_t AddFile(IOStream &rFile, + int64_t InDirectory, + int64_t ModificationTime, + int64_t AttributesHash, + int64_t DiffFromFileID, + const BackupStoreFilename &rFilename, + bool MarkFileWithSameNameAsOldVersions); + int64_t AddDirectory(int64_t InDirectory, + const BackupStoreFilename &rFilename, + const StreamableMemBlock &Attributes, + int64_t AttributesModTime, + int64_t ModificationTime, + bool &rAlreadyExists); void ChangeDirAttributes(int64_t Directory, const StreamableMemBlock &Attributes, int64_t AttributesModTime); bool ChangeFileAttributes(const BackupStoreFilename &rFilename, int64_t InDirectory, const StreamableMemBlock &Attributes, int64_t AttributesHash, int64_t &rObjectIDOut); bool DeleteFile(const BackupStoreFilename &rFilename, int64_t InDirectory, int64_t &rObjectIDOut); @@ -155,29 +176,32 @@ public: private: void MakeObjectFilename(int64_t ObjectID, std::string &rOutput, bool EnsureDirectoryExists = false); - BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID); - void SaveDirectory(BackupStoreDirectory &rDir, int64_t ObjectID); + BackupStoreDirectory &GetDirectoryInternal(int64_t ObjectID, + bool AllowFlushCache = true); + void SaveDirectory(BackupStoreDirectory &rDir); void RemoveDirectoryFromCache(int64_t ObjectID); - void DeleteDirectoryRecurse(int64_t ObjectID, int64_t &rBlocksDeletedOut, bool Undelete); + void ClearDirectoryCache(); + void DeleteDirectoryRecurse(int64_t ObjectID, bool Undelete); int64_t AllocateObjectID(); std::string mConnectionDetails; int32_t mClientID; - HousekeepingInterface &mrDaemon; + HousekeepingInterface *mpHousekeeping; int mProtocolPhase; bool mClientHasAccount; - std::string mStoreRoot; // has final directory separator + std::string mAccountRootDir; // has final directory separator int mStoreDiscSet; + bool mReadOnly; NamedLock mWriteLock; int mSaveStoreInfoDelay; // how many times to delay saving the store info - + // Store info std::auto_ptr<BackupStoreInfo> mapStoreInfo; // Refcount database std::auto_ptr<BackupStoreRefCountDatabase> mapRefCount; - + // Directory cache std::map<int64_t, BackupStoreDirectory*> mDirectoryCache; diff --git a/lib/backupstore/BackupStoreDirectory.cpp b/lib/backupstore/BackupStoreDirectory.cpp index 81126ede..6946f06e 100644 --- a/lib/backupstore/BackupStoreDirectory.cpp +++ b/lib/backupstore/BackupStoreDirectory.cpp @@ -1,7 +1,7 @@ // -------------------------------------------------------------------------- // // File -// Name: BackupStoreDirectory.h +// Name: BackupStoreDirectory.cpp // Purpose: Representation of a backup directory // Created: 2003/08/26 // @@ -36,11 +36,6 @@ typedef struct // Then a StreamableMemBlock for attributes } dir_StreamFormat; -typedef enum -{ - Option_DependencyInfoPresent = 1 -} dir_StreamFormatOptions; - typedef struct { uint64_t mModificationTime; @@ -75,9 +70,17 @@ END_STRUCTURE_PACKING_FOR_WIRE // // -------------------------------------------------------------------------- BackupStoreDirectory::BackupStoreDirectory() - : mRevisionID(0), mObjectID(0), mContainerID(0), mAttributesModTime(0), mUserInfo1(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mRevisionID(0), + mObjectID(0), + mContainerID(0), + mAttributesModTime(0), + mUserInfo1(0) { - ASSERT(sizeof(u_int64_t) == sizeof(box_time_t)); + ASSERT(sizeof(uint64_t) == sizeof(box_time_t)); } @@ -90,7 +93,15 @@ BackupStoreDirectory::BackupStoreDirectory() // // -------------------------------------------------------------------------- BackupStoreDirectory::BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID) - : mRevisionID(0), mObjectID(ObjectID), mContainerID(ContainerID), mAttributesModTime(0), mUserInfo1(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mRevisionID(0), + mObjectID(ObjectID), + mContainerID(ContainerID), + mAttributesModTime(0), + mUserInfo1(0) { } @@ -122,6 +133,7 @@ BackupStoreDirectory::~BackupStoreDirectory() // -------------------------------------------------------------------------- void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) { + ASSERT(!mInvalidated); // Compiled out of release builds // Get the header dir_StreamFormat hdr; if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -133,35 +145,35 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) if(OBJECTMAGIC_DIR_MAGIC_VALUE != ntohl(hdr.mMagicValue)) { THROW_EXCEPTION_MESSAGE(BackupStoreException, BadDirectoryFormat, - "Wrong magic number in directory object " << - BOX_FORMAT_OBJECTID(mObjectID) << ": expected " << + "Wrong magic number for directory: expected " << BOX_FORMAT_HEX32(OBJECTMAGIC_DIR_MAGIC_VALUE) << " but found " << - BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue))); + BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue)) << " in " << + rStream.ToString()); } - + // Get data mObjectID = box_ntoh64(hdr.mObjectID); mContainerID = box_ntoh64(hdr.mContainerID); mAttributesModTime = box_ntoh64(hdr.mAttributesModTime); - + // Options int32_t options = ntohl(hdr.mOptionsPresent); - + // Get attributes mAttributes.ReadFromStream(rStream, Timeout); - + // Decode count int count = ntohl(hdr.mNumEntries); - + // Clear existing list - for(std::vector<Entry*>::iterator i = mEntries.begin(); + for(std::vector<Entry*>::iterator i = mEntries.begin(); i != mEntries.end(); i++) { delete (*i); } mEntries.clear(); - + // Read them in! for(int c = 0; c < count; ++c) { @@ -170,7 +182,7 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) { // Read from stream pen->ReadFromStream(rStream, Timeout); - + // Add to list mEntries.push_back(pen); } @@ -180,7 +192,7 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) throw; } } - + // Read in dependency info? if(options & Option_DependencyInfoPresent) { @@ -202,6 +214,7 @@ void BackupStoreDirectory::ReadFromStream(IOStream &rStream, int Timeout) // -------------------------------------------------------------------------- void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet, bool StreamAttributes, bool StreamDependencyInfo) const { + ASSERT(!mInvalidated); // Compiled out of release builds // Get count of entries int32_t count = mEntries.size(); if(FlagsMustBeSet != Entry::Flags_INCLUDE_EVERYTHING || FlagsNotToBeSet != Entry::Flags_EXCLUDE_NOTHING) @@ -214,11 +227,11 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS count++; } } - + // Check that sensible IDs have been set ASSERT(mObjectID != 0); ASSERT(mContainerID != 0); - + // Need dependency info? bool dependencyInfoRequired = false; if(StreamDependencyInfo) @@ -231,9 +244,9 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS { dependencyInfoRequired = true; } - } + } } - + // Options int32_t options = 0; if(dependencyInfoRequired) options |= Option_DependencyInfoPresent; @@ -246,10 +259,10 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS hdr.mContainerID = box_hton64(mContainerID); hdr.mAttributesModTime = box_hton64(mAttributesModTime); hdr.mOptionsPresent = htonl(options); - + // Write header rStream.Write(&hdr, sizeof(hdr)); - + // Write the attributes? if(StreamAttributes) { @@ -268,7 +281,7 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS { pen->WriteToStream(rStream); } - + // Write dependency info? if(dependencyInfoRequired) { @@ -277,7 +290,7 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS while((pen = i.Next(FlagsMustBeSet, FlagsNotToBeSet)) != 0) { pen->WriteToStreamDependencyInfo(rStream); - } + } } } @@ -291,6 +304,7 @@ void BackupStoreDirectory::WriteToStream(IOStream &rStream, int16_t FlagsMustBeS // -------------------------------------------------------------------------- BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryToCopy) { + ASSERT(!mInvalidated); // Compiled out of release builds Entry *pnew = new Entry(rEntryToCopy); try { @@ -301,7 +315,7 @@ BackupStoreDirectory::Entry *BackupStoreDirectory::AddEntry(const Entry &rEntryT delete pnew; throw; } - + return pnew; } @@ -318,6 +332,7 @@ BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) { + ASSERT(!mInvalidated); // Compiled out of release builds Entry *pnew = new Entry(rName, ModificationTime, ObjectID, SizeInBlocks, Flags, AttributesHash); try @@ -329,7 +344,7 @@ BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, delete pnew; throw; } - + return pnew; } @@ -343,6 +358,7 @@ BackupStoreDirectory::AddEntry(const BackupStoreFilename &rName, // -------------------------------------------------------------------------- void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) { + ASSERT(!mInvalidated); // Compiled out of release builds for(std::vector<Entry*>::iterator i(mEntries.begin()); i != mEntries.end(); ++i) { @@ -356,9 +372,11 @@ void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) return; } } - + // Not found - THROW_EXCEPTION(BackupStoreException, CouldNotFindEntryInDirectory) + THROW_EXCEPTION_MESSAGE(BackupStoreException, CouldNotFindEntryInDirectory, + "Failed to find entry " << BOX_FORMAT_OBJECTID(ObjectID) << + " in directory " << BOX_FORMAT_OBJECTID(mObjectID)); } @@ -372,6 +390,7 @@ void BackupStoreDirectory::DeleteEntry(int64_t ObjectID) // -------------------------------------------------------------------------- BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectID) const { + ASSERT(!mInvalidated); // Compiled out of release builds for(std::vector<Entry*>::const_iterator i(mEntries.begin()); i != mEntries.end(); ++i) { @@ -396,15 +415,19 @@ BackupStoreDirectory::Entry *BackupStoreDirectory::FindEntryByID(int64_t ObjectI // // -------------------------------------------------------------------------- BackupStoreDirectory::Entry::Entry() - : mModificationTime(0), - mObjectID(0), - mSizeInBlocks(0), - mFlags(0), - mAttributesHash(0), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mModificationTime(0), + mObjectID(0), + mSizeInBlocks(0), + mFlags(0), + mAttributesHash(0), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) { } @@ -429,17 +452,21 @@ BackupStoreDirectory::Entry::~Entry() // // -------------------------------------------------------------------------- BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) - : mName(rToCopy.mName), - mModificationTime(rToCopy.mModificationTime), - mObjectID(rToCopy.mObjectID), - mSizeInBlocks(rToCopy.mSizeInBlocks), - mFlags(rToCopy.mFlags), - mAttributesHash(rToCopy.mAttributesHash), - mAttributes(rToCopy.mAttributes), - mMinMarkNumber(rToCopy.mMinMarkNumber), - mMarkNumber(rToCopy.mMarkNumber), - mDependsNewer(rToCopy.mDependsNewer), - mDependsOlder(rToCopy.mDependsOlder) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mName(rToCopy.mName), + mModificationTime(rToCopy.mModificationTime), + mObjectID(rToCopy.mObjectID), + mSizeInBlocks(rToCopy.mSizeInBlocks), + mFlags(rToCopy.mFlags), + mAttributesHash(rToCopy.mAttributesHash), + mAttributes(rToCopy.mAttributes), + mMinMarkNumber(rToCopy.mMinMarkNumber), + mMarkNumber(rToCopy.mMarkNumber), + mDependsNewer(rToCopy.mDependsNewer), + mDependsOlder(rToCopy.mDependsOlder) { } @@ -453,16 +480,20 @@ BackupStoreDirectory::Entry::Entry(const Entry &rToCopy) // // -------------------------------------------------------------------------- BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash) - : mName(rName), - mModificationTime(ModificationTime), - mObjectID(ObjectID), - mSizeInBlocks(SizeInBlocks), - mFlags(Flags), - mAttributesHash(AttributesHash), - mMinMarkNumber(0), - mMarkNumber(0), - mDependsNewer(0), - mDependsOlder(0) +: +#ifndef BOX_RELEASE_BUILD + mInvalidated(false), +#endif + mName(rName), + mModificationTime(ModificationTime), + mObjectID(ObjectID), + mSizeInBlocks(SizeInBlocks), + mFlags(Flags), + mAttributesHash(AttributesHash), + mMinMarkNumber(0), + mMarkNumber(0), + mDependsNewer(0), + mDependsOlder(0) { } @@ -478,19 +509,21 @@ BackupStoreDirectory::Entry::Entry(const BackupStoreFilename &rName, box_time_t // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) { + ASSERT(!mInvalidated); // Compiled out of release builds // Grab the raw bytes from the stream which compose the header en_StreamFormat entry; - if(!rStream.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout)) + if(!rStream.ReadFullBuffer(&entry, sizeof(entry), + 0 /* not interested in bytes read if this fails */, Timeout)) { THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) } // Do reading first before modifying the variables, to be more exception safe - + // Get the filename BackupStoreFilename name; name.ReadFromStream(rStream, Timeout); - + // Get the attributes mAttributes.ReadFromStream(rStream, Timeout); @@ -514,6 +547,7 @@ void BackupStoreDirectory::Entry::ReadFromStream(IOStream &rStream, int Timeout) // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const { + ASSERT(!mInvalidated); // Compiled out of release builds // Build a structure en_StreamFormat entry; entry.mModificationTime = box_hton64(mModificationTime); @@ -521,13 +555,13 @@ void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const entry.mSizeInBlocks = box_hton64(mSizeInBlocks); entry.mAttributesHash = box_hton64(mAttributesHash); entry.mFlags = htons(mFlags); - + // Write it rStream.Write(&entry, sizeof(entry)); - + // Write the filename mName.WriteToStream(rStream); - + // Write any attributes mAttributes.WriteToStream(rStream); } @@ -543,6 +577,7 @@ void BackupStoreDirectory::Entry::WriteToStream(IOStream &rStream) const // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout) { + ASSERT(!mInvalidated); // Compiled out of release builds // Grab the raw bytes from the stream which compose the header en_StreamFormatDepends depends; if(!rStream.ReadFullBuffer(&depends, sizeof(depends), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -566,13 +601,11 @@ void BackupStoreDirectory::Entry::ReadFromStreamDependencyInfo(IOStream &rStream // -------------------------------------------------------------------------- void BackupStoreDirectory::Entry::WriteToStreamDependencyInfo(IOStream &rStream) const { + ASSERT(!mInvalidated); // Compiled out of release builds // Build structure - en_StreamFormatDepends depends; + en_StreamFormatDepends depends; depends.mDependsNewer = box_hton64(mDependsNewer); depends.mDependsOlder = box_hton64(mDependsOlder); // Write rStream.Write(&depends, sizeof(depends)); } - - - diff --git a/lib/backupstore/BackupStoreDirectory.h b/lib/backupstore/BackupStoreDirectory.h index 1348f4e6..788a3ad0 100644 --- a/lib/backupstore/BackupStoreDirectory.h +++ b/lib/backupstore/BackupStoreDirectory.h @@ -29,9 +29,48 @@ class IOStream; // -------------------------------------------------------------------------- class BackupStoreDirectory { +private: +#ifndef BOX_RELEASE_BUILD + bool mInvalidated; +#endif + public: +#ifndef BOX_RELEASE_BUILD + void Invalidate() + { + mInvalidated = true; + for (std::vector<Entry*>::iterator i = mEntries.begin(); + i != mEntries.end(); i++) + { + (*i)->Invalidate(); + } + } +#endif + + typedef enum + { + Option_DependencyInfoPresent = 1 + } dir_StreamFormatOptions; + BackupStoreDirectory(); BackupStoreDirectory(int64_t ObjectID, int64_t ContainerID); + // Convenience constructor from a stream + BackupStoreDirectory(IOStream& rStream, + int Timeout = IOStream::TimeOutInfinite) +#ifndef BOX_RELEASE_BUILD + : mInvalidated(false) +#endif + { + ReadFromStream(rStream, Timeout); + } + BackupStoreDirectory(std::auto_ptr<IOStream> apStream, + int Timeout = IOStream::TimeOutInfinite) +#ifndef BOX_RELEASE_BUILD + : mInvalidated(false) +#endif + { + ReadFromStream(*apStream, Timeout); + } private: // Copying not allowed BackupStoreDirectory(const BackupStoreDirectory &rToCopy); @@ -40,40 +79,117 @@ public: class Entry { + private: +#ifndef BOX_RELEASE_BUILD + bool mInvalidated; +#endif + public: +#ifndef BOX_RELEASE_BUILD + void Invalidate() { mInvalidated = true; } +#endif + friend class BackupStoreDirectory; Entry(); ~Entry(); Entry(const Entry &rToCopy); Entry(const BackupStoreFilename &rName, box_time_t ModificationTime, int64_t ObjectID, int64_t SizeInBlocks, int16_t Flags, uint64_t AttributesHash); - + void ReadFromStream(IOStream &rStream, int Timeout); void WriteToStream(IOStream &rStream) const; - - const BackupStoreFilename &GetName() const {return mName;} - box_time_t GetModificationTime() const {return mModificationTime;} - int64_t GetObjectID() const {return mObjectID;} - int64_t GetSizeInBlocks() const {return mSizeInBlocks;} - int16_t GetFlags() const {return mFlags;} - void AddFlags(int16_t Flags) {mFlags |= Flags;} - void RemoveFlags(int16_t Flags) {mFlags &= ~Flags;} + + const BackupStoreFilename &GetName() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mName; + } + box_time_t GetModificationTime() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mModificationTime; + } + int64_t GetObjectID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mObjectID; + } + // SetObjectID is dangerous! It should only be used when + // creating a snapshot. + void SetObjectID(int64_t NewObjectID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mObjectID = NewObjectID; + } + int64_t GetSizeInBlocks() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mSizeInBlocks; + } + int16_t GetFlags() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mFlags; + } + void AddFlags(int16_t Flags) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mFlags |= Flags; + } + void RemoveFlags(int16_t Flags) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mFlags &= ~Flags; + } // Some things can be changed - void SetName(const BackupStoreFilename &rNewName) {mName = rNewName;} - void SetSizeInBlocks(int64_t SizeInBlocks) {mSizeInBlocks = SizeInBlocks;} + void SetName(const BackupStoreFilename &rNewName) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mName = rNewName; + } + void SetSizeInBlocks(int64_t SizeInBlocks) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mSizeInBlocks = SizeInBlocks; + } // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) {mAttributes.Set(rAttr); mAttributesHash = AttributesHash;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - uint64_t GetAttributesHash() const {return mAttributesHash;} - + bool HasAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return !mAttributes.IsEmpty(); + } + void SetAttributes(const StreamableMemBlock &rAttr, uint64_t AttributesHash) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mAttributes.Set(rAttr); + mAttributesHash = AttributesHash; + } + const StreamableMemBlock &GetAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributes; + } + uint64_t GetAttributesHash() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributesHash; + } + // Marks // The lowest mark number a version of a file of this name has ever had - uint32_t GetMinMarkNumber() const {return mMinMarkNumber;} + uint32_t GetMinMarkNumber() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mMinMarkNumber; + } // The mark number on this file - uint32_t GetMarkNumber() const {return mMarkNumber;} + uint32_t GetMarkNumber() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mMarkNumber; + } // Make sure these flags are synced with those in backupprocotol.txt // ListDirectory command @@ -94,41 +210,66 @@ public: // convenience methods bool inline IsDir() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_Dir; } bool inline IsFile() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_File; } bool inline IsOld() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_OldVersion; } bool inline IsDeleted() { + ASSERT(!mInvalidated); // Compiled out of release builds return GetFlags() & Flags_Deleted; } bool inline MatchesFlags(int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { + ASSERT(!mInvalidated); // Compiled out of release builds return ((FlagsMustBeSet == Flags_INCLUDE_EVERYTHING) || ((mFlags & FlagsMustBeSet) == FlagsMustBeSet)) && ((mFlags & FlagsNotToBeSet) == 0); }; // Get dependency info // new version this depends on - int64_t GetDependsNewer() const {return mDependsNewer;} - void SetDependsNewer(int64_t ObjectID) {mDependsNewer = ObjectID;} + int64_t GetDependsNewer() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mDependsNewer; + } + void SetDependsNewer(int64_t ObjectID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mDependsNewer = ObjectID; + } // older version which depends on this - int64_t GetDependsOlder() const {return mDependsOlder;} - void SetDependsOlder(int64_t ObjectID) {mDependsOlder = ObjectID;} + int64_t GetDependsOlder() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mDependsOlder; + } + void SetDependsOlder(int64_t ObjectID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mDependsOlder = ObjectID; + } // Dependency info saving - bool HasDependencies() {return mDependsNewer != 0 || mDependsOlder != 0;} + bool HasDependencies() + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mDependsNewer != 0 || mDependsOlder != 0; + } void ReadFromStreamDependencyInfo(IOStream &rStream, int Timeout); void WriteToStreamDependencyInfo(IOStream &rStream) const; private: - BackupStoreFilename mName; + BackupStoreFilename mName; box_time_t mModificationTime; int64_t mObjectID; int64_t mSizeInBlocks; @@ -137,11 +278,18 @@ public: StreamableMemBlock mAttributes; uint32_t mMinMarkNumber; uint32_t mMarkNumber; - + uint64_t mDependsNewer; // new version this depends on uint64_t mDependsOlder; // older version which depends on this }; - + +#ifndef BOX_RELEASE_BUILD + bool IsInvalidated() + { + return mInvalidated; + } +#endif // !BOX_RELEASE_BUILD + void ReadFromStream(IOStream &rStream, int Timeout); void WriteToStream(IOStream &rStream, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, @@ -155,28 +303,78 @@ public: uint64_t AttributesHash); void DeleteEntry(int64_t ObjectID); Entry *FindEntryByID(int64_t ObjectID) const; - - int64_t GetObjectID() const {return mObjectID;} - int64_t GetContainerID() const {return mContainerID;} - + + int64_t GetObjectID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mObjectID; + } + + int64_t GetContainerID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mContainerID; + } // Need to be able to update the container ID when moving objects - void SetContainerID(int64_t ContainerID) {mContainerID = ContainerID;} + void SetContainerID(int64_t ContainerID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mContainerID = ContainerID; + } + + // Purely for use of server -- not serialised into streams + int64_t GetRevisionID() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mRevisionID; + } + void SetRevisionID(int64_t RevisionID) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mRevisionID = RevisionID; + } - // Purely for use of server -- not serialised into streams - int64_t GetRevisionID() const {return mRevisionID;} - void SetRevisionID(int64_t RevisionID) {mRevisionID = RevisionID;} - - unsigned int GetNumberOfEntries() const {return mEntries.size();} + unsigned int GetNumberOfEntries() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mEntries.size(); + } // User info -- not serialised into streams - int64_t GetUserInfo1_SizeInBlocks() const {return mUserInfo1;} - void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) {mUserInfo1 = UserInfo1;} + int64_t GetUserInfo1_SizeInBlocks() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mUserInfo1; + } + void SetUserInfo1_SizeInBlocks(int64_t UserInfo1) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mUserInfo1 = UserInfo1; + } // Attributes - bool HasAttributes() const {return !mAttributes.IsEmpty();} - void SetAttributes(const StreamableMemBlock &rAttr, box_time_t AttributesModTime) {mAttributes.Set(rAttr); mAttributesModTime = AttributesModTime;} - const StreamableMemBlock &GetAttributes() const {return mAttributes;} - box_time_t GetAttributesModTime() const {return mAttributesModTime;} + bool HasAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return !mAttributes.IsEmpty(); + } + void SetAttributes(const StreamableMemBlock &rAttr, + box_time_t AttributesModTime) + { + ASSERT(!mInvalidated); // Compiled out of release builds + mAttributes.Set(rAttr); + mAttributesModTime = AttributesModTime; + } + const StreamableMemBlock &GetAttributes() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributes; + } + box_time_t GetAttributesModTime() const + { + ASSERT(!mInvalidated); // Compiled out of release builds + return mAttributesModTime; + } class Iterator { @@ -184,10 +382,12 @@ public: Iterator(const BackupStoreDirectory &rDir) : mrDir(rDir), i(rDir.mEntries.begin()) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds } - + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds // Skip over things which don't match the required flags while(i != mrDir.mEntries.end() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) { @@ -207,6 +407,7 @@ public: // In a looping situation, cache the decrypted filenames in another memory structure. BackupStoreDirectory::Entry *FindMatchingClearName(const BackupStoreFilenameClear &rFilename, int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds // Skip over things which don't match the required flags or filename while( (i != mrDir.mEntries.end()) && ( (!(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) @@ -227,7 +428,7 @@ public: const BackupStoreDirectory &mrDir; std::vector<Entry*>::const_iterator i; }; - + friend class Iterator; class ReverseIterator @@ -236,10 +437,12 @@ public: ReverseIterator(const BackupStoreDirectory &rDir) : mrDir(rDir), i(rDir.mEntries.rbegin()) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds } - + BackupStoreDirectory::Entry *Next(int16_t FlagsMustBeSet = Entry::Flags_INCLUDE_EVERYTHING, int16_t FlagsNotToBeSet = Entry::Flags_EXCLUDE_NOTHING) { + ASSERT(!mrDir.mInvalidated); // Compiled out of release builds // Skip over things which don't match the required flags while(i != mrDir.mEntries.rend() && !(*i)->MatchesFlags(FlagsMustBeSet, FlagsNotToBeSet)) { @@ -253,12 +456,12 @@ public: // Return entry, and increment return (*(i++)); } - + private: const BackupStoreDirectory &mrDir; std::vector<Entry*>::const_reverse_iterator i; }; - + friend class ReverseIterator; // For recovery of the store @@ -286,4 +489,3 @@ private: }; #endif // BACKUPSTOREDIRECTORY__H - diff --git a/lib/backupstore/BackupStoreException.txt b/lib/backupstore/BackupStoreException.txt index ece772c0..efdbcf68 100644 --- a/lib/backupstore/BackupStoreException.txt +++ b/lib/backupstore/BackupStoreException.txt @@ -70,3 +70,7 @@ DiffFromIDNotFoundInDirectory 66 When uploading via a diff, the diff from file m PatchChainInfoBadInDirectory 67 A directory contains inconsistent information. Run bbstoreaccounts check to fix it. UnknownObjectRefCountRequested 68 A reference count was requested for an object whose reference count is not known. MultiplyReferencedObject 69 Attempted to modify an object with multiple references, should be uncloned first +CorruptReferenceCountDatabase 70 The account's refcount database is corrupt and must be rebuilt by housekeeping. +CancelledByBackgroundTask 71 The current task was cancelled on request by the background task. +ObjectDoesNotExist 72 The specified object ID does not exist in the store. +AccountAlreadyExists 73 Tried to create an account that already exists. diff --git a/lib/backupstore/BackupStoreFile.cpp b/lib/backupstore/BackupStoreFile.cpp index 519305ff..99562685 100644 --- a/lib/backupstore/BackupStoreFile.cpp +++ b/lib/backupstore/BackupStoreFile.cpp @@ -22,29 +22,30 @@ #include <stdio.h> #endif +#include "BackupClientFileAttributes.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" #include "BackupStoreFile.h" -#include "BackupStoreFileWire.h" #include "BackupStoreFileCryptVar.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreFileWire.h" #include "BackupStoreFilename.h" -#include "BackupStoreException.h" -#include "IOStream.h" -#include "Guards.h" -#include "FileModificationTime.h" -#include "FileStream.h" -#include "BackupClientFileAttributes.h" +#include "BackupStoreInfo.h" #include "BackupStoreObjectMagic.h" -#include "Compress.h" -#include "CipherContext.h" -#include "CipherBlowfish.h" #include "CipherAES.h" -#include "BackupStoreConstants.h" +#include "CipherBlowfish.h" +#include "CipherContext.h" #include "CollectInBufferStream.h" -#include "RollingChecksum.h" +#include "Compress.h" +#include "FileModificationTime.h" +#include "FileStream.h" +#include "Guards.h" +#include "IOStream.h" +#include "Logging.h" #include "MD5Digest.h" -#include "ReadGatherStream.h" #include "Random.h" -#include "BackupStoreFileEncodeStream.h" -#include "Logging.h" +#include "ReadGatherStream.h" +#include "RollingChecksum.h" #include "MemLeakFindOn.h" @@ -78,7 +79,8 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) + RunStatusProvider* pRunStatusProvider, + BackgroundTask* pBackgroundTask) { // Create the stream std::auto_ptr<BackupStoreFileEncodeStream> stream( @@ -86,8 +88,9 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( // Do the initial setup stream->Setup(Filename, 0 /* no recipe, just encode */, ContainerID, - rStoreFilename, pModificationTime, pLogger, pRunStatusProvider); - + rStoreFilename, pModificationTime, pLogger, pRunStatusProvider, + pBackgroundTask); + // Return the stream for the caller return stream; } @@ -100,7 +103,15 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFile( // requirements. Doesn't verify that the data is intact // and can be decoded. Optionally returns the ID of the // file which it is diffed from, and the (original) -// container ID. +// container ID. This is more efficient than +// BackupStoreFile::VerifyStream() when the file data +// already exists on disk and we can Seek() around in +// it, but less efficient if we are reading the stream +// from the network and not intending to Write() it to +// a file first, so we need both unfortunately. +// TODO FIXME: use a modified VerifyStream() which +// repositions the file pointer and Close()s early to +// deduplicate this code. // Created: 2003/08/28 // // -------------------------------------------------------------------------- @@ -120,7 +131,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read header return false; } - + // Check magic number if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -130,7 +141,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro { return false; } - + // Get a filename, see if it loads OK try { @@ -142,7 +153,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // an error occured while reading it, so that's not good return false; } - + // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not try { @@ -163,10 +174,10 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Get current position in file -- the end of the header int64_t headerEnd = rFile.GetPosition(); - + // Get number of blocks int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - + // Calculate where the block index will be, check it's reasonable int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); if(blockIndexLoc < headerEnd) @@ -183,7 +194,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read block index header -- assume bad file return false; } - + // Check header if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -195,10 +206,10 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Bad header -- either magic value or number of blocks is wrong return false; } - + // Flag for recording whether a block is referenced from another file bool blockFromOtherFileReferenced = false; - + // Read the index, checking that the length values all make sense int64_t currentBlockStart = headerEnd; for(int64_t b = 0; b < numBlocks; ++b) @@ -210,7 +221,7 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Couldn't read block index entry -- assume bad file return false; } - + // Check size and location int64_t blkSize = box_ntoh64(blk.mEncodedSize); if(blkSize <= 0) @@ -226,19 +237,20 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Encoded size makes the block run over the index return false; } - - // Move the current block start ot the end of this block + + // Move the current block start to the end of this block currentBlockStart += blkSize; } } - + // Check that there's no empty space if(currentBlockStart != blockIndexLoc) { return false; } - - // Check that if another block is references, then the ID is there, and if one isn't there is no ID. + + // Check that if another file is referenced, then the ID is there, and if one + // isn't then there is no ID. int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); if((otherID != 0 && blockFromOtherFileReferenced == false) || (otherID == 0 && blockFromOtherFileReferenced == true)) @@ -246,13 +258,13 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro // Doesn't look good! return false; } - + // Does the caller want the other ID? if(pDiffFromObjectIDOut) { *pDiffFromObjectIDOut = otherID; } - + // Does the caller want the container ID? if(pContainerIDOut) { @@ -263,6 +275,346 @@ bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFro return true; } + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyStream::Write() +// Purpose: Handles writes to the verifying stream. If the write +// completes the current unit, then verify it, copy it +// to mpCopyToStream if not NULL, and move on to the +// next unit, otherwise throw an exception. +// Created: 2015/08/07 +// +// -------------------------------------------------------------------------- + +void BackupStoreFile::VerifyStream::Write(const void *pBuffer, int NBytes, int Timeout) +{ + // Check that we haven't already written too much to the current unit + size_t BytesToAdd; + if(mState == State_Blocks) + { + // We don't know how many bytes to expect + ASSERT(mCurrentUnitSize == 0); + BytesToAdd = NBytes; + } + else + { + ASSERT(mCurrentUnitData.GetSize() < mCurrentUnitSize); + size_t BytesLeftInCurrentUnit = mCurrentUnitSize - + mCurrentUnitData.GetSize(); + // Add however many bytes are needed/available to the current unit's buffer. + BytesToAdd = std::min(BytesLeftInCurrentUnit, (size_t)NBytes); + } + + // We must make progress here, or we could have infinite recursion. + ASSERT(BytesToAdd > 0); + + CollectInBufferStream* pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Write(pBuffer, BytesToAdd, Timeout); + if(mpCopyToStream) + { + mpCopyToStream->Write(pBuffer, BytesToAdd, Timeout); + } + + pBuffer = (uint8_t *)pBuffer + BytesToAdd; + NBytes -= BytesToAdd; + mCurrentPosition += BytesToAdd; + + if(mState == State_Blocks) + { + // The block index follows the blocks themselves, but without seeing the + // index we don't know how big the blocks are. So we just have to keep + // reading, holding the last mBlockIndexSize bytes in two buffers, until + // we reach the end of the file (when Close() is called) when we can look + // back over those buffers and extract the block index from them. + if(pCurrentBuffer->GetSize() >= mBlockIndexSize) + { + // Time to swap buffers, and clear the one we're about to + // overwrite. + mCurrentBufferIsAlternate = !mCurrentBufferIsAlternate; + pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Reset(); + } + + // We don't want to move data into the finished buffer while we're in this + // state, and we don't need to call ourselves recursively, so just return. + return; + } + + ASSERT(mState != State_Blocks); + + // If the current unit is not complete, just return now. + if(mCurrentUnitData.GetSize() < mCurrentUnitSize) + { + return; + } + + ASSERT(mCurrentUnitData.GetSize() == mCurrentUnitSize); + mCurrentUnitData.SetForReading(); + CollectInBufferStream finished(mCurrentUnitData); + + // This should leave mCurrentUnitData empty in write phase + ASSERT(!mCurrentUnitData.StreamClosed()); + ASSERT(mCurrentUnitData.GetSize() == 0); + + // Advance automatically to next state (to reduce code duplication) if the current + // state is anything but State_Blocks. We remain in that state for more than one + // read (processing one block at a time), and exit it manually. + int oldState = mState; + if(mState != State_Blocks) + { + mState++; + } + + // Process and complete the current unit, depending what it is. + if(oldState == State_Header) + { + // Get the header... + file_StreamFormat hdr; + ASSERT(finished.GetSize() == sizeof(hdr)); + memcpy(&hdr, finished.GetBuffer(), sizeof(hdr)); + + // Check magic number + if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid header magic in stream: expected " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_MAGIC_VALUE_V1) << + " or " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_MAGIC_VALUE_V0) << + " but found " << + BOX_FORMAT_HEX32(ntohl(hdr.mMagicValue))); + } + + mNumBlocks = box_ntoh64(hdr.mNumBlocks); + mBlockIndexSize = (mNumBlocks * sizeof(file_BlockIndexEntry)) + + sizeof(file_BlockIndexHeader); + mContainerID = box_ntoh64(hdr.mContainerID); + + ASSERT(mState == State_FilenameHeader); + mCurrentUnitSize = 2; + } + else if(oldState == State_FilenameHeader) + { + // Check that the encoding is an accepted value. + unsigned int encoding = + BACKUPSTOREFILENAME_GET_ENCODING( + (uint8_t *)finished.GetBuffer()); + if(encoding < BackupStoreFilename::Encoding_Min || + encoding > BackupStoreFilename::Encoding_Max) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid encoding in filename: " << encoding); + } + + ASSERT(mState == State_Filename); + mCurrentUnitSize = BACKUPSTOREFILENAME_GET_SIZE( + (uint8_t *)finished.GetBuffer()); + // Copy the first two bytes back into the new buffer, to be used by the + // completed filename. + finished.CopyStreamTo(mCurrentUnitData); + } + else if(oldState == State_Filename) + { + BackupStoreFilename fn; + fn.ReadFromStream(finished, IOStream::TimeOutInfinite); + ASSERT(mState == State_AttributesSize); + mCurrentUnitSize = sizeof(int32_t); + } + else if(oldState == State_AttributesSize) + { + ASSERT(mState == State_Attributes); + mCurrentUnitSize = ntohl(*(int32_t *)finished.GetBuffer()); + } + else if(oldState == State_Attributes) + { + // Skip the attributes -- because they're encrypted, the server can't tell + // whether they're OK or not. + ASSERT(mState == State_Blocks); + mBlockDataPosition = mCurrentPosition; + mCurrentUnitSize = 0; + } + else if(oldState == State_Blocks) + { + // The block index follows the blocks themselves, but without seeing the + // index we don't know how big the blocks are. So we just have to keep + // reading, holding the last mBlockIndexSize bytes in two buffers, until + // we reach the end of the file (when Close() is called) when we can look + // back over those buffers and extract the block index from them. + ASSERT(mState == State_Blocks); + if(pCurrentBuffer->GetSize() >= mBlockIndexSize) + { + // Time to swap buffers, and clear the one we're about to + // overwrite. + mCurrentBufferIsAlternate = !mCurrentBufferIsAlternate; + pCurrentBuffer = (mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData); + pCurrentBuffer->Reset(); + } + } + + if(NBytes > 0) + { + // Still some data to process, so call recursively to deal with it. + Write(pBuffer, NBytes, Timeout); + } +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupStoreFile::VerifyStream::Write() +// Purpose: Handles closing the verifying stream, which tells us +// that there is no more incoming data, at which point +// we can extract the block index from the data most +// recently read, and verify it. +// Created: 2015/08/07 +// +// -------------------------------------------------------------------------- + + +void BackupStoreFile::VerifyStream::Close(bool CloseCopyStream) +{ + if(mpCopyToStream && CloseCopyStream) + { + mpCopyToStream->Close(); + } + + if(mState != State_Blocks) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Stream closed too early to be a valid BackupStoreFile: was in " + "state " << mState << " instead of " << State_Blocks); + } + + // Find the last mBlockIndexSize bytes. + CollectInBufferStream finished; + + CollectInBufferStream* pCurrentBuffer = mCurrentBufferIsAlternate ? + &mAlternateData : &mCurrentUnitData; + CollectInBufferStream* pPreviousBuffer = mCurrentBufferIsAlternate ? + &mCurrentUnitData : &mAlternateData; + + int64_t BytesFromCurrentBuffer = std::min(mBlockIndexSize, + (int64_t)pCurrentBuffer->GetSize()); + int64_t BytesFromPreviousBuffer = mBlockIndexSize - BytesFromCurrentBuffer; + + if(pPreviousBuffer->GetSize() < BytesFromPreviousBuffer) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Not enough bytes for block index: expected " << + mBlockIndexSize << " but had collected " << + (pPreviousBuffer->GetSize() + pCurrentBuffer->GetSize()) << + " at " << mCurrentPosition << " bytes into file"); + } + + size_t PreviousBufferOffset = pPreviousBuffer->GetSize() - + BytesFromPreviousBuffer; + size_t CurrentBufferOffset = pCurrentBuffer->GetSize() - + BytesFromCurrentBuffer; + + file_BlockIndexHeader blkhdr; + finished.Write((uint8_t *)pPreviousBuffer->GetBuffer() + PreviousBufferOffset, + BytesFromPreviousBuffer); + finished.Write((uint8_t *)pCurrentBuffer->GetBuffer() + CurrentBufferOffset, + BytesFromCurrentBuffer); + ASSERT(finished.GetSize() == mBlockIndexSize); + + // Load the block index header + memcpy(&blkhdr, finished.GetBuffer(), sizeof(blkhdr)); + + if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 +#ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE + && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0 +#endif + ) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index magic in stream: expected " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1) << + " or " << + BOX_FORMAT_HEX32(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0) << + " but found " << + BOX_FORMAT_HEX32(ntohl(blkhdr.mMagicValue))); + } + + if((int64_t)box_ntoh64(blkhdr.mNumBlocks) != mNumBlocks) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index size in stream: expected " << + BOX_FORMAT_OBJECTID(mNumBlocks) << + " but found " << + BOX_FORMAT_OBJECTID(box_ntoh64(blkhdr.mNumBlocks))); + } + + // Flag for recording whether a block is referenced from another file + mBlockFromOtherFileReferenced = false; + int64_t blockIndexLoc = mCurrentPosition - mBlockIndexSize; + + // Read the index, checking that the length values all make sense + int64_t currentBlockStart = mBlockDataPosition; + file_BlockIndexEntry* pBlk = (file_BlockIndexEntry *) + ((uint8_t *)finished.GetBuffer() + sizeof(blkhdr)); + for(int64_t b = 0; b < mNumBlocks; ++b) + { + // Check size and location + int64_t blkSize = box_ntoh64(pBlk[b].mEncodedSize); + if(blkSize <= 0) + { + // Mark that this file references another file + mBlockFromOtherFileReferenced = true; + } + else + { + // This block is actually in this file + if((currentBlockStart + blkSize) > blockIndexLoc) + { + // Encoded size makes the block run over the index + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index: entry " << b << " makes " + "total size of block data exceed the available " + "space"); + } + + // Move the current block start to the end of this block + currentBlockStart += blkSize; + } + } + + // Check that there's no empty space + if(currentBlockStart != blockIndexLoc) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index: total size of blocks does not match the " + "actual amount of block data in the stream: expected " << + (blockIndexLoc - mBlockDataPosition) << " but found " << + (currentBlockStart - mBlockDataPosition)); + } + + // Check that if another file is referenced, then the ID is there, and if one + // isn't then there is no ID. + int64_t otherID = box_ntoh64(blkhdr.mOtherFileID); + if((otherID != 0 && mBlockFromOtherFileReferenced == false) + || (otherID == 0 && mBlockFromOtherFileReferenced == true)) + { + // Doesn't look good! + THROW_EXCEPTION_MESSAGE(BackupStoreException, BadBackupStoreFile, + "Invalid block index header: actual dependency status does not " + "match the file header: expected to depend on file object " << + BOX_FORMAT_OBJECTID(otherID)); + } + + mDiffFromObjectID = otherID; +} + + // -------------------------------------------------------------------------- // // Function @@ -279,7 +631,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile { THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists) } - + // Try, delete output file if error try { @@ -288,7 +640,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile // Get the decoding stream std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr)); - + // Is it a symlink? if(!stream->IsSymLink()) { @@ -311,7 +663,7 @@ void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFile // of the block index. I hope that reading an extra byte // doesn't hurt! // ASSERT(drained == 0); - + // Write the attributes try { @@ -348,10 +700,10 @@ std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream( { // Create stream std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout)); - + // Get it ready stream->Setup(pAlterativeAttr); - + // Return to caller return stream; } @@ -431,7 +783,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - bool inFileOrder = true; + bool inFileOrder = true; switch(ntohl(magic)) { #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -455,7 +807,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl default: THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // If not in file order, then the index list must be read now if(!inFileOrder) { @@ -484,7 +836,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - } + } // Check magic number if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1 @@ -498,7 +850,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Get the filename mFilename.ReadFromStream(mrEncodedFile, mTimeout); - + // Get the attributes (either from stream, or supplied attributes) if(pAlterativeAttr != 0) { @@ -514,7 +866,7 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Read the attributes from the stream mAttributes.ReadFromStream(mrEncodedFile, mTimeout); } - + // If it is in file order, go and read the file attributes // Requires that the stream can seek if(inFileOrder) @@ -524,30 +876,30 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl { THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) } - + // Store current location (beginning of encoded blocks) int64_t endOfHeaderPos = mrEncodedFile.GetPosition(); - + // Work out where the index is int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); - + // Seek to that position mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute); - + // Read the block index - ReadBlockIndex(false /* magic number still to be read */); - + ReadBlockIndex(false /* magic number still to be read */); + // Seek back to the end of header position, ready for reading the chunks mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute); } - + // Check view of blocks from block header and file header match if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks)) { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Need to allocate some memory for the two blocks for reading encoded data, and clear data if(mNumBlocks > 0) { @@ -560,11 +912,11 @@ void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAl // Get the clear and encoded size int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize); ASSERT(encodedSize > 0); - + // Larger? if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize; } - + // Allocate those blocks! mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32); @@ -589,7 +941,7 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) { // Header file_BlockIndexHeader blkhdr; - + // Read it in -- way depends on how whether the magic number has already been read if(MagicAlreadyRead) { @@ -609,7 +961,7 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - + // Check magic value if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE @@ -620,26 +972,26 @@ void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead) THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } } - + // Get the number of blocks out of the header mNumBlocks = box_ntoh64(blkhdr.mNumBlocks); - + // Read the IV base mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase); - + // Load the block entries in? if(mNumBlocks > 0) { // How big is the index? int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks; - + // Allocate some memory mpBlockIndex = ::malloc(indexSize); if(mpBlockIndex == 0) { throw std::bad_alloc(); } - + // Read it in if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout)) { @@ -676,7 +1028,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) int bytesToRead = NBytes; uint8_t *output = (uint8_t*)pBuffer; - + while(bytesToRead > 0 && mCurrentBlock < mNumBlocks) { // Anything left in the current block? @@ -685,16 +1037,16 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // Copy data out of this buffer int s = mCurrentBlockClearSize - mPositionInCurrentBlock; if(s > bytesToRead) s = bytesToRead; // limit to requested data - + // Copy ::memcpy(output, mpClearData + mPositionInCurrentBlock, s); - + // Update positions output += s; mPositionInCurrentBlock += s; bytesToRead -= s; } - + // Need to get some more data? if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize) { @@ -705,7 +1057,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // Stop now! break; } - + // Get the size from the block index const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex; int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize); @@ -716,14 +1068,14 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // It needs to be combined with the previous version first. THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining) } - + // Load in next block if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout)) { // Couldn't read header THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt) } - + // Decode the data mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize); @@ -734,7 +1086,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) // platforms with different endiannesses. iv = box_hton64(iv); sBlowfishDecryptBlockEntry.SetIV(&iv); - + // Decrypt the encrypted section file_BlockIndexEntryEnc entryEnc; int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), @@ -779,7 +1131,7 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) #endif } - + // Check the digest MD5Digest md5; md5.Add(mpClearData, mCurrentBlockClearSize); @@ -788,12 +1140,12 @@ int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck) } - + // Set vars to say what's happening mPositionInCurrentBlock = 0; } } - + ASSERT(bytesToRead >= 0); ASSERT(bytesToRead <= NBytes); @@ -816,14 +1168,14 @@ bool BackupStoreFile::DecodedStream::IsSymLink() { return false; } - + // So the attributes think it is a symlink. // Consistency check... if(mNumBlocks != 0) { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + return true; } @@ -836,7 +1188,8 @@ bool BackupStoreFile::DecodedStream::IsSymLink() // Created: 9/12/03 // // -------------------------------------------------------------------------- -void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes) +void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream) } @@ -916,7 +1269,7 @@ void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength) sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); sAESDecrypt.Reset(); sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength)); - + // Set encryption to use this key, instead of the "default" blowfish key spEncrypt = &sAESEncrypt; sEncryptCipherType = HEADER_AES_ENCODING; @@ -961,9 +1314,9 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi { rOutput.Reallocate(256); } - + // Check alignment of the block - ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + ASSERT((((uint64_t)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); // Want to compress it? bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE); @@ -981,10 +1334,10 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi const void *iv = spEncrypt->SetRandomIV(ivLen); ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen); outOffset += ivLen; - + // Start encryption process spEncrypt->Begin(); - + #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \ { \ if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \ @@ -992,13 +1345,13 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \ } \ } - + // Encode the chunk if(compressChunk) { // buffer to compress into uint8_t buffer[2048]; - + // Set compressor with all the chunk as an input Compress<true> compress; compress.Input(Chunk, ChunkSize); @@ -1011,7 +1364,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi if(s > 0) { ENCODECHUNK_CHECK_SPACE(s) - outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); + outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s); } else { @@ -1031,7 +1384,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi ENCODECHUNK_CHECK_SPACE(16) outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset); } - + ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check return outOffset; @@ -1051,7 +1404,7 @@ int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFi int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize) { // Check alignment of the encoded block - ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); + ASSERT((((uint64_t)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET); // First check if(EncodedSize < 1) @@ -1060,7 +1413,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } const uint8_t *input = (uint8_t*)Encoded; - + // Get header, make checks, etc uint8_t header = input[0]; bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED; @@ -1069,7 +1422,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out { THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding) } - + #ifndef HAVE_OLD_SSL // Choose cipher CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt); @@ -1081,7 +1434,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } CipherContext &cipher(sBlowfishDecrypt); #endif - + // Check enough space for header, an IV and one byte of input int ivLen = cipher.GetIVLength(); if(EncodedSize < (1 + ivLen + 1)) @@ -1092,7 +1445,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out // Set IV in decrypt context, and start cipher.SetIV(input + 1); cipher.Begin(); - + // Setup vars for code int inOffset = 1 + ivLen; uint8_t *output = (uint8_t*)Output; @@ -1104,10 +1457,10 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out // Do things in chunks uint8_t buffer[2048]; int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer)); - + // Decompressor Compress<false> decompress; - + while(inOffset < EncodedSize) { // Decrypt a block @@ -1115,7 +1468,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl); inOffset += bl; - + // Decompress the decrypted data if(s > 0) { @@ -1126,7 +1479,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out os = decompress.Output(output + outOffset, OutputSize - outOffset); outOffset += os; } while(os > 0); - + // Check that there's space left in the output buffer -- there always should be if(outOffset >= OutputSize) { @@ -1134,7 +1487,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out } } } - + // Get any compressed data remaining in the cipher context and compression int s = cipher.Final(buffer, sizeof(buffer)); decompress.Input(buffer, s); @@ -1157,7 +1510,7 @@ int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Out outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset); outOffset += cipher.Final(output + outOffset, OutputSize - outOffset); } - + return outOffset; } @@ -1209,10 +1562,10 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr { THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Get number of blocks int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); - + // Calculate where the block index will be, check it's reasonable int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader)); int64_t blockIndexLoc = fileSize - blockIndexSize; @@ -1221,10 +1574,10 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr // Doesn't look good! THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile) } - + // Build a reordered stream std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership)); - + // Set it up... ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get())); int component = rreordered.AddComponent(pStream); @@ -1232,7 +1585,7 @@ std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStr rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc); // And then the rest of the file rreordered.AddBlock(component, blockIndexLoc, true, 0); - + return reordered; } @@ -1287,7 +1640,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, { in.reset(new FileStream(Filename)); } - + // Read header file_BlockIndexHeader hdr; if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout)) @@ -1313,17 +1666,17 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, // Get basic information int64_t numBlocks = box_ntoh64(hdr.mNumBlocks); uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase); - + //TODO: Verify that these sizes look reasonable - + // setup void *data = 0; int32_t dataSize = -1; bool matches = true; int64_t totalSizeInBlockIndex = 0; - + try - { + { for(int64_t b = 0; b < numBlocks; ++b) { // Read an entry from the stream @@ -1332,8 +1685,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, { // Couldn't read entry THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream) - } - + } + // Calculate IV for this entry uint64_t iv = entryIVBase; iv += b; @@ -1345,8 +1698,8 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, iv = box_swap64(iv); } #endif - sBlowfishDecryptBlockEntry.SetIV(&iv); - + sBlowfishDecryptBlockEntry.SetIV(&iv); + // Decrypt the encrypted section file_BlockIndexEntryEnc entryEnc; int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc), @@ -1381,7 +1734,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } dataSize = blockClearSize + 128; } - + // Load in the block from the file, if it's not a symlink if(!sourceIsSymlink) { @@ -1403,7 +1756,7 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } } } - + // Keep on going regardless, to make sure the entire block index stream is read // -- must always be consistent about what happens with the stream. } @@ -1418,14 +1771,14 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, } throw; } - + // free block if(data != 0) { ::free(data); data = 0; } - + // Check for data left over if it's not a symlink if(!sourceIsSymlink) { @@ -1436,13 +1789,13 @@ bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, matches = false; } } - + // Symlinks must have zero size on server if(sourceIsSymlink) { matches = (totalSizeInBlockIndex == 0); } - + return matches; } @@ -1522,10 +1875,10 @@ void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize) } // Copy data ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize); - + // Free old BackupStoreFile::CodingChunkFree(mpBuffer); - + // Store new buffer mpBuffer = buffer; mBufferSize = NewSize; @@ -1554,5 +1907,44 @@ DiffTimer::DiffTimer() // // -------------------------------------------------------------------------- DiffTimer::~DiffTimer() -{ +{ +} + +// Shortcut interface +int64_t BackupStoreFile::QueryStoreFileDiff(BackupProtocolCallable& protocol, + const std::string& LocalFilename, int64_t DirectoryObjectID, + int64_t DiffFromFileID, int64_t AttributesHash, + const BackupStoreFilenameClear& StoreFilename, int Timeout, + DiffTimer *pDiffTimer, ReadLoggingStream::Logger* pLogger, + RunStatusProvider* pRunStatusProvider) +{ + int64_t ModificationTime; + std::auto_ptr<BackupStoreFileEncodeStream> pStream; + + if(DiffFromFileID) + { + // Fetch the block index for this one + std::auto_ptr<BackupProtocolSuccess> getblockindex = + protocol.QueryGetBlockIndexByName(DirectoryObjectID, + StoreFilename); + ASSERT(getblockindex->GetObjectID() == DiffFromFileID); + std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream()); + + pStream = EncodeFileDiff(LocalFilename, + DirectoryObjectID, StoreFilename, DiffFromFileID, + *(blockIndexStream.get()), Timeout, pDiffTimer, + &ModificationTime, NULL // pIsCompletelyDifferent + ); + } + else + { + pStream = BackupStoreFile::EncodeFile(LocalFilename, + DirectoryObjectID, StoreFilename, &ModificationTime); + } + + std::auto_ptr<IOStream> upload(pStream.release()); + return protocol.QueryStoreFile(DirectoryObjectID, + ModificationTime, AttributesHash, DiffFromFileID, + StoreFilename, upload)->GetObjectID(); } + diff --git a/lib/backupstore/BackupStoreFile.h b/lib/backupstore/BackupStoreFile.h index 7c72e010..fe69caeb 100644 --- a/lib/backupstore/BackupStoreFile.h +++ b/lib/backupstore/BackupStoreFile.h @@ -14,8 +14,11 @@ #include <memory> #include <cstdlib> +#include "autogen_BackupProtocol.h" #include "BackupClientFileAttributes.h" +#include "BackupStoreFileWire.h" #include "BackupStoreFilename.h" +#include "CollectInBufferStream.h" #include "IOStream.h" #include "ReadLoggingStream.h" @@ -26,6 +29,7 @@ typedef struct int64_t mTotalFileStreamSize; } BackupStoreFileStats; +class BackgroundTask; class RunStatusProvider; // Uncomment to disable backwards compatibility @@ -40,6 +44,8 @@ class RunStatusProvider; // Have some memory allocation commands, note closing "Off" at end of file. #include "MemLeakFindOn.h" +class BackupStoreFileEncodeStream; + // -------------------------------------------------------------------------- // // Class @@ -60,8 +66,6 @@ public: virtual bool IsManaged() = 0; }; -class BackupStoreFileEncodeStream; - // -------------------------------------------------------------------------- // // Class @@ -83,9 +87,10 @@ public: public: ~DecodedStream(); - // Stream functions + // Stream functions virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); @@ -119,6 +124,64 @@ public: #endif }; + class VerifyStream : public IOStream + { + private: + enum + { + State_Header = 0, + State_FilenameHeader, + State_Filename, + State_AttributesSize, + State_Attributes, + State_Blocks, + }; + + int mState; + IOStream* mpCopyToStream; + CollectInBufferStream mCurrentUnitData; + size_t mCurrentUnitSize; + int64_t mNumBlocks; + int64_t mBlockIndexSize; + int64_t mCurrentPosition; + int64_t mBlockDataPosition; + bool mCurrentBufferIsAlternate; + CollectInBufferStream mAlternateData; + bool mBlockFromOtherFileReferenced; + int64_t mContainerID; + int64_t mDiffFromObjectID; + + public: + VerifyStream(IOStream* pCopyToStream = NULL) + : mState(State_Header), + mpCopyToStream(pCopyToStream), + mCurrentUnitSize(sizeof(file_StreamFormat)), + mNumBlocks(0), + mBlockIndexSize(0), + mCurrentPosition(0), + mBlockDataPosition(0), + mCurrentBufferIsAlternate(false), + mBlockFromOtherFileReferenced(false), + mContainerID(0), + mDiffFromObjectID(0) + { } + virtual int Read(void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + virtual void Close(bool CloseCopyStream = true); + virtual bool StreamDataLeft() + { + THROW_EXCEPTION(CommonException, NotSupported); + } + virtual bool StreamClosed() + { + THROW_EXCEPTION(CommonException, NotSupported); + } + }; // Main interface static std::auto_ptr<BackupStoreFileEncodeStream> EncodeFile @@ -127,7 +190,8 @@ public: int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime = 0, ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL + RunStatusProvider* pRunStatusProvider = NULL, + BackgroundTask* pBackgroundTask = NULL ); static std::auto_ptr<BackupStoreFileEncodeStream> EncodeFileDiff ( @@ -137,8 +201,19 @@ public: int Timeout, DiffTimer *pDiffTimer, int64_t *pModificationTime = 0, - bool *pIsCompletelyDifferent = 0 + bool *pIsCompletelyDifferent = 0, + BackgroundTask* pBackgroundTask = NULL ); + // Shortcut interface + static int64_t QueryStoreFileDiff(BackupProtocolCallable& protocol, + const std::string& LocalFilename, int64_t DirectoryObjectID, + int64_t DiffFromFileID, int64_t AttributesHash, + const BackupStoreFilenameClear& StoreFilename, + int Timeout = IOStream::TimeOutInfinite, + DiffTimer *pDiffTimer = NULL, + ReadLoggingStream::Logger* pLogger = NULL, + RunStatusProvider* pRunStatusProvider = NULL); + static bool VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut = 0, int64_t *pContainerIDOut = 0); static void CombineFile(IOStream &rDiff, IOStream &rDiff2, IOStream &rFrom, IOStream &rOut); static void CombineDiffs(IOStream &rDiff1, IOStream &rDiff2, IOStream &rDiff2b, IOStream &rOut); diff --git a/lib/backupstore/BackupStoreFileCmbIdx.cpp b/lib/backupstore/BackupStoreFileCmbIdx.cpp index c8bcc3b9..0eec3872 100644 --- a/lib/backupstore/BackupStoreFileCmbIdx.cpp +++ b/lib/backupstore/BackupStoreFileCmbIdx.cpp @@ -32,7 +32,8 @@ public: ~BSFCombinedIndexStream(); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); virtual void Initialise(IOStream &rFrom); @@ -289,7 +290,8 @@ int BSFCombinedIndexStream::Read(void *pBuffer, int NBytes, int Timeout) // Created: 8/7/04 // // -------------------------------------------------------------------------- -void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes) +void BSFCombinedIndexStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures) } diff --git a/lib/backupstore/BackupStoreFileDiff.cpp b/lib/backupstore/BackupStoreFileDiff.cpp index fa8cb892..e6df11a6 100644 --- a/lib/backupstore/BackupStoreFileDiff.cpp +++ b/lib/backupstore/BackupStoreFileDiff.cpp @@ -16,7 +16,7 @@ #ifdef HAVE_TIME_H #include <time.h> -#elif HAVE_SYS_TIME_H +#elif defined HAVE_SYS_TIME_H #include <sys/time.h> #endif @@ -114,21 +114,21 @@ void BackupStoreFile::MoveStreamPositionToBlockIndex(IOStream &rStream) // Function // Name: BackupStoreFile::EncodeFileDiff(const char *, int64_t, const BackupStoreFilename &, int64_t, IOStream &, int64_t *) // Purpose: Similar to EncodeFile, but takes the object ID of the file it's -// diffing from, and the index of the blocks in a stream. It'll then -// calculate which blocks can be reused from that old file. -// The timeout is the timeout value for reading the diff block index. -// If pIsCompletelyDifferent != 0, it will be set to true if the -// the two files are completely different (do not share any block), false otherwise. -// +// diffing from, and the index of the blocks in a stream. It'll then +// calculate which blocks can be reused from that old file. +// The timeout is the timeout value for reading the diff block index. +// If pIsCompletelyDifferent != 0, it will be set to true if the +// the two files are completely different (do not share any block), false otherwise. // Created: 12/1/04 // // -------------------------------------------------------------------------- std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff ( const std::string& Filename, int64_t ContainerID, - const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, - IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, - int64_t *pModificationTime, bool *pIsCompletelyDifferent) + const BackupStoreFilename &rStoreFilename, int64_t DiffFromObjectID, + IOStream &rDiffFromBlockIndex, int Timeout, DiffTimer *pDiffTimer, + int64_t *pModificationTime, bool *pIsCompletelyDifferent, + BackgroundTask* pBackgroundTask) { // Is it a symlink? { @@ -144,7 +144,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff { *pIsCompletelyDifferent = true; } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + return EncodeFile(Filename, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); // BackgroundTask } } @@ -162,7 +166,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff { *pIsCompletelyDifferent = true; } - return EncodeFile(Filename, ContainerID, rStoreFilename, pModificationTime); + return EncodeFile(Filename, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); // BackgroundTask } // Pointer to recipe we're going to create @@ -210,7 +218,11 @@ std::auto_ptr<BackupStoreFileEncodeStream> BackupStoreFile::EncodeFileDiff new BackupStoreFileEncodeStream); // Do the initial setup - stream->Setup(Filename, precipe, ContainerID, rStoreFilename, pModificationTime); + stream->Setup(Filename, precipe, ContainerID, rStoreFilename, + pModificationTime, + NULL, // ReadLoggingStream::Logger + NULL, // RunStatusProvider + pBackgroundTask); precipe = 0; // Stream has taken ownership of this // Tell user about completely different status? @@ -515,7 +527,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> // Search for each block size in turn // NOTE: Do the smallest size first, so that the scheme for adding - // entries in the found list works as expected and replaces smallers block + // entries in the found list works as expected and replaces smaller blocks // with larger blocks when it finds matches at the same offset in the file. for(int s = BACKUP_FILE_DIFF_MAX_BLOCK_SIZES - 1; s >= 0; --s) { @@ -629,7 +641,7 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> { if(SecondStageMatch(phashTable[hash], rolling, beginnings, endings, offset, Sizes[s], fileBlockNumber, pIndex, rFoundBlocks)) { - BOX_TRACE("Found block match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); + BOX_TRACE("Found block match of " << Sizes[s] << " bytes with hash " << hash << " at offset " << fileOffset); goodnessOfFit[fileOffset] = Sizes[s]; // Block matched, roll the checksum forward to the next block without doing @@ -657,7 +669,8 @@ static void SearchForMatchingBlocks(IOStream &rFile, std::map<int64_t, int64_t> } else { - BOX_TRACE("False alarm match for " << hash << " of " << Sizes[s] << " bytes at offset " << fileOffset); + // Too many to log + // BOX_TRACE("False alarm match of " << Sizes[s] << " bytes with hash " << hash << " at offset " << fileOffset); } int64_t NumBlocksFound = static_cast<int64_t>( @@ -1029,9 +1042,11 @@ static void GenerateRecipe(BackupStoreFileEncodeStream::Recipe &rRecipe, BlocksA { char b[64]; #ifdef WIN32 - sprintf(b, "%8I64d", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); + snprintf(b, sizeof(b), "%8I64d", (int64_t) + (rRecipe[e].mpStartBlock - pIndex)); #else - sprintf(b, "%8lld", (int64_t)(rRecipe[e].mpStartBlock - pIndex)); + snprintf(b, sizeof(b), "%8lld", (int64_t) + (rRecipe[e].mpStartBlock - pIndex)); #endif BOX_TRACE(std::setw(8) << rRecipe[e].mSpaceBefore << diff --git a/lib/backupstore/BackupStoreFileEncodeStream.cpp b/lib/backupstore/BackupStoreFileEncodeStream.cpp index b53c4c26..83333a5b 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.cpp +++ b/lib/backupstore/BackupStoreFileEncodeStream.cpp @@ -11,6 +11,7 @@ #include <string.h> +#include "BackgroundTask.h" #include "BackupClientFileAttributes.h" #include "BackupStoreConstants.h" #include "BackupStoreException.h" @@ -40,25 +41,28 @@ using namespace BackupStoreFileCryptVar; // // -------------------------------------------------------------------------- BackupStoreFileEncodeStream::BackupStoreFileEncodeStream() - : mpRecipe(0), - mpFile(0), - mpLogging(0), - mpRunStatusProvider(NULL), - mStatus(Status_Header), - mSendData(true), - mTotalBlocks(0), - mAbsoluteBlockNumber(-1), - mInstructionNumber(-1), - mNumBlocks(0), - mCurrentBlock(-1), - mCurrentBlockEncodedSize(0), - mPositionInCurrentBlock(0), - mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), - mLastBlockSize(0), - mTotalBytesSent(0), - mpRawBuffer(0), - mAllocatedBufferSize(0), - mEntryIVBase(0) +: mpRecipe(0), + mpFile(0), + mpLogging(0), + mpRunStatusProvider(NULL), + mpBackgroundTask(NULL), + mStatus(Status_Header), + mSendData(true), + mTotalBlocks(0), + mBytesToUpload(0), + mBytesUploaded(0), + mAbsoluteBlockNumber(-1), + mInstructionNumber(-1), + mNumBlocks(0), + mCurrentBlock(-1), + mCurrentBlockEncodedSize(0), + mPositionInCurrentBlock(0), + mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE), + mLastBlockSize(0), + mTotalBytesSent(0), + mpRawBuffer(0), + mAllocatedBufferSize(0), + mEntryIVBase(0) { } @@ -78,21 +82,21 @@ BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream() ::free(mpRawBuffer); mpRawBuffer = 0; } - + // Close the file, which we might have open if(mpFile) { delete mpFile; mpFile = 0; } - + // Clear up logging stream if(mpLogging) { delete mpLogging; mpLogging = 0; } - + // Free the recipe if(mpRecipe != 0) { @@ -115,7 +119,8 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, BackupStoreFileEncodeStream::Recipe *pRecipe, int64_t ContainerID, const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger, - RunStatusProvider* pRunStatusProvider) + RunStatusProvider* pRunStatusProvider, + BackgroundTask* pBackgroundTask) { // Pointer to a blank recipe which we might create BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0; @@ -128,27 +133,27 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, BackupClientFileAttributes attr; attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime, 0 /* not interested in attr mod time */, &fileSize); - + // Might need to create a blank recipe... if(pRecipe == 0) { pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0); - + BackupStoreFileEncodeStream::RecipeInstruction instruction; instruction.mSpaceBefore = fileSize; // whole file instruction.mBlocks = 0; // no blocks instruction.mpStartBlock = 0; // no block - pblankRecipe->push_back(instruction); + pblankRecipe->push_back(instruction); pRecipe = pblankRecipe; } - + // Tell caller? if(pModificationTime != 0) { *pModificationTime = modTime; } - + // Go through each instruction in the recipe and work out how many blocks // it will add, and the max clear size of these blocks int maxBlockClearSize = 0; @@ -162,14 +167,15 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize); // Add to accumlated total mTotalBlocks += numBlocks; + mBytesToUpload += (*pRecipe)[inst].mSpaceBefore; // Update maximum clear size if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize; if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize; } - + // Add number of blocks copied from the previous file mTotalBlocks += (*pRecipe)[inst].mBlocks; - + // Check for bad things if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0)) { @@ -182,7 +188,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize; } } - + // Send data? (symlinks don't have any data in them) mSendData = !attr.IsSymLink(); @@ -191,7 +197,7 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, { maxBlockClearSize = 0; } - + // Header file_StreamFormat hdr; hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1); @@ -201,16 +207,16 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, // add a bit to make it harder to tell what's going on -- try not to give away too much info about file size hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128); hdr.mOptions = 0; // no options defined yet - + // Write header to stream mData.Write(&hdr, sizeof(hdr)); - + // Write filename to stream rStoreFilename.WriteToStream(mData); - + // Write attributes to stream attr.WriteToStream(mData); - + // Allocate some buffers for writing data if(mSendData) { @@ -229,10 +235,10 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, mpLogging = mpFile; mpFile = NULL; } - + // Work out the largest possible block required for the encoded data mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize); - + // Then allocate two blocks of this size mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize); if(mpRawBuffer == 0) @@ -256,13 +262,13 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, blkhdr.mNumBlocks = box_hton64(0); mData.Write(&blkhdr, sizeof(blkhdr)); } - + // Ready for reading mData.SetForReading(); - + // Update stats BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize; - + // Finally, store the pointer to the recipe, when we know exceptions won't occur mpRecipe = pRecipe; } @@ -276,8 +282,9 @@ void BackupStoreFileEncodeStream::Setup(const std::string& Filename, } throw; } - + mpRunStatusProvider = pRunStatusProvider; + mpBackgroundTask = pBackgroundTask; } @@ -296,14 +303,14 @@ void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t do { rBlockSizeOut *= 2; - + rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut; - + } while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER); - + // Last block size rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut); - + // Avoid small blocks? if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN && rNumBlocksOut > 1) @@ -312,7 +319,7 @@ void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t --rNumBlocksOut; rLastBlockSizeOut += rBlockSizeOut; } - + // checks! ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize); //TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut); @@ -335,26 +342,39 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { return 0; } - + if(mpRunStatusProvider && mpRunStatusProvider->StopRun()) { THROW_EXCEPTION(BackupStoreException, SignalReceived); } + if(mpBackgroundTask) + { + BackgroundTask::State state = (mpRecipe->at(0).mBlocks == 0) + ? BackgroundTask::Uploading_Full + : BackgroundTask::Uploading_Patch; + if(!mpBackgroundTask->RunBackgroundTask(state, mBytesUploaded, + mBytesToUpload)) + { + THROW_EXCEPTION(BackupStoreException, + CancelledByBackgroundTask); + } + } + int bytesToRead = NBytes; uint8_t *buffer = (uint8_t*)pBuffer; - + while(bytesToRead > 0 && mStatus != Status_Finished) { if(mStatus == Status_Header || mStatus == Status_BlockListing) { // Header or block listing phase -- send from the buffered stream - + // Send bytes from the data buffer int b = mData.Read(buffer, bytesToRead, Timeout); bytesToRead -= b; buffer += b; - + // Check to see if all the data has been used from this stream if(!mData.StreamDataLeft()) { @@ -367,7 +387,7 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { // Reset the buffer so it can be used for the next phase mData.Reset(); - + // Get buffer ready for index? if(mStatus == Status_Header) { @@ -377,14 +397,14 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) ASSERT(mpRecipe != 0); blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID()); blkhdr.mNumBlocks = box_hton64(mTotalBlocks); - + // Generate the IV base Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase)); blkhdr.mEntryIVBase = box_hton64(mEntryIVBase); - + mData.Write(&blkhdr, sizeof(blkhdr)); } - + ++mStatus; } } @@ -392,7 +412,7 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) else if(mStatus == Status_Blocks) { // Block sending phase - + if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize) { // Next block! @@ -405,10 +425,10 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) { SkipPreviousBlocksInInstruction(); } - + // Is there another instruction to go? ++mInstructionNumber; - + // Skip instructions which don't contain any data while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size()) && (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0) @@ -416,12 +436,12 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) SkipPreviousBlocksInInstruction(); ++mInstructionNumber; } - + if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size())) { // End of blocks, go to next phase ++mStatus; - + // Set the data to reading so the index can be written mData.SetForReading(); } @@ -438,17 +458,17 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) EncodeCurrentBlock(); } } - + // Send data from the current block (if there's data to send) if(mPositionInCurrentBlock < mCurrentBlockEncodedSize) { // How much data to put in the buffer? int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock; if(s > bytesToRead) s = bytesToRead; - + // Copy it in ::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s); - + // Update variables bytesToRead -= s; buffer += s; @@ -461,11 +481,11 @@ int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout) ASSERT(false); } } - + // Add encoded size to stats BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead); mTotalBytesSent += (NBytes - bytesToRead); - + // Return size of data to caller return NBytes - bytesToRead; } @@ -490,27 +510,27 @@ void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction() // Index of the first block in old file (being diffed from) int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock); - + int64_t sizeToSkip = 0; for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b) { // Update stats BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; - + // Store the entry StoreBlockIndexEntry(0 - (firstIndex + b), (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize, (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum, - (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); + (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum); // Increment the absolute block number -- kept encryption IV in sync ++mAbsoluteBlockNumber; - + // Add the size of this block to the size to skip sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize; } - + // Move forward in the stream mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative); } @@ -528,7 +548,7 @@ void BackupStoreFileEncodeStream::SetForInstruction() { // Calculate block sizes CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize); - + // Set variables mCurrentBlock = 0; mCurrentBlockEncodedSize = 0; @@ -561,7 +581,7 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() // File should be open, but isn't. So logical error. THROW_EXCEPTION(BackupStoreException, Internal) } - + // Read the data in if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize, 0 /* not interested in size if failure */)) @@ -571,13 +591,15 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() THROW_EXCEPTION(BackupStoreException, Temp_FileEncodeStreamDidntReadBuffer) } - + // Encode it mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer, blockRawSize, mEncodedBuffer); - + + mBytesUploaded += blockRawSize; + //TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize); - + // Create block listing data -- generate checksums RollingChecksum weakChecksum(mpRawBuffer, blockRawSize); MD5Digest strongChecksum; @@ -587,7 +609,7 @@ void BackupStoreFileEncodeStream::EncodeCurrentBlock() // Add entry to the index StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize, weakChecksum.GetChecksum(), strongChecksum.DigestAsData()); - + // Set vars to reading this block mPositionInCurrentBlock = 0; } @@ -611,7 +633,7 @@ void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex // Then the clear section file_BlockIndexEntry entry; entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex)); - + // Then encrypt the encryted section // Generate the IV from the block number if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase)) @@ -645,7 +667,8 @@ void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex // Created: 8/12/03 // // -------------------------------------------------------------------------- -void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes) +void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes, + int Timeout) { THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream) } @@ -686,11 +709,12 @@ bool BackupStoreFileEncodeStream::StreamClosed() // Created: 15/1/04 // // -------------------------------------------------------------------------- -BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, - int64_t NumBlocksInIndex, int64_t OtherFileID) - : mpBlockIndex(pBlockIndex), - mNumBlocksInIndex(NumBlocksInIndex), - mOtherFileID(OtherFileID) +BackupStoreFileEncodeStream::Recipe::Recipe( + BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex, + int64_t NumBlocksInIndex, int64_t OtherFileID) +: mpBlockIndex(pBlockIndex), + mNumBlocksInIndex(NumBlocksInIndex), + mOtherFileID(OtherFileID) { ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) } diff --git a/lib/backupstore/BackupStoreFileEncodeStream.h b/lib/backupstore/BackupStoreFileEncodeStream.h index a169e036..5b9b4a61 100644 --- a/lib/backupstore/BackupStoreFileEncodeStream.h +++ b/lib/backupstore/BackupStoreFileEncodeStream.h @@ -79,14 +79,20 @@ public: const BackupStoreFilename &rStoreFilename, int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger = NULL, - RunStatusProvider* pRunStatusProvider = NULL); + RunStatusProvider* pRunStatusProvider = NULL, + BackgroundTask* pBackgroundTask = NULL); virtual int Read(void *pBuffer, int NBytes, int Timeout); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); + int64_t GetBytesToUpload() { return mBytesToUpload; } int64_t GetTotalBytesSent() { return mTotalBytesSent; } + static void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, + int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); + private: enum { @@ -95,28 +101,29 @@ private: Status_BlockListing = 2, Status_Finished = 3 }; - -private: + void EncodeCurrentBlock(); - void CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut); void SkipPreviousBlocksInInstruction(); void SetForInstruction(); void StoreBlockIndexEntry(int64_t WncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum); -private: Recipe *mpRecipe; IOStream *mpFile; // source file CollectInBufferStream mData; // buffer for header and index entries IOStream *mpLogging; RunStatusProvider* mpRunStatusProvider; + BackgroundTask* mpBackgroundTask; int mStatus; bool mSendData; // true if there's file data to send (ie not a symlink) int64_t mTotalBlocks; // Total number of blocks in the file + int64_t mBytesToUpload; // Total number of clear bytes to encode and upload + int64_t mBytesUploaded; // Total number of clear bytes already encoded + // excluding reused blocks already on the server. int64_t mAbsoluteBlockNumber; // The absolute block number currently being output // Instruction number int64_t mInstructionNumber; // All the below are within the current instruction - int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases + int64_t mNumBlocks; // number of blocks. Last one will be a different size to the rest in most cases int64_t mCurrentBlock; int32_t mCurrentBlockEncodedSize; int32_t mPositionInCurrentBlock; // for reading out diff --git a/lib/backupstore/BackupStoreFilenameClear.h b/lib/backupstore/BackupStoreFilenameClear.h index 595d1158..b7cf555f 100644 --- a/lib/backupstore/BackupStoreFilenameClear.h +++ b/lib/backupstore/BackupStoreFilenameClear.h @@ -42,7 +42,7 @@ public: #endif void SetClearFilename(const std::string &rToEncode); - // Setup for encryption of filenames + // Setup for encryption of filenames static void SetBlowfishKey(const void *pKey, int KeyLength, const void *pIV, int IVLength); static void SetEncodingMethod(int Method); diff --git a/lib/backupstore/BackupStoreInfo.cpp b/lib/backupstore/BackupStoreInfo.cpp index b6714709..efe3f7bb 100644 --- a/lib/backupstore/BackupStoreInfo.cpp +++ b/lib/backupstore/BackupStoreInfo.cpp @@ -34,22 +34,24 @@ // // -------------------------------------------------------------------------- BackupStoreInfo::BackupStoreInfo() - : mAccountID(-1), - mDiscSet(-1), - mReadOnly(true), - mIsModified(false), - mClientStoreMarker(0), - mLastObjectIDUsed(-1), - mBlocksUsed(0), - mBlocksInCurrentFiles(0), - mBlocksInOldFiles(0), - mBlocksInDeletedFiles(0), - mBlocksInDirectories(0), - mNumFiles(0), - mNumOldFiles(0), - mNumDeletedFiles(0), - mNumDirectories(0), - mAccountEnabled(true) +: mAccountID(-1), + mDiscSet(-1), + mReadOnly(true), + mIsModified(false), + mClientStoreMarker(0), + mLastObjectIDUsed(-1), + mBlocksUsed(0), + mBlocksInCurrentFiles(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0), + mBlocksSoftLimit(0), + mBlocksHardLimit(0), + mNumCurrentFiles(0), + mNumOldFiles(0), + mNumDeletedFiles(0), + mNumDirectories(0), + mAccountEnabled(true) { } @@ -92,6 +94,31 @@ void BackupStoreInfo::CreateNew(int32_t AccountID, const std::string &rRootDir, info.Save(false); } +BackupStoreInfo::BackupStoreInfo(int32_t AccountID, const std::string &FileName, + int64_t BlockSoftLimit, int64_t BlockHardLimit) +: mAccountID(AccountID), + mDiscSet(-1), + mFilename(FileName), + mReadOnly(false), + mIsModified(false), + mClientStoreMarker(0), + mLastObjectIDUsed(0), + mBlocksUsed(0), + mBlocksInCurrentFiles(0), + mBlocksInOldFiles(0), + mBlocksInDeletedFiles(0), + mBlocksInDirectories(0), + mBlocksSoftLimit(BlockSoftLimit), + mBlocksHardLimit(BlockHardLimit), + mNumCurrentFiles(0), + mNumOldFiles(0), + mNumDeletedFiles(0), + mNumDirectories(0), + mAccountEnabled(true) +{ + mExtraData.SetForReading(); // extra data is empty in this case +} + // -------------------------------------------------------------------------- // // Function @@ -108,16 +135,31 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { // Generate the filename std::string fn(rRootDir + INFO_FILENAME); - + // Open the file for reading (passing on optional request for revision ID) std::auto_ptr<RaidFileRead> rf(RaidFileRead::Open(DiscSet, fn, pRevisionID)); + std::auto_ptr<BackupStoreInfo> info = Load(*rf, fn, ReadOnly); + + // Check it + if(info->GetAccountID() != AccountID) + { + THROW_FILE_ERROR("Found wrong account ID in store info", + fn, BackupStoreException, BadStoreInfoOnLoad); + } + + info->mDiscSet = DiscSet; + return info; +} +std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(IOStream& rStream, + const std::string FileName, bool ReadOnly) +{ // Read in format and version int32_t magic; - if(!rf->ReadFullBuffer(&magic, sizeof(magic), 0)) + if(!rStream.ReadFullBuffer(&magic, sizeof(magic), 0)) { THROW_FILE_ERROR("Failed to read store info file: " - "short read of magic number", fn, + "short read of magic number", FileName, BackupStoreException, CouldNotLoadStoreInfo); } @@ -135,16 +177,14 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { THROW_FILE_ERROR("Failed to read store info file: " "unknown magic " << BOX_FORMAT_HEX32(ntohl(magic)), - fn, BackupStoreException, BadStoreInfoOnLoad); + FileName, BackupStoreException, BadStoreInfoOnLoad); } // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); - + // Put in basic location info - info->mAccountID = AccountID; - info->mDiscSet = DiscSet; - info->mFilename = fn; + info->mFilename = FileName; info->mReadOnly = ReadOnly; int64_t numDelObj = 0; @@ -152,23 +192,17 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, { // Read in a header info_StreamFormat_1 hdr; - rf->Seek(0, IOStream::SeekType_Absolute); + rStream.Seek(0, IOStream::SeekType_Absolute); - if(!rf->ReadFullBuffer(&hdr, sizeof(hdr), + if(!rStream.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) { THROW_FILE_ERROR("Failed to read store info header", - fn, BackupStoreException, CouldNotLoadStoreInfo); - } - - // Check it - if((int32_t)ntohl(hdr.mAccountID) != AccountID) - { - THROW_FILE_ERROR("Found wrong account ID in store info", - fn, BackupStoreException, BadStoreInfoOnLoad); + FileName, BackupStoreException, CouldNotLoadStoreInfo); } - + // Insert info from file + info->mAccountID = ntohl(hdr.mAccountID); info->mClientStoreMarker = box_ntoh64(hdr.mClientStoreMarker); info->mLastObjectIDUsed = box_ntoh64(hdr.mLastObjectIDUsed); info->mBlocksUsed = box_ntoh64(hdr.mBlocksUsed); @@ -177,24 +211,16 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, info->mBlocksInDirectories = box_ntoh64(hdr.mBlocksInDirectories); info->mBlocksSoftLimit = box_ntoh64(hdr.mBlocksSoftLimit); info->mBlocksHardLimit = box_ntoh64(hdr.mBlocksHardLimit); - + // Load up array of deleted objects numDelObj = box_ntoh64(hdr.mNumberDeletedDirectories); } else if(v2) { - Archive archive(*rf, IOStream::TimeOutInfinite); + Archive archive(rStream, IOStream::TimeOutInfinite); // Check it - int32_t FileAccountID; - archive.Read(FileAccountID); - if (FileAccountID != AccountID) - { - THROW_FILE_ERROR("Found wrong account ID in store " - "info: " << BOX_FORMAT_HEX32(FileAccountID), - fn, BackupStoreException, BadStoreInfoOnLoad); - } - + archive.Read(info->mAccountID); archive.Read(info->mAccountName); archive.Read(info->mClientStoreMarker); archive.Read(info->mLastObjectIDUsed); @@ -205,35 +231,35 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, archive.Read(info->mBlocksInDirectories); archive.Read(info->mBlocksSoftLimit); archive.Read(info->mBlocksHardLimit); - archive.Read(info->mNumFiles); - archive.Read(info->mNumOldFiles); - archive.Read(info->mNumDeletedFiles); - archive.Read(info->mNumDirectories); - archive.Read(numDelObj); + archive.Read(info->mNumCurrentFiles); + archive.Read(info->mNumOldFiles); + archive.Read(info->mNumDeletedFiles); + archive.Read(info->mNumDirectories); + archive.Read(numDelObj); } // Then load the list of deleted directories if(numDelObj > 0) { int64_t objs[NUM_DELETED_DIRS_BLOCK]; - + int64_t toload = numDelObj; while(toload > 0) { // How many in this one? int b = (toload > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(toload)); - - if(!rf->ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */)) + + if(!rStream.ReadFullBuffer(objs, b * sizeof(int64_t), 0 /* not interested in bytes read if this fails */)) { THROW_EXCEPTION(BackupStoreException, CouldNotLoadStoreInfo) } - + // Add them for(int t = 0; t < b; ++t) { info->mDeletedDirectories.push_back(box_ntoh64(objs[t])); } - + // Number loaded toload -= b; } @@ -247,24 +273,24 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::Load(int32_t AccountID, if(v2) { - Archive archive(*rf, IOStream::TimeOutInfinite); + Archive archive(rStream, IOStream::TimeOutInfinite); archive.ReadIfPresent(info->mAccountEnabled, true); } else { info->mAccountEnabled = true; } - + // If there's any data left in the info file, from future additions to // the file format, then we need to load it so that it won't be lost when // we resave the file. - IOStream::pos_type bytesLeft = rf->BytesLeftToRead(); + IOStream::pos_type bytesLeft = rStream.BytesLeftToRead(); if (bytesLeft > 0) { - rf->CopyStreamTo(info->mExtraData); + rStream.CopyStreamTo(info->mExtraData); } info->mExtraData.SetForReading(); - + // return it to caller return info; } @@ -289,10 +315,10 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( { // Generate the filename std::string fn(rRootDir + INFO_FILENAME); - + // Make new object std::auto_ptr<BackupStoreInfo> info(new BackupStoreInfo); - + // Put in basic info info->mAccountID = AccountID; info->mAccountName = rAccountName; @@ -311,7 +337,7 @@ std::auto_ptr<BackupStoreInfo> BackupStoreInfo::CreateForRegeneration( info->mBlocksSoftLimit = BlockSoftLimit; info->mBlocksHardLimit = BlockHardLimit; info->mAccountEnabled = AccountEnabled; - + ExtraData.CopyStreamTo(info->mExtraData); info->mExtraData.SetForReading(); @@ -341,15 +367,22 @@ void BackupStoreInfo::Save(bool allowOverwrite) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + // Then... open a write file RaidFileWrite rf(mDiscSet, mFilename); rf.Open(allowOverwrite); - + Save(rf); + + // Commit it to disc, converting it to RAID now + rf.Commit(true); +} + +void BackupStoreInfo::Save(IOStream& rOutStream) +{ // Make header int32_t magic = htonl(INFO_MAGIC_VALUE_2); - rf.Write(&magic, sizeof(magic)); - Archive archive(rf, IOStream::TimeOutInfinite); + rOutStream.Write(&magic, sizeof(magic)); + Archive archive(rOutStream, IOStream::TimeOutInfinite); archive.Write(mAccountID); archive.Write(mAccountName); @@ -362,7 +395,7 @@ void BackupStoreInfo::Save(bool allowOverwrite) archive.Write(mBlocksInDirectories); archive.Write(mBlocksSoftLimit); archive.Write(mBlocksHardLimit); - archive.Write(mNumFiles); + archive.Write(mNumCurrentFiles); archive.Write(mNumOldFiles); archive.Write(mNumDeletedFiles); archive.Write(mNumDirectories); @@ -374,14 +407,14 @@ void BackupStoreInfo::Save(bool allowOverwrite) if(mDeletedDirectories.size() > 0) { int64_t objs[NUM_DELETED_DIRS_BLOCK]; - + int tosave = mDeletedDirectories.size(); std::vector<int64_t>::iterator i(mDeletedDirectories.begin()); while(tosave > 0) { // How many in this one? int b = (tosave > NUM_DELETED_DIRS_BLOCK)?NUM_DELETED_DIRS_BLOCK:((int)(tosave)); - + // Add them for(int t = 0; t < b; ++t) { @@ -390,23 +423,20 @@ void BackupStoreInfo::Save(bool allowOverwrite) i++; } - // Write - rf.Write(objs, b * sizeof(int64_t)); - + // Write + rOutStream.Write(objs, b * sizeof(int64_t)); + // Number saved tosave -= b; } } - + archive.Write(mAccountEnabled); - + mExtraData.Seek(0, IOStream::SeekType_Absolute); - mExtraData.CopyStreamTo(rf); + mExtraData.CopyStreamTo(rOutStream); mExtraData.Seek(0, IOStream::SeekType_Absolute); - // Commit it to disc, converting it to RAID now - rf.Commit(true); - // Mark is as not modified mIsModified = false; } @@ -426,7 +456,6 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) COMPARE(AccountID); COMPARE(AccountName); - COMPARE(LastObjectIDUsed); COMPARE(BlocksUsed); COMPARE(BlocksInCurrentFiles); COMPARE(BlocksInOldFiles); @@ -434,32 +463,47 @@ int BackupStoreInfo::ReportChangesTo(BackupStoreInfo& rOldInfo) COMPARE(BlocksInDirectories); COMPARE(BlocksSoftLimit); COMPARE(BlocksHardLimit); - COMPARE(NumFiles); + COMPARE(NumCurrentFiles); COMPARE(NumOldFiles); COMPARE(NumDeletedFiles); COMPARE(NumDirectories); #undef COMPARE + if (rOldInfo.GetLastObjectIDUsed() != GetLastObjectIDUsed()) + { + BOX_NOTICE("LastObjectIDUsed changed from " << + rOldInfo.GetLastObjectIDUsed() << " to " << + GetLastObjectIDUsed()); + // Not important enough to be an error + // numChanges++; + } + return numChanges; } -#define APPLY_DELTA(field, delta) \ - if(mReadOnly) \ - { \ - THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) \ - } \ - \ - if((field + delta) < 0) \ - { \ - THROW_EXCEPTION_MESSAGE(BackupStoreException, \ - StoreInfoBlockDeltaMakesValueNegative, \ - "Failed to reduce " << #field << " from " << \ - field << " by " << delta); \ - } \ - \ - field += delta; \ +void BackupStoreInfo::ApplyDelta(int64_t& field, const std::string& field_name, + const int64_t delta) +{ + if(mReadOnly) + { + THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly); + } + + if((field + delta) < 0) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, + StoreInfoBlockDeltaMakesValueNegative, + "Failed to reduce " << field_name << " from " << + field << " by " << delta); + } + + field += delta; mIsModified = true; +} + +#define APPLY_DELTA(field, delta) \ + ApplyDelta(field, #field, delta) // -------------------------------------------------------------------------- // @@ -527,9 +571,9 @@ void BackupStoreInfo::ChangeBlocksInDirectories(int64_t Delta) APPLY_DELTA(mBlocksInDirectories, Delta); } -void BackupStoreInfo::AdjustNumFiles(int64_t increase) +void BackupStoreInfo::AdjustNumCurrentFiles(int64_t increase) { - APPLY_DELTA(mNumFiles, increase); + APPLY_DELTA(mNumCurrentFiles, increase); } void BackupStoreInfo::AdjustNumOldFiles(int64_t increase) @@ -563,13 +607,13 @@ void BackupStoreInfo::CorrectAllUsedValues(int64_t Used, int64_t InOldFiles, int { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + // Set the values mBlocksUsed = Used; mBlocksInOldFiles = InOldFiles; mBlocksInDeletedFiles = InDeletedFiles; mBlocksInDirectories = InDirectories; - + mIsModified = true; } @@ -588,9 +632,8 @@ void BackupStoreInfo::AddDeletedDirectory(int64_t DirID) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + mDeletedDirectories.push_back(DirID); - mIsModified = true; } @@ -608,14 +651,14 @@ void BackupStoreInfo::RemovedDeletedDirectory(int64_t DirID) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + std::vector<int64_t>::iterator i(std::find(mDeletedDirectories.begin(), mDeletedDirectories.end(), DirID)); if(i == mDeletedDirectories.end()) { THROW_EXCEPTION(BackupStoreException, StoreInfoDirNotInList) } + mDeletedDirectories.erase(i); - mIsModified = true; } @@ -636,7 +679,7 @@ void BackupStoreInfo::ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimi mBlocksSoftLimit = BlockSoftLimit; mBlocksHardLimit = BlockHardLimit; - + mIsModified = true; } @@ -655,15 +698,16 @@ int64_t BackupStoreInfo::AllocateObjectID() { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } + if(mLastObjectIDUsed < 0) { THROW_EXCEPTION(BackupStoreException, StoreInfoNotInitialised) } - + + mIsModified = true; + // Return the next object ID return ++mLastObjectIDUsed; - - mIsModified = true; } @@ -682,9 +726,8 @@ void BackupStoreInfo::SetClientStoreMarker(int64_t ClientStoreMarker) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + mClientStoreMarker = ClientStoreMarker; - mIsModified = true; } @@ -703,9 +746,8 @@ void BackupStoreInfo::SetAccountName(const std::string& rName) { THROW_EXCEPTION(BackupStoreException, StoreInfoIsReadOnly) } - + mAccountName = rName; - mIsModified = true; } diff --git a/lib/backupstore/BackupStoreInfo.h b/lib/backupstore/BackupStoreInfo.h index 752cc44a..ec6cd2cb 100644 --- a/lib/backupstore/BackupStoreInfo.h +++ b/lib/backupstore/BackupStoreInfo.h @@ -77,20 +77,28 @@ private: BackupStoreInfo(); // No copying allowed BackupStoreInfo(const BackupStoreInfo &); - + public: // Create a New account, saving a blank info object to the disc - static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, int64_t BlockSoftLimit, int64_t BlockHardLimit); - + static void CreateNew(int32_t AccountID, const std::string &rRootDir, int DiscSet, + int64_t BlockSoftLimit, int64_t BlockHardLimit); + BackupStoreInfo(int32_t AccountID, const std::string &FileName, + int64_t BlockSoftLimit, int64_t BlockHardLimit); + // Load it from the store static std::auto_ptr<BackupStoreInfo> Load(int32_t AccountID, const std::string &rRootDir, int DiscSet, bool ReadOnly, int64_t *pRevisionID = 0); - + + // Load it from a stream (file or RaidFile) + static std::auto_ptr<BackupStoreInfo> Load(IOStream& rStream, + const std::string FileName, bool ReadOnly); + // Has info been modified? bool IsModified() const {return mIsModified;} - + // Save modified infomation back to store void Save(bool allowOverwrite = true); - + void Save(IOStream& rOutStream); + // Data access functions int32_t GetAccountID() const {return mAccountID;} int64_t GetLastObjectIDUsed() const {return mLastObjectIDUsed;} @@ -102,7 +110,7 @@ public: const std::vector<int64_t> &GetDeletedDirectories() const {return mDeletedDirectories;} int64_t GetBlocksSoftLimit() const {return mBlocksSoftLimit;} int64_t GetBlocksHardLimit() const {return mBlocksHardLimit;} - int64_t GetNumFiles() const {return mNumFiles;} + int64_t GetNumCurrentFiles() const {return mNumCurrentFiles;} int64_t GetNumOldFiles() const {return mNumOldFiles;} int64_t GetNumDeletedFiles() const {return mNumDeletedFiles;} int64_t GetNumDirectories() const {return mNumDirectories;} @@ -111,7 +119,7 @@ public: int GetDiscSetNumber() const {return mDiscSet;} int ReportChangesTo(BackupStoreInfo& rOldInfo); - + // Data modification functions void ChangeBlocksUsed(int64_t Delta); void ChangeBlocksInCurrentFiles(int64_t Delta); @@ -122,14 +130,14 @@ public: void AddDeletedDirectory(int64_t DirID); void RemovedDeletedDirectory(int64_t DirID); void ChangeLimits(int64_t BlockSoftLimit, int64_t BlockHardLimit); - void AdjustNumFiles(int64_t increase); + void AdjustNumCurrentFiles(int64_t increase); void AdjustNumOldFiles(int64_t increase); void AdjustNumDeletedFiles(int64_t increase); void AdjustNumDirectories(int64_t increase); - + // Object IDs int64_t AllocateObjectID(); - + // Client marker set and get int64_t GetClientStoreMarker() const {return mClientStoreMarker;} void SetClientStoreMarker(int64_t ClientStoreMarker); @@ -152,6 +160,20 @@ public: int64_t BlockSoftLimit, int64_t BlockHardLimit, bool AccountEnabled, IOStream& ExtraData); + typedef struct + { + int64_t mLastObjectIDUsed; + int64_t mBlocksUsed; + int64_t mBlocksInCurrentFiles; + int64_t mBlocksInOldFiles; + int64_t mBlocksInDeletedFiles; + int64_t mBlocksInDirectories; + int64_t mNumCurrentFiles; + int64_t mNumOldFiles; + int64_t mNumDeletedFiles; + int64_t mNumDirectories; + } Adjustment; + private: // Location information // Be VERY careful about changing types of these values, as @@ -162,10 +184,10 @@ private: std::string mFilename; bool mReadOnly; bool mIsModified; - + // Client infomation int64_t mClientStoreMarker; - + // Account information int64_t mLastObjectIDUsed; int64_t mBlocksUsed; @@ -175,13 +197,16 @@ private: int64_t mBlocksInDirectories; int64_t mBlocksSoftLimit; int64_t mBlocksHardLimit; - int64_t mNumFiles; + int64_t mNumCurrentFiles; int64_t mNumOldFiles; int64_t mNumDeletedFiles; int64_t mNumDirectories; std::vector<int64_t> mDeletedDirectories; bool mAccountEnabled; CollectInBufferStream mExtraData; + + void ApplyDelta(int64_t& field, const std::string& field_name, + const int64_t delta); }; #endif // BACKUPSTOREINFO__H diff --git a/lib/backupstore/BackupStoreRefCountDatabase.cpp b/lib/backupstore/BackupStoreRefCountDatabase.cpp index 26f9acca..b2ea1abd 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.cpp +++ b/lib/backupstore/BackupStoreRefCountDatabase.cpp @@ -9,6 +9,8 @@ #include "Box.h" +#include <stdio.h> + #include <algorithm> #include "BackupStoreRefCountDatabase.h" @@ -34,12 +36,85 @@ // // -------------------------------------------------------------------------- BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const - BackupStoreAccountDatabase::Entry& rAccount) + BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly, + bool Temporary, std::auto_ptr<FileStream> apDatabaseFile) : mAccount(rAccount), - mFilename(GetFilename(rAccount)), - mReadOnly(true), - mIsModified(false) + mFilename(GetFilename(rAccount, Temporary)), + mReadOnly(ReadOnly), + mIsModified(false), + mIsTemporaryFile(Temporary), + mapDatabaseFile(apDatabaseFile) +{ + ASSERT(!(ReadOnly && Temporary)); // being both doesn't make sense +} + +void BackupStoreRefCountDatabase::Commit() { + if (!mIsTemporaryFile) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Cannot commit a permanent reference count database"); + } + + if (!mapDatabaseFile.get()) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Reference count database is already closed"); + } + + mapDatabaseFile->Close(); + mapDatabaseFile.reset(); + + std::string Final_Filename = GetFilename(mAccount, false); + + #ifdef WIN32 + if(FileExists(Final_Filename) && unlink(Final_Filename.c_str()) != 0) + { + THROW_EMU_FILE_ERROR("Failed to delete old permanent refcount " + "database file", mFilename, CommonException, + OSFileError); + } + #endif + + if(rename(mFilename.c_str(), Final_Filename.c_str()) != 0) + { + THROW_EMU_ERROR("Failed to rename temporary refcount database " + "file from " << mFilename << " to " << + Final_Filename, CommonException, OSFileError); + } + + mFilename = Final_Filename; + mIsModified = false; + mIsTemporaryFile = false; +} + +void BackupStoreRefCountDatabase::Discard() +{ + if (!mIsTemporaryFile) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Cannot discard a permanent reference count database"); + } + + // Under normal conditions, we should know whether the file is still + // open or not, and not Discard it unless it's open. However if the + // final rename() fails during Commit(), the file will already be + // closed, and we don't want to blow up here in that case. + if (mapDatabaseFile.get()) + { + mapDatabaseFile->Close(); + mapDatabaseFile.reset(); + } + + if(unlink(mFilename.c_str()) != 0) + { + THROW_EMU_FILE_ERROR("Failed to delete temporary refcount " + "database file", mFilename, CommonException, + OSFileError); + } + + mIsModified = false; + mIsTemporaryFile = false; } // -------------------------------------------------------------------------- @@ -52,16 +127,35 @@ BackupStoreRefCountDatabase::BackupStoreRefCountDatabase(const // -------------------------------------------------------------------------- BackupStoreRefCountDatabase::~BackupStoreRefCountDatabase() { + if (mIsTemporaryFile) + { + // Don't throw exceptions in a destructor. + BOX_ERROR("BackupStoreRefCountDatabase destroyed without " + "explicit commit or discard"); + try + { + Discard(); + } + catch(BoxException &e) + { + BOX_LOG_SYS_ERROR("Failed to discard BackupStoreRefCountDatabase " + "in destructor: " << e.what()); + } + } } std::string BackupStoreRefCountDatabase::GetFilename(const - BackupStoreAccountDatabase::Entry& rAccount) + BackupStoreAccountDatabase::Entry& rAccount, bool Temporary) { std::string RootDir = BackupStoreAccounts::GetAccountRoot(rAccount); ASSERT(RootDir[RootDir.size() - 1] == '/' || RootDir[RootDir.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR); - std::string fn(RootDir + REFCOUNT_FILENAME ".db"); + std::string fn(RootDir + REFCOUNT_FILENAME ".rdb"); + if(Temporary) + { + fn += "X"; + } RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(rAccount.GetDiscSet())); return RaidFileUtil::MakeWriteFileName(rdiscSet, fn); @@ -72,40 +166,53 @@ std::string BackupStoreRefCountDatabase::GetFilename(const // Function // Name: BackupStoreRefCountDatabase::Create(int32_t, // const std::string &, int, bool) -// Purpose: Create a new database, overwriting an existing -// one only if AllowOverwrite is true. +// Purpose: Create a blank database, using a temporary file that +// you must Discard() or Commit() to make permanent. // Created: 2003/08/28 // // -------------------------------------------------------------------------- -void BackupStoreRefCountDatabase::Create(const - BackupStoreAccountDatabase::Entry& rAccount, bool AllowOverwrite) +std::auto_ptr<BackupStoreRefCountDatabase> + BackupStoreRefCountDatabase::Create + (const BackupStoreAccountDatabase::Entry& rAccount) { // Initial header refcount_StreamFormat hdr; hdr.mMagicValue = htonl(REFCOUNT_MAGIC_VALUE); hdr.mAccountID = htonl(rAccount.GetID()); - // Generate the filename - std::string Filename = GetFilename(rAccount); + std::string Filename = GetFilename(rAccount, true); // temporary // Open the file for writing - if (FileExists(Filename) && !AllowOverwrite) - { - THROW_FILE_ERROR("Failed to overwrite refcount database: " - "not allowed here", Filename, RaidFileException, - CannotOverwriteExistingFile); - } - - int flags = O_CREAT | O_BINARY | O_RDWR; - if (!AllowOverwrite) + if (FileExists(Filename)) { - flags |= O_EXCL; + BOX_WARNING(BOX_FILE_MESSAGE(Filename, "Overwriting existing " + "temporary reference count database")); + if (unlink(Filename.c_str()) != 0) + { + THROW_SYS_FILE_ERROR("Failed to delete old temporary " + "reference count database file", Filename, + CommonException, OSFileError); + } } + int flags = O_CREAT | O_BINARY | O_RDWR | O_EXCL; std::auto_ptr<FileStream> DatabaseFile(new FileStream(Filename, flags)); // Write header DatabaseFile->Write(&hdr, sizeof(hdr)); + + // Make new object + std::auto_ptr<BackupStoreRefCountDatabase> refcount( + new BackupStoreRefCountDatabase(rAccount, false, true, + DatabaseFile)); + + // The root directory must always have one reference for a database + // to be valid, so set that now on the new database. This will leave + // mIsModified set to true. + refcount->SetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); + + // return it to caller + return refcount; } // -------------------------------------------------------------------------- @@ -122,12 +229,13 @@ void BackupStoreRefCountDatabase::Create(const std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly) { - // Generate the filename - std::string filename = GetFilename(rAccount); + // Generate the filename. Cannot open a temporary database, so it must + // be a permanent one. + std::string Filename = GetFilename(rAccount, false); int flags = ReadOnly ? O_RDONLY : O_RDWR; // Open the file for read/write - std::auto_ptr<FileStream> dbfile(new FileStream(filename, + std::auto_ptr<FileStream> dbfile(new FileStream(Filename, flags | O_BINARY)); // Read in a header @@ -135,7 +243,7 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( if(!dbfile->ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */)) { THROW_FILE_ERROR("Failed to read refcount database: " - "short read", filename, BackupStoreException, + "short read", Filename, BackupStoreException, CouldNotLoadStoreInfo); } @@ -144,16 +252,14 @@ std::auto_ptr<BackupStoreRefCountDatabase> BackupStoreRefCountDatabase::Load( (int32_t)ntohl(hdr.mAccountID) != rAccount.GetID()) { THROW_FILE_ERROR("Failed to read refcount database: " - "bad magic number", filename, BackupStoreException, + "bad magic number", Filename, BackupStoreException, BadStoreInfoOnLoad); } // Make new object - std::auto_ptr<BackupStoreRefCountDatabase> refcount(new BackupStoreRefCountDatabase(rAccount)); - - // Put in basic location info - refcount->mReadOnly = ReadOnly; - refcount->mapDatabaseFile = dbfile; + std::auto_ptr<BackupStoreRefCountDatabase> refcount( + new BackupStoreRefCountDatabase(rAccount, ReadOnly, false, + dbfile)); // return it to caller return refcount; @@ -229,6 +335,7 @@ void BackupStoreRefCountDatabase::SetRefCount(int64_t ObjectID, mapDatabaseFile->Seek(offset, SEEK_SET); refcount_t RefCountNetOrder = htonl(NewRefCount); mapDatabaseFile->Write(&RefCountNetOrder, sizeof(RefCountNetOrder)); + mIsModified = true; } bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) @@ -240,3 +347,31 @@ bool BackupStoreRefCountDatabase::RemoveReference(int64_t ObjectID) return (refcount > 0); } +int BackupStoreRefCountDatabase::ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs) +{ + int ErrorCount = 0; + int64_t MaxOldObjectId = rOldRefs.GetLastObjectIDUsed(); + int64_t MaxNewObjectId = GetLastObjectIDUsed(); + + for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID; + ObjectID < std::max(MaxOldObjectId, MaxNewObjectId); + ObjectID++) + { + typedef BackupStoreRefCountDatabase::refcount_t refcount_t; + refcount_t OldRefs = (ObjectID <= MaxOldObjectId) ? + rOldRefs.GetRefCount(ObjectID) : 0; + refcount_t NewRefs = (ObjectID <= MaxNewObjectId) ? + this->GetRefCount(ObjectID) : 0; + + if (OldRefs != NewRefs) + { + BOX_WARNING("Reference count of object " << + BOX_FORMAT_OBJECTID(ObjectID) << + " changed from " << OldRefs << + " to " << NewRefs); + ErrorCount++; + } + } + + return ErrorCount; +} diff --git a/lib/backupstore/BackupStoreRefCountDatabase.h b/lib/backupstore/BackupStoreRefCountDatabase.h index 93c79afb..915653a4 100644 --- a/lib/backupstore/BackupStoreRefCountDatabase.h +++ b/lib/backupstore/BackupStoreRefCountDatabase.h @@ -15,6 +15,7 @@ #include <vector> #include "BackupStoreAccountDatabase.h" +#include "BackupStoreConstants.h" #include "FileStream.h" class BackupStoreCheck; @@ -59,48 +60,39 @@ public: private: // Creation through static functions only BackupStoreRefCountDatabase(const BackupStoreAccountDatabase::Entry& - rAccount); + rAccount, bool ReadOnly, bool Temporary, + std::auto_ptr<FileStream> apDatabaseFile); // No copying allowed BackupStoreRefCountDatabase(const BackupStoreRefCountDatabase &); public: - // Create a new database for a new account. This method will refuse - // to overwrite any existing file. - static void CreateNew(const BackupStoreAccountDatabase::Entry& rAccount) - { - Create(rAccount, false); - } - + // Create a blank database, using a temporary file that you must + // Discard() or Commit() to make permanent. + static std::auto_ptr<BackupStoreRefCountDatabase> Create + (const BackupStoreAccountDatabase::Entry& rAccount); + void Commit(); + void Discard(); + // Load it from the store static std::auto_ptr<BackupStoreRefCountDatabase> Load(const BackupStoreAccountDatabase::Entry& rAccount, bool ReadOnly); - + typedef uint32_t refcount_t; // Data access functions refcount_t GetRefCount(int64_t ObjectID) const; int64_t GetLastObjectIDUsed() const; - + // Data modification functions void AddReference(int64_t ObjectID); // RemoveReference returns false if refcount drops to zero bool RemoveReference(int64_t ObjectID); + int ReportChangesTo(BackupStoreRefCountDatabase& rOldRefs); private: - // Create a new database for an existing account. Used during - // account checking if opening the old database throws an exception. - // This method will overwrite any existing file. - static void CreateForRegeneration(const - BackupStoreAccountDatabase::Entry& rAccount) - { - Create(rAccount, true); - } - - static void Create(const BackupStoreAccountDatabase::Entry& rAccount, - bool AllowOverwrite); - static std::string GetFilename(const BackupStoreAccountDatabase::Entry& - rAccount); + rAccount, bool Temporary); + IOStream::pos_type GetSize() const { return mapDatabaseFile->GetPosition() + @@ -122,7 +114,13 @@ private: std::string mFilename; bool mReadOnly; bool mIsModified; + bool mIsTemporaryFile; std::auto_ptr<FileStream> mapDatabaseFile; + + bool NeedsCommitOrDiscard() + { + return mapDatabaseFile.get() && mIsModified && mIsTemporaryFile; + } }; #endif // BACKUPSTOREREFCOUNTDATABASE__H diff --git a/lib/backupstore/HousekeepStoreAccount.cpp b/lib/backupstore/HousekeepStoreAccount.cpp index 75feda7f..d5acf62c 100644 --- a/lib/backupstore/HousekeepStoreAccount.cpp +++ b/lib/backupstore/HousekeepStoreAccount.cpp @@ -2,7 +2,7 @@ // // File // Name: HousekeepStoreAccount.cpp -// Purpose: +// Purpose: // Created: 11/12/03 // // -------------------------------------------------------------------------- @@ -51,6 +51,7 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, mDeletionSizeTarget(0), mPotentialDeletionsTotalSize(0), mMaxSizeInPotentialDeletions(0), + mErrorCount(0), mBlocksUsed(0), mBlocksInOldFiles(0), mBlocksInDeletedFiles(0), @@ -61,8 +62,6 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, mBlocksInDirectoriesDelta(0), mFilesDeleted(0), mEmptyDirectoriesDeleted(0), - mSuppressRefCountChangeWarnings(false), - mRefCountsAdjusted(0), mCountUntilNextInterprocessMsgCheck(POLL_INTERPROCESS_MSG_CHECK_FREQUENCY) { std::ostringstream tag; @@ -80,6 +79,20 @@ HousekeepStoreAccount::HousekeepStoreAccount(int AccountID, // -------------------------------------------------------------------------- HousekeepStoreAccount::~HousekeepStoreAccount() { + if(mapNewRefs.get()) + { + // Discard() can throw exception, but destructors aren't supposed to do that, so + // just catch and log them. + try + { + mapNewRefs->Discard(); + } + catch(BoxException &e) + { + BOX_ERROR("Failed to destroy housekeeper: discarding the refcount " + "database threw an exception: " << e.what()); + } + } } // -------------------------------------------------------------------------- @@ -105,7 +118,7 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) { if(KeepTryingForever) { - BOX_WARNING("Failed to lock account for housekeeping, " + BOX_INFO("Failed to lock account for housekeeping, " "still trying..."); while(!writeLock.TryAndGetLock(writeLockFilename, 0600 /* restrictive file permissions */)) @@ -143,204 +156,108 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) mDeletionSizeTarget = 0; } - // initialise the refcount database - mNewRefCounts.clear(); - // try to pre-allocate as much memory as we need - mNewRefCounts.reserve(info->GetLastObjectIDUsed()); - // initialise the refcount of the root entry - mNewRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1, 0); - mNewRefCounts[BACKUPSTORE_ROOT_DIRECTORY_ID] = 1; + BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); + mapNewRefs = BackupStoreRefCountDatabase::Create(account); // Scan the directory for potential things to delete // This will also remove eligible items marked with RemoveASAP - bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID); + bool continueHousekeeping = ScanDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, + *info); - // If scan directory stopped for some reason, probably parent - // instructed to terminate, stop now. if(!continueHousekeeping) { - // If any files were marked "delete now", then update - // the size of the store. - if(mBlocksUsedDelta != 0 || - mBlocksInOldFilesDelta != 0 || - mBlocksInDeletedFilesDelta != 0) - { - info->ChangeBlocksUsed(mBlocksUsedDelta); - info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); - info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - - // Save the store info back - info->ReportChangesTo(*pOldInfo); - info->Save(); - } - - return false; + // The scan was incomplete, so the new block counts are + // incorrect, we can't rely on them. It's better to discard + // the new info and adjust the old one instead. + info = pOldInfo; + + // We're about to reset counters and exit, so report what + // happened now. + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " removed " << + (0 - mBlocksUsedDelta) << " blocks (" << + mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs) and the directory " + "scan was interrupted"); } - // Log any difference in opinion between the values recorded in - // the store info, and the values just calculated for space usage. - // BLOCK - { - int64_t used = info->GetBlocksUsed(); - int64_t usedOld = info->GetBlocksInOldFiles(); - int64_t usedDeleted = info->GetBlocksInDeletedFiles(); - int64_t usedDirectories = info->GetBlocksInDirectories(); - - // If the counts were wrong, taking into account RemoveASAP - // items deleted, log a message - if((used + mBlocksUsedDelta) != mBlocksUsed - || (usedOld + mBlocksInOldFilesDelta) != mBlocksInOldFiles - || (usedDeleted + mBlocksInDeletedFilesDelta) != mBlocksInDeletedFiles - || usedDirectories != mBlocksInDirectories) - { - // Log this - BOX_ERROR("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " found " - "and fixed wrong block counts: " - "used (" << - (used + mBlocksUsedDelta) << "," << - mBlocksUsed << "), old (" << - (usedOld + mBlocksInOldFilesDelta) << "," << - mBlocksInOldFiles << "), deleted (" << - (usedDeleted + mBlocksInDeletedFilesDelta) << - "," << mBlocksInDeletedFiles << "), dirs (" << - usedDirectories << "," << mBlocksInDirectories - << ")"); - } - - // If the current values don't match, store them - if(used != mBlocksUsed - || usedOld != mBlocksInOldFiles - || usedDeleted != mBlocksInDeletedFiles - || usedDirectories != (mBlocksInDirectories + mBlocksInDirectoriesDelta)) - { - // Set corrected values in store info - info->CorrectAllUsedValues(mBlocksUsed, - mBlocksInOldFiles, mBlocksInDeletedFiles, - mBlocksInDirectories + mBlocksInDirectoriesDelta); - - info->ReportChangesTo(*pOldInfo); - info->Save(); - } - } - - // Reset the delta counts for files, as they will include - // RemoveASAP flagged files deleted during the initial scan. + // If housekeeping made any changes, such as deleting RemoveASAP files, + // the differences in block counts will be recorded in the deltas. + info->ChangeBlocksUsed(mBlocksUsedDelta); + info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); + info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); - // keep for reporting + // Reset the delta counts for files, as they will include + // RemoveASAP flagged files deleted during the initial scan. + // keep removeASAPBlocksUsedDelta for reporting int64_t removeASAPBlocksUsedDelta = mBlocksUsedDelta; - mBlocksUsedDelta = 0; mBlocksInOldFilesDelta = 0; mBlocksInDeletedFilesDelta = 0; - - // Go and delete items from the accounts - bool deleteInterrupted = DeleteFiles(); - - // If that wasn't interrupted, remove any empty directories which - // are also marked as deleted in their containing directory - if(!deleteInterrupted) - { - deleteInterrupted = DeleteEmptyDirectories(); - } - - // Log deletion if anything was deleted - if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) - { - BOX_INFO("Housekeeping on account " << - BOX_FORMAT_ACCOUNT(mAccountID) << " " - "removed " << - (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << - " blocks (" << mFilesDeleted << " files, " << - mEmptyDirectoriesDeleted << " dirs)" << - (deleteInterrupted?" and was interrupted":"")); - } + // If scan directory stopped for some reason, probably parent + // instructed to terminate, stop now. + // // We can only update the refcount database if we successfully // finished our scan of all directories, otherwise we don't actually // know which of the new counts are valid and which aren't - // (we might not have seen second references to some objects, etc.) + // (we might not have seen second references to some objects, etc.). - BackupStoreAccountDatabase::Entry account(mAccountID, mStoreDiscSet); - std::auto_ptr<BackupStoreRefCountDatabase> apReferences; + if(!continueHousekeeping) + { + mapNewRefs->Discard(); + info->Save(); + return false; + } + + // Report any UNexpected changes, and consider them to be errors. + // Do this before applying the expected changes below. + mErrorCount += info->ReportChangesTo(*pOldInfo); + info->Save(); + + // Try to load the old reference count database and check whether + // any counts have changed. We want to compare the mapNewRefs to + // apOldRefs before we delete any files, because that will also change + // the reference count in a way that's not an error. - // try to load the reference count database try { - apReferences = BackupStoreRefCountDatabase::Load(account, - false); + std::auto_ptr<BackupStoreRefCountDatabase> apOldRefs = + BackupStoreRefCountDatabase::Load(account, false); + mErrorCount += mapNewRefs->ReportChangesTo(*apOldRefs); } catch(BoxException &e) { - BOX_WARNING("Reference count database is missing or corrupted " - "during housekeeping, creating a new one."); - mSuppressRefCountChangeWarnings = true; - BackupStoreRefCountDatabase::CreateForRegeneration(account); - apReferences = BackupStoreRefCountDatabase::Load(account, - false); + BOX_WARNING("Reference count database was missing or " + "corrupted during housekeeping, cannot check it for " + "errors."); + mErrorCount++; } - int64_t LastUsedObjectIdOnDisk = apReferences->GetLastObjectIDUsed(); + // Go and delete items from the accounts + bool deleteInterrupted = DeleteFiles(*info); - for (int64_t ObjectID = BACKUPSTORE_ROOT_DIRECTORY_ID; - ObjectID < mNewRefCounts.size(); ObjectID++) + // If that wasn't interrupted, remove any empty directories which + // are also marked as deleted in their containing directory + if(!deleteInterrupted) { - if (ObjectID > LastUsedObjectIdOnDisk) - { - if (!mSuppressRefCountChangeWarnings) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " not found in database, added" - " with " << mNewRefCounts[ObjectID] << - " references"); - } - apReferences->SetRefCount(ObjectID, - mNewRefCounts[ObjectID]); - mRefCountsAdjusted++; - LastUsedObjectIdOnDisk = ObjectID; - continue; - } - - BackupStoreRefCountDatabase::refcount_t OldRefCount = - apReferences->GetRefCount(ObjectID); - - if (OldRefCount != mNewRefCounts[ObjectID]) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " changed from " << OldRefCount << - " to " << mNewRefCounts[ObjectID]); - apReferences->SetRefCount(ObjectID, - mNewRefCounts[ObjectID]); - mRefCountsAdjusted++; - } + deleteInterrupted = DeleteEmptyDirectories(*info); } - // zero excess references in the database - for (int64_t ObjectID = mNewRefCounts.size(); - ObjectID <= LastUsedObjectIdOnDisk; ObjectID++) + // Log deletion if anything was deleted + if(mFilesDeleted > 0 || mEmptyDirectoriesDeleted > 0) { - BackupStoreRefCountDatabase::refcount_t OldRefCount = - apReferences->GetRefCount(ObjectID); - BackupStoreRefCountDatabase::refcount_t NewRefCount = 0; - - if (OldRefCount != NewRefCount) - { - BOX_WARNING("Reference count of object " << - BOX_FORMAT_OBJECTID(ObjectID) << - " changed from " << OldRefCount << - " to " << NewRefCount << " (not found)"); - apReferences->SetRefCount(ObjectID, NewRefCount); - mRefCountsAdjusted++; - } + BOX_INFO("Housekeeping on account " << + BOX_FORMAT_ACCOUNT(mAccountID) << " " + "removed " << + (0 - (mBlocksUsedDelta + removeASAPBlocksUsedDelta)) << + " blocks (" << mFilesDeleted << " files, " << + mEmptyDirectoriesDeleted << " dirs)" << + (deleteInterrupted?" and was interrupted":"")); } - // force file to be saved and closed before releasing the lock below - apReferences.reset(); - - // Make sure the delta's won't cause problems if the counts are - // really wrong, and it wasn't fixed because the store was + // Make sure the delta's won't cause problems if the counts are + // really wrong, and it wasn't fixed because the store was // updated during the scan. if(mBlocksUsedDelta < (0 - info->GetBlocksUsed())) { @@ -348,28 +265,31 @@ bool HousekeepStoreAccount::DoHousekeeping(bool KeepTryingForever) } if(mBlocksInOldFilesDelta < (0 - info->GetBlocksInOldFiles())) { - mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); + mBlocksInOldFilesDelta = (0 - info->GetBlocksInOldFiles()); } if(mBlocksInDeletedFilesDelta < (0 - info->GetBlocksInDeletedFiles())) { - mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); + mBlocksInDeletedFilesDelta = (0 - info->GetBlocksInDeletedFiles()); } if(mBlocksInDirectoriesDelta < (0 - info->GetBlocksInDirectories())) { mBlocksInDirectoriesDelta = (0 - info->GetBlocksInDirectories()); } - + // Update the usage counts in the store info->ChangeBlocksUsed(mBlocksUsedDelta); info->ChangeBlocksInOldFiles(mBlocksInOldFilesDelta); info->ChangeBlocksInDeletedFiles(mBlocksInDeletedFilesDelta); info->ChangeBlocksInDirectories(mBlocksInDirectoriesDelta); - + // Save the store info back - info->ReportChangesTo(*pOldInfo); info->Save(); - - // Explicity release the lock (would happen automatically on + + // force file to be saved and closed before releasing the lock below + mapNewRefs->Commit(); + mapNewRefs.reset(); + + // Explicity release the lock (would happen automatically on // going out of scope, included for code clarity) writeLock.ReleaseLock(); @@ -405,7 +325,8 @@ void HousekeepStoreAccount::MakeObjectFilename(int64_t ObjectID, std::string &rF // Created: 11/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) +bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID, + BackupStoreInfo& rBackupStoreInfo) { #ifndef WIN32 if((--mCountUntilNextInterprocessMsgCheck) <= 0) @@ -430,18 +351,19 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) // Open it. std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, objectFilename)); - + // Add the size of the directory on disc to the size being calculated int64_t originalDirSizeInBlocks = dirStream->GetDiscUsageInBlocks(); mBlocksInDirectories += originalDirSizeInBlocks; mBlocksUsed += originalDirSizeInBlocks; - + // Read the directory in BackupStoreDirectory dir; BufferedStream buf(*dirStream); dir.ReadFromStream(buf, IOStream::TimeOutInfinite); + dir.SetUserInfo1_SizeInBlocks(originalDirSizeInBlocks); dirStream->Close(); - + // Is it empty? if(dir.GetNumberOfEntries() == 0) { @@ -459,11 +381,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) while((en = i.Next()) != 0) { // This directory references this object - if (mNewRefCounts.size() <= en->GetObjectID()) - { - mNewRefCounts.resize(en->GetObjectID() + 1, 0); - } - mNewRefCounts[en->GetObjectID()]++; + mapNewRefs->AddReference(en->GetObjectID()); } } @@ -482,14 +400,15 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { int16_t enFlags = en->GetFlags(); if((enFlags & BackupStoreDirectory::Entry::Flags_RemoveASAP) != 0 - && (enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) + && (en->IsDeleted() || en->IsOld())) { // Delete this immediately. - DeleteFile(ObjectID, en->GetObjectID(), dir, objectFilename, originalDirSizeInBlocks); - + DeleteFile(ObjectID, en->GetObjectID(), dir, + objectFilename, rBackupStoreInfo); + // flag as having done something deletedSomething = true; - + // Must start the loop from the beginning again, as iterator is now // probably invalid. break; @@ -497,7 +416,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } } while(deletedSomething); } - + // BLOCK { // Add files to the list of potential deletions @@ -517,9 +436,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) int16_t enFlags = en->GetFlags(); int64_t enSizeInBlocks = en->GetSizeInBlocks(); mBlocksUsed += enSizeInBlocks; - if(enFlags & BackupStoreDirectory::Entry::Flags_OldVersion) mBlocksInOldFiles += enSizeInBlocks; - if(enFlags & BackupStoreDirectory::Entry::Flags_Deleted) mBlocksInDeletedFiles += enSizeInBlocks; - + if(en->IsOld()) mBlocksInOldFiles += enSizeInBlocks; + if(en->IsDeleted()) mBlocksInDeletedFiles += enSizeInBlocks; + // Work out ages of this version from the last mark int32_t enVersionAge = 0; std::map<version_t, int32_t>::iterator enVersionAgeI( @@ -536,9 +455,9 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) markVersionAges[version_t(en->GetName().GetEncodedFilename(), en->GetMarkNumber())] = enVersionAge; } // enVersionAge is now the age of this version. - + // Potentially add it to the list if it's deleted, if it's an old version or deleted - if((enFlags & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) != 0) + if(en->IsOld() || en->IsDeleted()) { // Is deleted / old version. DelEn d; @@ -547,17 +466,15 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) d.mSizeInBlocks = en->GetSizeInBlocks(); d.mMarkNumber = en->GetMarkNumber(); d.mVersionAgeWithinMark = enVersionAge; - d.mIsFlagDeleted = (enFlags & - BackupStoreDirectory::Entry::Flags_Deleted) - ? true : false; - + d.mIsFlagDeleted = en->IsDeleted(); + // Add it to the list mPotentialDeletions.insert(d); - + // Update various counts mPotentialDeletionsTotalSize += d.mSizeInBlocks; if(d.mSizeInBlocks > mMaxSizeInPotentialDeletions) mMaxSizeInPotentialDeletions = d.mSizeInBlocks; - + // Too much in the list of potential deletions? // (check against the deletion target + the max size in deletions, so that we never delete things // and take the total size below the deletion size target) @@ -565,7 +482,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) { int64_t sizeToRemove = mPotentialDeletionsTotalSize - (mDeletionSizeTarget + mMaxSizeInPotentialDeletions); bool recalcMaxSize = false; - + while(sizeToRemove > 0) { // Make iterator for the last element, while checking that there's something there in the first place. @@ -577,7 +494,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } // Make this into an iterator pointing to the last element in the set --i; - + // Delete this one? if(sizeToRemove > i->mSizeInBlocks) { @@ -595,7 +512,7 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) break; } } - + if(recalcMaxSize) { // Because an object which was the maximum size recorded was deleted from the set @@ -615,23 +532,22 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) } } + // Recurse into subdirectories { - // Recurse into subdirectories BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next(BackupStoreDirectory::Entry::Flags_Dir)) != 0) { - // Next level - ASSERT((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) == BackupStoreDirectory::Entry::Flags_Dir); - - if(!ScanDirectory(en->GetObjectID())) + ASSERT(en->IsDir()); + + if(!ScanDirectory(en->GetObjectID(), rBackupStoreInfo)) { // Halting operation return false; } } } - + return true; } @@ -648,11 +564,11 @@ bool HousekeepStoreAccount::ScanDirectory(int64_t ObjectID) bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount::DelEn &x, const HousekeepStoreAccount::DelEn &y) { // STL spec says this: - // A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. + // A Strict Weak Ordering is a Binary Predicate that compares two objects, returning true if the first precedes the second. // The sort order here is intended to preserve the entries of most value, that is, the newest objects // which are on a mark boundary. - + // Reverse order age, so oldest goes first if(x.mVersionAgeWithinMark > y.mVersionAgeWithinMark) { @@ -687,7 +603,7 @@ bool HousekeepStoreAccount::DelEnCompare::operator()(const HousekeepStoreAccount // Created: 15/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::DeleteFiles() +bool HousekeepStoreAccount::DeleteFiles(BackupStoreInfo& rBackupStoreInfo) { // Only delete files if the deletion target is greater than zero // (otherwise we delete one file each time round, which gradually deletes the old versions) @@ -718,21 +634,33 @@ bool HousekeepStoreAccount::DeleteFiles() // Get the filename std::string dirFilename; BackupStoreDirectory dir; - int64_t dirSizeInBlocksOrig = 0; { MakeObjectFilename(i->mInDirectory, dirFilename); std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(mStoreDiscSet, dirFilename)); - dirSizeInBlocksOrig = dirStream->GetDiscUsageInBlocks(); dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + dir.SetUserInfo1_SizeInBlocks(dirStream->GetDiscUsageInBlocks()); } - + // Delete the file - DeleteFile(i->mInDirectory, i->mObjectID, dir, dirFilename, dirSizeInBlocksOrig); - BOX_INFO("Housekeeping removed " << - (i->mIsFlagDeleted ? "deleted" : "old") << - " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << - " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); - + BackupStoreRefCountDatabase::refcount_t refs = + DeleteFile(i->mInDirectory, i->mObjectID, dir, + dirFilename, rBackupStoreInfo); + if(refs == 0) + { + BOX_INFO("Housekeeping removed " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " from dir " << BOX_FORMAT_OBJECTID(i->mInDirectory)); + } + else + { + BOX_TRACE("Housekeeping preserved " << + (i->mIsFlagDeleted ? "deleted" : "old") << + " file " << BOX_FORMAT_OBJECTID(i->mObjectID) << + " in dir " << BOX_FORMAT_OBJECTID(i->mInDirectory) << + " with " << refs << " references"); + } + // Stop if the deletion target has been matched or exceeded // (checking here rather than at the beginning will tend to reduce the // space to slightly less than the soft limit, which will allow the backup @@ -754,11 +682,17 @@ bool HousekeepStoreAccount::DeleteFiles() // BackupStoreDirectory &, const std::string &, int64_t) // Purpose: Delete a file. Takes the directory already loaded // in and the filename, for efficiency in both the -// usage scenarios. +// usage scenarios. Returns the number of references +// remaining. If it's zero, the file was removed from +// disk as unused. // Created: 15/7/04 // // -------------------------------------------------------------------------- -void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks) + +BackupStoreRefCountDatabase::refcount_t HousekeepStoreAccount::DeleteFile( + int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, + const std::string &rDirectoryFilename, + BackupStoreInfo& rBackupStoreInfo) { // Find the entry inside the directory bool wasDeleted = false; @@ -768,6 +702,9 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba std::auto_ptr<RaidFileWrite> padjustedEntry; // BLOCK { + BackupStoreRefCountDatabase::refcount_t refs = + mapNewRefs->GetRefCount(ObjectID); + BackupStoreDirectory::Entry *pentry = rDirectory.FindEntryByID(ObjectID); if(pentry == 0) { @@ -775,26 +712,47 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BOX_FORMAT_ACCOUNT(mAccountID) << " " "found error: object " << BOX_FORMAT_OBJECTID(ObjectID) << " " - "not found in dir " << + "not found in dir " << BOX_FORMAT_OBJECTID(InDirectory) << ", " "indicates logic error/corruption? Run " "bbstoreaccounts check <accid> fix"); - return; + mErrorCount++; + return refs; } - + // Record the flags it's got set - wasDeleted = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0); - wasOldVersion = ((pentry->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0); + wasDeleted = pentry->IsDeleted(); + wasOldVersion = pentry->IsOld(); // Check this should be deleted if(!wasDeleted && !wasOldVersion) { - // Things changed size we were last around - return; + // Things changed since we were last around + return refs; } - + // Record size deletedFileSizeInBlocks = pentry->GetSizeInBlocks(); - + + if(refs > 1) + { + // Not safe to merge patches if someone else has a + // reference to this object, so just remove the + // directory entry and return. + rDirectory.DeleteEntry(ObjectID); + if(wasDeleted) + { + rBackupStoreInfo.AdjustNumDeletedFiles(-1); + } + + if(wasOldVersion) + { + rBackupStoreInfo.AdjustNumOldFiles(-1); + } + + mapNewRefs->RemoveReference(ObjectID); + return refs - 1; + } + // If the entry is involved in a chain of patches, it needs to be handled // a bit more carefully. if(pentry->GetDependsNewer() != 0 && pentry->GetDependsOlder() == 0) @@ -826,7 +784,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba else { // This entry is in the middle of a chain, and two patches need combining. - + // First, adjust the directory entries BackupStoreDirectory::Entry *pnewer = rDirectory.FindEntryByID(pentry->GetDependsNewer()); if(pnewer == 0 || pnewer->GetDependsOlder() != ObjectID @@ -838,7 +796,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba pnewer->SetDependsOlder(pentry->GetDependsOlder()); polder->SetDependsNewer(pentry->GetDependsNewer()); } - + // COMMON CODE to both cases // Generate the filename of the older version @@ -853,7 +811,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba std::auto_ptr<RaidFileRead> pobjectBeingDeleted(RaidFileRead::Open(mStoreDiscSet, objFilename)); // And open a write file to overwrite the other directory entry padjustedEntry.reset(new RaidFileWrite(mStoreDiscSet, - objFilenameOlder, mNewRefCounts[ObjectID])); + objFilenameOlder, mapNewRefs->GetRefCount(ObjectID))); padjustedEntry->Open(true /* allow overwriting */); if(pentry->GetDependsNewer() == 0) @@ -867,84 +825,86 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba BackupStoreFile::CombineDiffs(*pobjectBeingDeleted, *pdiff, *pdiff2, *padjustedEntry); } // The file will be committed later when the directory is safely commited. - + // Work out the adjusted size int64_t newSize = padjustedEntry->GetDiscUsageInBlocks(); int64_t sizeDelta = newSize - polder->GetSizeInBlocks(); mBlocksUsedDelta += sizeDelta; - if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0) + if(polder->IsDeleted()) { mBlocksInDeletedFilesDelta += sizeDelta; } - if((polder->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) != 0) + if(polder->IsOld()) { mBlocksInOldFilesDelta += sizeDelta; } polder->SetSizeInBlocks(newSize); } - + // pentry no longer valid } - + // Delete it from the directory rDirectory.DeleteEntry(ObjectID); - + // Save directory back to disc // BLOCK - int64_t dirRevisedSize = 0; { RaidFileWrite writeDir(mStoreDiscSet, rDirectoryFilename, - mNewRefCounts[InDirectory]); + mapNewRefs->GetRefCount(InDirectory)); writeDir.Open(true /* allow overwriting */); rDirectory.WriteToStream(writeDir); - // get the disc usage (must do this before commiting it) - dirRevisedSize = writeDir.GetDiscUsageInBlocks(); + // Get the disc usage (must do this before commiting it) + int64_t new_size = writeDir.GetDiscUsageInBlocks(); // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - // adjust usage counts for this directory - if(dirRevisedSize > 0) - { - int64_t adjust = dirRevisedSize - OriginalDirSizeInBlocks; - mBlocksUsedDelta += adjust; - mBlocksInDirectoriesDelta += adjust; - } + // Adjust block counts if the directory itself changed in size + int64_t original_size = rDirectory.GetUserInfo1_SizeInBlocks(); + int64_t adjust = new_size - original_size; + mBlocksUsedDelta += adjust; + mBlocksInDirectoriesDelta += adjust; + + UpdateDirectorySize(rDirectory, new_size); } // Commit any new adjusted entry if(padjustedEntry.get() != 0) { padjustedEntry->Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); - padjustedEntry.reset(); // delete it now + padjustedEntry.reset(); // delete it now } - // Drop reference count by one. If it reaches zero, delete the file. - if(--mNewRefCounts[ObjectID] == 0) + // Drop reference count by one. Must now be zero, to delete the file. + bool remaining_refs = mapNewRefs->RemoveReference(ObjectID); + ASSERT(!remaining_refs); + + // Delete from disc + BOX_TRACE("Removing unreferenced object " << + BOX_FORMAT_OBJECTID(ObjectID)); + std::string objFilename; + MakeObjectFilename(ObjectID, objFilename); + RaidFileWrite del(mStoreDiscSet, objFilename, mapNewRefs->GetRefCount(ObjectID)); + del.Delete(); + + // Adjust counts for the file + ++mFilesDeleted; + mBlocksUsedDelta -= deletedFileSizeInBlocks; + + if(wasDeleted) { - // Delete from disc - BOX_TRACE("Removing unreferenced object " << - BOX_FORMAT_OBJECTID(ObjectID)); - std::string objFilename; - MakeObjectFilename(ObjectID, objFilename); - RaidFileWrite del(mStoreDiscSet, objFilename, - mNewRefCounts[ObjectID]); - del.Delete(); + mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; + rBackupStoreInfo.AdjustNumDeletedFiles(-1); } - else + + if(wasOldVersion) { - BOX_TRACE("Preserving object " << - BOX_FORMAT_OBJECTID(ObjectID) << " with " << - mNewRefCounts[ObjectID] << " references"); + mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; + rBackupStoreInfo.AdjustNumOldFiles(-1); } - // Adjust counts for the file - ++mFilesDeleted; - mBlocksUsedDelta -= deletedFileSizeInBlocks; - if(wasDeleted) mBlocksInDeletedFilesDelta -= deletedFileSizeInBlocks; - if(wasOldVersion) mBlocksInOldFilesDelta -= deletedFileSizeInBlocks; - // Delete the directory? // Do this if... dir has zero entries, and is marked as deleted in it's containing directory if(rDirectory.GetNumberOfEntries() == 0) @@ -952,8 +912,81 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba // Candidate for deletion mEmptyDirectories.push_back(InDirectory); } + + return 0; } +// -------------------------------------------------------------------------- +// +// Function +// Name: HousekeepStoreAccount::UpdateDirectorySize( +// BackupStoreDirectory& rDirectory, +// IOStream::pos_type new_size_in_blocks) +// Purpose: Update the directory size, modifying the parent +// directory's entry for this directory if necessary. +// Created: 05/03/14 +// +// -------------------------------------------------------------------------- + +void HousekeepStoreAccount::UpdateDirectorySize( + BackupStoreDirectory& rDirectory, + IOStream::pos_type new_size_in_blocks) +{ +#ifndef BOX_RELEASE_BUILD + { + std::string dirFilename; + MakeObjectFilename(rDirectory.GetObjectID(), dirFilename); + std::auto_ptr<RaidFileRead> dirStream( + RaidFileRead::Open(mStoreDiscSet, dirFilename)); + ASSERT(new_size_in_blocks == dirStream->GetDiscUsageInBlocks()); + } +#endif + + IOStream::pos_type old_size_in_blocks = + rDirectory.GetUserInfo1_SizeInBlocks(); + + if(new_size_in_blocks == old_size_in_blocks) + { + return; + } + + rDirectory.SetUserInfo1_SizeInBlocks(new_size_in_blocks); + + if (rDirectory.GetObjectID() == BACKUPSTORE_ROOT_DIRECTORY_ID) + { + return; + } + + std::string parentFilename; + MakeObjectFilename(rDirectory.GetContainerID(), parentFilename); + std::auto_ptr<RaidFileRead> parentStream( + RaidFileRead::Open(mStoreDiscSet, parentFilename)); + BackupStoreDirectory parent(*parentStream); + parentStream.reset(); + + BackupStoreDirectory::Entry* en = + parent.FindEntryByID(rDirectory.GetObjectID()); + ASSERT(en); + + if (en->GetSizeInBlocks() != old_size_in_blocks) + { + BOX_WARNING("Directory " << + BOX_FORMAT_OBJECTID(rDirectory.GetObjectID()) << + " entry in directory " << + BOX_FORMAT_OBJECTID(rDirectory.GetContainerID()) << + " had incorrect size " << en->GetSizeInBlocks() << + ", should have been " << old_size_in_blocks); + mErrorCount++; + } + + en->SetSizeInBlocks(new_size_in_blocks); + + RaidFileWrite writeDir(mStoreDiscSet, parentFilename, + mapNewRefs->GetRefCount(rDirectory.GetContainerID())); + writeDir.Open(true /* allow overwriting */); + parent.WriteToStream(writeDir); + writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); +} // -------------------------------------------------------------------------- // @@ -964,7 +997,7 @@ void HousekeepStoreAccount::DeleteFile(int64_t InDirectory, int64_t ObjectID, Ba // Created: 15/12/03 // // -------------------------------------------------------------------------- -bool HousekeepStoreAccount::DeleteEmptyDirectories() +bool HousekeepStoreAccount::DeleteEmptyDirectories(BackupStoreInfo& rBackupStoreInfo) { while(mEmptyDirectories.size() > 0) { @@ -992,7 +1025,7 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() continue; } - DeleteEmptyDirectory(*i, toExamine); + DeleteEmptyDirectory(*i, toExamine, rBackupStoreInfo); } // Remove contents of empty directories @@ -1000,13 +1033,13 @@ bool HousekeepStoreAccount::DeleteEmptyDirectories() // Swap in new, so it's examined next time round mEmptyDirectories.swap(toExamine); } - + // Not interrupted return false; } void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, - std::vector<int64_t>& rToExamine) + std::vector<int64_t>& rToExamine, BackupStoreInfo& rBackupStoreInfo) { // Load up the directory to potentially delete std::string dirFilename; @@ -1046,16 +1079,18 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, std::auto_ptr<RaidFileRead> containingDirStream( RaidFileRead::Open(mStoreDiscSet, containingDirFilename)); - containingDirSizeInBlocksOrig = + containingDirSizeInBlocksOrig = containingDirStream->GetDiscUsageInBlocks(); containingDir.ReadFromStream(*containingDirStream, IOStream::TimeOutInfinite); + containingDir.SetUserInfo1_SizeInBlocks(containingDirSizeInBlocksOrig); } // Find the entry - BackupStoreDirectory::Entry *pdirentry = + BackupStoreDirectory::Entry *pdirentry = containingDir.FindEntryByID(dir.GetObjectID()); - if((pdirentry != 0) && ((pdirentry->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) != 0)) + // TODO FIXME invert test and reduce indentation + if((pdirentry != 0) && pdirentry->IsDeleted()) { // Should be deleted containingDir.DeleteEntry(dir.GetObjectID()); @@ -1068,7 +1103,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Write revised parent directory RaidFileWrite writeDir(mStoreDiscSet, containingDirFilename, - mNewRefCounts[containingDir.GetObjectID()]); + mapNewRefs->GetRefCount(containingDir.GetObjectID())); writeDir.Open(true /* allow overwriting */); containingDir.WriteToStream(writeDir); @@ -1077,6 +1112,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Commit directory writeDir.Commit(BACKUP_STORE_CONVERT_TO_RAID_IMMEDIATELY); + UpdateDirectorySize(containingDir, dirSize); // adjust usage counts for this directory if(dirSize > 0) @@ -1086,17 +1122,22 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, mBlocksInDirectoriesDelta += adjust; } - - if (--mNewRefCounts[dir.GetObjectID()] == 0) + if (mapNewRefs->RemoveReference(dir.GetObjectID())) { - // Delete the directory itself - RaidFileWrite del(mStoreDiscSet, dirFilename, - mNewRefCounts[dir.GetObjectID()]); - del.Delete(); + // Still referenced + BOX_TRACE("Housekeeping spared empty deleted dir " << + BOX_FORMAT_OBJECTID(dirId) << " due to " << + mapNewRefs->GetRefCount(dir.GetObjectID()) << + "remaining references"); + return; } - BOX_INFO("Housekeeping removed empty deleted dir " << + // Delete the directory itself + BOX_INFO("Housekeeping removing empty deleted dir " << BOX_FORMAT_OBJECTID(dirId)); + RaidFileWrite del(mStoreDiscSet, dirFilename, + mapNewRefs->GetRefCount(dir.GetObjectID())); + del.Delete(); // And adjust usage counts for the directory that's // just been deleted @@ -1105,6 +1146,7 @@ void HousekeepStoreAccount::DeleteEmptyDirectory(int64_t dirId, // Update count ++mEmptyDirectoriesDeleted; + rBackupStoreInfo.AdjustNumDirectories(-1); } } diff --git a/lib/backupstore/HousekeepStoreAccount.h b/lib/backupstore/HousekeepStoreAccount.h index cfa05a2e..ff9e9ffe 100644 --- a/lib/backupstore/HousekeepStoreAccount.h +++ b/lib/backupstore/HousekeepStoreAccount.h @@ -14,6 +14,8 @@ #include <set> #include <vector> +#include "BackupStoreRefCountDatabase.h" + class BackupStoreDirectory; class HousekeepingCallback @@ -39,20 +41,25 @@ public: ~HousekeepStoreAccount(); bool DoHousekeeping(bool KeepTryingForever = false); - int GetRefCountsAdjusted() { return mRefCountsAdjusted; } + int GetErrorCount() { return mErrorCount; } private: // utility functions void MakeObjectFilename(int64_t ObjectID, std::string &rFilenameOut); - bool ScanDirectory(int64_t ObjectID); - bool DeleteFiles(); - bool DeleteEmptyDirectories(); - void DeleteEmptyDirectory(int64_t dirId, - std::vector<int64_t>& rToExamine); - void DeleteFile(int64_t InDirectory, int64_t ObjectID, BackupStoreDirectory &rDirectory, const std::string &rDirectoryFilename, int64_t OriginalDirSizeInBlocks); + bool ScanDirectory(int64_t ObjectID, BackupStoreInfo& rBackupStoreInfo); + bool DeleteFiles(BackupStoreInfo& rBackupStoreInfo); + bool DeleteEmptyDirectories(BackupStoreInfo& rBackupStoreInfo); + void DeleteEmptyDirectory(int64_t dirId, std::vector<int64_t>& rToExamine, + BackupStoreInfo& rBackupStoreInfo); + BackupStoreRefCountDatabase::refcount_t DeleteFile(int64_t InDirectory, + int64_t ObjectID, + BackupStoreDirectory &rDirectory, + const std::string &rDirectoryFilename, + BackupStoreInfo& rBackupStoreInfo); + void UpdateDirectorySize(BackupStoreDirectory &rDirectory, + IOStream::pos_type new_size_in_blocks); -private: typedef struct { int64_t mObjectID; @@ -81,6 +88,9 @@ private: // List of directories which are empty, and might be good for deleting std::vector<int64_t> mEmptyDirectories; + + // Count of errors found and fixed + int64_t mErrorCount; // The re-calculated blocks used stats int64_t mBlocksUsed; @@ -99,9 +109,7 @@ private: int64_t mEmptyDirectoriesDeleted; // New reference count list - std::vector<uint32_t> mNewRefCounts; - bool mSuppressRefCountChangeWarnings; - int mRefCountsAdjusted; + std::auto_ptr<BackupStoreRefCountDatabase> mapNewRefs; // Poll frequency int mCountUntilNextInterprocessMsgCheck; diff --git a/lib/backupstore/Makefile.extra b/lib/backupstore/Makefile.extra index c55fd549..6f181abd 100644 --- a/lib/backupstore/Makefile.extra +++ b/lib/backupstore/Makefile.extra @@ -1,9 +1,9 @@ MAKEPROTOCOL = ../../lib/server/makeprotocol.pl -GEN_CMD = $(MAKEPROTOCOL) backupprotocol.txt +GEN_CMD = $(MAKEPROTOCOL) BackupProtocol.txt # AUTOGEN SEEDING -autogen_BackupProtocol.cpp autogen_BackupProtocol.h: $(MAKEPROTOCOL) backupprotocol.txt +autogen_BackupProtocol.cpp autogen_BackupProtocol.h: $(MAKEPROTOCOL) BackupProtocol.txt $(_PERL) $(GEN_CMD) diff --git a/lib/backupstore/StoreTestUtils.cpp b/lib/backupstore/StoreTestUtils.cpp new file mode 100644 index 00000000..2b773cb1 --- /dev/null +++ b/lib/backupstore/StoreTestUtils.cpp @@ -0,0 +1,300 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreTestUtils.cpp +// Purpose: Utilities for housekeeping and checking a test store +// Created: 18/02/14 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include <cstdio> +#include <vector> + +#include "autogen_BackupProtocol.h" +#include "BoxPortsAndFiles.h" +#include "BackupStoreAccounts.h" +#include "BackupStoreAccountDatabase.h" +#include "BackupStoreConfigVerify.h" +#include "BackupStoreConstants.h" +#include "BackupStoreInfo.h" +#include "HousekeepStoreAccount.h" +#include "Logging.h" +#include "ServerControl.h" +#include "SocketStreamTLS.h" +#include "StoreTestUtils.h" +#include "TLSContext.h" +#include "Test.h" + +bool create_account(int soft, int hard) +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + + Logger::LevelGuard guard(Logging::GetConsole(), Log::WARNING); + int result = control.CreateAccount(0x01234567, 0, soft, hard); + TEST_EQUAL(0, result); + return (result == 0); +} + +bool delete_account() +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + Logger::LevelGuard guard(Logging::GetConsole(), Log::WARNING); + TEST_THAT_THROWONFAIL(control.DeleteAccount(0x01234567, false) == 0); + return true; +} + +std::vector<uint32_t> ExpectedRefCounts; +int bbstored_pid = 0, bbackupd_pid = 0; + +void set_refcount(int64_t ObjectID, uint32_t RefCount) +{ + if ((int64_t)ExpectedRefCounts.size() <= ObjectID) + { + ExpectedRefCounts.resize(ObjectID + 1, 0); + } + ExpectedRefCounts[ObjectID] = RefCount; + for (size_t i = ExpectedRefCounts.size() - 1; i >= 1; i--) + { + if (ExpectedRefCounts[i] == 0) + { + // BackupStoreCheck and housekeeping will both + // regenerate the refcount DB without any missing + // items at the end, so we need to prune ourselves + // of all items with no references to match. + ExpectedRefCounts.resize(i); + } + } +} + +void init_context(TLSContext& rContext) +{ + rContext.Initialise(false /* client */, + "testfiles/clientCerts.pem", + "testfiles/clientPrivKey.pem", + "testfiles/clientTrustedCAs.pem"); +} + +std::auto_ptr<SocketStream> open_conn(const char *hostname, + TLSContext& rContext) +{ + init_context(rContext); + std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS); + conn->Open(rContext, Socket::TypeINET, hostname, + BOX_PORT_BBSTORED_TEST); + return static_cast<std::auto_ptr<SocketStream> >(conn); +} + +std::auto_ptr<BackupProtocolCallable> connect_to_bbstored(TLSContext& rContext) +{ + // Make a protocol + std::auto_ptr<BackupProtocolCallable> protocol(new + BackupProtocolClient(open_conn("localhost", rContext))); + + // Check the version + std::auto_ptr<BackupProtocolVersion> serverVersion( + protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION)); + TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + + return protocol; +} + +std::auto_ptr<BackupProtocolCallable> connect_and_login(TLSContext& rContext, + int flags) +{ + // Make a protocol + std::auto_ptr<BackupProtocolCallable> protocol = + connect_to_bbstored(rContext); + + // Login + protocol->QueryLogin(0x01234567, flags); + + return protocol; +} + +bool check_num_files(int files, int old, int deleted, int dirs) +{ + std::auto_ptr<BackupStoreInfo> apInfo = + BackupStoreInfo::Load(0x1234567, + "backup/01234567/", 0, true); + TEST_EQUAL_LINE(files, apInfo->GetNumCurrentFiles(), + "current files"); + TEST_EQUAL_LINE(old, apInfo->GetNumOldFiles(), + "old files"); + TEST_EQUAL_LINE(deleted, apInfo->GetNumDeletedFiles(), + "deleted files"); + TEST_EQUAL_LINE(dirs, apInfo->GetNumDirectories(), + "directories"); + + return (files == apInfo->GetNumCurrentFiles() && + old == apInfo->GetNumOldFiles() && + deleted == apInfo->GetNumDeletedFiles() && + dirs == apInfo->GetNumDirectories()); +} + +bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, + int Deleted, int Dirs, int Total) +{ + std::auto_ptr<BackupProtocolAccountUsage2> usage = + Client.QueryGetAccountUsage2(); + TEST_EQUAL_LINE(Total, usage->GetBlocksUsed(), "wrong BlocksUsed"); + TEST_EQUAL_LINE(Current, usage->GetBlocksInCurrentFiles(), + "wrong BlocksInCurrentFiles"); + TEST_EQUAL_LINE(Old, usage->GetBlocksInOldFiles(), + "wrong BlocksInOldFiles"); + TEST_EQUAL_LINE(Deleted, usage->GetBlocksInDeletedFiles(), + "wrong BlocksInDeletedFiles"); + TEST_EQUAL_LINE(Dirs, usage->GetBlocksInDirectories(), + "wrong BlocksInDirectories"); + return (Total == usage->GetBlocksUsed() && + Current == usage->GetBlocksInCurrentFiles() && + Old == usage->GetBlocksInOldFiles() && + Deleted == usage->GetBlocksInDeletedFiles() && + Dirs == usage->GetBlocksInDirectories()); +} + +bool change_account_limits(const char* soft, const char* hard) +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + int result = control.SetLimit(0x01234567, soft, hard); + TEST_EQUAL(0, result); + return (result == 0); +} + +int check_account_for_errors(Log::Level log_level) +{ + Logger::LevelGuard guard(Logging::GetConsole(), log_level); + Logging::Tagger tag("check fix", true); + Logging::ShowTagOnConsole show; + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify("testfiles/bbstored.conf", + &BackupConfigFileVerify, errs)); + BackupStoreAccountsControl control(*config); + int errors_fixed = control.CheckAccount(0x01234567, + true, // FixErrors + false, // Quiet + true); // ReturnNumErrorsFound + return errors_fixed; +} + +bool check_account(Log::Level log_level) +{ + int errors_fixed = check_account_for_errors(log_level); + TEST_EQUAL(0, errors_fixed); + return (errors_fixed == 0); +} + +int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount) +{ + std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount); + int discSet = rAccount.GetDiscSet(); + + // Do housekeeping on this account + HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir, + discSet, NULL); + TEST_THAT(housekeeping.DoHousekeeping(true /* keep trying forever */)); + return housekeeping.GetErrorCount(); +} + +// Run housekeeping (for which we need to disconnect ourselves) and check +// that it doesn't change the numbers of files. +// +// Also check that bbstoreaccounts doesn't change anything + +bool run_housekeeping_and_check_account() +{ + int error_count; + + { + Logging::Tagger tag("", true); + Logging::ShowTagOnConsole show; + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + BackupStoreAccountDatabase::Entry account = + apAccounts->GetEntry(0x1234567); + error_count = run_housekeeping(account); + } + + TEST_EQUAL_LINE(0, error_count, "housekeeping errors"); + + bool check_account_is_ok = check_account(); + TEST_THAT(check_account_is_ok); + + return error_count == 0 && check_account_is_ok; +} + +bool check_reference_counts() +{ + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + BackupStoreAccountDatabase::Entry account = + apAccounts->GetEntry(0x1234567); + + std::auto_ptr<BackupStoreRefCountDatabase> apReferences( + BackupStoreRefCountDatabase::Load(account, true)); + TEST_EQUAL(ExpectedRefCounts.size(), + apReferences->GetLastObjectIDUsed() + 1); + + bool counts_ok = true; + + for (unsigned int i = BackupProtocolListDirectory::RootDirectory; + i < ExpectedRefCounts.size(); i++) + { + TEST_EQUAL_LINE(ExpectedRefCounts[i], + apReferences->GetRefCount(i), + "object " << BOX_FORMAT_OBJECTID(i)); + if (ExpectedRefCounts[i] != apReferences->GetRefCount(i)) + { + counts_ok = false; + } + } + + return counts_ok; +} + +bool StartServer() +{ + bbstored_pid = StartDaemon(bbstored_pid, + BBSTORED " " + bbstored_args + " testfiles/bbstored.conf", + "testfiles/bbstored.pid"); + return bbstored_pid != 0; +} + +bool StopServer(bool wait_for_process) +{ + bool result = StopDaemon(bbstored_pid, "testfiles/bbstored.pid", + "bbstored.memleaks", wait_for_process); + bbstored_pid = 0; + return result; +} + +bool StartClient(const std::string& bbackupd_conf_file) +{ + bbackupd_pid = StartDaemon(bbackupd_pid, + BBACKUPD " " + bbackupd_args + " " + bbackupd_conf_file, + "testfiles/bbackupd.pid"); + return bbackupd_pid != 0; +} + +bool StopClient(bool wait_for_process) +{ + bool result = StopDaemon(bbackupd_pid, "testfiles/bbackupd.pid", + "bbackupd.memleaks", wait_for_process); + bbackupd_pid = 0; + return result; +} + diff --git a/lib/backupstore/StoreTestUtils.h b/lib/backupstore/StoreTestUtils.h new file mode 100644 index 00000000..b3faebb5 --- /dev/null +++ b/lib/backupstore/StoreTestUtils.h @@ -0,0 +1,118 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: StoreTestUtils.h +// Purpose: Utilities for housekeeping and checking a test store +// Created: 18/02/14 +// +// -------------------------------------------------------------------------- + +#ifndef STORETESTUTILS__H +#define STORETESTUTILS__H + +#include "Test.h" + +class BackupProtocolCallable; +class BackupProtocolClient; +class SocketStreamTLS; +class TLSContext; + +//! Holds the expected reference counts of each object. +extern std::vector<uint32_t> ExpectedRefCounts; + +//! Holds the PID of the currently running bbstored test server. +extern int bbstored_pid, bbackupd_pid; + +//! Sets the expected refcount of an object, resizing vector if necessary. +void set_refcount(int64_t ObjectID, uint32_t RefCount = 1); + +//! Initialises a TLSContext object using the standard certficate filenames. +void init_context(TLSContext& rContext); + +//! Opens a connection to the server (bbstored). +std::auto_ptr<SocketStream> open_conn(const char *hostname, + TLSContext& rContext); + +//! Opens a connection to the server (bbstored) without logging in. +std::auto_ptr<BackupProtocolCallable> connect_to_bbstored(TLSContext& rContext); + +//! Opens a connection to the server (bbstored) and logs in. +std::auto_ptr<BackupProtocolCallable> connect_and_login(TLSContext& rContext, + int flags = 0); + +//! Checks the number of files of each type in the store against expectations. +bool check_num_files(int files, int old, int deleted, int dirs); + +//! Checks the number of blocks in files of each type against expectations. +bool check_num_blocks(BackupProtocolCallable& Client, int Current, int Old, + int Deleted, int Dirs, int Total); + +//! Change the soft and hard limits on the test account. +bool change_account_limits(const char* soft, const char* hard); + +//! Checks an account for errors, returning the number of errors found and fixed. +int check_account_for_errors(Log::Level log_level = Log::WARNING); + +//! Checks an account for errors, returning true if it's OK, for use in assertions. +bool check_account(Log::Level log_level = Log::WARNING); + +//! Runs housekeeping on an account, to remove old and deleted files if necessary. +int64_t run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount); + +//! Runs housekeeping and checks the account, returning true if it's OK. +bool run_housekeeping_and_check_account(); + +//! Tests that all object reference counts have the expected values. +bool check_reference_counts(); + +//! Starts the bbstored test server running, which must not already be running. +bool StartServer(); + +//! Stops the currently running bbstored test server. +bool StopServer(bool wait_for_process = false); + +//! Starts the bbackupd client running, which must not already be running. +bool StartClient(const std::string& bbackupd_conf_file = "testfiles/bbackupd.conf"); + +//! Stops the currently running bbackupd client. +bool StopClient(bool wait_for_process = false); + +//! Creates the standard test account, for example after delete_account(). +bool create_account(int soft, int hard); + +//! Deletes the standard test account, for testing behaviour with no account. +bool delete_account(); + +#define TEST_PROTOCOL_ERROR_OR(protocol, error, or_statements) \ + { \ + int type, subtype; \ + (protocol).GetLastError(type, subtype); \ + if (type == BackupProtocolError::ErrorType) \ + { \ + TEST_EQUAL_LINE(BackupProtocolError::error, subtype, \ + "command returned error: " << \ + BackupProtocolError::GetMessage(subtype)); \ + if (subtype != BackupProtocolError::error) \ + { \ + or_statements; \ + } \ + } \ + else \ + { \ + TEST_FAIL_WITH_MESSAGE("command did not return an error, but a " \ + "response of type " << type << ", subtype " << subtype << \ + " instead"); \ + or_statements; \ + } \ + } + +#define TEST_COMMAND_RETURNS_ERROR_OR(protocol, command, error, or_statements) \ + TEST_CHECK_THROWS_AND_OR((protocol) . command, ConnectionException, \ + Protocol_UnexpectedReply, /* and_command */, or_statements); \ + TEST_PROTOCOL_ERROR_OR(protocol, error, or_statements) + +#define TEST_COMMAND_RETURNS_ERROR(protocol, command, error) \ + TEST_COMMAND_RETURNS_ERROR_OR(protocol, command, error,) + +#endif // STORETESTUTILS__H + diff --git a/bin/bbackupd/BackupClientContext.cpp b/lib/bbackupd/BackupClientContext.cpp index 26df04be..4c0b01ce 100644 --- a/bin/bbackupd/BackupClientContext.cpp +++ b/lib/bbackupd/BackupClientContext.cpp @@ -42,11 +42,11 @@ // -------------------------------------------------------------------------- BackupClientContext::BackupClientContext ( - LocationResolver &rResolver, - TLSContext &rTLSContext, + LocationResolver &rResolver, + TLSContext &rTLSContext, const std::string &rHostname, int Port, - uint32_t AccountNumber, + uint32_t AccountNumber, bool ExtendedLogging, bool ExtendedLogToFile, std::string ExtendedLogFile, @@ -73,7 +73,8 @@ BackupClientContext::BackupClientContext mKeepAliveTimer(0, "KeepAliveTime"), mbIsManaged(false), mrProgressNotifier(rProgressNotifier), - mTcpNiceMode(TcpNiceMode) + mTcpNiceMode(TcpNiceMode), + mpNice(NULL) { } @@ -106,52 +107,56 @@ BackupClientContext::~BackupClientContext() // Created: 2003/10/08 // // -------------------------------------------------------------------------- -BackupProtocolClient &BackupClientContext::GetConnection() +BackupProtocolCallable &BackupClientContext::GetConnection() { // Already got it? Just return it. if(mapConnection.get()) { return *mapConnection; } - - // there shouldn't be a connection open - ASSERT(mapSocket.get() == 0); + // Defensive. Must close connection before releasing any old socket. mapConnection.reset(); - mapSocket.reset(new SocketStreamTLS); - + + std::auto_ptr<SocketStream> apSocket(new SocketStreamTLS); + try { // Defensive. mapConnection.reset(); - + // Log intention - BOX_INFO("Opening connection to server '" << - mHostname << "'..."); + BOX_INFO("Opening connection to server '" << mHostname << + "'..."); // Connect! - ((SocketStreamTLS *)(mapSocket.get()))->Open(mrTLSContext, + ((SocketStreamTLS *)(apSocket.get()))->Open(mrTLSContext, Socket::TypeINET, mHostname, mPort); - + if(mTcpNiceMode) { - // Pass control of mapSocket to NiceSocketStream, + // Pass control of apSocket to NiceSocketStream, // which will take care of destroying it for us. - mapNice.reset(new NiceSocketStream(mapSocket)); - mapConnection.reset(new BackupProtocolClient(*mapNice)); - } - else - { - mapConnection.reset(new BackupProtocolClient(*mapSocket)); + // But we need to hang onto a pointer to the nice + // socket, so we can enable and disable nice mode. + // This is scary, it could be deallocated under us. + mpNice = new NiceSocketStream(apSocket); + apSocket.reset(mpNice); } - + + // We need to call some methods that aren't defined in + // BackupProtocolCallable, so we need to hang onto a more + // strongly typed pointer (to avoid far too many casts). + BackupProtocolClient *pClient = new BackupProtocolClient(apSocket); + mapConnection.reset(pClient); + // Set logging option - mapConnection->SetLogToSysLog(mExtendedLogging); - + pClient->SetLogToSysLog(mExtendedLogging); + if (mExtendedLogToFile) { ASSERT(mpExtendedLogFileHandle == NULL); - + mpExtendedLogFileHandle = fopen( mExtendedLogFile.c_str(), "a+"); @@ -162,13 +167,13 @@ BackupProtocolClient &BackupClientContext::GetConnection() } else { - mapConnection->SetLogToFile(mpExtendedLogFileHandle); + pClient->SetLogToFile(mpExtendedLogFileHandle); } } - + // Handshake - mapConnection->Handshake(); - + pClient->Handshake(); + // Check the version of the server { std::auto_ptr<BackupProtocolVersion> serverVersion( @@ -182,7 +187,7 @@ BackupProtocolClient &BackupClientContext::GetConnection() // Login -- if this fails, the Protocol will exception std::auto_ptr<BackupProtocolLoginConfirmed> loginConf( mapConnection->QueryLogin(mAccountNumber, 0 /* read/write */)); - + // Check that the client store marker is the one we expect if(mClientStoreMarker != ClientStoreMarker_NotKnown) { @@ -192,19 +197,33 @@ BackupProtocolClient &BackupClientContext::GetConnection() try { mapConnection->QueryFinished(); - mapNice.reset(); - mapSocket.reset(); } catch(...) { // IGNORE } - + // Then throw an exception about this - THROW_EXCEPTION(BackupStoreException, ClientMarkerNotAsExpected) + THROW_EXCEPTION_MESSAGE(BackupStoreException, + ClientMarkerNotAsExpected, + "Expected " << mClientStoreMarker << + " but found " << loginConf->GetClientStoreMarker() << + ": is someone else writing to the " + "same account?"); } } - + else // mClientStoreMarker == ClientStoreMarker_NotKnown + { + // Yes, choose one, the current time will do + box_time_t marker = GetCurrentBoxTime(); + + // Set it on the store + mapConnection->QuerySetClientStoreMarker(marker); + + // Record it so that it can be picked up later. + mClientStoreMarker = marker; + } + // Log success BOX_INFO("Connection made, login successful"); @@ -222,14 +241,17 @@ BackupProtocolClient &BackupClientContext::GetConnection() { // Clean up. mapConnection.reset(); - mapNice.reset(); - mapSocket.reset(); throw; } return *mapConnection; } +BackupProtocolCallable* BackupClientContext::GetOpenConnection() const +{ + return mapConnection.get(); +} + // -------------------------------------------------------------------------- // // Function @@ -240,46 +262,23 @@ BackupProtocolClient &BackupClientContext::GetConnection() // -------------------------------------------------------------------------- void BackupClientContext::CloseAnyOpenConnection() { - if(mapConnection.get()) + BackupProtocolCallable* pConnection(GetOpenConnection()); + if(pConnection) { try { - // Need to set a client store marker? - if(mClientStoreMarker == ClientStoreMarker_NotKnown) - { - // Yes, choose one, the current time will do - box_time_t marker = GetCurrentBoxTime(); - - // Set it on the store - mapConnection->QuerySetClientStoreMarker(marker); - - // Record it so that it can be picked up later. - mClientStoreMarker = marker; - } - // Quit nicely - mapConnection->QueryFinished(); + pConnection->QueryFinished(); } catch(...) { // Ignore errors here } - + // Delete it anyway. mapConnection.reset(); } - try - { - // Be nice about closing the socket - mapNice.reset(); - mapSocket.reset(); - } - catch(...) - { - // Ignore errors - } - // Delete any pending list if(mpDeleteList != 0) { @@ -306,9 +305,10 @@ void BackupClientContext::CloseAnyOpenConnection() // -------------------------------------------------------------------------- int BackupClientContext::GetTimeout() const { - if(mapConnection.get()) + BackupProtocolCallable* pConnection(GetOpenConnection()); + if(pConnection) { - return mapConnection->GetTimeout(); + return pConnection->GetTimeout(); } return (15*60*1000); @@ -414,7 +414,7 @@ bool BackupClientContext::FindFilename(int64_t ObjectID, int64_t ContainingDirec bool &rIsCurrentVersionOut, box_time_t *pModTimeOnServer, box_time_t *pAttributesHashOnServer, BackupStoreFilenameClear *pLeafname) { // Make a connection to the server - BackupProtocolClient &connection(GetConnection()); + BackupProtocolCallable &connection(GetConnection()); // Request filenames from the server, in a "safe" manner to ignore errors properly { @@ -550,7 +550,8 @@ void BackupClientContext::UnManageDiffProcess() // -------------------------------------------------------------------------- void BackupClientContext::DoKeepAlive() { - if (!mapConnection.get()) + BackupProtocolCallable* pConnection(GetOpenConnection()); + if (!pConnection) { return; } @@ -566,7 +567,7 @@ void BackupClientContext::DoKeepAlive() } BOX_TRACE("KeepAliveTime reached, sending keep-alive message"); - mapConnection->QueryGetIsAlive(); + pConnection->QueryGetIsAlive(); mKeepAliveTimer.Reset(mKeepAliveTime * MILLI_SEC_IN_SEC); } diff --git a/bin/bbackupd/BackupClientContext.h b/lib/bbackupd/BackupClientContext.h index 1fcc6ede..df43a232 100644 --- a/bin/bbackupd/BackupClientContext.h +++ b/lib/bbackupd/BackupClientContext.h @@ -42,11 +42,11 @@ class BackupClientContext : public DiffTimer public: BackupClientContext ( - LocationResolver &rResolver, - TLSContext &rTLSContext, + LocationResolver &rResolver, + TLSContext &rTLSContext, const std::string &rHostname, int32_t Port, - uint32_t AccountNumber, + uint32_t AccountNumber, bool ExtendedLogging, bool ExtendedLogToFile, std::string ExtendedLogFile, @@ -54,16 +54,18 @@ public: bool TcpNiceMode ); virtual ~BackupClientContext(); + private: BackupClientContext(const BackupClientContext &); -public: - BackupProtocolClient &GetConnection(); - +public: + // GetConnection() will open a connection if none is currently open. + virtual BackupProtocolCallable& GetConnection(); + // GetOpenConnection() will not open a connection, just return NULL if there is + // no connection already open. + virtual BackupProtocolCallable* GetOpenConnection() const; void CloseAnyOpenConnection(); - int GetTimeout() const; - BackupClientDeleteList &GetDeleteList(); void PerformDeletions(); @@ -74,7 +76,7 @@ public: void SetClientStoreMarker(int64_t ClientStoreMarker) {mClientStoreMarker = ClientStoreMarker;} int64_t GetClientStoreMarker() const {return mClientStoreMarker;} - + bool StorageLimitExceeded() {return mStorageLimitExceeded;} void SetStorageLimitExceeded() {mStorageLimitExceeded = true;} @@ -170,7 +172,7 @@ public: // Created: 04/19/2005 // // -------------------------------------------------------------------------- - void SetKeepAliveTime(int iSeconds); + virtual void SetKeepAliveTime(int iSeconds); // -------------------------------------------------------------------------- // @@ -214,7 +216,7 @@ public: { if(mTcpNiceMode) { - mapNice->SetEnabled(enabled); + mpNice->SetEnabled(enabled); } } @@ -226,9 +228,7 @@ private: std::string mHostname; int mPort; uint32_t mAccountNumber; - std::auto_ptr<SocketStream> mapSocket; - std::auto_ptr<NiceSocketStream> mapNice; - std::auto_ptr<BackupProtocolClient> mapConnection; + std::auto_ptr<BackupProtocolCallable> mapConnection; bool mExtendedLogging; bool mExtendedLogToFile; std::string mExtendedLogFile; @@ -246,6 +246,7 @@ private: int mMaximumDiffingTime; ProgressNotifier &mrProgressNotifier; bool mTcpNiceMode; + NiceSocketStream *mpNice; }; #endif // BACKUPCLIENTCONTEXT__H diff --git a/bin/bbackupd/BackupClientDeleteList.cpp b/lib/bbackupd/BackupClientDeleteList.cpp index c0414b78..ce5e6264 100644 --- a/bin/bbackupd/BackupClientDeleteList.cpp +++ b/lib/bbackupd/BackupClientDeleteList.cpp @@ -132,7 +132,7 @@ void BackupClientDeleteList::PerformDeletions(BackupClientContext &rContext) } // Get a connection - BackupProtocolClient &connection(rContext.GetConnection()); + BackupProtocolCallable &connection(rContext.GetConnection()); // Do the deletes for(std::vector<DirToDelete>::iterator i(mDirectoryList.begin()); diff --git a/bin/bbackupd/BackupClientDeleteList.h b/lib/bbackupd/BackupClientDeleteList.h index b0fbf51a..b0fbf51a 100644 --- a/bin/bbackupd/BackupClientDeleteList.h +++ b/lib/bbackupd/BackupClientDeleteList.h diff --git a/bin/bbackupd/BackupClientDirectoryRecord.cpp b/lib/bbackupd/BackupClientDirectoryRecord.cpp index 90caf2e7..94cb7965 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.cpp +++ b/lib/bbackupd/BackupClientDirectoryRecord.cpp @@ -56,7 +56,6 @@ BackupClientDirectoryRecord::BackupClientDirectoryRecord(int64_t ObjectID, const mSubDirName(rSubDirName), mInitialSyncDone(false), mSyncDone(false), - mSuppressMultipleLinksWarning(false), mpPendingEntries(0) { ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); @@ -208,8 +207,8 @@ void BackupClientDirectoryRecord::SyncDirectory( { BackupClientInodeToIDMap &idMap( rParams.mrContext.GetNewIDMap()); - idMap.AddToMap(dest_st.st_ino, mObjectID, - ContainingDirectoryID); + idMap.AddToMap(dest_st.st_ino, mObjectID, ContainingDirectoryID, + ConvertVssPathToRealPath(rLocalPath, rBackupLocation)); } // Add attributes to checksum currentStateChecksum.Add(&dest_st.st_mode, @@ -238,24 +237,8 @@ void BackupClientDirectoryRecord::SyncDirectory( std::vector<std::string> files; bool downloadDirectoryRecordBecauseOfFutureFiles = false; - EMU_STRUCT_STAT link_st; - if(EMU_LSTAT(rLocalPath.c_str(), &link_st) != 0) - { - // Report the error (logs and - // eventual email to administrator) - rNotifier.NotifyFileStatFailed(this, - ConvertVssPathToRealPath(rLocalPath, rBackupLocation), - strerror(errno)); - - // FIXME move to NotifyFileStatFailed() - SetErrorWhenReadingFilesystemObject(rParams, rLocalPath); - - // This shouldn't happen, so we'd better not continue - THROW_EXCEPTION(CommonException, OSFileError) - } - // BLOCK - { + { // read the contents... DIR *dirHandle = 0; try @@ -267,8 +250,7 @@ void BackupClientDirectoryRecord::SyncDirectory( dirHandle = ::opendir(rLocalPath.c_str()); if(dirHandle == 0) { - // Report the error (logs and - // eventual email to administrator) + // Report the error (logs and eventual email to administrator) if (errno == EACCES) { rNotifier.NotifyDirListFailed(this, @@ -277,11 +259,11 @@ void BackupClientDirectoryRecord::SyncDirectory( } else { - rNotifier.NotifyDirListFailed(this, + rNotifier.NotifyDirListFailed(this, nonVssDirPath, strerror(errno)); } - + // Report the error (logs and eventual email // to administrator) SetErrorWhenReadingFilesystemObject(rParams, @@ -289,251 +271,29 @@ void BackupClientDirectoryRecord::SyncDirectory( // Ignore this directory for now. return; } - - // Basic structure for checksum info - struct { - box_time_t mModificationTime; - box_time_t mAttributeModificationTime; - int64_t mSize; - // And then the name follows - } checksum_info; - // Be paranoid about structure packing - ::memset(&checksum_info, 0, sizeof(checksum_info)); - + struct dirent *en = 0; - EMU_STRUCT_STAT file_st; - std::string filename; + int num_entries_found = 0; + while((en = ::readdir(dirHandle)) != 0) { + num_entries_found++; rParams.mrContext.DoKeepAlive(); - - // Don't need to use - // LinuxWorkaround_FinishDirentStruct(en, - // rLocalPath.c_str()); - // on Linux, as a stat is performed to - // get all this info - - if(en->d_name[0] == '.' && - (en->d_name[1] == '\0' || (en->d_name[1] == '.' && en->d_name[2] == '\0'))) + if(rParams.mpBackgroundTask) { - // ignore, it's . or .. - continue; + rParams.mpBackgroundTask->RunBackgroundTask( + BackgroundTask::Scanning_Dirs, + num_entries_found, 0); } - // Stat file to get info - filename = MakeFullPath(rLocalPath, en->d_name); - std::string realFileName = ConvertVssPathToRealPath(filename, - rBackupLocation); - - #ifdef WIN32 - // Don't stat the file just yet, to ensure - // that users can exclude unreadable files - // to suppress warnings that they are - // not accessible. - // - // Our emulated readdir() abuses en->d_type, - // which would normally contain DT_REG, - // DT_DIR, etc, but we only use it here and - // prefer to have the full file attributes. - int type; - if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) - { - type = S_IFDIR; - } - else + if (!SyncDirectoryEntry(rParams, rNotifier, + rBackupLocation, rLocalPath, + currentStateChecksum, en, dest_st, dirs, + files, downloadDirectoryRecordBecauseOfFutureFiles)) { - type = S_IFREG; - } - - #else // !WIN32 - if(EMU_LSTAT(filename.c_str(), &file_st) != 0) - { - if(!(rParams.mrContext.ExcludeDir( - filename))) - { - // Report the error (logs and - // eventual email to - // administrator) - rNotifier.NotifyFileStatFailed( - this, filename, - strerror(errno)); - - // FIXME move to - // NotifyFileStatFailed() - SetErrorWhenReadingFilesystemObject(rParams, filename); - } - - // Ignore this entry for now. - continue; - } - - int type = file_st.st_mode & S_IFMT; - - // ecryptfs reports nlink > 1 for directories - // with contents, but no filesystem supports - // hardlinking directories? so we can ignore - // this if the entry is a directory. - if(file_st.st_nlink != 1 && type == S_IFDIR) - { - BOX_INFO("Ignoring apparent hard link " - "count on directory: " << - filename << ", nlink=" << - file_st.st_nlink); - } - else if(file_st.st_nlink > 1) - { - if(!mSuppressMultipleLinksWarning) - { - BOX_WARNING("File is hard linked, this may " - "cause rename tracking to fail and " - "move files incorrectly in your " - "backup! " << filename << - ", nlink=" << file_st.st_nlink << - " (suppressing further warnings)"); - mSuppressMultipleLinksWarning = true; - } - SetErrorWhenReadingFilesystemObject(rParams, filename); - } - - BOX_TRACE("Stat entry '" << filename << "' " - "found device/inode " << - file_st.st_dev << "/" << - file_st.st_ino); - - /* Workaround for apparent btrfs bug, where - symlinks appear to be on a different filesystem - than their containing directory, thanks to - Toke Hoiland-Jorgensen */ - if(type == S_IFDIR && - file_st.st_dev != dest_st.st_dev) - { - if(!(rParams.mrContext.ExcludeDir( - filename))) - { - rNotifier.NotifyMountPointSkipped( - this, filename); - } + // This entry is not to be backed up. continue; } - #endif - - if(type == S_IFREG || type == S_IFLNK) - { - // File or symbolic link - - // Exclude it? - if(rParams.mrContext.ExcludeFile(realFileName)) - { - rNotifier.NotifyFileExcluded(this, realFileName); - // Next item! - continue; - } - - // Store on list - files.push_back(std::string(en->d_name)); - } - else if(type == S_IFDIR) - { - // Directory - - // Exclude it? - if(rParams.mrContext.ExcludeDir(realFileName)) - { - rNotifier.NotifyDirExcluded(this, realFileName); - - // Next item! - continue; - } - - #ifdef WIN32 - // exclude reparse points, as Application Data points to the - // parent directory under Vista and later, and causes an - // infinite loop: - // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ - if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) - { - rNotifier.NotifyMountPointSkipped(this, realFileName); - continue; - } - #endif - - // Store on list - dirs.push_back(std::string(en->d_name)); - } - else // not a file or directory, what is it? - { - if (type == S_IFSOCK -# ifndef WIN32 - || type == S_IFIFO -# endif - ) - { - // removed notification for these types - // see Debian bug 479145, no objections - } - else if(rParams.mrContext.ExcludeFile(realFileName)) - { - rNotifier.NotifyFileExcluded(this, realFileName); - } - else - { - rNotifier.NotifyUnsupportedFileType(this, - realFileName); - SetErrorWhenReadingFilesystemObject(rParams, - realFileName); - } - - continue; - } - - // Here if the object is something to back up (file, symlink or dir, not excluded) - // So make the information for adding to the checksum - - #ifdef WIN32 - // We didn't stat the file before, - // but now we need the information. - if(emu_stat(filename.c_str(), &file_st) != 0) - { - rNotifier.NotifyFileStatFailed(this, - ConvertVssPathToRealPath(filename, rBackupLocation), - strerror(errno)); - - // Report the error (logs and - // eventual email to administrator) - SetErrorWhenReadingFilesystemObject(rParams, filename); - - // Ignore this entry for now. - continue; - } - - if(file_st.st_dev != link_st.st_dev) - { - rNotifier.NotifyMountPointSkipped(this, - ConvertVssPathToRealPath(filename, rBackupLocation)); - continue; - } - #endif - - checksum_info.mModificationTime = FileModificationTime(file_st); - checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st); - checksum_info.mSize = file_st.st_size; - currentStateChecksum.Add(&checksum_info, sizeof(checksum_info)); - currentStateChecksum.Add(en->d_name, strlen(en->d_name)); - - // If the file has been modified madly into the future, download the - // directory record anyway to ensure that it doesn't get uploaded - // every single time the disc is scanned. - if(checksum_info.mModificationTime > rParams.mUploadAfterThisTimeInTheFuture) - { - downloadDirectoryRecordBecauseOfFutureFiles = true; - // Log that this has happened - if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) - { - rNotifier.NotifyFileModifiedInFuture(this, - ConvertVssPathToRealPath(filename, rBackupLocation)); - rParams.mHaveLoggedWarningAboutFutureFileTimes = true; - } - } } if(::closedir(dirHandle) != 0) @@ -562,7 +322,7 @@ void BackupClientDirectoryRecord::SyncDirectory( } // Pointer to potentially downloaded store directory info - BackupStoreDirectory *pdirOnStore = 0; + std::auto_ptr<BackupStoreDirectory> apDirOnStore; try { @@ -570,33 +330,35 @@ void BackupClientDirectoryRecord::SyncDirectory( if(ThisDirHasJustBeenCreated) { // Avoid sending another command to the server when we know it's empty - pdirOnStore = new BackupStoreDirectory(mObjectID, ContainingDirectoryID); + apDirOnStore.reset(new BackupStoreDirectory(mObjectID, + ContainingDirectoryID)); } - else + // Consider asking the store for it + else if(!mInitialSyncDone || checksumDifferent || + downloadDirectoryRecordBecauseOfFutureFiles) { - // Consider asking the store for it - if(!mInitialSyncDone || checksumDifferent || downloadDirectoryRecordBecauseOfFutureFiles) - { - pdirOnStore = FetchDirectoryListing(rParams); - } + apDirOnStore = FetchDirectoryListing(rParams); } - - // Make sure the attributes are up to date -- if there's space on the server - // and this directory has not just been created (because it's attributes will be correct in this case) - // and the checksum is different, implying they *MIGHT* be different. - if((!ThisDirHasJustBeenCreated) && checksumDifferent && (!rParams.mrContext.StorageLimitExceeded())) + + // Make sure the attributes are up to date -- if there's space + // on the server and this directory has not just been created + // (because it's attributes will be correct in this case) and + // the checksum is different, implying they *MIGHT* be + // different. + if((!ThisDirHasJustBeenCreated) && checksumDifferent && + !rParams.mrContext.StorageLimitExceeded()) { - UpdateAttributes(rParams, pdirOnStore, rLocalPath); + UpdateAttributes(rParams, apDirOnStore.get(), rLocalPath); } // Create the list of pointers to directory entries std::vector<BackupStoreDirectory::Entry *> entriesLeftOver; - if(pdirOnStore) + if(apDirOnStore.get()) { - entriesLeftOver.resize(pdirOnStore->GetNumberOfEntries(), 0); - BackupStoreDirectory::Iterator i(*pdirOnStore); + entriesLeftOver.resize(apDirOnStore->GetNumberOfEntries(), 0); + BackupStoreDirectory::Iterator i(*apDirOnStore); // Copy in pointers to all the entries - for(unsigned int l = 0; l < pdirOnStore->GetNumberOfEntries(); ++l) + for(unsigned int l = 0; l < apDirOnStore->GetNumberOfEntries(); ++l) { entriesLeftOver[l] = i.Next(); } @@ -604,14 +366,17 @@ void BackupClientDirectoryRecord::SyncDirectory( // Do the directory reading bool updateCompleteSuccess = UpdateItems(rParams, rLocalPath, - rRemotePath, rBackupLocation, pdirOnStore, entriesLeftOver, files, dirs); + rRemotePath, rBackupLocation, apDirOnStore.get(), + entriesLeftOver, files, dirs); // LAST THING! (think exception safety) - // Store the new checksum -- don't fetch things unnecessarily in the future - // But... only if 1) the storage limit isn't exceeded -- make sure things are done again if - // the directory is modified later - // and 2) All the objects within the directory were stored successfully. - if(!rParams.mrContext.StorageLimitExceeded() && updateCompleteSuccess) + // Store the new checksum -- don't fetch things unnecessarily + // in the future But... only if 1) the storage limit isn't + // exceeded -- make sure things are done again if the directory + // is modified later and 2) All the objects within the + // directory were stored successfully. + if(!rParams.mrContext.StorageLimitExceeded() && + updateCompleteSuccess) { currentStateChecksum.CopyDigestTo(mStateChecksum); } @@ -619,25 +384,12 @@ void BackupClientDirectoryRecord::SyncDirectory( catch(...) { // Bad things have happened -- clean up - if(pdirOnStore != 0) - { - delete pdirOnStore; - pdirOnStore = 0; - } - // Set things so that we get a full go at stuff later ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); throw; } - // Clean up directory on store - if(pdirOnStore != 0) - { - delete pdirOnStore; - pdirOnStore = 0; - } - // Flag things as having happened. mInitialSyncDone = true; mSyncDone = true; @@ -646,54 +398,281 @@ void BackupClientDirectoryRecord::SyncDirectory( // -------------------------------------------------------------------------- // // Function -// Name: BackupClientDirectoryRecord::FetchDirectoryListing(BackupClientDirectoryRecord::SyncParams &) -// Purpose: Fetch the directory listing of this directory from the store. -// Created: 2003/10/09 +// Name: BackupClientDirectoryRecord::SyncDirectorEntry( +// BackupClientDirectoryRecord::SyncParams &, +// int64_t, const std::string &, +// const std::string &, bool) +// Purpose: Recursively synchronise a local directory +// with the server. +// Created: 2003/10/08 // // -------------------------------------------------------------------------- -BackupStoreDirectory *BackupClientDirectoryRecord::FetchDirectoryListing(BackupClientDirectoryRecord::SyncParams &rParams) +bool BackupClientDirectoryRecord::SyncDirectoryEntry( + BackupClientDirectoryRecord::SyncParams &rParams, + ProgressNotifier& rNotifier, + const Location& rBackupLocation, + const std::string &rDirLocalPath, + MD5Digest& currentStateChecksum, + struct dirent *en, + EMU_STRUCT_STAT dir_st, + std::vector<std::string>& rDirs, + std::vector<std::string>& rFiles, + bool& rDownloadDirectoryRecordBecauseOfFutureFiles) { - BackupStoreDirectory *pdir = 0; + std::string entry_name = en->d_name; + if(entry_name == "." || entry_name == "..") + { + // ignore parent directory entries + return false; + } + + // Stat file to get info + std::string filename = MakeFullPath(rDirLocalPath, entry_name); + std::string realFileName = ConvertVssPathToRealPath(filename, + rBackupLocation); + EMU_STRUCT_STAT file_st; + +#ifdef WIN32 + // Don't stat the file just yet, to ensure that users can exclude + // unreadable files to suppress warnings that they are not accessible. + // + // Our emulated readdir() abuses en->d_type, which would normally + // contain DT_REG, DT_DIR, etc, but we only use it here and prefer to + // have the full file attributes. + + int type; + if (en->d_type & FILE_ATTRIBUTE_DIRECTORY) + { + type = S_IFDIR; + } + else + { + type = S_IFREG; + } +#else // !WIN32 + if(EMU_LSTAT(filename.c_str(), &file_st) != 0) + { + // We don't know whether it's a file or a directory, so check + // both. This only affects whether a warning message is + // displayed; the file is not backed up in either case. + if(!(rParams.mrContext.ExcludeFile(filename)) && + !(rParams.mrContext.ExcludeDir(filename))) + { + // Report the error (logs and eventual email to + // administrator) + rNotifier.NotifyFileStatFailed(this, filename, + strerror(errno)); + + // FIXME move to NotifyFileStatFailed() + SetErrorWhenReadingFilesystemObject(rParams, filename); + } + + // Ignore this entry for now. + return false; + } + + BOX_TRACE("Stat entry '" << filename << "' found device/inode " << + file_st.st_dev << "/" << file_st.st_ino); + + // Workaround for apparent btrfs bug, where symlinks appear to be on + // a different filesystem than their containing directory, thanks to + // Toke Hoiland-Jorgensen. + + int type = file_st.st_mode & S_IFMT; + if(type == S_IFDIR && file_st.st_dev != dir_st.st_dev) + { + if(!(rParams.mrContext.ExcludeDir(filename))) + { + rNotifier.NotifyMountPointSkipped(this, filename); + } + return false; + } +#endif + + if(type == S_IFREG || type == S_IFLNK) + { + // File or symbolic link + + // Exclude it? + if(rParams.mrContext.ExcludeFile(realFileName)) + { + rNotifier.NotifyFileExcluded(this, realFileName); + // Next item! + return false; + } + } + else if(type == S_IFDIR) + { + // Directory + + // Exclude it? + if(rParams.mrContext.ExcludeDir(realFileName)) + { + rNotifier.NotifyDirExcluded(this, realFileName); + + // Next item! + return false; + } + + #ifdef WIN32 + // exclude reparse points, as Application Data points to the + // parent directory under Vista and later, and causes an + // infinite loop: + // http://social.msdn.microsoft.com/forums/en-US/windowscompatibility/thread/05d14368-25dd-41c8-bdba-5590bf762a68/ + if (en->d_type & FILE_ATTRIBUTE_REPARSE_POINT) + { + rNotifier.NotifyMountPointSkipped(this, realFileName); + return false; + } + #endif + } + else // not a file or directory, what is it? + { + if (type == S_IFSOCK +#ifndef WIN32 + || type == S_IFIFO +#endif + ) + { + // removed notification for these types + // see Debian bug 479145, no objections + } + else if(rParams.mrContext.ExcludeFile(realFileName)) + { + rNotifier.NotifyFileExcluded(this, realFileName); + } + else + { + rNotifier.NotifyUnsupportedFileType(this, realFileName); + SetErrorWhenReadingFilesystemObject(rParams, + realFileName); + } + + return false; + } + + // The object should be backed up (file, symlink or dir, not excluded). + // So make the information for adding to the checksum. + + #ifdef WIN32 + // We didn't stat the file before, but now we need the information. + if(emu_stat(filename.c_str(), &file_st) != 0) + { + rNotifier.NotifyFileStatFailed(this, + ConvertVssPathToRealPath(filename, rBackupLocation), + strerror(errno)); + + // Report the error (logs and eventual email to administrator) + SetErrorWhenReadingFilesystemObject(rParams, filename); + + // Ignore this entry for now. + return false; + } + + if(file_st.st_dev != dir_st.st_dev) + { + rNotifier.NotifyMountPointSkipped(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); + return false; + } + #endif + + // Basic structure for checksum info + struct { + box_time_t mModificationTime; + box_time_t mAttributeModificationTime; + int64_t mSize; + // And then the name follows + } checksum_info; + + // Be paranoid about structure packing + ::memset(&checksum_info, 0, sizeof(checksum_info)); + + checksum_info.mModificationTime = FileModificationTime(file_st); + checksum_info.mAttributeModificationTime = FileAttrModificationTime(file_st); + checksum_info.mSize = file_st.st_size; + currentStateChecksum.Add(&checksum_info, sizeof(checksum_info)); + currentStateChecksum.Add(en->d_name, strlen(en->d_name)); - try + // If the file has been modified madly into the future, download the + // directory record anyway to ensure that it doesn't get uploaded + // every single time the disc is scanned. + if(checksum_info.mModificationTime > rParams.mUploadAfterThisTimeInTheFuture) { - // Get connection to store - BackupProtocolClient &connection(rParams.mrContext.GetConnection()); - - // Query the directory - std::auto_ptr<BackupProtocolSuccess> dirreply(connection.QueryListDirectory( - mObjectID, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, // both files and directories - BackupProtocolListDirectory::Flags_Deleted | - BackupProtocolListDirectory::Flags_OldVersion, // exclude old/deleted stuff - true /* want attributes */)); - - // Retrieve the directory from the stream following - pdir = new BackupStoreDirectory; - ASSERT(pdir != 0); - std::auto_ptr<IOStream> dirstream(connection.ReceiveStream()); - pdir->ReadFromStream(*dirstream, connection.GetTimeout()); + rDownloadDirectoryRecordBecauseOfFutureFiles = true; + // Log that this has happened + if(!rParams.mHaveLoggedWarningAboutFutureFileTimes) + { + rNotifier.NotifyFileModifiedInFuture(this, + ConvertVssPathToRealPath(filename, rBackupLocation)); + rParams.mHaveLoggedWarningAboutFutureFileTimes = true; + } } - catch(...) + + // We've decided to back it up, so add to file or directory list. + if(type == S_IFREG || type == S_IFLNK) { - delete pdir; - pdir = 0; - throw; + rFiles.push_back(entry_name); } + else if(type == S_IFDIR) + { + rDirs.push_back(entry_name); + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupClientDirectoryRecord::FetchDirectoryListing( +// BackupClientDirectoryRecord::SyncParams &) +// Purpose: Fetch the directory listing of this directory from +// the store. +// Created: 2003/10/09 +// +// -------------------------------------------------------------------------- +std::auto_ptr<BackupStoreDirectory> +BackupClientDirectoryRecord::FetchDirectoryListing( + BackupClientDirectoryRecord::SyncParams &rParams) +{ + std::auto_ptr<BackupStoreDirectory> apDir; - return pdir; + // Get connection to store + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Query the directory + std::auto_ptr<BackupProtocolSuccess> dirreply(connection.QueryListDirectory( + mObjectID, + // both files and directories + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + // exclude old/deleted stuff + BackupProtocolListDirectory::Flags_Deleted | + BackupProtocolListDirectory::Flags_OldVersion, + true /* want attributes */)); + + // Retrieve the directory from the stream following + apDir.reset(new BackupStoreDirectory(connection.ReceiveStream(), + connection.GetTimeout())); + return apDir; } // -------------------------------------------------------------------------- // // Function -// Name: BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord::SyncParams &, const std::string &) -// Purpose: Sets the attributes of the directory on the store, if necessary +// Name: BackupClientDirectoryRecord::UpdateAttributes( +// BackupClientDirectoryRecord::SyncParams &, +// const std::string &) +// Purpose: Sets the attributes of the directory on the store, +// if necessary. // Created: 2003/10/09 // // -------------------------------------------------------------------------- -void BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord::SyncParams &rParams, BackupStoreDirectory *pDirOnStore, const std::string &rLocalPath) +void BackupClientDirectoryRecord::UpdateAttributes( + BackupClientDirectoryRecord::SyncParams &rParams, + BackupStoreDirectory *pDirOnStore, + const std::string &rLocalPath) { // Get attributes for the directory BackupClientFileAttributes attr; @@ -725,7 +704,7 @@ void BackupClientDirectoryRecord::UpdateAttributes(BackupClientDirectoryRecord:: if(updateAttr) { // Get connection to store - BackupProtocolClient &connection(rParams.mrContext.GetConnection()); + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); // Exception thrown if this doesn't work std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); @@ -802,7 +781,7 @@ bool BackupClientDirectoryRecord::UpdateItems( std::string filenameClear; try { - filenameClear = DecryptFilename(en, + filenameClear = DecryptFilename(en, rRemotePath); decryptedEntries[filenameClear] = en; } @@ -841,7 +820,7 @@ bool BackupClientDirectoryRecord::UpdateItems( rNotifier.NotifyFileStatFailed(this, nonVssFilePath, strerror(errno)); - // Report the error (logs and + // Report the error (logs and // eventual email to administrator) SetErrorWhenReadingFilesystemObject(rParams, nonVssFilePath); @@ -900,7 +879,7 @@ bool BackupClientDirectoryRecord::UpdateItems( if(rContext.FindFilename(renameObjectID, renameInDirectory, localPotentialOldName, isDir, isCurrentVersion, &srvModTime, &srvAttributesHash, &oldLeafname)) - { + { // Only interested if it's a file and the latest version if(!isDir && isCurrentVersion) { @@ -911,8 +890,8 @@ bool BackupClientDirectoryRecord::UpdateItems( // Doesn't exist locally, but does exist on the server. // Therefore we can safely rename it to this new file. - // Get the connection to the server - BackupProtocolClient &connection(rContext.GetConnection()); + // Get the connection to the server + BackupProtocolCallable &connection(rContext.GetConnection()); // Only do this step if there is room on the server. // This step will be repeated later when there is space available @@ -975,12 +954,12 @@ bool BackupClientDirectoryRecord::UpdateItems( // Condition for upload: // modification time within sync period // if it's been seen before but not uploaded, is the time from this first sight longer than the MaxUploadWait - // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store + // and if we know about it from a directory listing, that it hasn't got the same upload time as on the store bool doUpload = false; std::string decisionReason = "unknown"; - // Only upload a file if the mod time locally is + // Only upload a file if the mod time locally is // different to that on the server. if (en == 0 || en->GetModificationTime() != modTime) @@ -1003,32 +982,32 @@ bool BackupClientDirectoryRecord::UpdateItems( } } - // However, just in case things are continually + // However, just in case things are continually // modified, we check the first seen time. - // The two compares of syncPeriodEnd and - // pendingFirstSeenTime are because the values + // The two compares of syncPeriodEnd and + // pendingFirstSeenTime are because the values // are unsigned. - if (!doUpload && + if (!doUpload && pendingFirstSeenTime != 0 && rParams.mSyncPeriodEnd > pendingFirstSeenTime && - (rParams.mSyncPeriodEnd - pendingFirstSeenTime) + (rParams.mSyncPeriodEnd - pendingFirstSeenTime) > rParams.mMaxUploadWait) { doUpload = true; decisionReason = "continually modified"; } - // Then make sure that if files are added with a + // Then make sure that if files are added with a // time less than the sync period start - // (which can easily happen on file server), it + // (which can easily happen on file server), it // gets uploaded. The directory contents checksum - // will pick up the fact it has been added, so the + // will pick up the fact it has been added, so the // store listing will be available when this happens. if (!doUpload && - modTime <= rParams.mSyncPeriodStart && - en != 0 && + modTime <= rParams.mSyncPeriodStart && + en != 0 && en->GetModificationTime() != modTime) { doUpload = true; @@ -1039,7 +1018,7 @@ bool BackupClientDirectoryRecord::UpdateItems( // the future for file server clients, // just upload the file if it's madly in the future. - if (!doUpload && modTime > + if (!doUpload && modTime > rParams.mUploadAfterThisTimeInTheFuture) { doUpload = true; @@ -1060,14 +1039,14 @@ bool BackupClientDirectoryRecord::UpdateItems( int age = BoxTimeToSeconds(now - modTime); std::ostringstream s; - s << "modified too recently: " - "only " << age << " seconds ago"; + s << "modified too recently: only " << + age << " seconds ago"; decisionReason = s.str(); } else { std::ostringstream s; - s << "mod time is " << modTime << + s << "mod time is " << modTime << " which is outside sync window, " << rParams.mSyncPeriodStart << " to " << rParams.mSyncPeriodEnd; @@ -1098,7 +1077,7 @@ bool BackupClientDirectoryRecord::UpdateItems( { // Upload the file to the server, recording the // object ID it returns - bool noPreviousVersionOnServer = + bool noPreviousVersionOnServer = ((pDirOnStore != 0) && (en == 0)); // Surround this in a try/catch block, to @@ -1109,6 +1088,7 @@ bool BackupClientDirectoryRecord::UpdateItems( latestObjectID = UploadFile(rParams, filename, nonVssFilePath, + rRemotePath + "/" + *f, storeFilename, fileSize, modTime, attributesHash, @@ -1141,7 +1121,7 @@ bool BackupClientDirectoryRecord::UpdateItems( if (e.GetType() == BackupStoreException::ExceptionType && e.GetSubType() == BackupStoreException::SignalReceived) { - // abort requested, pass the + // abort requested, pass the // exception on up. throw; } @@ -1182,7 +1162,7 @@ bool BackupClientDirectoryRecord::UpdateItems( // been downloaded, and the entry will be available. // Get connection - BackupProtocolClient &connection(rContext.GetConnection()); + BackupProtocolCallable &connection(rContext.GetConnection()); // Only do this step if there is room on the server. // This step will be repeated later when there is @@ -1247,17 +1227,7 @@ bool BackupClientDirectoryRecord::UpdateItems( BackupClientInodeToIDMap &idMap(rContext.GetNewIDMap()); // Need to get an ID from somewhere... - if(latestObjectID != 0) - { - // Use this one - BOX_TRACE("Storing uploaded file ID " << - inodeNum << " (" << nonVssFilePath << ") " - "in ID map as object " << - latestObjectID << " with parent " << - mObjectID); - idMap.AddToMap(inodeNum, latestObjectID, mObjectID /* containing directory */); - } - else + if(latestObjectID == 0) { // Don't know it -- haven't sent anything to the store, and didn't get a listing. // Look it up in the current map, and if it's there, use that. @@ -1283,15 +1253,25 @@ bool BackupClientDirectoryRecord::UpdateItems( // into it. However, in a long running process this may happen occasionally and // not indicate anything wrong. // Run the release version for real life use, where this check is not made. - BOX_TRACE("Storing found file ID " << inodeNum << - " (" << nonVssFilePath << ") in ID map as " - "object " << objid << " with parent " << mObjectID); - idMap.AddToMap(inodeNum, objid, - mObjectID /* containing directory */); + + latestObjectID = objid; } } + + if(latestObjectID != 0) + { + BOX_TRACE("Storing uploaded file ID " << + inodeNum << " (" << nonVssFilePath << ") " + "in ID map as object " << + latestObjectID << " with parent " << + mObjectID); + idMap.AddToMap(inodeNum, latestObjectID, + mObjectID /* containing directory */, + nonVssFilePath); + } + } - + if (fileSynced) { rNotifier.NotifyFileSynchronised(this, nonVssFilePath, @@ -1302,7 +1282,7 @@ bool BackupClientDirectoryRecord::UpdateItems( // Erase contents of files to save space when recursing rFiles.clear(); - // Delete the pending entries, if the map is entry + // Delete the pending entries, if the map is empty if(mpPendingEntries != 0 && mpPendingEntries->size() == 0) { BOX_TRACE("Deleting mpPendingEntries from dir ID " << @@ -1340,7 +1320,7 @@ bool BackupClientDirectoryRecord::UpdateItems( { // Entry exists, but is not a directory. Bad. // Get rid of it. - BackupProtocolClient &connection(rContext.GetConnection()); + BackupProtocolCallable &connection(rContext.GetConnection()); connection.QueryDeleteFile(mObjectID /* in directory */, storeFilename); std::string filenameClear = DecryptFilename(en, @@ -1352,6 +1332,19 @@ bool BackupClientDirectoryRecord::UpdateItems( en = 0; } + // Zero pointer in rEntriesLeftOver, if we have a pointer to zero + if(en != 0) + { + for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) + { + if(rEntriesLeftOver[l] == en) + { + rEntriesLeftOver[l] = 0; + break; + } + } + } + // Flag for having created directory, so can optimise the // recursive call not to read it again, because we know // it's empty. @@ -1367,12 +1360,12 @@ bool BackupClientDirectoryRecord::UpdateItems( // In the list, just use this pointer psubDirRecord = e->second; } - else + else { // Note: if we have exceeded our storage limit, then // we should not upload any more data, nor create any // DirectoryRecord representing data that would have - // been uploaded. This step will be repeated when + // been uploaded. This step will be repeated when // there is some space available. bool doCreateDirectoryRecord = true; @@ -1383,134 +1376,38 @@ bool BackupClientDirectoryRecord::UpdateItems( // No. Exists on the server, and we know about it from the listing. subDirObjectID = en->GetObjectID(); } - else if(rContext.StorageLimitExceeded()) + else if(rContext.StorageLimitExceeded()) // know we've got a connection if we get this far, // as dir will have been modified. { doCreateDirectoryRecord = false; - } + } else { // Yes, creation required! - // It is known that the it doesn't exist: - // if pDirOnStore == 0, then the directory has had an initial sync, and hasn't been modified. - // so it has definately been created already. - // if en == 0 but pDirOnStore != 0, well... obviously it doesn't exist. - - // Get attributes - box_time_t attrModTime = 0; - InodeRefType inodeNum = 0; - BackupClientFileAttributes attr; - bool failedToReadAttributes = false; - - try - { - attr.ReadAttributes(dirname.c_str(), - true /* directories have zero mod times */, - 0 /* not interested in mod time */, - &attrModTime, 0 /* not file size */, - &inodeNum); - } - catch (BoxException &e) - { - BOX_WARNING("Failed to read attributes " - "of directory, cannot check " - "for rename, assuming new: '" - << nonVssDirPath << "'"); - failedToReadAttributes = true; - } - - // Check to see if the directory been renamed - // First, do we have a record in the ID map? - int64_t renameObjectID = 0, renameInDirectory = 0; - bool renameDir = false; - const BackupClientInodeToIDMap &idMap( - rContext.GetCurrentIDMap()); - - if(!failedToReadAttributes && idMap.Lookup(inodeNum, - renameObjectID, renameInDirectory)) - { - // Look up on the server to get the name, to build the local filename - std::string localPotentialOldName; - bool isDir = false; - bool isCurrentVersion = false; - if(rContext.FindFilename(renameObjectID, - renameInDirectory, localPotentialOldName, - isDir, isCurrentVersion)) - { - // Only interested if it's a directory - if(isDir && isCurrentVersion) - { - // Check that the object doesn't exist already - EMU_STRUCT_STAT st; - if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && errno == ENOENT) - { - // Doesn't exist locally, but does exist on the server. - // Therefore we can safely rename it. - renameDir = true; - } - } - } - } - - // Get connection - BackupProtocolClient &connection(rContext.GetConnection()); - - // Don't do a check for storage limit exceeded here, because if we get to this - // stage, a connection will have been opened, and the status known, so the check - // in the else if(...) above will be correct. - - // Build attribute stream for sending - std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); - - if(renameDir) - { - // Rename the existing directory on the server - connection.QueryMoveObject(renameObjectID, - renameInDirectory, - mObjectID /* move to this directory */, - BackupProtocolMoveObject::Flags_MoveAllWithSameName | - BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, - storeFilename); - - // Put the latest attributes on it - connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream); - - // Stop it being deleted later - BackupClientDeleteList &rdelList( - rContext.GetDeleteList()); - rdelList.StopDirectoryDeletion(renameObjectID); - - // This is the ID for the renamed directory - subDirObjectID = renameObjectID; - } - else - { - // Create a new directory - std::auto_ptr<BackupProtocolSuccess> dirCreate( - connection.QueryCreateDirectory( - mObjectID, attrModTime, - storeFilename, attrStream)); - subDirObjectID = dirCreate->GetObjectID(); - - // Flag as having done this for optimisation later - haveJustCreatedDirOnServer = true; - - std::string filenameClear = - DecryptFilename(storeFilename, - subDirObjectID, - rRemotePath); - rNotifier.NotifyDirectoryCreated( - subDirObjectID, filenameClear, - nonVssDirPath); - } + // It is known that it doesn't exist: + // + // if en == 0 and pDirOnStore == 0, then the + // directory has had an initial sync, and + // hasn't been modified (Really? then why + // are we here? TODO FIXME) + // so it has definitely been created already + // (so why create it again?) + // + // if en == 0 but pDirOnStore != 0, well... obviously it doesn't exist. + // + subDirObjectID = CreateRemoteDir(dirname, + nonVssDirPath, rRemotePath + "/" + *d, + storeFilename, &haveJustCreatedDirOnServer, + rParams); + doCreateDirectoryRecord = (subDirObjectID != 0); } if (doCreateDirectoryRecord) - { + { // New an object for this psubDirRecord = new BackupClientDirectoryRecord(subDirObjectID, *d); - + // Store in list try { @@ -1524,9 +1421,13 @@ bool BackupClientDirectoryRecord::UpdateItems( } } } - - ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded()); - + + // ASSERT(psubDirRecord != 0 || rContext.StorageLimitExceeded()); + // There's another possible reason now: the directory no longer + // existed when we finally got around to checking its + // attributes. See for example Brendon Baumgartner's reported + // error with Wordpress cache directories. + if(psubDirRecord) { // Sync this sub directory too @@ -1534,28 +1435,15 @@ bool BackupClientDirectoryRecord::UpdateItems( rRemotePath + "/" + *d, rBackupLocation, haveJustCreatedDirOnServer); } - - // Zero pointer in rEntriesLeftOver, if we have a pointer to zero - if(en != 0) - { - for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) - { - if(rEntriesLeftOver[l] == en) - { - rEntriesLeftOver[l] = 0; - break; - } - } - } } - + // Delete everything which is on the store, but not on disc for(unsigned int l = 0; l < rEntriesLeftOver.size(); ++l) { if(rEntriesLeftOver[l] != 0) { BackupStoreDirectory::Entry *en = rEntriesLeftOver[l]; - + // These entries can't be deleted immediately, as it would prevent // renaming and moving of objects working properly. So we add them // to a list, which is actually deleted at the very end of the session. @@ -1582,7 +1470,7 @@ bool BackupClientDirectoryRecord::UpdateItems( filenameClear); std::string nonVssLocalName = ConvertVssPathToRealPath(localName, rBackupLocation); - + // Delete this entry -- file or directory? if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_File) != 0) { @@ -1596,7 +1484,7 @@ bool BackupClientDirectoryRecord::UpdateItems( rdel.AddDirectoryDelete(en->GetObjectID(), localName); - // If there's a directory record for it in + // If there's a directory record for it in // the sub directory map, delete it now BackupStoreFilenameClear dirname(en->GetName()); std::map<std::string, BackupClientDirectoryRecord *>::iterator @@ -1610,7 +1498,7 @@ bool BackupClientDirectoryRecord::UpdateItems( BOX_TRACE("Deleted directory record for " << nonVssLocalName); - } + } } } } @@ -1619,6 +1507,143 @@ bool BackupClientDirectoryRecord::UpdateItems( return allUpdatedSuccessfully; } +int64_t BackupClientDirectoryRecord::CreateRemoteDir(const std::string& localDirPath, + const std::string& nonVssDirPath, const std::string& remoteDirPath, + BackupStoreFilenameClear& storeFilename, bool* pHaveJustCreatedDirOnServer, + BackupClientDirectoryRecord::SyncParams &rParams) +{ + // Get attributes + box_time_t attrModTime = 0; + InodeRefType inodeNum = 0; + BackupClientFileAttributes attr; + *pHaveJustCreatedDirOnServer = false; + ProgressNotifier& rNotifier(rParams.mrContext.GetProgressNotifier()); + + try + { + attr.ReadAttributes(localDirPath, + true /* directories have zero mod times */, + 0 /* not interested in mod time */, + &attrModTime, 0 /* not file size */, + &inodeNum); + } + catch (BoxException &e) + { + // We used to try to recover from this, but we'd need an + // attributes block to upload to the server, so we have to + // skip creating the directory instead. + BOX_WARNING("Failed to read attributes of directory, " + "ignoring it for now: " << nonVssDirPath); + return 0; // no object ID + } + + // Check to see if the directory been renamed + // First, do we have a record in the ID map? + int64_t renameObjectID = 0, renameInDirectory = 0; + bool renameDir = false; + const BackupClientInodeToIDMap &idMap(rParams.mrContext.GetCurrentIDMap()); + + if(idMap.Lookup(inodeNum, renameObjectID, renameInDirectory)) + { + // Look up on the server to get the name, to build the local filename + std::string localPotentialOldName; + bool isDir = false; + bool isCurrentVersion = false; + if(rParams.mrContext.FindFilename(renameObjectID, renameInDirectory, + localPotentialOldName, isDir, isCurrentVersion)) + { + // Only interested if it's a directory + if(isDir && isCurrentVersion) + { + // Check that the object doesn't exist already + EMU_STRUCT_STAT st; + if(EMU_STAT(localPotentialOldName.c_str(), &st) != 0 && + errno == ENOENT) + { + // Doesn't exist locally, but does exist + // on the server. Therefore we can + // safely rename it. + renameDir = true; + } + } + } + } + + // Get connection + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); + + // Don't do a check for storage limit exceeded here, because if we get to this + // stage, a connection will have been opened, and the status known, so the check + // in the else if(...) above will be correct. + + // Build attribute stream for sending + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); + + if(renameDir) + { + // Rename the existing directory on the server + connection.QueryMoveObject(renameObjectID, + renameInDirectory, + mObjectID /* move to this directory */, + BackupProtocolMoveObject::Flags_MoveAllWithSameName | + BackupProtocolMoveObject::Flags_AllowMoveOverDeletedObject, + storeFilename); + + // Put the latest attributes on it + connection.QueryChangeDirAttributes(renameObjectID, attrModTime, attrStream); + + // Stop it being deleted later + BackupClientDeleteList &rdelList( + rParams.mrContext.GetDeleteList()); + rdelList.StopDirectoryDeletion(renameObjectID); + + // This is the ID for the renamed directory + return renameObjectID; + } + else + { + int64_t subDirObjectID = 0; // no object ID + + // Create a new directory + try + { + subDirObjectID = connection.QueryCreateDirectory( + mObjectID, attrModTime, storeFilename, + attrStream)->GetObjectID(); + // Flag as having done this for optimisation later + *pHaveJustCreatedDirOnServer = true; + } + catch(BoxException &e) + { + int type, subtype; + connection.GetLastError(type, subtype); + rNotifier.NotifyFileUploadServerError(this, nonVssDirPath, + type, subtype); + if(e.GetType() == ConnectionException::ExceptionType && + e.GetSubType() == ConnectionException::Protocol_UnexpectedReply && + type == BackupProtocolError::ErrorType && + subtype == BackupProtocolError::Err_StorageLimitExceeded) + { + // The hard limit was exceeded on the server, notify! + rParams.mrContext.SetStorageLimitExceeded(); + rParams.mrSysadminNotifier.NotifySysadmin( + SysadminNotifier::StoreFull); + } + else + { + throw; + } + } + + if(*pHaveJustCreatedDirOnServer) + { + rNotifier.NotifyDirectoryCreated(subDirObjectID, + nonVssDirPath, remoteDirPath); + } + + return subDirObjectID; + } +} // -------------------------------------------------------------------------- // @@ -1636,7 +1661,7 @@ void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( const std::string &rFilename) { // First, delete the directory - BackupProtocolClient &connection(rParams.mrContext.GetConnection()); + BackupProtocolCallable &connection(rParams.mrContext.GetConnection()); connection.QueryDeleteDirectory(pEntry->GetObjectID()); BackupStoreFilenameClear clear(pEntry->GetName()); @@ -1675,9 +1700,10 @@ void BackupClientDirectoryRecord::RemoveDirectoryInPlaceOfFile( // -------------------------------------------------------------------------- int64_t BackupClientDirectoryRecord::UploadFile( BackupClientDirectoryRecord::SyncParams &rParams, - const std::string &rFilename, + const std::string &rLocalPath, const std::string &rNonVssFilePath, - const BackupStoreFilename &rStoreFilename, + const std::string &rRemotePath, + const BackupStoreFilenameClear &rStoreFilename, int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, @@ -1687,7 +1713,7 @@ int64_t BackupClientDirectoryRecord::UploadFile( ProgressNotifier& rNotifier(rContext.GetProgressNotifier()); // Get the connection - BackupProtocolClient &connection(rContext.GetConnection()); + BackupProtocolCallable &connection(rContext.GetConnection()); // Info int64_t objID = 0; @@ -1712,8 +1738,6 @@ int64_t BackupClientDirectoryRecord::UploadFile( if(diffFromID != 0) { // Found an old version - rNotifier.NotifyFileUploadingPatch(this, - rNonVssFilePath); // Get the index std::auto_ptr<IOStream> blockIndexStream(connection.ReceiveStream()); @@ -1727,33 +1751,40 @@ int64_t BackupClientDirectoryRecord::UploadFile( bool isCompletelyDifferent = false; apStreamToUpload = BackupStoreFile::EncodeFileDiff( - rFilename, + rLocalPath, mObjectID, /* containing directory */ rStoreFilename, diffFromID, *blockIndexStream, - connection.GetTimeout(), + connection.GetTimeout(), &rContext, // DiffTimer implementation 0 /* not interested in the modification time */, - &isCompletelyDifferent); + &isCompletelyDifferent, + rParams.mpBackgroundTask); if(isCompletelyDifferent) { diffFromID = 0; } - + rContext.UnManageDiffProcess(); - } + } } - - if(!apStreamToUpload.get()) // No patch upload, so do a normal upload + + if(apStreamToUpload.get()) + { + rNotifier.NotifyFileUploadingPatch(this, rNonVssFilePath, + apStreamToUpload->GetBytesToUpload()); + } + else // No patch upload, so do a normal upload { // below threshold or nothing to diff from, so upload whole rNotifier.NotifyFileUploading(this, rNonVssFilePath); // Prepare to upload, getting a stream which will encode the file as we go along apStreamToUpload = BackupStoreFile::EncodeFile( - rFilename, mObjectID, /* containing directory */ - rStoreFilename, NULL, &rParams, - &(rParams.mrRunStatusProvider)); + rLocalPath, mObjectID, /* containing directory */ + rStoreFilename, NULL, &rParams, + &(rParams.mrRunStatusProvider), + rParams.mpBackgroundTask); } rContext.SetNiceMode(true); @@ -1776,15 +1807,13 @@ int64_t BackupClientDirectoryRecord::UploadFile( // Send to store std::auto_ptr<BackupProtocolSuccess> stored( - connection.QueryStoreFile( - mObjectID, ModificationTime, - AttributesHash, - diffFromID, - rStoreFilename, apWrappedStream)); + connection.QueryStoreFile(mObjectID, ModificationTime, + AttributesHash, diffFromID, rStoreFilename, + apWrappedStream)); rContext.SetNiceMode(false); - // Get object ID from the result + // Get object ID from the result objID = stored->GetObjectID(); uploadedSize = apStreamToUpload->GetTotalBytesSent(); } @@ -1821,7 +1850,7 @@ int64_t BackupClientDirectoryRecord::UploadFile( } rNotifier.NotifyFileUploaded(this, rNonVssFilePath, FileSize, - uploadedSize); + uploadedSize, objID); // Return the new object ID of this file return objID; @@ -1831,9 +1860,10 @@ int64_t BackupClientDirectoryRecord::UploadFile( // -------------------------------------------------------------------------- // // Function -// Name: BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject(SyncParams &, const char *) -// Purpose: Sets the error state when there were problems reading an object -// from the filesystem. +// Name: BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( +// SyncParams &, const char *) +// Purpose: Sets the error state when there were problems +// reading an object from the filesystem. // Created: 29/3/04 // // -------------------------------------------------------------------------- @@ -1844,11 +1874,12 @@ void BackupClientDirectoryRecord::SetErrorWhenReadingFilesystemObject( // Zero hash, so it gets synced properly next time round. ::memset(mStateChecksum, 0, sizeof(mStateChecksum)); - // Log the error - already done by caller - /* - rParams.GetProgressNotifier().NotifyFileReadFailed(this, - Filename, strerror(errno)); - */ + // More detailed logging was already done by the caller, but if we + // have a read error reported, we need to be able to search the logs + // to find out which file it was, so we need to log a consistent and + // clear error message. + BOX_WARNING("Failed to backup file, see above for details: " << + rFilename); // Mark that an error occured in the parameters object rParams.mReadErrorsOnFilesystemObjects = true; @@ -1868,13 +1899,15 @@ BackupClientDirectoryRecord::SyncParams::SyncParams( RunStatusProvider &rRunStatusProvider, SysadminNotifier &rSysadminNotifier, ProgressNotifier &rProgressNotifier, - BackupClientContext &rContext) + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask) : mSyncPeriodStart(0), mSyncPeriodEnd(0), mMaxUploadWait(0), mMaxFileTimeInFuture(99999999999999999LL), mFileTrackingSizeThreshold(16*1024), mDiffingUploadSizeThreshold(16*1024), + mpBackgroundTask(pBackgroundTask), mrRunStatusProvider(rRunStatusProvider), mrSysadminNotifier(rSysadminNotifier), mrProgressNotifier(rProgressNotifier), @@ -2083,11 +2116,8 @@ void BackupClientDirectoryRecord::Serialize(Archive & rArchive) const // // -------------------------------------------------------------------------- Location::Location() - : mIDMapIndex(0), - mpExcludeFiles(0), - mpExcludeDirs(0) -{ -} +: mIDMapIndex(0) +{ } // -------------------------------------------------------------------------- // @@ -2098,19 +2128,7 @@ Location::Location() // // -------------------------------------------------------------------------- Location::~Location() -{ - // Clean up exclude locations - if(mpExcludeDirs != 0) - { - delete mpExcludeDirs; - mpExcludeDirs = 0; - } - if(mpExcludeFiles != 0) - { - delete mpExcludeFiles; - mpExcludeFiles = 0; - } -} +{ } // -------------------------------------------------------------------------- // @@ -2134,7 +2152,7 @@ void Location::Serialize(Archive & rArchive) const // // // - if(mpDirectoryRecord.get() == NULL) + if(!mapDirectoryRecord.get()) { int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; rArchive.Write(aMagicMarker); @@ -2144,13 +2162,13 @@ void Location::Serialize(Archive & rArchive) const int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows rArchive.Write(aMagicMarker); - mpDirectoryRecord->Serialize(rArchive); + mapDirectoryRecord->Serialize(rArchive); } // // // - if(!mpExcludeFiles) + if(!mapExcludeFiles.get()) { int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; rArchive.Write(aMagicMarker); @@ -2160,13 +2178,13 @@ void Location::Serialize(Archive & rArchive) const int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows rArchive.Write(aMagicMarker); - mpExcludeFiles->Serialize(rArchive); + mapExcludeFiles->Serialize(rArchive); } // // // - if(!mpExcludeDirs) + if(!mapExcludeDirs.get()) { int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_NOOP; rArchive.Write(aMagicMarker); @@ -2176,7 +2194,7 @@ void Location::Serialize(Archive & rArchive) const int64_t aMagicMarker = ARCHIVE_MAGIC_VALUE_RECURSE; // be explicit about whether recursion follows rArchive.Write(aMagicMarker); - mpExcludeDirs->Serialize(rArchive); + mapExcludeDirs->Serialize(rArchive); } } @@ -2194,17 +2212,9 @@ void Location::Deserialize(Archive &rArchive) // // // - mpDirectoryRecord.reset(NULL); - if(mpExcludeFiles) - { - delete mpExcludeFiles; - mpExcludeFiles = NULL; - } - if(mpExcludeDirs) - { - delete mpExcludeDirs; - mpExcludeDirs = NULL; - } + mapDirectoryRecord.reset(); + mapExcludeFiles.reset(); + mapExcludeDirs.reset(); // // @@ -2231,8 +2241,8 @@ void Location::Deserialize(Archive &rArchive) throw std::bad_alloc(); } - mpDirectoryRecord.reset(pSubRecord); - mpDirectoryRecord->Deserialize(rArchive); + mapDirectoryRecord.reset(pSubRecord); + mapDirectoryRecord->Deserialize(rArchive); } else { @@ -2251,13 +2261,13 @@ void Location::Deserialize(Archive &rArchive) } else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) { - mpExcludeFiles = new ExcludeList; - if(!mpExcludeFiles) + mapExcludeFiles.reset(new ExcludeList); + if(!mapExcludeFiles.get()) { throw std::bad_alloc(); } - mpExcludeFiles->Deserialize(rArchive); + mapExcludeFiles->Deserialize(rArchive); } else { @@ -2276,13 +2286,13 @@ void Location::Deserialize(Archive &rArchive) } else if(aMagicMarker == ARCHIVE_MAGIC_VALUE_RECURSE) { - mpExcludeDirs = new ExcludeList; - if(!mpExcludeDirs) + mapExcludeDirs.reset(new ExcludeList); + if(!mapExcludeDirs.get()) { throw std::bad_alloc(); } - mpExcludeDirs->Deserialize(rArchive); + mapExcludeDirs->Deserialize(rArchive); } else { diff --git a/bin/bbackupd/BackupClientDirectoryRecord.h b/lib/bbackupd/BackupClientDirectoryRecord.h index fb9d7fc8..865fc747 100644 --- a/bin/bbackupd/BackupClientDirectoryRecord.h +++ b/lib/bbackupd/BackupClientDirectoryRecord.h @@ -14,6 +14,7 @@ #include <map> #include <memory> +#include "BackgroundTask.h" #include "BackupClientFileAttributes.h" #include "BackupDaemonInterface.h" #include "BackupStoreDirectory.h" @@ -47,7 +48,7 @@ class BackupClientDirectoryRecord { public: BackupClientDirectoryRecord(int64_t ObjectID, const std::string &rSubDirName); - ~BackupClientDirectoryRecord(); + virtual ~BackupClientDirectoryRecord(); void Deserialize(Archive & rArchive); void Serialize(Archive & rArchive) const; @@ -76,7 +77,8 @@ public: RunStatusProvider &rRunStatusProvider, SysadminNotifier &rSysadminNotifier, ProgressNotifier &rProgressNotifier, - BackupClientContext &rContext); + BackupClientContext &rContext, + BackgroundTask *pBackgroundTask); ~SyncParams(); private: // No copying @@ -91,6 +93,7 @@ public: box_time_t mMaxFileTimeInFuture; int32_t mFileTrackingSizeThreshold; int32_t mDiffingUploadSizeThreshold; + BackgroundTask *mpBackgroundTask; RunStatusProvider &mrRunStatusProvider; SysadminNotifier &mrSysadminNotifier; ProgressNotifier &mrProgressNotifier; @@ -103,11 +106,11 @@ public: bool mHaveLoggedWarningAboutFutureFileTimes; bool StopRun() { return mrRunStatusProvider.StopRun(); } - void NotifySysadmin(SysadminNotifier::EventCode Event) + void NotifySysadmin(SysadminNotifier::EventCode Event) { - mrSysadminNotifier.NotifySysadmin(Event); + mrSysadminNotifier.NotifySysadmin(Event); } - ProgressNotifier& GetProgressNotifier() const + ProgressNotifier& GetProgressNotifier() const { return mrProgressNotifier; } @@ -138,6 +141,17 @@ public: const Location& rBackupLocation, bool ThisDirHasJustBeenCreated = false); + bool SyncDirectoryEntry(SyncParams &rParams, + ProgressNotifier& rNotifier, + const Location& rBackupLocation, + const std::string &rDirLocalPath, + MD5Digest& currentStateChecksum, + struct dirent *en, + EMU_STRUCT_STAT dir_st, + std::vector<std::string>& rDirs, + std::vector<std::string>& rFiles, + bool& rDownloadDirectoryRecordBecauseOfFutureFiles); + std::string ConvertVssPathToRealPath(const std::string &rVssPath, const Location& rBackupLocation); @@ -145,21 +159,31 @@ public: private: void DeleteSubDirectories(); - BackupStoreDirectory *FetchDirectoryListing(SyncParams &rParams); + std::auto_ptr<BackupStoreDirectory> FetchDirectoryListing(SyncParams &rParams); void UpdateAttributes(SyncParams &rParams, BackupStoreDirectory *pDirOnStore, const std::string &rLocalPath); - bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath, +protected: // to allow tests to hook in before UpdateItems() runs + virtual bool UpdateItems(SyncParams &rParams, + const std::string &rLocalPath, const std::string &rRemotePath, const Location& rBackupLocation, BackupStoreDirectory *pDirOnStore, std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, std::vector<std::string> &rFiles, const std::vector<std::string> &rDirs); +private: + int64_t CreateRemoteDir(const std::string& localDirPath, + const std::string& nonVssDirPath, + const std::string& remoteDirPath, + BackupStoreFilenameClear& storeFilename, + bool* pHaveJustCreatedDirOnServer, + BackupClientDirectoryRecord::SyncParams &rParams); int64_t UploadFile(SyncParams &rParams, const std::string &rFilename, const std::string &rNonVssFilePath, - const BackupStoreFilename &rStoreFilename, + const std::string &rRemotePath, + const BackupStoreFilenameClear &rStoreFilename, int64_t FileSize, box_time_t ModificationTime, box_time_t AttributesHash, bool NoPreviousVersionOnServer); void SetErrorWhenReadingFilesystemObject(SyncParams &rParams, @@ -178,7 +202,6 @@ private: std::string mSubDirName; bool mInitialSyncDone; bool mSyncDone; - bool mSuppressMultipleLinksWarning; // Checksum of directory contents and attributes, used to detect changes uint8_t mStateChecksum[MD5Digest::DigestLength]; @@ -204,10 +227,10 @@ private: public: std::string mName; std::string mPath; - std::auto_ptr<BackupClientDirectoryRecord> mpDirectoryRecord; + std::auto_ptr<BackupClientDirectoryRecord> mapDirectoryRecord; + std::auto_ptr<ExcludeList> mapExcludeFiles; + std::auto_ptr<ExcludeList> mapExcludeDirs; int mIDMapIndex; - ExcludeList *mpExcludeFiles; - ExcludeList *mpExcludeDirs; #ifdef ENABLE_VSS bool mIsSnapshotCreated; diff --git a/bin/bbackupd/BackupClientInodeToIDMap.cpp b/lib/bbackupd/BackupClientInodeToIDMap.cpp index 8240d62c..6eaf7394 100644 --- a/bin/bbackupd/BackupClientInodeToIDMap.cpp +++ b/lib/bbackupd/BackupClientInodeToIDMap.cpp @@ -16,15 +16,16 @@ #include "BackupClientInodeToIDMap.h" #undef BACKIPCLIENTINODETOIDMAP_IMPLEMENTATION +#include "Archive.h" #include "BackupStoreException.h" +#include "CollectInBufferStream.h" +#include "MemBlockStream.h" +#include "autogen_CommonException.h" #include "MemLeakFindOn.h" -typedef struct -{ - int64_t mObjectID; - int64_t mInDirectory; -} IDBRecord; +#define BOX_DBM_INODE_DB_VERSION_KEY "BackupClientInodeToIDMap.Version" +#define BOX_DBM_INODE_DB_VERSION_CURRENT 2 #define BOX_DBM_MESSAGE(stuff) stuff << " (qdbm): " << dperrmsg(dpecode) @@ -118,12 +119,58 @@ void BackupClientInodeToIDMap::Open(const char *Filename, bool ReadOnly, if(!mpDepot) { - BOX_WARNING(BOX_DBM_MESSAGE("Failed to open inode " - "database: " << mFilename)); THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, BOX_DBM_MESSAGE("Failed to open inode database: " << mFilename)); } + + const char* version_key = BOX_DBM_INODE_DB_VERSION_KEY; + int32_t version = 0; + + if(CreateNew) + { + version = BOX_DBM_INODE_DB_VERSION_CURRENT; + + int ret = dpput(mpDepot, version_key, strlen(version_key), + (char *)(&version), sizeof(version), DP_DKEEP); + + if(!ret) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + BOX_DBM_MESSAGE("Failed to write version number to inode " + "database: " << mFilename)); + } + } + else + { + int ret = dpgetwb(mpDepot, version_key, strlen(version_key), 0, + sizeof(version), (char *)(&version)); + + if(ret == -1) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Missing version number in inode database. Perhaps it " + "needs to be recreated: " << mFilename); + } + + if(ret != sizeof(version)) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong size version number in inode database: expected " + << sizeof(version) << " bytes but found " << ret); + } + + if(version != BOX_DBM_INODE_DB_VERSION_CURRENT) + { + THROW_EXCEPTION_MESSAGE(BackupStoreException, BerkelyDBFailure, + "Wrong version number in inode database: expected " << + BOX_DBM_INODE_DB_VERSION_CURRENT << " but found " << + version << ". Perhaps it needs to be recreated: " << + mFilename); + } + + // By this point the version number has been checked and is OK. + } // Read only flag mReadOnly = ReadOnly; @@ -175,7 +222,7 @@ void BackupClientInodeToIDMap::Close() // // -------------------------------------------------------------------------- void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, - int64_t InDirectory) + int64_t InDirectory, const std::string& LocalPath) { if(mReadOnly) { @@ -190,12 +237,15 @@ void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, ASSERT_DBM_OPEN(); // Setup structures - IDBRecord rec; - rec.mObjectID = ObjectID; - rec.mInDirectory = InDirectory; + CollectInBufferStream buf; + Archive arc(buf, IOStream::TimeOutInfinite); + arc.WriteExact((uint64_t)ObjectID); + arc.WriteExact((uint64_t)InDirectory); + arc.Write(LocalPath); + buf.SetForReading(); ASSERT_DBM_OK(dpput(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), - (const char *)&rec, sizeof(rec), DP_DOVER), + (const char *)buf.GetBuffer(), buf.GetSize(), DP_DOVER), "Failed to add record to inode database", mFilename, BackupStoreException, BerkelyDBFailure); } @@ -207,12 +257,12 @@ void BackupClientInodeToIDMap::AddToMap(InodeRefType InodeRef, int64_t ObjectID, // int64_t &, int64_t &) const // Purpose: Looks up an inode in the map, returning true if it // exists, and the object ids of it and the directory -// it's in the reference arguments. +// it's in the reference arguments. // Created: 11/11/03 // // -------------------------------------------------------------------------- -bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, - int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const +bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut) const { if(mEmpty) { @@ -226,19 +276,44 @@ bool BackupClientInodeToIDMap::Lookup(InodeRefType InodeRef, } ASSERT_DBM_OPEN(); - - IDBRecord rec; - - if(dpgetwb(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), - 0, sizeof(IDBRecord), (char *)&rec) == -1) + int size; + char* data = dpget(mpDepot, (const char *)&InodeRef, sizeof(InodeRef), + 0, -1, &size); + if(data == NULL) { // key not in file return false; } - + + // Free data automatically when the guard goes out of scope. + MemoryBlockGuard<char *> guard(data); + MemBlockStream stream(data, size); + Archive arc(stream, IOStream::TimeOutInfinite); + // Return data - rObjectIDOut = rec.mObjectID; - rInDirectoryOut = rec.mInDirectory; + try + { + arc.Read(rObjectIDOut); + arc.Read(rInDirectoryOut); + if(pLocalPathOut) + { + arc.Read(*pLocalPathOut); + } + } + catch(CommonException &e) + { + if(e.GetSubType() == CommonException::ArchiveBlockIncompleteRead) + { + THROW_FILE_ERROR("Failed to lookup record in inode database: " + << InodeRef << ": not enough data in record", mFilename, + BackupStoreException, BerkelyDBFailure); + // Need to throw precisely that exception to ensure that the + // invalid database is deleted, so that we don't hit the same + // error next time. + } + + throw; + } // Found return true; diff --git a/bin/bbackupd/BackupClientInodeToIDMap.h b/lib/bbackupd/BackupClientInodeToIDMap.h index fbe45114..4bb1e085 100644 --- a/bin/bbackupd/BackupClientInodeToIDMap.h +++ b/lib/bbackupd/BackupClientInodeToIDMap.h @@ -40,8 +40,10 @@ public: void Open(const char *Filename, bool ReadOnly, bool CreateNew); void OpenEmpty(); - void AddToMap(InodeRefType InodeRef, int64_t ObjectID, int64_t InDirectory); - bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, int64_t &rInDirectoryOut) const; + void AddToMap(InodeRefType InodeRef, int64_t ObjectID, + int64_t InDirectory, const std::string& LocalPath); + bool Lookup(InodeRefType InodeRef, int64_t &rObjectIDOut, + int64_t &rInDirectoryOut, std::string* pLocalPathOut = NULL) const; void Close(); diff --git a/bin/bbackupd/BackupDaemon.cpp b/lib/bbackupd/BackupDaemon.cpp index 39bb98e3..996c1919 100644 --- a/bin/bbackupd/BackupDaemon.cpp +++ b/lib/bbackupd/BackupDaemon.cpp @@ -53,6 +53,7 @@ #include "autogen_BackupProtocol.h" #include "autogen_ClientException.h" +#include "autogen_CommonException.h" #include "autogen_ConversionException.h" #include "Archive.h" #include "BackupClientContext.h" @@ -61,6 +62,7 @@ #include "BackupClientFileAttributes.h" #include "BackupClientInodeToIDMap.h" #include "BackupClientMakeExcludeList.h" +#include "BackupConstants.h" #include "BackupDaemon.h" #include "BackupDaemonConfigVerify.h" #include "BackupStoreConstants.h" @@ -163,6 +165,39 @@ } } # endif + + // Mutex support by Achim: see https://www.boxbackup.org/ticket/67 + + // Creates the two mutexes checked for by the installer/uninstaller to + // see if the program is still running. One of the mutexes is created + // in the global name space (which makes it possible to access the + // mutex across user sessions in Windows XP); the other is created in + // the session name space (because versions of Windows NT prior to + // 4.0 TSE don't have a global name space and don't support the + // 'Global\' prefix). + + void CreateMutexes(const std::string& rName) + { + SECURITY_DESCRIPTOR SecurityDesc; + SECURITY_ATTRIBUTES SecurityAttr; + + /* By default on Windows NT, created mutexes are accessible only by the user + running the process. We need our mutexes to be accessible to all users, so + that the mutex detection can work across user sessions in Windows XP. To + do this we use a security descriptor with a null DACL. + */ + + InitializeSecurityDescriptor(&SecurityDesc, SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&SecurityDesc, TRUE, NULL, FALSE); + SecurityAttr.nLength = sizeof(SecurityAttr); + SecurityAttr.lpSecurityDescriptor = &SecurityDesc; + SecurityAttr.bInheritHandle = FALSE; + // We don't care if this succeeds or fails. It's only used to + // ensure that an installer can detect if Box Backup is running. + CreateMutexA(&SecurityAttr, FALSE, rName.c_str()); + std::string global_name = "Global\\" + rName; + CreateMutexA(&SecurityAttr, FALSE, global_name.c_str()); + } #endif #include "MemLeakFindOn.h" @@ -202,7 +237,8 @@ BackupDaemon::BackupDaemon() mpProgressNotifier(this), mpLocationResolver(this), mpRunStatusProvider(this), - mpSysadminNotifier(this) + mpSysadminNotifier(this), + mapCommandSocketPollTimer(NULL) #ifdef WIN32 , mInstallService(false), mRemoveService(false), @@ -286,7 +322,6 @@ const ConfigurationVerify *BackupDaemon::GetConfigVerify() const return &BackupDaemonConfigVerify; } -#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET // -------------------------------------------------------------------------- // // Function @@ -299,6 +334,37 @@ const ConfigurationVerify *BackupDaemon::GetConfigVerify() const // -------------------------------------------------------------------------- void BackupDaemon::SetupInInitialProcess() { + const Configuration& config(GetConfiguration()); + + // These keys may or may not be required, depending on the configured + // store type (e.g. not when using Amazon S3 stores), so they can't be + // verified by BackupDaemonConfigVerify. + std::vector<std::string> requiredKeys; + requiredKeys.push_back("StoreHostname"); + requiredKeys.push_back("AccountNumber"); + requiredKeys.push_back("CertificateFile"); + requiredKeys.push_back("PrivateKeyFile"); + requiredKeys.push_back("TrustedCAsFile"); + bool missingRequiredKeys = false; + + for(std::vector<std::string>::const_iterator i = requiredKeys.begin(); + i != requiredKeys.end(); i++) + { + if(!config.KeyExists(*i)) + { + BOX_ERROR("Missing required configuration key: " << *i); + missingRequiredKeys = true; + } + } + + if(missingRequiredKeys) + { + THROW_EXCEPTION_MESSAGE(CommonException, InvalidConfiguration, + "Some required configuration keys are missing in " << + GetConfigFileName()); + } + +#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET // Print a warning on this platform if the CommandSocket is used. if(GetConfiguration().KeyExists("CommandSocket")) { @@ -311,8 +377,8 @@ void BackupDaemon::SetupInInitialProcess() "==============================================================================\n" ); } -} #endif +} // -------------------------------------------------------------------------- @@ -408,6 +474,8 @@ int BackupDaemon::Main(const std::string &rConfigFileName) } #endif + CreateMutexes("__boxbackup_mutex__"); + int returnCode; if (mRunAsService) @@ -427,7 +495,7 @@ int BackupDaemon::Main(const std::string &rConfigFileName) return returnCode; } -#endif +#endif // WIN32 // -------------------------------------------------------------------------- // @@ -442,6 +510,9 @@ void BackupDaemon::Run() // initialise global timer mechanism Timers::Init(); + mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, + "CommandSocketPollTimer")); + #ifndef WIN32 // Ignore SIGPIPE so that if a command connection is broken, // the daemon doesn't terminate. @@ -473,25 +544,23 @@ void BackupDaemon::Run() } catch(...) { - if(mapCommandSocketInfo.get()) + try { - try - { - mapCommandSocketInfo.reset(); - } - catch(std::exception &e) - { - BOX_WARNING("Internal error while " - "closing command socket after " - "another exception: " << e.what()); - } - catch(...) - { - BOX_WARNING("Error closing command socket " - "after exception, ignored."); - } + mapCommandSocketInfo.reset(); + } + catch(std::exception &e) + { + BOX_WARNING("Internal error while closing command " + "socket after another exception, ignored: " << + e.what()); + } + catch(...) + { + BOX_WARNING("Error closing command socket after " + "exception, ignored."); } + mapCommandSocketPollTimer.reset(); Timers::Cleanup(); throw; @@ -499,6 +568,7 @@ void BackupDaemon::Run() // Clean up mapCommandSocketInfo.reset(); + mapCommandSocketPollTimer.reset(); Timers::Cleanup(); } @@ -533,6 +603,7 @@ void BackupDaemon::Run2() // How often to connect to the store (approximate) mUpdateStoreInterval = SecondsToBoxTime( conf.GetKeyValueInt("UpdateStoreInterval")); + mBackupErrorDelay = conf.GetKeyValueInt("BackupErrorDelay"); // But are we connecting automatically? bool automaticBackup = conf.GetKeyValueBool("AutomaticBackup"); @@ -543,10 +614,10 @@ void BackupDaemon::Run2() // When the last sync started (only updated if the store was not full when the sync ended) mLastSyncTime = 0; - // -------------------------------------------------------------------------------------------- + // -------------------------------------------------------------------------------------------- - mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo( - mLastSyncTime, mNextSyncTime); + mDeleteStoreObjectInfoFile = DeserializeStoreObjectInfo(mLastSyncTime, + mNextSyncTime); // -------------------------------------------------------------------------------------------- @@ -564,115 +635,146 @@ void BackupDaemon::Run2() bool doSync = false; bool mDoSyncForcedByCommand = false; + // Check whether we should be stopping, and if so, + // don't hang around waiting on the command socket. + if(StopRun()) + { + BOX_INFO("Skipping command socket polling " + "due to shutdown request"); + break; + } + // Is a delay necessary? - box_time_t currentTime; + box_time_t currentTime = GetCurrentBoxTime(); + box_time_t requiredDelay = (mNextSyncTime < currentTime) + ? (0) : (mNextSyncTime - currentTime); + mNextSyncTime = currentTime + requiredDelay; - do + if (mDoSyncForcedByPreviousSyncError) { - // Check whether we should be stopping, - // and don't run a sync if so. - if(StopRun()) break; - - currentTime = GetCurrentBoxTime(); - - // Pause a while, but no more than - // MAX_SLEEP_TIME seconds (use the conditional - // because times are unsigned) - box_time_t requiredDelay = - (mNextSyncTime < currentTime) - ? (0) - : (mNextSyncTime - currentTime); - - // If there isn't automatic backup happening, - // set a long delay. And limit delays at the - // same time. - if(!automaticBackup && !mDoSyncForcedByPreviousSyncError) - { - requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); - } - else if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) - { - requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); - } + BOX_INFO("Last backup was not successful, " + "next one starting at " << + FormatTime(mNextSyncTime, false, true)); + } + else if (automaticBackup) + { + BOX_INFO("Automatic backups are enabled, " + "next one starting at " << + FormatTime(mNextSyncTime, false, true)); + } + else + { + BOX_INFO("No automatic backups, waiting for " + "bbackupctl snapshot command"); + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); + } - // Only delay if necessary - if(requiredDelay > 0) - { - // Sleep somehow. There are choices - // on how this should be done, - // depending on the state of the - // control connection - if(mapCommandSocketInfo.get() != 0) - { - // A command socket exists, - // so sleep by waiting on it - WaitOnCommandSocket(requiredDelay, - doSync, mDoSyncForcedByCommand); - } - else - { - // No command socket or - // connection, just do a - // normal sleep - time_t sleepSeconds = - BoxTimeToSeconds(requiredDelay); - ::sleep((sleepSeconds <= 0) - ? 1 : sleepSeconds); - } - } - - if ((automaticBackup || mDoSyncForcedByPreviousSyncError) - && currentTime >= mNextSyncTime) - { - doSync = true; - } + if(requiredDelay > SecondsToBoxTime(MAX_SLEEP_TIME)) + { + requiredDelay = SecondsToBoxTime(MAX_SLEEP_TIME); } - while(!doSync && !StopRun()); - // Time of sync start, and if it's time for another sync - // (and we're doing automatic syncs), set the flag - mCurrentSyncStartTime = GetCurrentBoxTime(); - if((automaticBackup || mDoSyncForcedByPreviousSyncError) && - mCurrentSyncStartTime >= mNextSyncTime) + // Only delay if necessary + if(requiredDelay == 0) { - doSync = true; + // No sleep necessary, so don't listen on the command + // socket at all right now. } - + else if(mapCommandSocketInfo.get() != 0) + { + // A command socket exists, so sleep by waiting for a + // connection or command on it. + WaitOnCommandSocket(requiredDelay, doSync, + mDoSyncForcedByCommand); + } + else + { + // No command socket or connection, just do a normal + // sleep. + time_t sleepSeconds = + BoxTimeToSeconds(requiredDelay); + ::sleep((sleepSeconds <= 0) + ? 1 : sleepSeconds); + } + + // We have now slept, so if automaticBackup is enabled then + // it's time for a backup now. + + if(StopRun()) + { + BOX_INFO("Stopping idle loop due to shutdown request"); + break; + } + else if(doSync) + { + BOX_INFO("Starting a backup immediately due to " + "bbackupctl sync command"); + } + else if(GetCurrentBoxTime() < mNextSyncTime) + { + BOX_TRACE("Deadline not reached, sleeping again"); + continue; + } + else if(mDoSyncForcedByPreviousSyncError) + { + BOX_INFO("Last backup was not successful, next one " + "starting now"); + } + else if(!automaticBackup) + { + BOX_TRACE("Sleeping again because automatic backups " + "are not enabled"); + continue; + } + else + { + BOX_INFO("Automatic backups are enabled, next one " + "starting now"); + } + + // If we pass this point, or exit the loop, we should have + // logged something at INFO level or higher to explain why. + // Use a script to see if sync is allowed now? - if(!mDoSyncForcedByCommand && doSync && !StopRun()) + if(mDoSyncForcedByCommand) + { + BOX_INFO("Skipping SyncAllowScript due to bbackupctl " + "force-sync command"); + } + else { int d = UseScriptToSeeIfSyncAllowed(); if(d > 0) { // Script has asked for a delay - mNextSyncTime = GetCurrentBoxTime() + + mNextSyncTime = GetCurrentBoxTime() + SecondsToBoxTime(d); - doSync = false; + BOX_INFO("Impending backup stopped by " + "SyncAllowScript, next attempt " + "scheduled for " << + FormatTime(mNextSyncTime, false)); + continue; } } - // Ready to sync? (but only if we're not supposed - // to be stopping) - if(doSync && !StopRun()) - { - RunSyncNowWithExceptionHandling(); - } + mCurrentSyncStartTime = GetCurrentBoxTime(); + RunSyncNowWithExceptionHandling(); // Set state SetState(storageLimitExceeded?State_StorageLimitExceeded:State_Idle); - - } while(!StopRun()); + } + while(!StopRun()); // Make sure we have a clean start next time round (if restart) DeleteAllLocations(); DeleteAllIDMaps(); } -void BackupDaemon::RunSyncNowWithExceptionHandling() +std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNowWithExceptionHandling() { bool errorOccurred = false; int errorCode = 0, errorSubCode = 0; - const char* errorString = "unknown"; + std::string errorString = "unknown"; try { @@ -702,13 +804,13 @@ void BackupDaemon::RunSyncNowWithExceptionHandling() // do not retry immediately without a good reason mDoSyncForcedByPreviousSyncError = false; - + + // Is it a berkely db failure? + bool isBerkelyDbFailure = false; + // Notify system administrator about the final state of the backup if(errorOccurred) { - // Is it a berkely db failure? - bool isBerkelyDbFailure = false; - if (errorCode == BackupStoreException::ExceptionType && errorSubCode == BackupStoreException::BerkelyDBFailure) { @@ -726,11 +828,10 @@ void BackupDaemon::RunSyncNowWithExceptionHandling() // Handle restart? if(StopRun()) { - BOX_NOTICE("Exception (" << errorCode - << "/" << errorSubCode - << ") due to signal"); + BOX_NOTICE("Exception (" << errorCode << "/" << + errorSubCode << ") due to signal"); OnBackupFinish(); - return; + return mapClientContext; // releases mapClientContext } NotifySysadmin(SysadminNotifier::BackupError); @@ -754,9 +855,9 @@ void BackupDaemon::RunSyncNowWithExceptionHandling() " " << errorCode << "/" << errorSubCode << "), reset state and waiting to retry..."); ::sleep(10); - mNextSyncTime = mCurrentSyncStartTime + - SecondsToBoxTime(100) + - Random::RandomInt(mUpdateStoreInterval >> + mNextSyncTime = GetCurrentBoxTime() + + SecondsToBoxTime(mBackupErrorDelay) + + Random::RandomInt(mUpdateStoreInterval >> SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); } } @@ -779,9 +880,10 @@ void BackupDaemon::RunSyncNowWithExceptionHandling() // If we were retrying after an error, and this backup succeeded, // then now would be a good time to stop :-) - mDoSyncForcedByPreviousSyncError = errorOccurred; + mDoSyncForcedByPreviousSyncError = errorOccurred && !isBerkelyDbFailure; OnBackupFinish(); + return mapClientContext; // releases mapClientContext } void BackupDaemon::ResetCachedState() @@ -794,17 +896,40 @@ void BackupDaemon::ResetCachedState() DeleteAllIDMaps(); } -void BackupDaemon::RunSyncNow() +std::auto_ptr<BackupClientContext> BackupDaemon::GetNewContext +( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode +) +{ + std::auto_ptr<BackupClientContext> context(new BackupClientContext( + rResolver, rTLSContext, rHostname, Port, AccountNumber, + ExtendedLogging, ExtendedLogToFile, ExtendedLogFile, + rProgressNotifier, TcpNiceMode)); + return context; +} + +// Returns the BackupClientContext so that tests can use it to hold the +// connection open and prevent housekeeping from running. Otherwise don't use +// it, let it be destroyed and close the connection. +std::auto_ptr<BackupClientContext> BackupDaemon::RunSyncNow() { // Delete the serialised store object file, // so that we don't try to reload it after a // partially completed backup - if(mDeleteStoreObjectInfoFile && - !DeleteStoreObjectInfo()) + if(mDeleteStoreObjectInfoFile && !DeleteStoreObjectInfo()) { BOX_ERROR("Failed to delete the StoreObjectInfoFile, " "backup cannot continue safely."); - THROW_EXCEPTION(ClientException, + THROW_EXCEPTION(ClientException, FailedToDeleteStoreObjectInfoFile); } @@ -819,14 +944,21 @@ void BackupDaemon::RunSyncNow() if (conf.KeyExists("LogFile")) { + bool overwrite = false; + if (conf.KeyExists("LogFileOverwrite")) + { + overwrite = conf.GetKeyValueBool("LogFileOverwrite"); + } + Log::Level level = Log::INFO; if (conf.KeyExists("LogFileLevel")) { level = Logging::GetNamedLevel( conf.GetKeyValue("LogFileLevel")); } + fileLogger.reset(new FileLogger(conf.GetKeyValue("LogFile"), - level)); + level, !overwrite)); } std::string extendedLogFile; @@ -834,28 +966,27 @@ void BackupDaemon::RunSyncNow() { extendedLogFile = conf.GetKeyValue("ExtendedLogFile"); } - + if (conf.KeyExists("LogAllFileAccess")) { mLogAllFileAccess = conf.GetKeyValueBool("LogAllFileAccess"); } - - // Then create a client context object (don't + + // Then create a client context object (don't // just connect, as this may be unnecessary) - BackupClientContext clientContext - ( - *mpLocationResolver, - mTlsContext, + mapClientContext = GetNewContext( + *mpLocationResolver, + mTlsContext, conf.GetKeyValue("StoreHostname"), conf.GetKeyValueInt("StorePort"), - conf.GetKeyValueUint32("AccountNumber"), + conf.GetKeyValueUint32("AccountNumber"), conf.GetKeyValueBool("ExtendedLogging"), conf.KeyExists("ExtendedLogFile"), extendedLogFile, *mpProgressNotifier, conf.GetKeyValueBool("TcpNice") ); - + // The minimum age a file needs to be before it will be // considered for uploading box_time_t minimumFileAge = SecondsToBoxTime( @@ -887,38 +1018,37 @@ void BackupDaemon::RunSyncNow() // than minimumFileAge after the last sync. Deal with it by // moving back syncPeriodStart, which should not do any // damage. - syncPeriodStart = syncPeriodEnd - - SecondsToBoxTime(1); + syncPeriodStart = syncPeriodEnd - SecondsToBoxTime(1); } if(syncPeriodStart >= syncPeriodEnd) { - BOX_ERROR("Invalid (negative) sync period: " - "perhaps your clock is going " - "backwards (" << syncPeriodStart << - " to " << syncPeriodEnd << ")"); - THROW_EXCEPTION(ClientException, - ClockWentBackwards); + BOX_ERROR("Invalid (negative) sync period: perhaps your clock " + "is going backwards? (" << syncPeriodStart << " to " << + syncPeriodEnd << ")"); + THROW_EXCEPTION(ClientException, ClockWentBackwards); } // Check logic ASSERT(syncPeriodEnd > syncPeriodStart); // Paranoid check on sync times - if(syncPeriodStart >= syncPeriodEnd) return; + if(syncPeriodStart >= syncPeriodEnd) + { + return mapClientContext; // releases mapClientContext + } - // Adjust syncPeriodEnd to emulate snapshot - // behaviour properly + // Adjust syncPeriodEnd to emulate snapshot behaviour properly box_time_t syncPeriodEndExtended = syncPeriodEnd; // Using zero min file age? if(minimumFileAge == 0) { // Add a year on to the end of the end time, - // to make sure we sync files which are + // to make sure we sync files which are // modified after the scan run started. - // Of course, they may be eligible to be + // Of course, they may be eligible to be // synced again the next time round, - // but this should be OK, because the changes + // but this should be OK, because the changes // only upload should upload no data. syncPeriodEndExtended += SecondsToBoxTime( (time_t)(356*24*3600)); @@ -926,16 +1056,16 @@ void BackupDaemon::RunSyncNow() // Set up the sync parameters BackupClientDirectoryRecord::SyncParams params(*mpRunStatusProvider, - *mpSysadminNotifier, *mpProgressNotifier, clientContext); + *mpSysadminNotifier, *mpProgressNotifier, *mapClientContext, this); params.mSyncPeriodStart = syncPeriodStart; params.mSyncPeriodEnd = syncPeriodEndExtended; // use potentially extended end time params.mMaxUploadWait = maxUploadWait; - params.mFileTrackingSizeThreshold = + params.mFileTrackingSizeThreshold = conf.GetKeyValueInt("FileTrackingSizeThreshold"); - params.mDiffingUploadSizeThreshold = + params.mDiffingUploadSizeThreshold = conf.GetKeyValueInt("DiffingUploadSizeThreshold"); - params.mMaxFileTimeInFuture = + params.mMaxFileTimeInFuture = SecondsToBoxTime(conf.GetKeyValueInt("MaxFileTimeInFuture")); mNumFilesUploaded = 0; mNumDirsCreated = 0; @@ -969,37 +1099,36 @@ void BackupDaemon::RunSyncNow() keepAliveTime = conf.GetKeyValueInt("KeepAliveTime"); } - clientContext.SetMaximumDiffingTime(maximumDiffingTime); - clientContext.SetKeepAliveTime(keepAliveTime); - + mapClientContext->SetMaximumDiffingTime(maximumDiffingTime); + mapClientContext->SetKeepAliveTime(keepAliveTime); + // Set store marker - clientContext.SetClientStoreMarker(mClientStoreMarker); - - // Set up the locations, if necessary -- - // need to do it here so we have a - // (potential) connection to use + mapClientContext->SetClientStoreMarker(mClientStoreMarker); + + // Set up the locations, if necessary -- need to do it here so we have + // a (potential) connection to use. { const Configuration &locations( conf.GetSubConfiguration( "BackupLocations")); - + // Make sure all the directory records // are set up - SetupLocations(clientContext, locations); + SetupLocations(*mapClientContext, locations); } - - mpProgressNotifier->NotifyIDMapsSetup(clientContext); - + + mpProgressNotifier->NotifyIDMapsSetup(*mapClientContext); + // Get some ID maps going SetupIDMapsForSync(); // Delete any unused directories? - DeleteUnusedRootDirEntries(clientContext); + DeleteUnusedRootDirEntries(*mapClientContext); #ifdef ENABLE_VSS CreateVssBackupComponents(); #endif - + // Go through the records, syncing them for(Locations::const_iterator i(mLocations.begin()); @@ -1007,14 +1136,14 @@ void BackupDaemon::RunSyncNow() { // Set current and new ID map pointers // in the context - clientContext.SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], + mapClientContext->SetIDMaps(mCurrentIDMaps[(*i)->mIDMapIndex], mNewIDMaps[(*i)->mIDMapIndex]); // Set exclude lists (context doesn't // take ownership) - clientContext.SetExcludeLists( - (*i)->mpExcludeFiles, - (*i)->mpExcludeDirs); + mapClientContext->SetExcludeLists( + (*i)->mapExcludeFiles.get(), + (*i)->mapExcludeDirs.get()); // Sync the directory std::string locationPath = (*i)->mPath; @@ -1025,29 +1154,26 @@ void BackupDaemon::RunSyncNow() } #endif - (*i)->mpDirectoryRecord->SyncDirectory(params, + (*i)->mapDirectoryRecord->SyncDirectory(params, BackupProtocolListDirectory::RootDirectory, locationPath, std::string("/") + (*i)->mName, **i); // Unset exclude lists (just in case) - clientContext.SetExcludeLists(0, 0); + mapClientContext->SetExcludeLists(0, 0); } - + // Perform any deletions required -- these are // delayed until the end to allow renaming to // happen neatly. - clientContext.PerformDeletions(); - - // Close any open connection - clientContext.CloseAnyOpenConnection(); + mapClientContext->PerformDeletions(); #ifdef ENABLE_VSS CleanupVssBackupComponents(); #endif // Get the new store marker - mClientStoreMarker = clientContext.GetClientStoreMarker(); - mStorageLimitExceeded = clientContext.StorageLimitExceeded(); + mClientStoreMarker = mapClientContext->GetClientStoreMarker(); + mStorageLimitExceeded = mapClientContext->StorageLimitExceeded(); mReadErrorsOnFilesystemObjects |= params.mReadErrorsOnFilesystemObjects; @@ -1064,8 +1190,8 @@ void BackupDaemon::RunSyncNow() CommitIDMapsAfterSync(); // Calculate when the next sync run should be - mNextSyncTime = mCurrentSyncStartTime + - mUpdateStoreInterval + + mNextSyncTime = mCurrentSyncStartTime + + mUpdateStoreInterval + Random::RandomInt(mUpdateStoreInterval >> SYNC_PERIOD_RANDOM_EXTRA_TIME_SHIFT_BY); @@ -1075,11 +1201,12 @@ void BackupDaemon::RunSyncNow() // info. If we save successfully, we must // delete the file next time we start a backup - mDeleteStoreObjectInfoFile = - SerializeStoreObjectInfo(mLastSyncTime, - mNextSyncTime); + mDeleteStoreObjectInfoFile = + SerializeStoreObjectInfo(mLastSyncTime, mNextSyncTime); // -------------------------------------------------------------------------------------------- + + return mapClientContext; // releases mapClientContext } #ifdef ENABLE_VSS @@ -1136,16 +1263,6 @@ bool BackupDaemon::CallAndWaitForAsync(AsyncMethod method, return WaitForAsync(pAsync, description); } -void FreeSnapshotProp(VSS_SNAPSHOT_PROP *pSnap) -{ - CoTaskMemFree(pSnap->m_pwszSnapshotDeviceObject); - CoTaskMemFree(pSnap->m_pwszOriginalVolumeName); - CoTaskMemFree(pSnap->m_pwszOriginatingMachine); - CoTaskMemFree(pSnap->m_pwszServiceMachine); - CoTaskMemFree(pSnap->m_pwszExposedName); - CoTaskMemFree(pSnap->m_pwszExposedPath); -} - void BackupDaemon::CreateVssBackupComponents() { std::map<char, VSS_ID> volumesIncluded; @@ -1351,21 +1468,6 @@ void BackupDaemon::CreateVssBackupComponents() " to snapshot set"); volumesIncluded[path[0]] = newVolumeId; rLocation.mSnapshotVolumeId = newVolumeId; - rLocation.mIsSnapshotCreated = true; - - // If the snapshot path starts with the volume root - // (drive letter), because the path is absolute (as - // it should be), then remove it so that the - // resulting snapshot path can be appended to the - // snapshot device object to make a real path, - // without a spurious drive letter in it. - - if (path.substr(0, volumeRoot.length()) == volumeRoot) - { - path = path.substr(volumeRoot.length()); - } - - rLocation.mSnapshotPath = path; } else { @@ -1380,8 +1482,23 @@ void BackupDaemon::CreateVssBackupComponents() BOX_TRACE("VSS: Skipping already included volume " << volumeRoot << " for backup location " << path); rLocation.mSnapshotVolumeId = i->second; - rLocation.mIsSnapshotCreated = true; } + + rLocation.mIsSnapshotCreated = true; + + // If the snapshot path starts with the volume root + // (drive letter), because the path is absolute (as it + // should be), then remove it so that the resulting + // snapshot path can be appended to the snapshot device + // object to make a real path, without a spurious drive + // letter in it. + + if (path.substr(0, volumeRoot.length()) == volumeRoot) + { + path = path.substr(volumeRoot.length()); + } + + rLocation.mSnapshotPath = path; } else { @@ -1502,7 +1619,7 @@ void BackupDaemon::CreateVssBackupComponents() rLocation.mSnapshotPath = WideStringToString(prop.m_pwszSnapshotDeviceObject) + DIRECTORY_SEPARATOR + rLocation.mSnapshotPath; - FreeSnapshotProp(&prop); + VssFreeSnapshotProperties(&prop); BOX_INFO("VSS: Location " << rLocation.mPath << " using " "snapshot path " << rLocation.mSnapshotPath); @@ -1600,7 +1717,7 @@ void BackupDaemon::CreateVssBackupComponents() } BOX_TRACE("VSS: Snapshot status: " << status); - FreeSnapshotProp(pSnap); + VssFreeSnapshotProperties(pSnap); } } @@ -1640,6 +1757,8 @@ void BackupDaemon::CleanupVssBackupComponents() void BackupDaemon::OnBackupStart() { + ResetLogFile(); + // Touch a file to record times in filesystem TouchFileInWorkingDir("last_sync_start"); @@ -1652,6 +1771,10 @@ void BackupDaemon::OnBackupStart() // Notify administrator NotifySysadmin(SysadminNotifier::BackupStart); + // Setup timer for polling the command socket + mapCommandSocketPollTimer.reset(new Timer(COMMAND_SOCKET_POLL_INTERVAL, + "CommandSocketPollTimer")); + // Set state and log start SetState(State_Connected); BOX_NOTICE("Beginning scan of local files"); @@ -1680,6 +1803,10 @@ void BackupDaemon::OnBackupFinish() // Notify administrator NotifySysadmin(SysadminNotifier::BackupFinish); + // Stop the timer for polling the command socket, + // to prevent needless alarms while sleeping. + mapCommandSocketPollTimer.reset(); + // Tell anything connected to the command socket SendSyncStartOrFinish(false /* finish */); @@ -1774,7 +1901,8 @@ int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, if(delay == "") { - BOX_ERROR("SyncAllowScript output an empty line"); + BOX_ERROR("SyncAllowScript output an empty line, sleeping for " + << waitInSeconds << " seconds (" << script << ")"); return waitInSeconds; } @@ -1785,7 +1913,7 @@ int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, waitInSeconds = -1; BOX_NOTICE("SyncAllowScript requested a backup now " - << "(" << script << ")"); + "(" << script << ")"); } else { @@ -1802,9 +1930,8 @@ int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, throw; } - BOX_NOTICE("SyncAllowScript requested a delay of " << - waitInSeconds << " seconds due to SyncAllowScript " - << "(" << script << ")"); + BOX_NOTICE("SyncAllowScript requested a delay of " << + waitInSeconds << " seconds (" << script << ")"); } if(iss.eof()) @@ -1845,14 +1972,68 @@ int BackupDaemon::ParseSyncAllowScriptOutput(const std::string& script, // -------------------------------------------------------------------------- // // Function +// Name: BackupDaemon::RunBackgroundTask() +// Purpose: Checks for connections or commands on the command +// socket and handles them with minimal delay. Polled +// during lengthy operations such as file uploads. +// Created: 07/04/14 +// +// -------------------------------------------------------------------------- +bool BackupDaemon::RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum) +{ + BOX_TRACE("BackupDaemon::RunBackgroundTask: state = " << state << + ", progress = " << progress << "/" << maximum); + + if(!mapCommandSocketPollTimer.get()) + { + return true; // no background task + } + + if(mapCommandSocketPollTimer->HasExpired()) + { + mapCommandSocketPollTimer->Reset(COMMAND_SOCKET_POLL_INTERVAL); + } + else + { + // Do no more work right now + return true; + } + + if(mapCommandSocketInfo.get()) + { + BOX_TRACE("BackupDaemon::RunBackgroundTask: polling command socket"); + + bool sync_flag_out, sync_is_forced_out; + + WaitOnCommandSocket(0, // RequiredDelay + sync_flag_out, sync_is_forced_out); + + if(sync_flag_out) + { + BOX_WARNING("Ignoring request to sync while " + "already syncing."); + } + } + + return true; +} + +// -------------------------------------------------------------------------- +// +// Function // Name: BackupDaemon::WaitOnCommandSocket(box_time_t, bool &, bool &) -// Purpose: Waits on a the command socket for a time of UP TO the required time -// but may be much less, and handles a command if necessary. +// Purpose: Waits on a the command socket for a time of UP TO +// the required time but may be much less, and handles +// a command if necessary. // Created: 18/2/04 // // -------------------------------------------------------------------------- void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFlagOut, bool &SyncIsForcedOut) { + DoSyncFlagOut = false; + SyncIsForcedOut = false; + ASSERT(mapCommandSocketInfo.get()); if(!mapCommandSocketInfo.get()) { @@ -1861,7 +2042,8 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla return; } - BOX_TRACE("Wait on command socket, delay = " << RequiredDelay); + BOX_TRACE("Wait on command socket, delay = " << + BOX_FORMAT_MICROSECONDS(RequiredDelay)); try { @@ -1874,8 +2056,13 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla // Wait for socket connection, or handle a command? if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { + // There should be no GetLine, as it would be holding onto a + // pointer to a dead mpConnectedSocket. + ASSERT(!mapCommandSocketInfo->mapGetLine.get()); + // No connection, listen for a new one - mapCommandSocketInfo->mpConnectedSocket.reset(mapCommandSocketInfo->mListeningSocket.Accept(timeout).release()); + mapCommandSocketInfo->mpConnectedSocket.reset( + mapCommandSocketInfo->mListeningSocket.Accept(timeout).release()); if(mapCommandSocketInfo->mpConnectedSocket.get() == 0) { @@ -1923,39 +2110,43 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla // Send a header line summarising the configuration and current state const Configuration &conf(GetConfiguration()); - char summary[256]; - int summarySize = sprintf(summary, "bbackupd: %d %d %d %d\nstate %d\n", - conf.GetKeyValueBool("AutomaticBackup"), - conf.GetKeyValueInt("UpdateStoreInterval"), - conf.GetKeyValueInt("MinimumFileAge"), - conf.GetKeyValueInt("MaxUploadWait"), - mState); - mapCommandSocketInfo->mpConnectedSocket->Write(summary, summarySize); + std::ostringstream hello; + hello << "bbackupd: " << + (conf.GetKeyValueBool("AutomaticBackup") ? 1 : 0) + << " " << + conf.GetKeyValueInt("UpdateStoreInterval") + << " " << + conf.GetKeyValueInt("MinimumFileAge") + << " " << + conf.GetKeyValueInt("MaxUploadWait") + << "\nstate " << mState << "\n"; + mapCommandSocketInfo->mpConnectedSocket->Write( + hello.str(), timeout); // Set the timeout to something very small, so we don't wait too long on waiting // for any incoming data timeout = 10; // milliseconds } } + + mapCommandSocketInfo->mapGetLine.reset( + new IOStreamGetLine( + *(mapCommandSocketInfo->mpConnectedSocket.get()))); } // So there must be a connection now. ASSERT(mapCommandSocketInfo->mpConnectedSocket.get() != 0); - - // Is there a getline object ready? - if(mapCommandSocketInfo->mpGetLine == 0) - { - // Create a new one - mapCommandSocketInfo->mpGetLine = new IOStreamGetLine(*(mapCommandSocketInfo->mpConnectedSocket.get())); - } + ASSERT(mapCommandSocketInfo->mapGetLine.get() != 0); // Ping the remote side, to provide errors which will mean the socket gets closed - mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5); + mapCommandSocketInfo->mpConnectedSocket->Write("ping\n", 5, + timeout); // Wait for a command or something on the socket std::string command; - while(mapCommandSocketInfo->mpGetLine != 0 && !mapCommandSocketInfo->mpGetLine->IsEOF() - && mapCommandSocketInfo->mpGetLine->GetLine(command, false /* no preprocessing */, timeout)) + while(mapCommandSocketInfo->mapGetLine.get() != 0 + && !mapCommandSocketInfo->mapGetLine->IsEOF() + && mapCommandSocketInfo->mapGetLine->GetLine(command, false /* no preprocessing */, timeout)) { BOX_TRACE("Receiving command '" << command << "' over command socket"); @@ -2000,7 +2191,9 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla // Send a response back? if(sendResponse) { - mapCommandSocketInfo->mpConnectedSocket->Write(sendOK?"ok\n":"error\n", sendOK?3:6); + std::string response = sendOK ? "ok\n" : "error\n"; + mapCommandSocketInfo->mpConnectedSocket->Write( + response, timeout); } // Set timeout to something very small, so this just checks for data which is waiting @@ -2008,7 +2201,8 @@ void BackupDaemon::WaitOnCommandSocket(box_time_t RequiredDelay, bool &DoSyncFla } // Close on EOF? - if(mapCommandSocketInfo->mpGetLine != 0 && mapCommandSocketInfo->mpGetLine->IsEOF()) + if(mapCommandSocketInfo->mapGetLine.get() != 0 && + mapCommandSocketInfo->mapGetLine->IsEOF()) { CloseCommandConnection(); } @@ -2084,12 +2278,7 @@ void BackupDaemon::CloseCommandConnection() try { BOX_TRACE("Closing command connection"); - - if(mapCommandSocketInfo->mpGetLine) - { - delete mapCommandSocketInfo->mpGetLine; - mapCommandSocketInfo->mpGetLine = 0; - } + mapCommandSocketInfo->mapGetLine.reset(); mapCommandSocketInfo->mpConnectedSocket.reset(); } catch(std::exception &e) @@ -2125,8 +2314,8 @@ void BackupDaemon::SendSyncStartOrFinish(bool SendStart) try { message += "\n"; - mapCommandSocketInfo->mpConnectedSocket->Write( - message.c_str(), message.size()); + mapCommandSocketInfo->mpConnectedSocket->Write(message, + 1); // short timeout, it's overlapped } catch(std::exception &e) { @@ -2331,15 +2520,15 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // Setup names in the location record pLoc->mName = *pLocName; pLoc->mPath = rConfig.GetKeyValue("Path"); - - // Read the exclude lists from the Configuration - pLoc->mpExcludeFiles = BackupClientMakeExcludeList_Files(rConfig); - pLoc->mpExcludeDirs = BackupClientMakeExcludeList_Dirs(rConfig); } + // Read the exclude lists from the Configuration + pLoc->mapExcludeFiles.reset(BackupClientMakeExcludeList_Files(rConfig)); + pLoc->mapExcludeDirs.reset(BackupClientMakeExcludeList_Dirs(rConfig)); + // Does this exist on the server? // Remove from dir object early, so that if we fail - // to stat the local directory, we still don't + // to stat the local directory, we still don't // consider to remote one for deletion. BackupStoreDirectory::Iterator iter(dir); BackupStoreFilenameClear dirname(pLoc->mName); // generate the filename @@ -2464,7 +2653,7 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con new MemBlockStream(attr)); std::auto_ptr<BackupProtocolSuccess> dirCreate(connection.QueryCreateDirectory( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory attrModTime, dirname, attrStream)); // Object ID for later creation @@ -2483,11 +2672,10 @@ void BackupDaemon::SetupLocations(BackupClientContext &rClientContext, const Con // Create and store the directory object for the root of this location ASSERT(oid != 0); - if(pLoc->mpDirectoryRecord.get() == NULL) + if(pLoc->mapDirectoryRecord.get() == NULL) { - BackupClientDirectoryRecord *precord = - new BackupClientDirectoryRecord(oid, *pLocName); - pLoc->mpDirectoryRecord.reset(precord); + pLoc->mapDirectoryRecord.reset( + new BackupClientDirectoryRecord(oid, *pLocName)); } // Remove it from the temporary list to avoid deletion @@ -2842,11 +3030,8 @@ void BackupDaemon::SetState(int State) // If there's a command socket connected, then inform it -- disconnecting from the // command socket if there's an error - char newState[64]; - sprintf(newState, "state %d", State); - std::string message = newState; - - message += "\n"; + std::ostringstream msg; + msg << "state " << State << "\n"; if(!mapCommandSocketInfo.get()) { @@ -2861,8 +3046,8 @@ void BackupDaemon::SetState(int State) // Something connected to the command socket, tell it about the new state try { - mapCommandSocketInfo->mpConnectedSocket->Write(message.c_str(), - message.length()); + mapCommandSocketInfo->mpConnectedSocket->Write(msg.str(), + 1); // very short timeout, it's overlapped anyway } catch(ConnectionException &ce) { @@ -2944,10 +3129,9 @@ void BackupDaemon::NotifySysadmin(SysadminNotifier::EventCode Event) if(Event < 0 || Event >= SysadminNotifier::MAX) { - BOX_ERROR("BackupDaemon::NotifySysadmin() called for " - "invalid event code " << Event); - THROW_EXCEPTION(BackupStoreException, - BadNotifySysadminEventCode); + THROW_EXCEPTION_MESSAGE(BackupStoreException, + BadNotifySysadminEventCode, "NotifySysadmin() called " + "for unknown event code " << Event); } BOX_TRACE("BackupDaemon::NotifySysadmin() called, event = " << @@ -3087,7 +3271,6 @@ typedef struct // // -------------------------------------------------------------------------- BackupDaemon::CommandSocketInfo::CommandSocketInfo() - : mpGetLine(0) { } @@ -3102,11 +3285,6 @@ BackupDaemon::CommandSocketInfo::CommandSocketInfo() // -------------------------------------------------------------------------- BackupDaemon::CommandSocketInfo::~CommandSocketInfo() { - if(mpGetLine) - { - delete mpGetLine; - mpGetLine = 0; - } } // -------------------------------------------------------------------------- @@ -3246,10 +3424,12 @@ bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, // if(!GetConfiguration().KeyExists("StoreObjectInfoFile")) { + BOX_NOTICE("Store object info file is not enabled. Will " + "download directory listings from store."); return false; } - std::string StoreObjectInfoFile = + std::string StoreObjectInfoFile = GetConfiguration().GetKeyValue("StoreObjectInfoFile"); if(StoreObjectInfoFile.size() <= 0) @@ -3257,161 +3437,168 @@ bool BackupDaemon::DeserializeStoreObjectInfo(box_time_t & theLastSyncTime, return false; } - try + int64_t fileSize; + if (!FileExists(StoreObjectInfoFile, &fileSize) || fileSize == 0) { - FileStream aFile(StoreObjectInfoFile.c_str(), O_RDONLY); - Archive anArchive(aFile, 0); + BOX_NOTICE(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file does not exist or is empty")); + } + else + { + try + { + FileStream aFile(StoreObjectInfoFile, O_RDONLY); + Archive anArchive(aFile, 0); - // - // see if the content looks like a valid serialised archive - // - int iMagicValue = 0; - anArchive.Read(iMagicValue); + // + // see if the content looks like a valid serialised archive + // + int iMagicValue = 0; + anArchive.Read(iMagicValue); - if(iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) - { - BOX_WARNING("Store object info file " - "is not a valid or compatible serialised " - "archive. Will re-cache from store. " - "(" << StoreObjectInfoFile << ")"); - return false; - } + if(iMagicValue != STOREOBJECTINFO_MAGIC_ID_VALUE) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is not a valid " + "or compatible serialised archive")); + return false; + } - // - // get a bit optimistic and read in a string identifier - // - std::string strMagicValue; - anArchive.Read(strMagicValue); + // + // get a bit optimistic and read in a string identifier + // + std::string strMagicValue; + anArchive.Read(strMagicValue); - if(strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) - { - BOX_WARNING("Store object info file " - "is not a valid or compatible serialised " - "archive. Will re-cache from store. " - "(" << StoreObjectInfoFile << ")"); - return false; - } + if(strMagicValue != STOREOBJECTINFO_MAGIC_ID_STRING) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is not a valid " + "or compatible serialised archive")); + return false; + } - // - // check if we are loading some future format - // version by mistake - // - int iVersion = 0; - anArchive.Read(iVersion); + // + // check if we are loading some future format + // version by mistake + // + int iVersion = 0; + anArchive.Read(iVersion); - if(iVersion != STOREOBJECTINFO_VERSION) - { - BOX_WARNING("Store object info file " - "version " << iVersion << " unsupported. " - "Will re-cache from store. " - "(" << StoreObjectInfoFile << ")"); - return false; - } + if(iVersion != STOREOBJECTINFO_VERSION) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file version " << + iVersion << " is not supported")); + return false; + } - // - // check if this state file is even valid - // for the loaded bbackupd.conf file - // - box_time_t lastKnownConfigModTime; - anArchive.Read(lastKnownConfigModTime); + // + // check if this state file is even valid + // for the loaded bbackupd.conf file + // + box_time_t lastKnownConfigModTime; + anArchive.Read(lastKnownConfigModTime); - if(lastKnownConfigModTime != GetLoadedConfigModifiedTime()) - { - BOX_WARNING("Store object info file " - "out of date. Will re-cache from store. " - "(" << StoreObjectInfoFile << ")"); - return false; - } + if(lastKnownConfigModTime != GetLoadedConfigModifiedTime()) + { + BOX_WARNING(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Store object info file is older than " + "configuration file")); + return false; + } - // - // this is it, go at it - // - anArchive.Read(mClientStoreMarker); - anArchive.Read(theLastSyncTime); - anArchive.Read(theNextSyncTime); + // + // this is it, go at it + // + anArchive.Read(mClientStoreMarker); + anArchive.Read(theLastSyncTime); + anArchive.Read(theNextSyncTime); - // - // - // - int64_t iCount = 0; - anArchive.Read(iCount); + // + // + // + int64_t iCount = 0; + anArchive.Read(iCount); - for(int v = 0; v < iCount; v++) - { - Location* pLocation = new Location; - if(!pLocation) + for(int v = 0; v < iCount; v++) { - throw std::bad_alloc(); + Location* pLocation = new Location; + if(!pLocation) + { + throw std::bad_alloc(); + } + + pLocation->Deserialize(anArchive); + mLocations.push_back(pLocation); } - pLocation->Deserialize(anArchive); - mLocations.push_back(pLocation); - } + // + // + // + iCount = 0; + anArchive.Read(iCount); - // - // - // - iCount = 0; - anArchive.Read(iCount); + for(int v = 0; v < iCount; v++) + { + std::string strItem; + anArchive.Read(strItem); - for(int v = 0; v < iCount; v++) - { - std::string strItem; - anArchive.Read(strItem); + mIDMapMounts.push_back(strItem); + } - mIDMapMounts.push_back(strItem); - } + // + // + // + iCount = 0; + anArchive.Read(iCount); - // - // - // - iCount = 0; - anArchive.Read(iCount); + for(int v = 0; v < iCount; v++) + { + int64_t anId; + anArchive.Read(anId); - for(int v = 0; v < iCount; v++) - { - int64_t anId; - anArchive.Read(anId); + std::string aName; + anArchive.Read(aName); - std::string aName; - anArchive.Read(aName); + mUnusedRootDirEntries.push_back(std::pair<int64_t, std::string>(anId, aName)); + } - mUnusedRootDirEntries.push_back(std::pair<int64_t, std::string>(anId, aName)); - } + if (iCount > 0) + anArchive.Read(mDeleteUnusedRootDirEntriesAfter); - if (iCount > 0) - anArchive.Read(mDeleteUnusedRootDirEntriesAfter); + // + // + // + aFile.Close(); - // - // - // - aFile.Close(); - BOX_INFO("Loaded store object info file version " << iVersion - << " (" << StoreObjectInfoFile << ")"); - - return true; - } - catch(std::exception &e) - { - BOX_ERROR("Internal error reading store object info file: " - << StoreObjectInfoFile << ": " << e.what()); - } - catch(...) - { - BOX_ERROR("Internal error reading store object info file: " - << StoreObjectInfoFile << ": unknown error"); + BOX_INFO(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Loaded store object info file version " << iVersion)); + return true; + } + catch(std::exception &e) + { + BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Internal error reading store object info " + "file: " << e.what())); + } + catch(...) + { + BOX_ERROR(BOX_FILE_MESSAGE(StoreObjectInfoFile, + "Internal error reading store object info " + "file: unknown error")); + } } + BOX_NOTICE("No usable cache, will download directory listings from " + "server."); + DeleteAllLocations(); mClientStoreMarker = BackupClientContext::ClientStoreMarker_NotKnown; theLastSyncTime = 0; theNextSyncTime = 0; - BOX_WARNING("Store object info file is missing, not accessible, " - "or inconsistent. Will re-cache from store. " - "(" << StoreObjectInfoFile << ")"); - return false; } diff --git a/bin/bbackupd/BackupDaemon.h b/lib/bbackupd/BackupDaemon.h index 1d3c991e..f9b8ba31 100644 --- a/bin/bbackupd/BackupDaemon.h +++ b/lib/bbackupd/BackupDaemon.h @@ -25,6 +25,7 @@ #include "TLSContext.h" #include "autogen_BackupProtocol.h" +#include "autogen_BackupStoreException.h" #ifdef WIN32 #include "WinNamedPipeListener.h" @@ -38,6 +39,8 @@ # include <VsBackup.h> #endif +#define COMMAND_SOCKET_POLL_INTERVAL 1000 + class BackupClientDirectoryRecord; class BackupClientContext; class Configuration; @@ -54,8 +57,8 @@ class Archive; // Created: 2003/10/08 // // -------------------------------------------------------------------------- -class BackupDaemon : public Daemon, ProgressNotifier, LocationResolver, -RunStatusProvider, SysadminNotifier +class BackupDaemon : public Daemon, public ProgressNotifier, public LocationResolver, +public RunStatusProvider, public SysadminNotifier, public BackgroundTask { public: BackupDaemon(); @@ -106,6 +109,25 @@ public: }; int GetState() {return mState;} + static std::string GetStateName(int state) + { + std::string stateName; + + #define STATE(x) case BackupDaemon::State_ ## x: stateName = #x; break; + switch (state) + { + STATE(Initialising); + STATE(Idle); + STATE(Connected); + STATE(Error); + STATE(StorageLimitExceeded); + default: + stateName = "unknown"; + } + #undef STATE + + return stateName; + } // Allow other classes to call this too void NotifySysadmin(SysadminNotifier::EventCode Event); @@ -115,8 +137,8 @@ private: public: void InitCrypto(); - void RunSyncNowWithExceptionHandling(); - void RunSyncNow(); + std::auto_ptr<BackupClientContext> RunSyncNowWithExceptionHandling(); + std::auto_ptr<BackupClientContext> RunSyncNow(); void ResetCachedState(); void OnBackupStart(); void OnBackupFinish(); @@ -124,6 +146,21 @@ public: // This does NOT constitute an API! void TouchFileInWorkingDir(const char *Filename); +protected: + virtual std::auto_ptr<BackupClientContext> GetNewContext + ( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode + ); + private: void DeleteAllLocations(); void SetupLocations(BackupClientContext &rClientContext, const Configuration &rLocationsConf); @@ -150,10 +187,8 @@ private: void DeleteUnusedRootDirEntries(BackupClientContext &rContext); -#ifdef PLATFORM_CANNOT_FIND_PEER_UID_OF_UNIX_SOCKET // For warning user about potential security hole virtual void SetupInInitialProcess(); -#endif int UseScriptToSeeIfSyncAllowed(); @@ -191,12 +226,12 @@ private: SocketListen<SocketStream, 1 /* listen backlog */> mListeningSocket; std::auto_ptr<SocketStream> mpConnectedSocket; #endif - IOStreamGetLine *mpGetLine; + std::auto_ptr<IOStreamGetLine> mapGetLine; }; - + // Using a socket? std::auto_ptr<CommandSocketInfo> mapCommandSocketInfo; - + // Stop notifications being repeated. SysadminNotifier::EventCode mLastNotifiedEvent; @@ -208,7 +243,8 @@ private: bool mStorageLimitExceeded; bool mReadErrorsOnFilesystemObjects; box_time_t mLastSyncTime, mNextSyncTime; - box_time_t mCurrentSyncStartTime, mUpdateStoreInterval; + box_time_t mCurrentSyncStartTime, mUpdateStoreInterval, + mBackupErrorDelay; TLSContext mTlsContext; bool mDeleteStoreObjectInfoFile; bool mDoSyncForcedByPreviousSyncError; @@ -217,7 +253,7 @@ private: public: int GetMaxBandwidthFromSyncAllowScript() { return mMaxBandwidthFromSyncAllowScript; } - bool StopRun() { return this->Daemon::StopRun(); } + bool StopRun() { return this->Daemon::StopRun(); } bool StorageLimitExceeded() { return mStorageLimitExceeded; } private: @@ -232,60 +268,70 @@ public: void SetLocationResolver (LocationResolver* p) { mpLocationResolver = p; } void SetRunStatusProvider(RunStatusProvider* p) { mpRunStatusProvider = p; } void SetSysadminNotifier (SysadminNotifier* p) { mpSysadminNotifier = p; } - + virtual bool RunBackgroundTask(State state, uint64_t progress, + uint64_t maximum); + private: ProgressNotifier* mpProgressNotifier; LocationResolver* mpLocationResolver; RunStatusProvider* mpRunStatusProvider; SysadminNotifier* mpSysadminNotifier; - - /* ProgressNotifier implementation */ + std::auto_ptr<Timer> mapCommandSocketPollTimer; + std::auto_ptr<BackupClientContext> mapClientContext; + + /* ProgressNotifier implementation */ public: virtual void NotifyIDMapsSetup(BackupClientContext& rContext) { } - virtual void NotifyScanDirectory( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) + virtual void NotifyScanDirectory( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) { if (mLogAllFileAccess) { BOX_INFO("Scanning directory: " << rLocalPath); - } + } + + if (!RunBackgroundTask(BackgroundTask::Scanning_Dirs, 0, 0)) + { + THROW_EXCEPTION(BackupStoreException, + CancelledByBackgroundTask); + } } - virtual void NotifyDirStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { + virtual void NotifyDirStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { BOX_WARNING("Failed to access directory: " << rLocalPath << ": " << rErrorMsg); - } - virtual void NotifyFileStatFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { + } + virtual void NotifyFileStatFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { BOX_WARNING("Failed to access file: " << rLocalPath << ": " << rErrorMsg); - } - virtual void NotifyDirListFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { + } + virtual void NotifyDirListFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { BOX_WARNING("Failed to list directory: " << rLocalPath << ": " << rErrorMsg); - } + } virtual void NotifyMountPointSkipped( const BackupClientDirectoryRecord* pDirRecord, const std::string& rLocalPath) { #ifdef WIN32 - BOX_WARNING("Ignored directory: " << rLocalPath << + BOX_WARNING("Ignored directory: " << rLocalPath << ": is an NTFS junction/reparse point; create " "a new location if you want to back it up"); #else - BOX_WARNING("Ignored directory: " << rLocalPath << + BOX_WARNING("Ignored directory: " << rLocalPath << ": is a mount point; create a new location " "if you want to back it up"); #endif @@ -297,7 +343,7 @@ public: if (mLogAllFileAccess) { BOX_INFO("Skipping excluded file: " << rLocalPath); - } + } } virtual void NotifyDirExcluded( const BackupClientDirectoryRecord* pDirRecord, @@ -306,7 +352,7 @@ public: if (mLogAllFileAccess) { BOX_INFO("Skipping excluded directory: " << rLocalPath); - } + } } virtual void NotifyUnsupportedFileType( const BackupClientDirectoryRecord* pDirRecord, @@ -314,161 +360,110 @@ public: { BOX_WARNING("Ignoring file of unknown type: " << rLocalPath); } - virtual void NotifyFileReadFailed( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const std::string& rErrorMsg) - { + virtual void NotifyFileReadFailed( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const std::string& rErrorMsg) + { BOX_WARNING("Error reading file: " << rLocalPath << ": " << rErrorMsg); - } - virtual void NotifyFileModifiedInFuture( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { + } + virtual void NotifyFileModifiedInFuture( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { BOX_WARNING("Some files have modification times excessively " "in the future. Check clock synchronisation. " "Example file (only one shown): " << rLocalPath); - } - virtual void NotifyFileSkippedServerFull( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) + } + virtual void NotifyFileSkippedServerFull( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) { BOX_WARNING("Skipped file: server is full: " << rLocalPath); } - virtual void NotifyFileUploadException( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - const BoxException& rException) - { + virtual void NotifyFileUploadException( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + const BoxException& rException) + { if (rException.GetType() == CommonException::ExceptionType && rException.GetSubType() == CommonException::AccessDenied) { - BOX_ERROR("Failed to upload file: " << rLocalPath + BOX_ERROR("Failed to upload file: " << rLocalPath << ": Access denied"); } else { - BOX_ERROR("Failed to upload file: " << rLocalPath - << ": caught exception: " << rException.what() + BOX_ERROR("Failed to upload file: " << rLocalPath + << ": caught exception: " << rException.what() << " (" << rException.GetType() << "/" << rException.GetSubType() << ")"); } - } - virtual void NotifyFileUploadServerError( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int type, int subtype) - { - std::ostringstream msgs; - if (type != BackupProtocolError::ErrorType) - { - msgs << "unknown error type " << type; - } - else - { - switch(subtype) - { - case BackupProtocolError::Err_WrongVersion: - msgs << "WrongVersion"; - break; - case BackupProtocolError::Err_NotInRightProtocolPhase: - msgs << "NotInRightProtocolPhase"; - break; - case BackupProtocolError::Err_BadLogin: - msgs << "BadLogin"; - break; - case BackupProtocolError::Err_CannotLockStoreForWriting: - msgs << "CannotLockStoreForWriting"; - break; - case BackupProtocolError::Err_SessionReadOnly: - msgs << "SessionReadOnly"; - break; - case BackupProtocolError::Err_FileDoesNotVerify: - msgs << "FileDoesNotVerify"; - break; - case BackupProtocolError::Err_DoesNotExist: - msgs << "DoesNotExist"; - break; - case BackupProtocolError::Err_DirectoryAlreadyExists: - msgs << "DirectoryAlreadyExists"; - break; - case BackupProtocolError::Err_CannotDeleteRoot: - msgs << "CannotDeleteRoot"; - break; - case BackupProtocolError::Err_TargetNameExists: - msgs << "TargetNameExists"; - break; - case BackupProtocolError::Err_StorageLimitExceeded: - msgs << "StorageLimitExceeded"; - break; - case BackupProtocolError::Err_DiffFromFileDoesNotExist: - msgs << "DiffFromFileDoesNotExist"; - break; - case BackupProtocolError::Err_DoesNotExistInDirectory: - msgs << "DoesNotExistInDirectory"; - break; - case BackupProtocolError::Err_PatchConsistencyError: - msgs << "PatchConsistencyError"; - break; - default: - msgs << "unknown error subtype " << subtype; - } - } - - BOX_ERROR("Failed to upload file: " << rLocalPath - << ": server error: " << msgs.str()); - } - virtual void NotifyFileUploading( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) - { + } + virtual void NotifyFileUploadServerError( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int type, int subtype) + { + BOX_ERROR("Failed to upload file: " << rLocalPath << + ": server error: " << + BackupProtocolError::GetMessage(subtype)); + } + virtual void NotifyFileUploading( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) + { if (mLogAllFileAccess) { BOX_NOTICE("Uploading complete file: " << rLocalPath); - } + } } - virtual void NotifyFileUploadingPatch( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) + virtual void NotifyFileUploadingPatch( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t EstimatedBytesToUpload) { if (mLogAllFileAccess) { - BOX_NOTICE("Uploading patch to file: " << rLocalPath); - } + BOX_NOTICE("Uploading patch to file: " << rLocalPath << + ", estimated upload size = " << + EstimatedBytesToUpload); + } } - virtual void NotifyFileUploadingAttributes( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) + virtual void NotifyFileUploadingAttributes( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath) { if (mLogAllFileAccess) { - BOX_NOTICE("Uploading new file attributes: " << + BOX_NOTICE("Uploading new file attributes: " << rLocalPath); - } + } } - virtual void NotifyFileUploaded( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize, int64_t UploadedSize) + virtual void NotifyFileUploaded( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) { if (mLogAllFileAccess) { - BOX_NOTICE("Uploaded file: " << rLocalPath << ", " - "total size = " << FileSize << ", " + BOX_NOTICE("Uploaded file: " << rLocalPath << + " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << + "): total size = " << FileSize << ", " "uploaded size = " << UploadedSize); } mNumFilesUploaded++; } - virtual void NotifyFileSynchronised( - const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath, - int64_t FileSize) + virtual void NotifyFileSynchronised( + const BackupClientDirectoryRecord* pDirRecord, + const std::string& rLocalPath, + int64_t FileSize) { if (mLogAllFileAccess) { BOX_INFO("Synchronised file: " << rLocalPath); - } + } } virtual void NotifyDirectoryCreated( int64_t ObjectID, @@ -477,7 +472,7 @@ public: { if (mLogAllFileAccess) { - BOX_NOTICE("Created directory: " << rRemotePath << + BOX_NOTICE("Created directory: " << rRemotePath << " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << ")"); } @@ -489,7 +484,7 @@ public: { if (mLogAllFileAccess) { - BOX_NOTICE("Deleted directory: " << rRemotePath << + BOX_NOTICE("Deleted directory: " << rRemotePath << " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << ")"); } @@ -500,7 +495,7 @@ public: { if (mLogAllFileAccess) { - BOX_NOTICE("Deleted file: " << rRemotePath << + BOX_NOTICE("Deleted file: " << rRemotePath << " (ID " << BOX_FORMAT_OBJECTID(ObjectID) << ")"); } @@ -508,19 +503,19 @@ public: virtual void NotifyReadProgress(int64_t readSize, int64_t offset, int64_t length, box_time_t elapsed, box_time_t finish) { - BOX_TRACE("Read " << readSize << " bytes at " << offset << + BOX_TRACE("Read " << readSize << " bytes at " << offset << ", " << (length - offset) << " remain, eta " << BoxTimeToSeconds(finish - elapsed) << "s"); } virtual void NotifyReadProgress(int64_t readSize, int64_t offset, int64_t length) { - BOX_TRACE("Read " << readSize << " bytes at " << offset << + BOX_TRACE("Read " << readSize << " bytes at " << offset << ", " << (length - offset) << " remain"); } virtual void NotifyReadProgress(int64_t readSize, int64_t offset) { - BOX_TRACE("Read " << readSize << " bytes at " << offset << + BOX_TRACE("Read " << readSize << " bytes at " << offset << ", unknown bytes remaining"); } diff --git a/bin/bbackupd/BackupDaemonInterface.h b/lib/bbackupd/BackupDaemonInterface.h index a847b264..d5c47c85 100644 --- a/bin/bbackupd/BackupDaemonInterface.h +++ b/lib/bbackupd/BackupDaemonInterface.h @@ -11,14 +11,8 @@ #define BACKUPDAEMONINTERFACE__H #include <string> -// #include <map> -// #include "BackupClientFileAttributes.h" -// #include "BackupStoreDirectory.h" #include "BoxTime.h" -// #include "MD5Digest.h" -// #include "ReadLoggingStream.h" -// #include "RunStatusProvider.h" class Archive; class BackupClientContext; @@ -122,14 +116,15 @@ class ProgressNotifier const std::string& rLocalPath) = 0; virtual void NotifyFileUploadingPatch( const BackupClientDirectoryRecord* pDirRecord, - const std::string& rLocalPath) = 0; + const std::string& rLocalPath, + int64_t EstimatedBytesToUpload) = 0; virtual void NotifyFileUploadingAttributes( const BackupClientDirectoryRecord* pDirRecord, const std::string& rLocalPath) = 0; virtual void NotifyFileUploaded( const BackupClientDirectoryRecord* pDirRecord, const std::string& rLocalPath, - int64_t FileSize, int64_t UploadedSize) = 0; + int64_t FileSize, int64_t UploadedSize, int64_t ObjectID) = 0; virtual void NotifyFileSynchronised( const BackupClientDirectoryRecord* pDirRecord, const std::string& rLocalPath, diff --git a/bin/bbackupd/Win32BackupService.cpp b/lib/bbackupd/Win32BackupService.cpp index 6d027abf..6d027abf 100644 --- a/bin/bbackupd/Win32BackupService.cpp +++ b/lib/bbackupd/Win32BackupService.cpp diff --git a/bin/bbackupd/Win32BackupService.h b/lib/bbackupd/Win32BackupService.h index e7f077f2..e7f077f2 100644 --- a/bin/bbackupd/Win32BackupService.h +++ b/lib/bbackupd/Win32BackupService.h diff --git a/bin/bbackupd/Win32ServiceFunctions.cpp b/lib/bbackupd/Win32ServiceFunctions.cpp index 2df914a7..2df914a7 100644 --- a/bin/bbackupd/Win32ServiceFunctions.cpp +++ b/lib/bbackupd/Win32ServiceFunctions.cpp diff --git a/bin/bbackupd/Win32ServiceFunctions.h b/lib/bbackupd/Win32ServiceFunctions.h index e04c368f..e04c368f 100644 --- a/bin/bbackupd/Win32ServiceFunctions.h +++ b/lib/bbackupd/Win32ServiceFunctions.h diff --git a/bin/bbackupquery/BackupQueries.cpp b/lib/bbackupquery/BackupQueries.cpp index b8b9525b..bcb1827e 100644 --- a/bin/bbackupquery/BackupQueries.cpp +++ b/lib/bbackupquery/BackupQueries.cpp @@ -24,6 +24,7 @@ #include <dirent.h> #endif +#include <algorithm> #include <cstring> #include <limits> #include <iostream> @@ -71,7 +72,7 @@ // Created: 2003/10/10 // // -------------------------------------------------------------------------- -BackupQueries::BackupQueries(BackupProtocolClient &rConnection, +BackupQueries::BackupQueries(BackupProtocolCallable &rConnection, const Configuration &rConfiguration, bool readWrite) : mReadWrite(readWrite), mrConnection(rConnection), @@ -114,7 +115,9 @@ void BackupQueries::DoCommand(ParsedCommand& rCommand) if(rCommand.mFailed) { - BOX_ERROR("Parse failed"); + BOX_ERROR("Parse failed: unknown command '" << + rCommand.mCmdElements[0] << "' or failed to convert " + "encoding of arguments"); return; } @@ -246,6 +249,19 @@ void BackupQueries::DoCommand(ParsedCommand& rCommand) } } +#define LIST_OPTION_TIMES_ATTRIBS 'a' +#define LIST_OPTION_SORT_NO_DIRS_FIRST 'D' +#define LIST_OPTION_NOFLAGS 'F' +#define LIST_OPTION_DISPLAY_HASH 'h' +#define LIST_OPTION_SORT_ID 'i' +#define LIST_OPTION_NOOBJECTID 'I' +#define LIST_OPTION_SORT_REVERSE 'r' +#define LIST_OPTION_RECURSIVE 'R' +#define LIST_OPTION_SIZEINBLOCKS 's' +#define LIST_OPTION_SORT_SIZE 'S' +#define LIST_OPTION_TIMES_LOCAL 't' +#define LIST_OPTION_TIMES_UTC 'T' +#define LIST_OPTION_SORT_NONE 'U' // -------------------------------------------------------------------------- // @@ -257,15 +273,6 @@ void BackupQueries::DoCommand(ParsedCommand& rCommand) // -------------------------------------------------------------------------- void BackupQueries::CommandList(const std::vector<std::string> &args, const bool *opts) { - #define LIST_OPTION_RECURSIVE 'r' - #define LIST_OPTION_NOOBJECTID 'I' - #define LIST_OPTION_NOFLAGS 'F' - #define LIST_OPTION_TIMES_LOCAL 't' - #define LIST_OPTION_TIMES_UTC 'T' - #define LIST_OPTION_TIMES_ATTRIBS 'a' - #define LIST_OPTION_SIZEINBLOCKS 's' - #define LIST_OPTION_DISPLAY_HASH 'h' - // default to using the current directory int64_t rootDir = GetCurrentDirectoryID(); @@ -284,8 +291,8 @@ void BackupQueries::CommandList(const std::vector<std::string> &args, const bool #endif // Attempt to find the directory - rootDir = FindDirectoryObjectID(storeDirEncoded, - opts[LIST_OPTION_ALLOWOLD], + rootDir = FindDirectoryObjectID(storeDirEncoded, + opts[LIST_OPTION_ALLOWOLD], opts[LIST_OPTION_ALLOWDELETED]); if(rootDir == 0) @@ -348,6 +355,75 @@ static std::string GetTimeString(BackupStoreDirectory::Entry& en, return out.str(); } +/* We need a way to pass options to sort functions for sorting. The algorithm + * doesn't seem to provide a way to do this, so I'm using a global variable. + * Which is not thread safe, but we don't currently use threads so that should + * be OK. Do not use threads without checking! + */ +const bool *gThreadUnsafeOptions; + +int DirsFirst(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + if (a->IsDir() && !b->IsDir()) + { + return -1; // a < b + } + else if (!a->IsDir() && b->IsDir()) + { + return 1; // b > a + } + else + { + return 0; // continue comparison + } +} + +#define MAYBE_DIRS_FIRST(a, b) \ + if (!gThreadUnsafeOptions[LIST_OPTION_SORT_NO_DIRS_FIRST]) \ + { \ + int result = DirsFirst(a, b); \ + if (result < 0) return true; /* a < b */ \ + else if (result > 0) return false; /* a > b */ \ + /* else: fall through */ \ + } + +#define MAYBE_REVERSE(result) \ + (result != gThreadUnsafeOptions[LIST_OPTION_SORT_REVERSE]) +// result is false, opts[reverse] is false => return false +// result is false, opts[reverse] is true => return true +// result is true, opts[reverse] is false => return true +// result is true, opts[reverse] is true => return false +// this is logical XOR, for which the boolean operator is !=. + +bool SortById(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + bool result = (a->GetObjectID() < b->GetObjectID()); + return MAYBE_REVERSE(result); +} + +bool SortBySize(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + bool result = (a->GetSizeInBlocks() < b->GetSizeInBlocks()); + return MAYBE_REVERSE(result); +} + +bool SortByName(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b) +{ + MAYBE_DIRS_FIRST(a, b); + BackupStoreFilenameClear afc(a->GetName()); + BackupStoreFilenameClear bfc(b->GetName()); + std::string an = afc.GetClearFilename(); + std::string bn = bfc.GetClearFilename(); + bool result = (an < bn); + return MAYBE_REVERSE(result); +} + // -------------------------------------------------------------------------- // // Function @@ -397,11 +473,51 @@ void BackupQueries::List(int64_t DirID, const std::string &rListRoot, std::auto_ptr<IOStream> dirstream(mrConnection.ReceiveStream()); dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); - // Then... display everything + // Store entry pointers in a std::vector for sorting BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; + std::vector<BackupStoreDirectory::Entry*> sorted_entries; while((en = i.Next()) != 0) { + sorted_entries.push_back(en); + } + + // Typedef to avoid mind-bending while dealing with pointers to functions. + typedef bool (EntryComparator_t)(BackupStoreDirectory::Entry* a, + BackupStoreDirectory::Entry* b); + // Default is no comparator, i.e. no sorting. + EntryComparator_t* pComparator = NULL; + + if (opts[LIST_OPTION_SORT_ID]) + { + pComparator = &SortById; + } + else if (opts[LIST_OPTION_SORT_SIZE]) + { + pComparator = &SortBySize; + } + else if (opts[LIST_OPTION_SORT_NONE]) + { + // do nothing + } + else // sort by name + { + pComparator = &SortByName; + } + + if (pComparator != NULL) + { + gThreadUnsafeOptions = opts; + sort(sorted_entries.begin(), sorted_entries.end(), + pComparator); + gThreadUnsafeOptions = NULL; + } + + for (std::vector<BackupStoreDirectory::Entry*>::const_iterator + i = sorted_entries.begin(); + i != sorted_entries.end(); i++) + { + en = *i; std::ostringstream buf; // Display this entry @@ -873,21 +989,27 @@ void BackupQueries::CommandGetObject(const std::vector<std::string> &args, const { // Request object std::auto_ptr<BackupProtocolSuccess> getobj(mrConnection.QueryGetObject(id)); - if(getobj->GetObjectID() != BackupProtocolGetObject::NoObject) - { - // Stream that object out to the file - std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); - objectStream->CopyStreamTo(out); + + // Stream that object out to the file + std::auto_ptr<IOStream> objectStream(mrConnection.ReceiveStream()); + objectStream->CopyStreamTo(out); - BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(id) << - " fetched successfully."); - } - else + BOX_INFO("Object ID " << BOX_FORMAT_OBJECTID(id) << + " fetched successfully."); + } + catch(ConnectionException &e) + { + if(mrConnection.GetLastErrorType() == BackupProtocolError::Err_DoesNotExist) { BOX_ERROR("Object ID " << BOX_FORMAT_OBJECTID(id) << " does not exist on store."); ::unlink(args[1].c_str()); } + else + { + BOX_ERROR("Error occured fetching object."); + ::unlink(args[1].c_str()); + } } catch(...) { @@ -1965,11 +2087,9 @@ void BackupQueries::CommandRestore(const std::vector<std::string> &args, const b // At TRACE level, we print a line for each file and // directory, so we don't need dots. - bool printDots = ! Logging::IsEnabled(Log::TRACE); - result = BackupClientRestore(mrConnection, dirID, storeDirEncoded.c_str(), localName.c_str(), - printDots /* print progress dots */, restoreDeleted, + true /* print progress dots */, restoreDeleted, false /* don't undelete after restore! */, opts['r'] /* resume? */, opts['f'] /* force continue after errors */); diff --git a/bin/bbackupquery/BackupQueries.h b/lib/bbackupquery/BackupQueries.h index 62ff231d..96df34f5 100644 --- a/bin/bbackupquery/BackupQueries.h +++ b/lib/bbackupquery/BackupQueries.h @@ -18,7 +18,7 @@ #include "BoxBackupCompareParams.h" #include "BackupStoreDirectory.h" -class BackupProtocolClient; +class BackupProtocolCallable; class Configuration; class ExcludeList; @@ -55,7 +55,7 @@ struct QueryCommandSpecification; class BackupQueries { public: - BackupQueries(BackupProtocolClient &rConnection, + BackupQueries(BackupProtocolCallable &rConnection, const Configuration &rConfiguration, bool readWrite); ~BackupQueries(); @@ -77,6 +77,7 @@ public: ParsedCommand(const std::string& Command, bool isFromCommandLine); bool IsEmpty() { return mCmdElements.empty(); } + bool IsFailed() { return mFailed; } }; void DoCommand(ParsedCommand& rCommand); @@ -271,6 +272,7 @@ public: virtual void NotifyDirComparing(const std::string& rLocalPath, const std::string& rRemotePath) { + BOX_INFO("Comparing directory: " << rLocalPath); } virtual void NotifyDirCompared( @@ -302,6 +304,7 @@ public: virtual void NotifyFileComparing(const std::string& rLocalPath, const std::string& rRemotePath) { + BOX_TRACE("Comparing file: " << rLocalPath); } virtual void NotifyFileCompared(const std::string& rLocalPath, @@ -393,7 +396,7 @@ private: private: bool mReadWrite; - BackupProtocolClient &mrConnection; + BackupProtocolCallable &mrConnection; const Configuration &mrConfiguration; bool mQuitNow; std::vector<std::pair<std::string, int64_t> > mDirStack; @@ -404,14 +407,14 @@ private: typedef std::vector<std::string> (*CompletionHandler) (BackupQueries::ParsedCommand& rCommand, const std::string& prefix, - BackupProtocolClient& rProtocol, const Configuration& rConfig, + BackupProtocolCallable& rProtocol, const Configuration& rConfig, BackupQueries& rQueries); std::vector<std::string> CompleteCommand(BackupQueries::ParsedCommand& rCommand, - const std::string& prefix, BackupProtocolClient& rProtocol, + const std::string& prefix, BackupProtocolCallable& rProtocol, const Configuration& rConfig, BackupQueries& rQueries); std::vector<std::string> CompleteOptions(BackupQueries::ParsedCommand& rCommand, - const std::string& prefix, BackupProtocolClient& rProtocol, + const std::string& prefix, BackupProtocolCallable& rProtocol, const Configuration& rConfig, BackupQueries& rQueries); #define MAX_COMPLETION_HANDLERS 4 diff --git a/bin/bbackupquery/BoxBackupCompareParams.h b/lib/bbackupquery/BoxBackupCompareParams.h index 655df947..655df947 100644 --- a/bin/bbackupquery/BoxBackupCompareParams.h +++ b/lib/bbackupquery/BoxBackupCompareParams.h diff --git a/bin/bbackupquery/CommandCompletion.cpp b/lib/bbackupquery/CommandCompletion.cpp index 93c4d3fd..761fc97e 100644 --- a/bin/bbackupquery/CommandCompletion.cpp +++ b/lib/bbackupquery/CommandCompletion.cpp @@ -45,7 +45,7 @@ std::vector<std::string> Complete ## name( \ BackupQueries::ParsedCommand& rCommand, \ const std::string& prefix, \ - BackupProtocolClient& rProtocol, const Configuration& rConfig, \ + BackupProtocolCallable& rProtocol, const Configuration& rConfig, \ BackupQueries& rQueries) \ { \ std::vector<std::string> completions; \ @@ -170,7 +170,7 @@ int16_t GetExcludeFlags(BackupQueries::ParsedCommand& rCommand) std::vector<std::string> CompleteRemoteFileOrDirectory( BackupQueries::ParsedCommand& rCommand, - const std::string& prefix, BackupProtocolClient& rProtocol, + const std::string& prefix, BackupProtocolCallable& rProtocol, BackupQueries& rQueries, int16_t includeFlags) { std::vector<std::string> completions; @@ -428,7 +428,7 @@ QueryCommandSpecification commands[] = { { "quit", "", Command_Quit, {} }, { "exit", "", Command_Quit, {} }, - { "list", "rodIFtTash", Command_List, {CompleteRemoteDir} }, + { "list", "adDFhiIorRsStTU", Command_List, {CompleteRemoteDir} }, { "pwd", "", Command_pwd, {} }, { "cd", "od", Command_cd, {CompleteRemoteDir} }, { "lcd", "", Command_lcd, {CompleteLocalDir} }, @@ -510,8 +510,10 @@ BackupQueries::ParsedCommand::ParsedCommand(const std::string& Command, { inQuoted = true; } - // Start of options? - else if(currentArg.empty() && *c == '-') + // Start of options? You can't have options if there's no + // command before them, so treat the options as a command (which + // doesn't exist, so it will fail to parse) in that case. + else if(currentArg.empty() && *c == '-' && !mCmdElements.empty()) { mInOptions = true; } diff --git a/bin/bbackupquery/documentation.txt b/lib/bbackupquery/Documentation.txt index 214fe218..b16a6f7c 100644 --- a/bin/bbackupquery/documentation.txt +++ b/lib/bbackupquery/Documentation.txt @@ -29,11 +29,11 @@ All directory names relative to a "current" directory, or from root if they start with '/'. The initial directory is always the root directory. -> list [options] [directory-name] +> ls [options] [directory-name] List contents of current directory, or specified directory. - -r -- recursively list all files + -R -- recursively list all files -d -- list deleted files/directories -o -- list old versions of files/directories -I -- don't display object ID @@ -45,13 +45,17 @@ start with '/'. The initial directory is always the root directory. -s -- show file size in blocks used on server (only very approximate indication of size locally) -h -- show file attributes hash + -D -- sort directories together with files (not dirs first) + -i -- sort by object ID (the old default) + -S -- sort by object size in blocks + -U -- don't sort the results (new default is to sort by name) -ls can be used as an alias. +list can be used as an alias. < -> ls +> list - Alias for 'list'. Type 'help list' for options. + Alias for 'ls'. Type 'help ls' for options. < > cd [options] <directory-name> diff --git a/lib/bbackupquery/Makefile.extra b/lib/bbackupquery/Makefile.extra new file mode 100644 index 00000000..5d37c09f --- /dev/null +++ b/lib/bbackupquery/Makefile.extra @@ -0,0 +1,6 @@ + +# AUTOGEN SEEDING +autogen_Documentation.cpp: makedocumentation.pl Documentation.txt + $(_PERL) makedocumentation.pl + + diff --git a/bin/bbackupquery/makedocumentation.pl.in b/lib/bbackupquery/makedocumentation.pl.in index 530c4ff6..503ac9c8 100755 --- a/bin/bbackupquery/makedocumentation.pl.in +++ b/lib/bbackupquery/makedocumentation.pl.in @@ -3,7 +3,7 @@ use strict; print "Creating built-in documentation for bbackupquery...\n"; -open DOC,"documentation.txt" or die "Can't open documentation.txt file"; +open DOC, "Documentation.txt" or die "Can't open Documentation.txt file: $!"; my $section; my %help; my @in_order; diff --git a/bin/bbstored/BBStoreDHousekeeping.cpp b/lib/bbstored/BBStoreDHousekeeping.cpp index 86d6409c..86d6409c 100644 --- a/bin/bbstored/BBStoreDHousekeeping.cpp +++ b/lib/bbstored/BBStoreDHousekeeping.cpp diff --git a/bin/bbstored/BackupStoreDaemon.cpp b/lib/bbstored/BackupStoreDaemon.cpp index 70ca1d67..8fddf125 100644 --- a/bin/bbstored/BackupStoreDaemon.cpp +++ b/lib/bbstored/BackupStoreDaemon.cpp @@ -272,11 +272,11 @@ void BackupStoreDaemon::Run() // Created: 2003/08/20 // // -------------------------------------------------------------------------- -void BackupStoreDaemon::Connection(SocketStreamTLS &rStream) +void BackupStoreDaemon::Connection(std::auto_ptr<SocketStreamTLS> apStream) { try { - Connection2(rStream); + Connection2(apStream); } catch(BoxException &e) { @@ -304,10 +304,10 @@ void BackupStoreDaemon::Connection(SocketStreamTLS &rStream) // Created: 2006/11/12 // // -------------------------------------------------------------------------- -void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) +void BackupStoreDaemon::Connection2(std::auto_ptr<SocketStreamTLS> apStream) { // Get the common name from the certificate - std::string clientCommonName(rStream.GetPeerCommonName()); + std::string clientCommonName(apStream->GetPeerCommonName()); // Log the name BOX_INFO("Client certificate CN: " << clientCommonName); @@ -329,7 +329,7 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) Logging::Tagger tagWithClientID(tag.str()); // Create a context, using this ID - BackupStoreContext context(id, *this, GetConnectionDetails()); + BackupStoreContext context(id, this, GetConnectionDetails()); if (mpTestHook) { @@ -346,7 +346,8 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) } // Handle a connection with the backup protocol - BackupProtocolServer server(rStream); + std::auto_ptr<SocketStream> apPlainStream(apStream); + BackupProtocolServer server(apPlainStream); server.SetLogToSysLog(mExtendedLogging); server.SetTimeout(BACKUP_STORE_TIMEOUT); try @@ -355,22 +356,22 @@ void BackupStoreDaemon::Connection2(SocketStreamTLS &rStream) } catch(...) { - LogConnectionStats(id, context.GetAccountName(), rStream); + LogConnectionStats(id, context.GetAccountName(), server); throw; } - LogConnectionStats(id, context.GetAccountName(), rStream); + LogConnectionStats(id, context.GetAccountName(), server); context.CleanUp(); } void BackupStoreDaemon::LogConnectionStats(uint32_t accountId, - const std::string& accountName, const SocketStreamTLS &s) + const std::string& accountName, const BackupProtocolServer &server) { // Log the amount of data transferred BOX_NOTICE("Connection statistics for " << BOX_FORMAT_ACCOUNT(accountId) << " " "(name=" << accountName << "):" - " IN=" << s.GetBytesRead() << - " OUT=" << s.GetBytesWritten() << - " NET_IN=" << (s.GetBytesRead() - s.GetBytesWritten()) << - " TOTAL=" << (s.GetBytesRead() + s.GetBytesWritten())); + " IN=" << server.GetBytesRead() << + " OUT=" << server.GetBytesWritten() << + " NET_IN=" << (server.GetBytesRead() - server.GetBytesWritten()) << + " TOTAL=" << (server.GetBytesRead() + server.GetBytesWritten())); } diff --git a/bin/bbstored/BackupStoreDaemon.h b/lib/bbstored/BackupStoreDaemon.h index ce538477..a2dab5e5 100644 --- a/bin/bbstored/BackupStoreDaemon.h +++ b/lib/bbstored/BackupStoreDaemon.h @@ -52,8 +52,8 @@ protected: virtual void Run(); - virtual void Connection(SocketStreamTLS &rStream); - void Connection2(SocketStreamTLS &rStream); + virtual void Connection(std::auto_ptr<SocketStreamTLS> apStream); + void Connection2(std::auto_ptr<SocketStreamTLS> apStream); virtual const char *DaemonName() const; virtual std::string DaemonBanner() const; @@ -64,7 +64,7 @@ protected: void HousekeepingProcess(); void LogConnectionStats(uint32_t accountId, - const std::string& accountName, const SocketStreamTLS &s); + const std::string& accountName, const BackupProtocolServer &server); public: // HousekeepingInterface implementation diff --git a/lib/common/Archive.h b/lib/common/Archive.h index 6d5ce88b..2b27b303 100644 --- a/lib/common/Archive.h +++ b/lib/common/Archive.h @@ -26,7 +26,7 @@ class Archive { public: Archive(IOStream &Stream, int Timeout) - : mrStream(Stream) + : mrStream(Stream) { mTimeout = Timeout; } @@ -38,6 +38,7 @@ public: ~Archive() { } + // // // @@ -46,21 +47,29 @@ public: Write((int) Item); } void WriteExact(uint32_t Item) { Write((int)Item); } + // TODO FIXME: use of "int" here is dangerous and deprecated. It can lead to + // incompatible serialisation on non-32-bit machines. Passing anything other + // than one of the specifically supported fixed size types should be forbidden. void Write(int Item) { int32_t privItem = htonl(Item); - mrStream.Write(&privItem, sizeof(privItem)); + mrStream.Write(&privItem, sizeof(privItem), mTimeout); } void Write(int64_t Item) { int64_t privItem = box_hton64(Item); - mrStream.Write(&privItem, sizeof(privItem)); + mrStream.Write(&privItem, sizeof(privItem), mTimeout); + } + void WriteInt16(uint16_t Item) + { + uint16_t privItem = htons(Item); + mrStream.Write(&privItem, sizeof(privItem), mTimeout); } void WriteExact(uint64_t Item) { Write(Item); } void Write(uint64_t Item) { uint64_t privItem = box_hton64(Item); - mrStream.Write(&privItem, sizeof(privItem)); + mrStream.Write(&privItem, sizeof(privItem), mTimeout); } void Write(uint8_t Item) { @@ -71,7 +80,7 @@ public: { int size = Item.size(); Write(size); - mrStream.Write(Item.c_str(), size); + mrStream.Write(Item.c_str(), size, mTimeout); } // // @@ -100,17 +109,29 @@ public: void Read(int &rItemOut) { int32_t privItem; - if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) + if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), + 0 /* not interested in bytes read if this fails */, + mTimeout)) { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead); } rItemOut = ntohl(privItem); } + void ReadFullBuffer(void* Buffer, size_t Size) + { + if(!mrStream.ReadFullBuffer(Buffer, Size, + 0 /* not interested in bytes read if this fails */, + mTimeout)) + { + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead); + } + } void ReadIfPresent(int &rItemOut, int ValueIfNotPresent) { int32_t privItem; int bytesRead; - if(mrStream.ReadFullBuffer(&privItem, sizeof(privItem), &bytesRead)) + if(mrStream.ReadFullBuffer(&privItem, sizeof(privItem), + &bytesRead, mTimeout)) { rItemOut = ntohl(privItem); } @@ -122,48 +143,70 @@ public: else { // bad number of remaining bytes - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) + THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead); } } void Read(int64_t &rItemOut) { int64_t privItem; - if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) - { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) - } + ReadFullBuffer(&privItem, sizeof(privItem)); rItemOut = box_ntoh64(privItem); } void ReadExact(uint64_t &rItemOut) { Read(rItemOut); } void Read(uint64_t &rItemOut) { uint64_t privItem; - if(!mrStream.ReadFullBuffer(&privItem, sizeof(privItem), 0 /* not interested in bytes read if this fails */)) - { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) - } + ReadFullBuffer(&privItem, sizeof(privItem)); rItemOut = box_ntoh64(privItem); } + void ReadInt16(uint16_t &rItemOut) + { + uint16_t privItem; + ReadFullBuffer(&privItem, sizeof(privItem)); + rItemOut = ntohs(privItem); + } void Read(uint8_t &rItemOut) { int privItem; Read(privItem); rItemOut = privItem; } + void ReadIfPresent(std::string &rItemOut, const std::string& ValueIfNotPresent) + { + ReadString(rItemOut, &ValueIfNotPresent); + } void Read(std::string &rItemOut) { + ReadString(rItemOut, NULL); + } +private: + void ReadString(std::string &rItemOut, const std::string* pValueIfNotPresent) + { int size; - Read(size); + int bytesRead; + if(!mrStream.ReadFullBuffer(&size, sizeof(size), &bytesRead, mTimeout)) + { + if(bytesRead == 0 && pValueIfNotPresent != NULL) + { + // item is simply not present + rItemOut = *pValueIfNotPresent; + return; + } + else + { + // bad number of remaining bytes + THROW_EXCEPTION(CommonException, + ArchiveBlockIncompleteRead) + } + } + size = ntohl(size); // Assume most strings are relatively small char buf[256]; if(size < (int) sizeof(buf)) { // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us - if(!mrStream.ReadFullBuffer(buf, size, 0 /* not interested in bytes read if this fails */, mTimeout)) - { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) - } + ReadFullBuffer(buf, size); // assign to this string, storing the header and the extra payload rItemOut.assign(buf, size); } @@ -174,10 +217,7 @@ public: char *ppayload = dataB; // Fetch rest of pPayload, relying on the Protocol to error on stupidly large sizes for us - if(!mrStream.ReadFullBuffer(ppayload, size, 0 /* not interested in bytes read if this fails */, mTimeout)) - { - THROW_EXCEPTION(CommonException, ArchiveBlockIncompleteRead) - } + ReadFullBuffer(ppayload, size); // assign to this string, storing the header and the extra pPayload rItemOut.assign(ppayload, size); } diff --git a/lib/common/BannerText.h b/lib/common/BannerText.h index f0772c9c..9ca0c11c 100644 --- a/lib/common/BannerText.h +++ b/lib/common/BannerText.h @@ -16,7 +16,7 @@ #define BANNER_TEXT(UtilityName) \ "Box " UtilityName " v" BOX_VERSION ", (c) Ben Summers and " \ - "contributors 2003-2011" + "contributors 2003-2014" #endif // BANNERTEXT__H diff --git a/lib/common/Box.h b/lib/common/Box.h index 316f4364..8ce2a625 100644 --- a/lib/common/Box.h +++ b/lib/common/Box.h @@ -116,16 +116,8 @@ { \ if((!HideExceptionMessageGuard::ExceptionsHidden() \ && !HideSpecificExceptionGuard::IsHidden( \ - type::ExceptionType, type::subtype)) \ - || Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ + type::ExceptionType, type::subtype))) \ { \ - std::auto_ptr<Logging::Guard> guard; \ - \ - if(Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ - { \ - guard.reset(new Logging::Guard(Log::EVERYTHING)); \ - } \ - \ OPTIONAL_DO_BACKTRACE \ BOX_WARNING("Exception thrown: " \ #type "(" #subtype ") " \ @@ -140,16 +132,8 @@ _box_throw_line << message; \ if((!HideExceptionMessageGuard::ExceptionsHidden() \ && !HideSpecificExceptionGuard::IsHidden( \ - type::ExceptionType, type::subtype)) \ - || Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ + type::ExceptionType, type::subtype))) \ { \ - std::auto_ptr<Logging::Guard> guard; \ - \ - if(Logging::Guard::IsGuardingFrom(Log::EVERYTHING)) \ - { \ - guard.reset(new Logging::Guard(Log::EVERYTHING)); \ - } \ - \ OPTIONAL_DO_BACKTRACE \ BOX_WARNING("Exception thrown: " \ #type "(" #subtype ") (" << \ diff --git a/lib/common/BoxConfig-MSVC.h b/lib/common/BoxConfig-MSVC.h index eeb25d2e..2ec2edd7 100644 --- a/lib/common/BoxConfig-MSVC.h +++ b/lib/common/BoxConfig-MSVC.h @@ -386,9 +386,6 @@ /* Define to empty if `const' does not conform to ANSI C. */ /* #undef const */ -/* Define to `int' if <sys/types.h> doesn't define. */ -#define gid_t int - /* Define to `int' if <sys/types.h> does not define. */ /* #undef mode_t */ @@ -400,6 +397,3 @@ /* Define to `unsigned' if <sys/types.h> does not define. */ /* #undef size_t */ - -/* Define to `int' if <sys/types.h> doesn't define. */ -#define uid_t int diff --git a/lib/common/BoxPlatform.h b/lib/common/BoxPlatform.h index 53a967e8..f7c74bfc 100644 --- a/lib/common/BoxPlatform.h +++ b/lib/common/BoxPlatform.h @@ -21,11 +21,13 @@ #define PLATFORM_DEV_NULL "/dev/null" -#ifdef _MSC_VER -#include "BoxConfig-MSVC.h" -#define NEED_BOX_VERSION_H +#if defined BOX_CMAKE +# include "BoxConfig.cmake.h" +#elif defined _MSC_VER +# include "BoxConfig-MSVC.h" +# define NEED_BOX_VERSION_H #else -#include "BoxConfig.h" +# include "BoxConfig.h" #endif #ifdef WIN32 @@ -40,9 +42,12 @@ #endif #endif +#include "emu.h" + #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif + #ifdef HAVE_INTTYPES_H #include <inttypes.h> #else @@ -93,66 +98,19 @@ #endif // Handle differing xattr APIs -#ifdef HAVE_SYS_XATTR_H - #if !defined(HAVE_LLISTXATTR) && defined(HAVE_LISTXATTR) && HAVE_DECL_XATTR_NOFOLLOW - #define llistxattr(a,b,c) listxattr(a,b,c,XATTR_NOFOLLOW) - #endif - #if !defined(HAVE_LGETXATTR) && defined(HAVE_GETXATTR) && HAVE_DECL_XATTR_NOFOLLOW - #define lgetxattr(a,b,c,d) getxattr(a,b,c,d,0,XATTR_NOFOLLOW) - #endif - #if !defined(HAVE_LSETXATTR) && defined(HAVE_SETXATTR) && HAVE_DECL_XATTR_NOFOLLOW - #define lsetxattr(a,b,c,d,e) setxattr(a,b,c,d,0,(e)|XATTR_NOFOLLOW) - #endif +#if !defined(HAVE_LLISTXATTR) && defined(HAVE_LISTXATTR) && HAVE_DECL_XATTR_NOFOLLOW + #define llistxattr(a,b,c) listxattr(a,b,c,XATTR_NOFOLLOW) + #define HAVE_LLISTXATTR #endif -#if defined WIN32 && !defined __MINGW32__ - typedef __int8 int8_t; - typedef __int16 int16_t; - typedef __int32 int32_t; - typedef __int64 int64_t; - - typedef unsigned __int8 u_int8_t; - typedef unsigned __int16 u_int16_t; - typedef unsigned __int32 u_int32_t; - typedef unsigned __int64 u_int64_t; - - #define HAVE_U_INT8_T - #define HAVE_U_INT16_T - #define HAVE_U_INT32_T - #define HAVE_U_INT64_T -#endif // WIN32 && !__MINGW32__ - -// Define missing types -#ifndef HAVE_UINT8_T - typedef u_int8_t uint8_t; +#if !defined(HAVE_LGETXATTR) && defined(HAVE_GETXATTR) && HAVE_DECL_XATTR_NOFOLLOW + #define lgetxattr(a,b,c,d) getxattr(a,b,c,d,0,XATTR_NOFOLLOW) + #define HAVE_LGETXATTR #endif -#ifndef HAVE_UINT16_T - typedef u_int16_t uint16_t; -#endif - -#ifndef HAVE_UINT32_T - typedef u_int32_t uint32_t; -#endif - -#ifndef HAVE_UINT64_T - typedef u_int64_t uint64_t; -#endif - -#ifndef HAVE_U_INT8_T - typedef uint8_t u_int8_t; -#endif - -#ifndef HAVE_U_INT16_T - typedef uint16_t u_int16_t; -#endif - -#ifndef HAVE_U_INT32_T - typedef uint32_t u_int32_t; -#endif - -#ifndef HAVE_U_INT64_T - typedef uint64_t u_int64_t; +#if !defined(HAVE_LSETXATTR) && defined(HAVE_SETXATTR) && HAVE_DECL_XATTR_NOFOLLOW + #define lsetxattr(a,b,c,d,e) setxattr(a,b,c,d,0,(e)|XATTR_NOFOLLOW) + #define HAVE_LSETXATTR #endif #if !HAVE_DECL_INFTIM @@ -173,7 +131,7 @@ #endif #ifdef WIN32 - typedef u_int64_t InodeRefType; + typedef uint64_t InodeRefType; #else typedef ino_t InodeRefType; #endif @@ -182,8 +140,6 @@ #define WIN32_LEAN_AND_MEAN #endif -#include "emu.h" - #ifdef WIN32 #define INVALID_FILE INVALID_HANDLE_VALUE typedef HANDLE tOSFileHandle; diff --git a/lib/common/BoxTime.cpp b/lib/common/BoxTime.cpp index f62b1c35..77daae6d 100644 --- a/lib/common/BoxTime.cpp +++ b/lib/common/BoxTime.cpp @@ -35,21 +35,30 @@ // -------------------------------------------------------------------------- box_time_t GetCurrentBoxTime() { - #ifdef HAVE_GETTIMEOFDAY - struct timeval tv; - if (gettimeofday(&tv, NULL) != 0) - { - BOX_LOG_SYS_ERROR("Failed to gettimeofday(), " - "dropping precision"); - } - else - { - box_time_t timeNow = (tv.tv_sec * MICRO_SEC_IN_SEC_LL) - + tv.tv_usec; - return timeNow; - } - #endif - +#ifdef HAVE_GETTIMEOFDAY + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) + { + BOX_LOG_SYS_ERROR("Failed to gettimeofday(), " + "dropping precision"); + } + else + { + box_time_t time_now = (tv.tv_sec * MICRO_SEC_IN_SEC_LL) + tv.tv_usec; + return time_now; + } +#elif WIN32 + // There's no Win32 API function that returns the current time as a UNIX timestamp with + // sub-second precision. So we use time(0) and add the fractional part from + // GetSystemTime() in the hope that the difference between these two (if any) is a whole + // number of seconds. + box_time_t time_now = SecondsToBoxTime(time(0)); + SYSTEMTIME system_time; + GetSystemTime(&system_time); + time_now += MilliSecondsToBoxTime(system_time.wMilliseconds); + return time_now; +#endif + return SecondsToBoxTime(time(0)); } @@ -83,7 +92,7 @@ std::string FormatTime(box_time_t time, bool includeDate, bool showMicros) if (showMicros) { - buf << "." << std::setw(6) << micros; + buf << "." << std::setw(3) << (int)(micros / 1000); } } else @@ -108,8 +117,7 @@ void ShortSleep(box_time_t duration, bool logDuration) { if(logDuration) { - BOX_TRACE("Sleeping for " << BoxTimeToMicroSeconds(duration) << - " microseconds"); + BOX_TRACE("Sleeping for " << BOX_FORMAT_MICROSECONDS(duration)); } #ifdef WIN32 @@ -118,7 +126,9 @@ void ShortSleep(box_time_t duration, bool logDuration) struct timespec ts; memset(&ts, 0, sizeof(ts)); ts.tv_sec = duration / MICRO_SEC_IN_SEC; - ts.tv_nsec = duration % MICRO_SEC_IN_SEC; + ts.tv_nsec = (duration % MICRO_SEC_IN_SEC) * 1000; + + box_time_t start_time = GetCurrentBoxTime(); while (nanosleep(&ts, &ts) == -1 && errno == EINTR) { @@ -140,6 +150,10 @@ void ShortSleep(box_time_t duration, bool logDuration) BOX_TRACE("nanosleep interrupted with " << remain_ns << " nanosecs remaining, sleeping again"); } + + box_time_t sleep_time = GetCurrentBoxTime() - start_time; + BOX_TRACE("Actually slept for " << BOX_FORMAT_MICROSECONDS(sleep_time) << + ", was aiming for " << BOX_FORMAT_MICROSECONDS(duration)); #endif } diff --git a/lib/common/BoxTime.h b/lib/common/BoxTime.h index 3108d809..6afaada3 100644 --- a/lib/common/BoxTime.h +++ b/lib/common/BoxTime.h @@ -10,8 +10,8 @@ #ifndef BOXTIME__H #define BOXTIME__H -// Time is presented as an unsigned 64 bit integer, in microseconds -typedef int64_t box_time_t; +// Time is presented as a signed 64 bit integer, in microseconds +typedef int64_t box_time_t; #define NANO_SEC_IN_SEC (1000000000LL) #define NANO_SEC_IN_USEC (1000) diff --git a/lib/common/BufferedStream.cpp b/lib/common/BufferedStream.cpp index b58253f3..847cf66c 100644 --- a/lib/common/BufferedStream.cpp +++ b/lib/common/BufferedStream.cpp @@ -96,7 +96,7 @@ IOStream::pos_type BufferedStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void BufferedStream::Write(const void *pBuffer, int NBytes) +void BufferedStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, NotSupported); } @@ -189,7 +189,9 @@ void BufferedStream::Close() // -------------------------------------------------------------------------- bool BufferedStream::StreamDataLeft() { - return mrSource.StreamDataLeft(); + // Return true if either the source has data left to read, or we have + // buffered data still to be read. + return mrSource.StreamDataLeft() || (mBufferPosition < mBufferSize); } // -------------------------------------------------------------------------- diff --git a/lib/common/BufferedStream.h b/lib/common/BufferedStream.h index 079c482a..3984aceb 100644 --- a/lib/common/BufferedStream.h +++ b/lib/common/BufferedStream.h @@ -25,7 +25,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); @@ -33,6 +34,10 @@ public: virtual bool StreamDataLeft(); virtual bool StreamClosed(); + virtual std::string ToString() const + { + return std::string("Buffered ") + mrSource.ToString(); + } private: BufferedStream(const BufferedStream &rToCopy) : mrSource(rToCopy.mrSource) { /* do not call */ } diff --git a/lib/common/BufferedWriteStream.cpp b/lib/common/BufferedWriteStream.cpp index 797be00d..8fbabe9b 100644 --- a/lib/common/BufferedWriteStream.cpp +++ b/lib/common/BufferedWriteStream.cpp @@ -64,7 +64,7 @@ IOStream::pos_type BufferedWriteStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void BufferedWriteStream::Write(const void *pBuffer, int NBytes) +void BufferedWriteStream::Write(const void *pBuffer, int NBytes, int Timeout) { int numBytesRemain = NBytes; diff --git a/lib/common/BufferedWriteStream.h b/lib/common/BufferedWriteStream.h index 7a1c8c17..5f6d5f19 100644 --- a/lib/common/BufferedWriteStream.h +++ b/lib/common/BufferedWriteStream.h @@ -25,7 +25,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Flush(int Timeout = IOStream::TimeOutInfinite); diff --git a/lib/common/CollectInBufferStream.cpp b/lib/common/CollectInBufferStream.cpp index 90e2e7bc..47b271f0 100644 --- a/lib/common/CollectInBufferStream.cpp +++ b/lib/common/CollectInBufferStream.cpp @@ -98,7 +98,7 @@ IOStream::pos_type CollectInBufferStream::BytesLeftToRead() // Created: 2003/08/26 // // -------------------------------------------------------------------------- -void CollectInBufferStream::Write(const void *pBuffer, int NBytes) +void CollectInBufferStream::Write(const void *pBuffer, int NBytes, int Timeout) { if(mInWritePhase != true) { THROW_EXCEPTION(CommonException, CollectInBufferStreamNotInCorrectPhase) } diff --git a/lib/common/CollectInBufferStream.h b/lib/common/CollectInBufferStream.h index d73af8db..297d2851 100644 --- a/lib/common/CollectInBufferStream.h +++ b/lib/common/CollectInBufferStream.h @@ -26,24 +26,31 @@ class CollectInBufferStream : public IOStream public: CollectInBufferStream(); ~CollectInBufferStream(); -private: - // No copying - CollectInBufferStream(const CollectInBufferStream &); - CollectInBufferStream(const IOStream &); -public: + + // Move constructor: + CollectInBufferStream(CollectInBufferStream& rOther) + : mBuffer(rOther.mBuffer.Release()), + mBufferSize(rOther.mBufferSize), + mBytesInBuffer(rOther.mBytesInBuffer), + mReadPosition(rOther.mReadPosition), + mInWritePhase(rOther.mInWritePhase) + { + rOther.Reset(); + } virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual bool StreamDataLeft(); virtual bool StreamClosed(); void SetForReading(); - + void Reset(); - + void *GetBuffer() const; int GetSize() const; bool IsSetForReading() const {return !mInWritePhase;} diff --git a/lib/common/CommonException.txt b/lib/common/CommonException.txt index 05da2709..610ba1a8 100644 --- a/lib/common/CommonException.txt +++ b/lib/common/CommonException.txt @@ -55,3 +55,5 @@ DatabaseRecordAlreadyExists 47 The database already contains a record with this DatabaseRecordBadSize 48 The database contains a record with an invalid size DatabaseIterateFailed 49 Failed to iterate over the database keys ReferenceNotFound 50 The database does not contain an expected reference +TimersNotInitialised 51 The timer framework should have been ready at this point +InvalidConfiguration 52 Some required values are missing or incorrect in the configuration file. diff --git a/lib/common/Configuration.cpp b/lib/common/Configuration.cpp index f49f3c6e..8ce8d389 100644 --- a/lib/common/Configuration.cpp +++ b/lib/common/Configuration.cpp @@ -34,6 +34,8 @@ inline bool iw(int c) static const char *sValueBooleanStrings[] = {"yes", "true", "no", "false", 0}; static const bool sValueBooleanValue[] = {true, true, false, false}; +const ConfigurationCategory ConfigurationVerify::VERIFY_ERROR("VerifyError"); + ConfigurationVerifyKey::ConfigurationVerifyKey ( std::string name, @@ -212,8 +214,8 @@ std::auto_ptr<Configuration> Configuration::LoadAndVerify( if(!rErrorMsg.empty()) { // An error occured, return now - BOX_ERROR("Error in Configuration::LoadInto: " << - rErrorMsg); + BOX_LOG_CATEGORY(Log::ERROR, ConfigurationVerify::VERIFY_ERROR, + "Error in Configuration::LoadInto: " << rErrorMsg); return std::auto_ptr<Configuration>(0); } @@ -222,8 +224,11 @@ std::auto_ptr<Configuration> Configuration::LoadAndVerify( { if(!apConfig->Verify(*pVerify, std::string(), rErrorMsg)) { - BOX_ERROR("Error verifying configuration: " << - rErrorMsg); + BOX_LOG_CATEGORY(Log::ERROR, + ConfigurationVerify::VERIFY_ERROR, + "Error verifying configuration: " << + rErrorMsg.substr(0, rErrorMsg.size() > 0 + ? rErrorMsg.size() - 1 : 0)); return std::auto_ptr<Configuration>(0); } } @@ -425,7 +430,8 @@ const std::string &Configuration::GetKeyValue(const std::string& rKeyName) const if(i == mKeys.end()) { - BOX_ERROR("Missing configuration key: " << rKeyName); + BOX_LOG_CATEGORY(Log::ERROR, ConfigurationVerify::VERIFY_ERROR, + "Missing configuration key: " << rKeyName); THROW_EXCEPTION(CommonException, ConfigNoKey) } else diff --git a/lib/common/Configuration.h b/lib/common/Configuration.h index 4828b315..e6498e80 100644 --- a/lib/common/Configuration.h +++ b/lib/common/Configuration.h @@ -27,6 +27,14 @@ enum ConfigTest_IsBool = 32 }; +class ConfigurationCategory : public Log::Category +{ + public: + ConfigurationCategory(const std::string& name) + : Log::Category(std::string("Configuration/") + name) + { } +}; + class ConfigurationVerifyKey { public: @@ -75,6 +83,7 @@ public: const ConfigurationVerifyKey *mpKeys; int Tests; void *TestFunction; // set to zero for now, will implement later + static const ConfigurationCategory VERIFY_ERROR; }; class FdGetLine; diff --git a/lib/common/DebugMemLeakFinder.cpp b/lib/common/DebugMemLeakFinder.cpp index 0b123675..58a82c0e 100644 --- a/lib/common/DebugMemLeakFinder.cpp +++ b/lib/common/DebugMemLeakFinder.cpp @@ -15,14 +15,19 @@ #undef realloc #undef free -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - +#include <limits.h> #include <signal.h> #include <stdio.h> #include <string.h> +#ifdef HAVE_PROCESS_H +# include <process.h> +#endif + +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + #include <cstdlib> // for std::atexit #include <map> #include <set> @@ -155,7 +160,17 @@ void *memleakfinder_malloc(size_t size, const char *file, int line) InternalAllocGuard guard; void *b = std::malloc(size); - if(!memleakfinder_global_enable) return b; + + if(!memleakfinder_global_enable) + { + // We may not be tracking this allocation, but if + // someone realloc()s the buffer later then it will + // trigger an untracked buffer warning, which we don't + // want to see either. + memleakfinder_notaleak(b); + return b; + } + if(!memleakfinder_initialised) return b; memleakfinder_malloc_add_block(b, size, file, line); @@ -176,25 +191,58 @@ void *memleakfinder_calloc(size_t blocks, size_t size, const char *file, int lin void *memleakfinder_realloc(void *ptr, size_t size) { + if(!ptr) + { + return memleakfinder_malloc(size, "realloc", 0); + } + + if(!size) + { + memleakfinder_free(ptr); + return NULL; + } + InternalAllocGuard guard; + ASSERT(ptr != NULL); + if(!ptr) return NULL; // defensive + if(!memleakfinder_global_enable || !memleakfinder_initialised) { - return std::realloc(ptr, size); + ptr = std::realloc(ptr, size); + if(!memleakfinder_global_enable) + { + // We may not be tracking this allocation, but if + // someone realloc()s the buffer later then it will + // trigger an untracked buffer warning, which we don't + // want to see either. + memleakfinder_notaleak(ptr); + } + return ptr; } // Check it's been allocated std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); - if(ptr && i == sMallocBlocks.end()) + std::set<void *>::iterator j(sNotLeaks.find(ptr)); + + if(i == sMallocBlocks.end() && j == sNotLeaks.end()) { BOX_WARNING("Block " << ptr << " realloc()ated, but not " "in list. Error? Or allocated in startup static " "objects?"); } + if(j != sNotLeaks.end()) + { + // It's in the list of not-leaks, so don't warn about it, + // but it's being reallocated, so remove it from the list too, + // in case it's reassigned, and add the new block below. + sNotLeaks.erase(j); + } + void *b = std::realloc(ptr, size); - if(ptr && i!=sMallocBlocks.end()) + if(i != sMallocBlocks.end()) { // Worked? if(b != 0) @@ -230,10 +278,19 @@ void memleakfinder_free(void *ptr) { // Check it's been allocated std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); + std::set<void *>::iterator j(sNotLeaks.find(ptr)); + if(i != sMallocBlocks.end()) { sMallocBlocks.erase(i); } + else if(j != sNotLeaks.end()) + { + // It's in the list of not-leaks, so don't warn + // about it, but it's being freed, so remove it + // from the list too, in case it's reassigned. + sNotLeaks.erase(j); + } else { BOX_WARNING("Block " << ptr << " freed, but not " @@ -284,7 +341,8 @@ void memleakfinder_notaleak(void *ptr) ASSERT(!sTrackingDataDestroyed); memleakfinder_notaleak_insert_pre(); - if(memleakfinder_global_enable && memleakfinder_initialised) + + if(memleakfinder_initialised) { sNotLeaks.insert(ptr); } @@ -294,7 +352,9 @@ void memleakfinder_notaleak(void *ptr) sizeof(sNotLeaksPre)/sizeof(*sNotLeaksPre) ) sNotLeaksPre[sNotLeaksPreNum++] = ptr; } -/* { + + /* + { std::map<void *, MallocBlockInfo>::iterator i(sMallocBlocks.find(ptr)); if(i != sMallocBlocks.end()) sMallocBlocks.erase(i); } @@ -531,9 +591,14 @@ extern "C" void memleakfinder_atexit() memleakfinder_reportleaks_appendfile(atexit_filename, atexit_markertext); } -void memleakfinder_setup_exit_report(const char *filename, const char *markertext) +void memleakfinder_setup_exit_report(const std::string& filename, + const char *markertext) { - ::strncpy(atexit_filename, filename, sizeof(atexit_filename)-1); + char buffer[PATH_MAX]; + std::string abs_filename = std::string(getcwd(buffer, sizeof(buffer))) + + DIRECTORY_SEPARATOR + filename; + ::strncpy(atexit_filename, abs_filename.c_str(), + sizeof(atexit_filename)-1); ::strncpy(atexit_markertext, markertext, sizeof(atexit_markertext)-1); atexit_filename[sizeof(atexit_filename)-1] = 0; atexit_markertext[sizeof(atexit_markertext)-1] = 0; @@ -544,9 +609,6 @@ void memleakfinder_setup_exit_report(const char *filename, const char *markertex } } - - - void add_object_block(void *block, size_t size, const char *file, int line, bool array) { InternalAllocGuard guard; @@ -557,6 +619,10 @@ void add_object_block(void *block, size_t size, const char *file, int line, bool if(block != 0) { + std::map<void *, ObjectInfo>::iterator j(sObjectBlocks.find(block)); + // The same block should not already be tracked! + ASSERT(j == sObjectBlocks.end()); + ObjectInfo i; i.size = size; i.file = file; @@ -637,7 +703,7 @@ void *operator new(size_t size) } */ -void *operator new[](size_t size) +void *operator new[](size_t size) throw (std::bad_alloc) { return internal_new(size, "standard libraries", 0); } diff --git a/lib/common/EventWatchFilesystemObject.cpp b/lib/common/EventWatchFilesystemObject.cpp deleted file mode 100644 index 43533fc8..00000000 --- a/lib/common/EventWatchFilesystemObject.cpp +++ /dev/null @@ -1,112 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: EventWatchFilesystemObject.cpp -// Purpose: WaitForEvent compatible object for watching directories -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- - -#include "Box.h" - -#include <errno.h> -#include <fcntl.h> - -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - -#include "EventWatchFilesystemObject.h" -#include "autogen_CommonException.h" -#include "Logging.h" - -#include "MemLeakFindOn.h" - - -// -------------------------------------------------------------------------- -// -// Function -// Name: EventWatchFilesystemObject::EventWatchFilesystemObject -// (const char *) -// Purpose: Constructor -- opens the file object -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -EventWatchFilesystemObject::EventWatchFilesystemObject(const char *Filename) -#ifdef HAVE_KQUEUE - : mDescriptor(::open(Filename, O_RDONLY /*O_EVTONLY*/, 0)) -#endif -{ -#ifdef HAVE_KQUEUE - if(mDescriptor == -1) - { - BOX_LOG_SYS_ERROR("EventWatchFilesystemObject: " - "Failed to open file '" << Filename << "'"); - THROW_EXCEPTION(CommonException, OSFileOpenError) - } -#else - THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) -#endif -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: EventWatchFilesystemObject::~EventWatchFilesystemObject() -// Purpose: Destructor -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -EventWatchFilesystemObject::~EventWatchFilesystemObject() -{ - if(mDescriptor != -1) - { - ::close(mDescriptor); - } -} - - -// -------------------------------------------------------------------------- -// -// Function -// Name: EventWatchFilesystemObject::EventWatchFilesystemObject -// (const EventWatchFilesystemObject &) -// Purpose: Copy constructor -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -EventWatchFilesystemObject::EventWatchFilesystemObject( - const EventWatchFilesystemObject &rToCopy) - : mDescriptor(::dup(rToCopy.mDescriptor)) -{ - if(mDescriptor == -1) - { - THROW_EXCEPTION(CommonException, OSFileError) - } -} - - -#ifdef HAVE_KQUEUE -// -------------------------------------------------------------------------- -// -// Function -// Name: EventWatchFilesystemObject::FillInKEvent(struct kevent &, int) -// Purpose: For WaitForEvent -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -void EventWatchFilesystemObject::FillInKEvent(struct kevent &rEvent, - int Flags) const -{ - EV_SET(&rEvent, mDescriptor, EVFILT_VNODE, EV_CLEAR, - NOTE_DELETE | NOTE_WRITE, 0, (void*)this); -} -#else -void EventWatchFilesystemObject::FillInPoll(int &fd, short &events, - int Flags) const -{ - THROW_EXCEPTION(CommonException, KQueueNotSupportedOnThisPlatform) -} -#endif - diff --git a/lib/common/EventWatchFilesystemObject.h b/lib/common/EventWatchFilesystemObject.h deleted file mode 100644 index f9175a49..00000000 --- a/lib/common/EventWatchFilesystemObject.h +++ /dev/null @@ -1,48 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: EventWatchFilesystemObject.h -// Purpose: WaitForEvent compatible object for watching directories -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- - -#ifndef EVENTWATCHFILESYSTEMOBJECT__H -#define EVENTWATCHFILESYSTEMOBJECT__H - -#ifdef HAVE_KQUEUE - #include <sys/event.h> -#endif - - -// -------------------------------------------------------------------------- -// -// Class -// Name: EventWatchFilesystemObject -// Purpose: WaitForEvent compatible object for watching files and directories -// Created: 12/3/04 -// -// -------------------------------------------------------------------------- -class EventWatchFilesystemObject -{ -public: - EventWatchFilesystemObject(const char *Filename); - ~EventWatchFilesystemObject(); - EventWatchFilesystemObject(const EventWatchFilesystemObject &rToCopy); -private: - // Assignment not allowed - EventWatchFilesystemObject &operator=(const EventWatchFilesystemObject &); -public: - -#ifdef HAVE_KQUEUE - void FillInKEvent(struct kevent &rEvent, int Flags = 0) const; -#else - void FillInPoll(int &fd, short &events, int Flags = 0) const; -#endif - -private: - int mDescriptor; -}; - -#endif // EventWatchFilesystemObject__H - diff --git a/lib/common/ExcludeList.cpp b/lib/common/ExcludeList.cpp index 213c4f8e..3f9f69ee 100644 --- a/lib/common/ExcludeList.cpp +++ b/lib/common/ExcludeList.cpp @@ -10,9 +10,9 @@ #include "Box.h" #ifdef HAVE_REGEX_SUPPORT - #ifdef HAVE_PCREPOSIX_H + #if defined HAVE_PCREPOSIX_H #include <pcreposix.h> - #else + #elif defined HAVE_REGEX_H #include <regex.h> #endif #define EXCLUDELIST_IMPLEMENTATION_REGEX_T_DEFINED @@ -139,9 +139,10 @@ void ExcludeList::AddDefiniteEntries(const std::string &rEntries) if (entry.size() > 0 && entry[entry.size() - 1] == DIRECTORY_SEPARATOR_ASCHAR) { - BOX_WARNING("Exclude entry ends in path " - "separator, will never match: " - << entry); + BOX_LOG_CATEGORY(Log::WARNING, + ConfigurationVerify::VERIFY_ERROR, + "Exclude entry ends in path separator, " + "will never match: " << entry); } mDefinite.insert(entry); @@ -198,9 +199,9 @@ void ExcludeList::AddRegexEntries(const std::string &rEntries) { char buf[1024]; regerror(errcode, pregex, buf, sizeof(buf)); - BOX_ERROR("Invalid regular expression: " << + THROW_EXCEPTION_MESSAGE(CommonException, BadRegularExpression, + "Invalid regular expression: " << entry << ": " << buf); - THROW_EXCEPTION(CommonException, BadRegularExpression) } // Store in list of regular expressions diff --git a/lib/common/FileModificationTime.cpp b/lib/common/FileModificationTime.cpp index 06fc7887..50f1fb62 100644 --- a/lib/common/FileModificationTime.cpp +++ b/lib/common/FileModificationTime.cpp @@ -18,11 +18,14 @@ box_time_t FileModificationTime(const EMU_STRUCT_STAT &st) { -#ifndef HAVE_STRUCT_STAT_ST_MTIMESPEC - box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); -#else +#if defined HAVE_STRUCT_STAT_ST_ATIM + box_time_t datamodified = (((int64_t)st.st_mtim.tv_nsec) / NANO_SEC_IN_USEC_LL) + + (((int64_t)st.st_mtim.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIMESPEC box_time_t datamodified = (((int64_t)st.st_mtimespec.tv_nsec) / NANO_SEC_IN_USEC_LL) + (((int64_t)st.st_mtimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#else + box_time_t datamodified = ((int64_t)st.st_mtime) * (MICRO_SEC_IN_SEC_LL); #endif return datamodified; @@ -31,7 +34,10 @@ box_time_t FileModificationTime(const EMU_STRUCT_STAT &st) box_time_t FileAttrModificationTime(const EMU_STRUCT_STAT &st) { box_time_t statusmodified = -#ifdef HAVE_STRUCT_STAT_ST_MTIMESPEC +#if defined HAVE_STRUCT_STAT_ST_ATIM + (((int64_t)st.st_ctim.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + + (((int64_t)st.st_ctim.tv_sec) * (MICRO_SEC_IN_SEC_LL)); +#elif defined HAVE_STRUCT_STAT_ST_ATIMESPEC (((int64_t)st.st_ctimespec.tv_nsec) / (NANO_SEC_IN_USEC_LL)) + (((int64_t)st.st_ctimespec.tv_sec) * (MICRO_SEC_IN_SEC_LL)); #elif defined HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC diff --git a/lib/common/FileStream.cpp b/lib/common/FileStream.cpp index fc0319da..51752f85 100644 --- a/lib/common/FileStream.cpp +++ b/lib/common/FileStream.cpp @@ -67,22 +67,29 @@ void FileStream::AfterOpen() { MEMLEAKFINDER_NOT_A_LEAK(this); - #ifdef WIN32 - BOX_LOG_WIN_WARNING_NUMBER("Failed to open file: " << - mFileName, winerrno); - #else - BOX_LOG_SYS_WARNING("Failed to open file: " << - mFileName); - #endif - +#ifdef WIN32 + if(errno == EACCES) + { + THROW_WIN_FILE_ERRNO("Failed to open file", mFileName, + winerrno, CommonException, AccessDenied); + } + else + { + THROW_WIN_FILE_ERRNO("Failed to open file", mFileName, + winerrno, CommonException, OSFileOpenError); + } +#else if(errno == EACCES) { - THROW_EXCEPTION(CommonException, AccessDenied) + THROW_SYS_FILE_ERROR("Failed to open file", mFileName, + CommonException, AccessDenied); } else { - THROW_EXCEPTION(CommonException, OSFileOpenError) + THROW_SYS_FILE_ERROR("Failed to open file", mFileName, + CommonException, OSFileOpenError); } +#endif } } @@ -244,9 +251,9 @@ IOStream::pos_type FileStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void FileStream::Write(const void *pBuffer, int NBytes) +void FileStream::Write(const void *pBuffer, int NBytes, int Timeout) { - if(mOSFileHandle == INVALID_FILE) + if(mOSFileHandle == INVALID_FILE) { THROW_EXCEPTION(CommonException, FileClosed) } diff --git a/lib/common/FileStream.h b/lib/common/FileStream.h index 9101a968..1426d8a2 100644 --- a/lib/common/FileStream.h +++ b/lib/common/FileStream.h @@ -23,7 +23,7 @@ class FileStream : public IOStream { public: - FileStream(const std::string& rFilename, + FileStream(const std::string& rFilename, int flags = (O_RDONLY | O_BINARY), int mode = (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)); @@ -40,7 +40,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); @@ -49,6 +50,11 @@ public: virtual bool StreamClosed(); bool CompareWith(IOStream& rOther, int Timeout = IOStream::TimeOutInfinite); + std::string ToString() const + { + return std::string("local file ") + mFileName; + } + const std::string GetFileName() const { return mFileName; } private: tOSFileHandle mOSFileHandle; diff --git a/lib/common/Guards.h b/lib/common/Guards.h index cd2e4628..46b6d2bd 100644 --- a/lib/common/Guards.h +++ b/lib/common/Guards.h @@ -37,9 +37,8 @@ public: { if(mOSFileHandle < 0) { - BOX_LOG_SYS_ERROR("FileHandleGuard: failed to open " - "file '" << rFilename << "'"); - THROW_EXCEPTION(CommonException, OSFileOpenError) + THROW_SYS_FILE_ERROR("Failed to open file", rFilename, + CommonException, OSFileOpenError); } } @@ -78,7 +77,17 @@ class MemoryBlockGuard { public: MemoryBlockGuard(int BlockSize) - : mpBlock(::malloc(BlockSize)) + : mpBlock(::malloc(BlockSize)), + mBlockSize(BlockSize) + { + if(mpBlock == 0) + { + throw std::bad_alloc(); + } + } + + MemoryBlockGuard(void *pBlock) + : mpBlock(pBlock) { if(mpBlock == 0) { @@ -110,9 +119,21 @@ public: } mpBlock = ptrn; } + + void* Release() + { + void* pBlock = mpBlock; + mpBlock = ::malloc(mBlockSize); + if(mpBlock == 0) + { + throw std::bad_alloc(); + } + return pBlock; + } private: void *mpBlock; + int mBlockSize; }; #include "MemLeakFindOff.h" diff --git a/lib/common/IOStream.cpp b/lib/common/IOStream.cpp index fc9d0bc3..3e126d3f 100644 --- a/lib/common/IOStream.cpp +++ b/lib/common/IOStream.cpp @@ -127,8 +127,8 @@ int IOStream::ConvertSeekTypeToOSWhence(int SeekType) // Function // Name: IOStream::ReadFullBuffer(void *, int, int) // Purpose: Reads bytes into buffer, returning whether or not it managed to -// get all the bytes required. Exception and abort use of stream -// if this returns false. +// get all the bytes required. Exception and abort use of stream +// if this returns false. // Created: 2003/08/26 // // -------------------------------------------------------------------------- @@ -165,7 +165,7 @@ bool IOStream::ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int T // Created: 2003/08/26 // // -------------------------------------------------------------------------- -void IOStream::WriteAllBuffered() +void IOStream::WriteAllBuffered(int Timeout) { } @@ -245,7 +245,30 @@ void IOStream::Flush(int Timeout) } } -void IOStream::Write(const char *pBuffer) +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::Write +// Purpose: Convenience method for writing a C++ string to a +// protocol buffer. +// +// -------------------------------------------------------------------------- +void IOStream::Write(const std::string& rBuffer, int Timeout) +{ + Write(rBuffer.c_str(), rBuffer.size(), Timeout); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: IOStream::ToString() +// Purpose: Returns a string which describes this stream. Useful +// when reporting exceptions about a stream of unknown +// origin, for example in BackupStoreDirectory(). +// Created: 2014/04/28 +// +// -------------------------------------------------------------------------- +std::string IOStream::ToString() const { - Write(pBuffer, strlen(pBuffer)); + return "unknown IOStream"; } diff --git a/lib/common/IOStream.h b/lib/common/IOStream.h index 0b1cedd3..df7216c3 100644 --- a/lib/common/IOStream.h +++ b/lib/common/IOStream.h @@ -47,27 +47,27 @@ public: typedef int64_t pos_type; virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite) = 0; virtual pos_type BytesLeftToRead(); // may return IOStream::SizeOfStreamUnknown (and will for most stream types) - virtual void Write(const void *pBuffer, int NBytes) = 0; - virtual void Write(const char *pBuffer); - virtual void WriteAllBuffered(); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) = 0; + virtual void Write(const std::string& rBuffer, + int Timeout = IOStream::TimeOutInfinite); + virtual void WriteAllBuffered(int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual void Close(); - + // Has all data that can be read been read? virtual bool StreamDataLeft() = 0; // Has the stream been closed (writing not possible) virtual bool StreamClosed() = 0; - + // Utility functions bool ReadFullBuffer(void *pBuffer, int NBytes, int *pNBytesRead, int Timeout = IOStream::TimeOutInfinite); bool CopyStreamTo(IOStream &rCopyTo, int Timeout = IOStream::TimeOutInfinite, int BufferSize = 1024); void Flush(int Timeout = IOStream::TimeOutInfinite); - + static int ConvertSeekTypeToOSWhence(int SeekType); + virtual std::string ToString() const; }; - #endif // IOSTREAM__H - - diff --git a/lib/common/IOStreamGetLine.h b/lib/common/IOStreamGetLine.h index c4289073..1b537031 100644 --- a/lib/common/IOStreamGetLine.h +++ b/lib/common/IOStreamGetLine.h @@ -33,16 +33,22 @@ private: public: bool GetLine(std::string &rOutput, bool Preprocess = false, int Timeout = IOStream::TimeOutInfinite); - + std::string GetLine() + { + std::string output; + GetLine(output); + return output; + } + // Call to detach, setting file pointer correctly to last bit read. // Only works for lseek-able file descriptors. void DetachFile(); - + virtual bool IsStreamDataLeft() { return mrStream.StreamDataLeft(); } - + // For doing interesting stuff with the remaining data... // Be careful with this! const void *GetBufferedData() const {return mBuffer + mBufferBegin;} @@ -52,7 +58,7 @@ public: protected: int ReadMore(int Timeout = IOStream::TimeOutInfinite); - + private: IOStream &mrStream; }; diff --git a/lib/common/InvisibleTempFileStream.cpp b/lib/common/InvisibleTempFileStream.cpp index abfcb5f6..1a9d6d5a 100644 --- a/lib/common/InvisibleTempFileStream.cpp +++ b/lib/common/InvisibleTempFileStream.cpp @@ -22,7 +22,8 @@ // Created: 2006/10/13 // // -------------------------------------------------------------------------- -InvisibleTempFileStream::InvisibleTempFileStream(const char *Filename, int flags, int mode) +InvisibleTempFileStream::InvisibleTempFileStream(const std::string& Filename, + int flags, int mode) #ifdef WIN32 : FileStream(Filename, flags | O_TEMPORARY, mode) #else @@ -30,7 +31,7 @@ InvisibleTempFileStream::InvisibleTempFileStream(const char *Filename, int flags #endif { #ifndef WIN32 - if(unlink(Filename) != 0) + if(unlink(Filename.c_str()) != 0) { MEMLEAKFINDER_NOT_A_LEAK(this); THROW_EXCEPTION(CommonException, OSFileOpenError) diff --git a/lib/common/InvisibleTempFileStream.h b/lib/common/InvisibleTempFileStream.h index a77d05e2..bb6c3954 100644 --- a/lib/common/InvisibleTempFileStream.h +++ b/lib/common/InvisibleTempFileStream.h @@ -16,7 +16,7 @@ class InvisibleTempFileStream : public FileStream { public: - InvisibleTempFileStream(const char *Filename, + InvisibleTempFileStream(const std::string& Filename, #ifdef WIN32 int flags = (O_RDONLY | O_BINARY), #else diff --git a/lib/common/Logging.cpp b/lib/common/Logging.cpp index 7ce0dd3f..0928a4d4 100644 --- a/lib/common/Logging.cpp +++ b/lib/common/Logging.cpp @@ -13,19 +13,19 @@ #include <time.h> #include <string.h> // for stderror -// c.f. http://bugs.debian.org/512510 -#include <cstdio> +#ifdef HAVE_PROCESS_H +# include <process.h> +#endif #ifdef HAVE_SYSLOG_H - #include <syslog.h> +# include <syslog.h> #endif + #ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif -#ifdef WIN32 - #include <process.h> +# include <unistd.h> #endif +#include <cstdio> #include <cstring> #include <iomanip> @@ -42,16 +42,14 @@ std::vector<Logger*> Logging::sLoggers; std::string Logging::sContext; Console* Logging::spConsole = NULL; Syslog* Logging::spSyslog = NULL; -Log::Level Logging::sGlobalLevel = Log::EVERYTHING; -Logging Logging::sGlobalLogging; //automatic initialisation +Logging Logging::sGlobalLogging; // automatic initialisation std::string Logging::sProgramName; +const Log::Category Logging::UNCATEGORISED("Uncategorised"); +std::auto_ptr<HideFileGuard> Logging::sapHideFileGuard; HideSpecificExceptionGuard::SuppressedExceptions_t HideSpecificExceptionGuard::sSuppressedExceptions; -int Logging::Guard::sGuardCount = 0; -Log::Level Logging::Guard::sOriginalLevel = Log::INVALID; - Logging::Logging() { ASSERT(!spConsole); @@ -139,14 +137,10 @@ void Logging::Remove(Logger* pOldLogger) } } -void Logging::Log(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage) +void Logging::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { - if (level > sGlobalLevel) - { - return; - } - std::string newMessage; if (sContextSet) @@ -154,12 +148,13 @@ void Logging::Log(Log::Level level, const std::string& rFile, newMessage += "[" + sContext + "] "; } - newMessage += rMessage; + newMessage += message; for (std::vector<Logger*>::iterator i = sLoggers.begin(); i != sLoggers.end(); i++) { - bool result = (*i)->Log(level, rFile, line, newMessage); + bool result = (*i)->Log(level, file, line, function, category, + newMessage); if (!result) { return; @@ -167,19 +162,15 @@ void Logging::Log(Log::Level level, const std::string& rFile, } } -void Logging::LogToSyslog(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage) +void Logging::LogToSyslog(Log::Level level, const std::string& rFile, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (!sLogToSyslog) { return; } - if (level > sGlobalLevel) - { - return; - } - std::string newMessage; if (sContextSet) @@ -187,9 +178,9 @@ void Logging::LogToSyslog(Log::Level level, const std::string& rFile, newMessage += "[" + sContext + "] "; } - newMessage += rMessage; + newMessage += message; - spSyslog->Log(level, rFile, line, newMessage); + spSyslog->Log(level, rFile, line, function, category, newMessage); } void Logging::SetContext(std::string context) @@ -255,8 +246,7 @@ Logger::~Logger() bool Logger::IsEnabled(Log::Level level) { - return Logging::IsEnabled(level) && - (int)mCurrentLevel >= (int)level; + return (int)mCurrentLevel >= (int)level; } bool Console::sShowTime = false; @@ -290,8 +280,9 @@ void Console::SetShowPID(bool enabled) sShowPID = enabled; } -bool Console::Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) +bool Console::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (level > GetLevel()) { @@ -299,12 +290,6 @@ bool Console::Log(Log::Level level, const std::string& rFile, } FILE* target = stdout; - - if (level <= Log::WARNING) - { - target = stderr; - } - std::ostringstream buf; if (sShowTime) @@ -354,7 +339,7 @@ bool Console::Log(Log::Level level, const std::string& rFile, buf << "TRACE: "; } - buf << rMessage; + buf << message; #ifdef WIN32 std::string output = buf.str(); @@ -376,8 +361,9 @@ bool Console::Log(Log::Level level, const std::string& rFile, return true; } -bool Syslog::Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) +bool Syslog::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (level > GetLevel()) { @@ -418,7 +404,7 @@ bool Syslog::Log(Log::Level level, const std::string& rFile, msg = "NOTICE: "; } - msg += rMessage; + msg += message; syslog(syslogLevel, "%s", msg.c_str()); @@ -432,6 +418,11 @@ Syslog::Syslog() : mFacility(LOG_LOCAL6) Syslog::~Syslog() { + Shutdown(); +} + +void Syslog::Shutdown() +{ ::closelog(); } @@ -467,8 +458,9 @@ int Syslog::GetNamedFacility(const std::string& rFacility) return LOG_LOCAL6; } -bool FileLogger::Log(Log::Level Level, const std::string& rFile, - int line, std::string& rMessage) +bool FileLogger::Log(Log::Level Level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (mLogFile.StreamClosed()) { @@ -515,7 +507,7 @@ bool FileLogger::Log(Log::Level Level, const std::string& rFile, buf << "[TRACE] "; } - buf << rMessage << "\n"; + buf << message << "\n"; std::string output = buf.str(); #ifdef WIN32 @@ -564,3 +556,211 @@ bool HideSpecificExceptionGuard::IsHidden(int type, int subtype) return false; } +// -------------------------------------------------------------------------- +// +// Function +// Name: Logging::OptionParser::GetOptionString() +// Purpose: Returns the valid Getopt command-line options +// that Logging::OptionParser::ProcessOption will handle. +// Created: 2014/04/09 +// +// -------------------------------------------------------------------------- +std::string Logging::OptionParser::GetOptionString() +{ + return "L:NPqQt:TUvVW:"; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Logging::OptionParser::ProcessOption(signed int option) +// Purpose: Processes the supplied option (equivalent to the +// return code from getopt()). Return zero if the +// option was handled successfully, or nonzero to +// abort the program with that return value. +// Created: 2007/09/18 +// +// -------------------------------------------------------------------------- +int Logging::OptionParser::ProcessOption(signed int option) +{ + switch(option) + { + case 'L': + { + if(sapHideFileGuard.get()) + { + sapHideFileGuard->Add(optarg); + } + else + { + sapHideFileGuard.reset(new HideFileGuard( + optarg, true)); // HideAllButSelected + } + } + break; + + case 'N': + { + mTruncateLogFile = true; + } + break; + + case 'P': + { + Console::SetShowPID(true); + } + break; + + case 'q': + { + if(mCurrentLevel == Log::NOTHING) + { + BOX_FATAL("Too many '-q': " + "Cannot reduce logging " + "level any more"); + return 2; + } + mCurrentLevel--; + } + break; + + case 'Q': + { + mCurrentLevel = Log::NOTHING; + } + break; + + case 't': + { + Logging::SetProgramName(optarg); + Console::SetShowTag(true); + } + break; + + case 'T': + { + Console::SetShowTime(true); + } + break; + + case 'U': + { + Console::SetShowTime(true); + Console::SetShowTimeMicros(true); + } + break; + + case 'v': + { + if(mCurrentLevel == Log::EVERYTHING) + { + BOX_FATAL("Too many '-v': " + "Cannot increase logging " + "level any more"); + return 2; + } + mCurrentLevel++; + } + break; + + case 'V': + { + mCurrentLevel = Log::EVERYTHING; + } + break; + + case 'W': + { + mCurrentLevel = Logging::GetNamedLevel(optarg); + if (mCurrentLevel == Log::INVALID) + { + BOX_FATAL("Invalid logging level: " << optarg); + return 2; + } + } + break; + + case '?': + { + BOX_FATAL("Unknown option on command line: " + << "'" << (char)optopt << "'"); + return 2; + } + break; + + default: + { + BOX_FATAL("Unknown error in getopt: returned " + << "'" << option << "'"); + return 1; + } + } + + // If we didn't explicitly return an error code above, then the option + // was fine, so return 0 to continue processing. + return 0; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: Logging::OptionParser::GetUsageString() +// Purpose: Returns a string suitable for displaying as part +// of a program's command-line usage help message, +// describing the logging options. +// Created: 2014/04/09 +// +// -------------------------------------------------------------------------- +std::string Logging::OptionParser::GetUsageString() +{ + return + " -L <file> Filter out log messages except from specified file, can repeat\n" + " (for example, -L " __FILE__ ")\n" + " -N Truncate log file at startup and on backup start\n" + " -P Show process ID (PID) in console output\n" + " -q Run more quietly, reduce verbosity level by one, can repeat\n" + " -Q Run at minimum verbosity, log nothing to console and system\n" + " -t <tag> Tag console output with specified marker\n" + " -T Timestamp console output\n" + " -U Timestamp console output with microseconds\n" + " -v Run more verbosely, increase verbosity level by one, can repeat\n" + " -V Run at maximum verbosity, log everything to console and system\n" + " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n"; +} + +bool HideCategoryGuard::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) +{ + std::list<Log::Category>::iterator i = std::find(mCategories.begin(), + mCategories.end(), category); + // Return false if category is in our list, to suppress further + // logging (thus, return true if it's not in our list, i.e. we + // found nothing, to allow it). + return (i == mCategories.end()); +} + +bool HideFileGuard::Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) +{ + std::list<std::string>::iterator i = std::find(mFileNames.begin(), + mFileNames.end(), file); + bool allow_log_message; + if(mHideAllButSelected) + { + // Return true if filename is in our list, to allow further + // logging (thus, return false if it's not in our list, i.e. we + // found nothing, to suppress it). + allow_log_message = (i != mFileNames.end()); + } + else + { + // Return false if filename is in our list, to suppress further + // logging (thus, return true if it's not in our list, i.e. we + // found nothing, to allow it). + allow_log_message = (i == mFileNames.end()); + } + return allow_log_message; +} + diff --git a/lib/common/Logging.h b/lib/common/Logging.h index 1074b7c3..3dc3e69c 100644 --- a/lib/common/Logging.h +++ b/lib/common/Logging.h @@ -12,9 +12,11 @@ #include <assert.h> +#include <algorithm> #include <cerrno> #include <cstring> #include <iomanip> +#include <list> #include <sstream> #include <vector> @@ -24,14 +26,24 @@ { \ std::ostringstream _box_log_line; \ _box_log_line << stuff; \ - Logging::Log(level, __FILE__, __LINE__, _box_log_line.str()); \ + Logging::Log(level, __FILE__, __LINE__, __FUNCTION__, \ + Logging::UNCATEGORISED, _box_log_line.str()); \ +} + +#define BOX_LOG_CATEGORY(level, category, stuff) \ +{ \ + std::ostringstream _box_log_line; \ + _box_log_line << stuff; \ + Logging::Log(level, __FILE__, __LINE__, __FUNCTION__, \ + category, _box_log_line.str()); \ } #define BOX_SYSLOG(level, stuff) \ { \ std::ostringstream _box_log_line; \ _box_log_line << stuff; \ - Logging::LogToSyslog(level, __FILE__, __LINE__, _box_log_line.str()); \ + Logging::LogToSyslog(level, __FILE__, __LINE__, __FUNCTION__, \ + Logging::UNCATEGORISED, _box_log_line.str()); \ } #define BOX_FATAL(stuff) BOX_LOG(Log::FATAL, stuff) @@ -39,9 +51,7 @@ #define BOX_WARNING(stuff) BOX_LOG(Log::WARNING, stuff) #define BOX_NOTICE(stuff) BOX_LOG(Log::NOTICE, stuff) #define BOX_INFO(stuff) BOX_LOG(Log::INFO, stuff) -#define BOX_TRACE(stuff) \ - if (Logging::IsEnabled(Log::TRACE)) \ - { BOX_LOG(Log::TRACE, stuff) } +#define BOX_TRACE(stuff) BOX_LOG(Log::TRACE, stuff) #define BOX_SYS_ERRNO_MESSAGE(error_number, stuff) \ stuff << ": " << std::strerror(error_number) << \ @@ -85,18 +95,20 @@ BOX_FILE_MESSAGE(filename, message)) #ifdef WIN32 + #define BOX_WIN_ERRNO_MESSAGE(error_number, stuff) \ + stuff << ": " << GetErrorMessage(error_number) + #define BOX_NATIVE_ERRNO_MESSAGE(error_number, stuff) \ + BOX_WIN_ERRNO_MESSAGE(error_number, stuff) #define BOX_LOG_WIN_ERROR(stuff) \ - BOX_ERROR(stuff << ": " << GetErrorMessage(GetLastError())) + BOX_ERROR(BOX_WIN_ERRNO_MESSAGE(GetLastError(), stuff)) #define BOX_LOG_WIN_WARNING(stuff) \ - BOX_WARNING(stuff << ": " << GetErrorMessage(GetLastError())) + BOX_WARNING(BOX_WIN_ERRNO_MESSAGE(GetLastError(), stuff)) #define BOX_LOG_WIN_ERROR_NUMBER(stuff, number) \ - BOX_ERROR(stuff << ": " << GetErrorMessage(number)) + BOX_ERROR(BOX_WIN_ERRNO_MESSAGE(number, stuff)) #define BOX_LOG_WIN_WARNING_NUMBER(stuff, number) \ - BOX_WARNING(stuff << ": " << GetErrorMessage(number)) + BOX_WARNING(BOX_WIN_ERRNO_MESSAGE(number, stuff)) #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_WIN_ERROR(stuff) #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_WIN_WARNING(stuff) - #define BOX_WIN_ERRNO_MESSAGE(error_number, stuff) \ - stuff << ": " << GetErrorMessage(error_number) #define THROW_WIN_ERROR_NUMBER(message, error_number, exception, subtype) \ THROW_EXCEPTION_MESSAGE(exception, subtype, \ BOX_WIN_ERRNO_MESSAGE(error_number, message)) @@ -106,19 +118,33 @@ #define THROW_WIN_FILE_ERROR(message, filename, exception, subtype) \ THROW_WIN_FILE_ERRNO(message, filename, GetLastError(), \ exception, subtype) + #define EMU_ERRNO winerrno + #define THROW_EMU_ERROR(message, exception, subtype) \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_NATIVE_ERRNO_MESSAGE(EMU_ERRNO, message)) #else + #define BOX_NATIVE_ERRNO_MESSAGE(error_number, stuff) \ + BOX_SYS_ERRNO_MESSAGE(error_number, stuff) #define BOX_LOG_NATIVE_ERROR(stuff) BOX_LOG_SYS_ERROR(stuff) #define BOX_LOG_NATIVE_WARNING(stuff) BOX_LOG_SYS_WARNING(stuff) + #define EMU_ERRNO errno + #define THROW_EMU_ERROR(message, exception, subtype) \ + THROW_EXCEPTION_MESSAGE(exception, subtype, \ + BOX_SYS_ERRNO_MESSAGE(EMU_ERRNO, message)) #endif +#define THROW_EMU_FILE_ERROR(message, filename, exception, subtype) \ + THROW_EMU_ERROR(BOX_FILE_MESSAGE(filename, message), \ + exception, subtype) + #ifdef WIN32 -# define BOX_LOG_SOCKET_ERROR(_type, _name, _port, stuff) \ - BOX_LOG_WIN_ERROR_NUMBER(stuff << " (type " << _type << ", name " << \ - _name << ", port " << _port << ")", WSAGetLastError()) +# define BOX_SOCKET_ERROR_MESSAGE(_type, _name, _port, stuff) \ + BOX_WIN_ERRNO_MESSAGE(WSAGetLastError(), stuff << " (type " << _type << \ + ", name " << _name << ", port " << _port << ")") #else -# define BOX_LOG_SOCKET_ERROR(_type, _name, _port, stuff) \ - BOX_LOG_NATIVE_ERROR(stuff << " (type " << _type << ", name " << \ - _name << ", port " << _port << ")") +# define BOX_SOCKET_ERROR_MESSAGE(_type, _name, _port, stuff) \ + BOX_SYS_ERROR_MESSAGE(stuff << " (type " << _type << ", name " << _name << \ + ", port " << _port << ")") #endif #define BOX_FORMAT_HEX32(number) \ @@ -141,14 +167,16 @@ #define BOX_FORMAT_TIMESPEC(timespec) \ timespec.tv_sec << \ + "." << \ std::setw(6) << \ + std::setfill('0') << \ timespec.tv_usec #define BOX_FORMAT_MICROSECONDS(t) \ (int)((t) / 1000000) << "." << \ - std::setw(6) << \ + std::setw(3) << \ std::setfill('0') << \ - (int)((t) % 1000000) << " seconds" + (int)((t % 1000000) / 1000) << " seconds" #undef ERROR @@ -166,6 +194,18 @@ namespace Log EVERYTHING, INVALID = -1 }; + + class Category { + private: + std::string mName; + + public: + Category(const std::string& name) + : mName(name) + { } + const std::string& ToString() { return mName; } + bool operator==(const Category& other) { return mName == other.mName; } + }; } // -------------------------------------------------------------------------- @@ -187,8 +227,9 @@ class Logger Logger(Log::Level level); virtual ~Logger(); - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) = 0; + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) = 0; void Filter(Log::Level level) { @@ -201,19 +242,23 @@ class Logger virtual void SetProgramName(const std::string& rProgramName) = 0; - class Guard + class LevelGuard { private: Logger& mLogger; Log::Level mOldLevel; public: - Guard(Logger& Logger) + LevelGuard(Logger& Logger, Log::Level newLevel = Log::INVALID) : mLogger(Logger) { mOldLevel = Logger.GetLevel(); + if (newLevel != Log::INVALID) + { + Logger.Filter(newLevel); + } } - ~Guard() + ~LevelGuard() { mLogger.Filter(mOldLevel); } @@ -239,8 +284,9 @@ class Console : public Logger static std::string sTag; public: - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage); + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); virtual const char* GetType() { return "Console"; } virtual void SetProgramName(const std::string& rProgramName); @@ -248,6 +294,33 @@ class Console : public Logger static void SetShowTime(bool enabled); static void SetShowTimeMicros(bool enabled); static void SetShowPID(bool enabled); + static bool GetShowTag() { return sShowTag; } + + class SettingsGuard + { + private: + bool mShowTag; + bool mShowTime; + bool mShowTimeMicros; + bool mShowPID; + std::string mTag; + public: + SettingsGuard() + : mShowTag(Console::sShowTag), + mShowTime(Console::sShowTime), + mShowTimeMicros(Console::sShowTimeMicros), + mShowPID(Console::sShowPID), + mTag(Console::sTag) + { } + ~SettingsGuard() + { + Console::SetShowTag(mShowTag); + Console::SetShowTime(mShowTime); + Console::SetShowTimeMicros(mShowTimeMicros); + Console::SetShowPID(mShowPID); + Console::sTag = mTag; + } + }; }; // -------------------------------------------------------------------------- @@ -269,17 +342,80 @@ class Syslog : public Logger Syslog(); virtual ~Syslog(); - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage); + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); virtual const char* GetType() { return "Syslog"; } virtual void SetProgramName(const std::string& rProgramName); virtual void SetFacility(int facility); + virtual void Shutdown(); static int GetNamedFacility(const std::string& rFacility); }; // -------------------------------------------------------------------------- // // Class +// Name: Capture +// Purpose: Keeps log messages for analysis in tests. +// Created: 2014/03/08 +// +// -------------------------------------------------------------------------- + +class Capture : public Logger +{ + public: + struct Message + { + Message(const Log::Category& category) + : mCategory(category) { } + Log::Level level; + std::string file; + int line; + std::string function; + Log::Category mCategory; + std::string message; + }; + + private: + std::vector<Message> mMessages; + + public: + virtual ~Capture() { } + + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) + { + Message msg(category); + msg.level = level; + msg.file = file; + msg.line = line; + msg.function = function; + msg.message = message; + mMessages.push_back(msg); + return true; + } + virtual const char* GetType() { return "Capture"; } + virtual void SetProgramName(const std::string& rProgramName) { } + const std::vector<Message>& GetMessages() const { return mMessages; } + std::string GetString() const + { + std::ostringstream oss; + for (std::vector<Message>::const_iterator i = mMessages.begin(); + i != mMessages.end(); i++) + { + oss << i->message << "\n"; + } + return oss.str(); + } +}; + +// Forward declaration +class HideFileGuard; + +// -------------------------------------------------------------------------- +// +// Class // Name: Logging // Purpose: Static logging helper, keeps track of enabled loggers // and distributes log messages to them. @@ -296,10 +432,10 @@ class Logging static bool sContextSet; static Console* spConsole; static Syslog* spSyslog; - static Log::Level sGlobalLevel; static Logging sGlobalLogging; static std::string sProgramName; - + static std::auto_ptr<HideFileGuard> sapHideFileGuard; + public: Logging (); ~Logging(); @@ -309,55 +445,35 @@ class Logging static void FilterConsole (Log::Level level); static void Add (Logger* pNewLogger); static void Remove (Logger* pOldLogger); - static void Log(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage); - static void LogToSyslog(Log::Level level, const std::string& rFile, - int line, const std::string& rMessage); + static void Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); + static void LogToSyslog(Log::Level level, const std::string& rFile, int line, + const std::string& function, const Log::Category& category, + const std::string& message); static void SetContext(std::string context); static void ClearContext(); - static void SetGlobalLevel(Log::Level level) { sGlobalLevel = level; } - static Log::Level GetGlobalLevel() { return sGlobalLevel; } static Log::Level GetNamedLevel(const std::string& rName); - static bool IsEnabled(Log::Level level) - { - return (int)sGlobalLevel >= (int)level; - } static void SetProgramName(const std::string& rProgramName); static std::string GetProgramName() { return sProgramName; } static void SetFacility(int facility); static Console& GetConsole() { return *spConsole; } static Syslog& GetSyslog() { return *spSyslog; } - class Guard + class ShowTagOnConsole { private: - Log::Level mOldLevel; - static int sGuardCount; - static Log::Level sOriginalLevel; - + bool mOldShowTag; + public: - Guard(Log::Level newLevel) - { - mOldLevel = Logging::GetGlobalLevel(); - if(sGuardCount == 0) - { - sOriginalLevel = mOldLevel; - } - sGuardCount++; - Logging::SetGlobalLevel(newLevel); - } - ~Guard() + ShowTagOnConsole() + : mOldShowTag(Console::GetShowTag()) { - sGuardCount--; - Logging::SetGlobalLevel(mOldLevel); + Console::SetShowTag(true); } - - static bool IsActive() { return (sGuardCount > 0); } - static Log::Level GetOriginalLevel() { return sOriginalLevel; } - static bool IsGuardingFrom(Log::Level originalLevel) + ~ShowTagOnConsole() { - return IsActive() && - (int)sOriginalLevel >= (int)originalLevel; + Console::SetShowTag(mOldShowTag); } }; @@ -365,16 +481,19 @@ class Logging { private: std::string mOldTag; + bool mReplace; public: Tagger() - : mOldTag(Logging::GetProgramName()) + : mOldTag(Logging::GetProgramName()), + mReplace(false) { } - Tagger(const std::string& rTempTag) - : mOldTag(Logging::GetProgramName()) + Tagger(const std::string& rTempTag, bool replace = false) + : mOldTag(Logging::GetProgramName()), + mReplace(replace) { - Logging::SetProgramName(mOldTag + " " + rTempTag); + Change(rTempTag); } ~Tagger() { @@ -383,9 +502,73 @@ class Logging void Change(const std::string& newTempTag) { - Logging::SetProgramName(mOldTag + " " + newTempTag); + if(mReplace || mOldTag.empty()) + { + Logging::SetProgramName(newTempTag); + } + else + { + Logging::SetProgramName(mOldTag + " " + newTempTag); + } + } + }; + + class TempLoggerGuard + { + private: + Logger* mpLogger; + + public: + TempLoggerGuard(Logger* pLogger) + : mpLogger(pLogger) + { + Logging::Add(mpLogger); + } + ~TempLoggerGuard() + { + Logging::Remove(mpLogger); + } + }; + + // Process global options + static std::string GetOptionString(); + static int ProcessOption(signed int option); + static std::string GetUsageString(); + + // -------------------------------------------------------------------------- + // + // Class + // Name: Logging::OptionParser + // Purpose: Process command-line options, some global, some local + // Created: 2014/04/09 + // + // -------------------------------------------------------------------------- + class OptionParser + { + public: + OptionParser(Log::Level InitialLevel = + #ifdef BOX_RELEASE_BUILD + Log::NOTICE + #else + Log::INFO + #endif + ) + : mCurrentLevel(InitialLevel), + mTruncateLogFile(false) + { } + + static std::string GetOptionString(); + int ProcessOption(signed int option); + static std::string GetUsageString(); + int mCurrentLevel; // need an int to do math with + bool mTruncateLogFile; + Log::Level GetCurrentLevel() + { + return (Log::Level) mCurrentLevel; } }; + + static const Log::Category UNCATEGORISED; }; class FileLogger : public Logger @@ -396,13 +579,14 @@ class FileLogger : public Logger : mLogFile("") { /* do not call */ } public: - FileLogger(const std::string& rFileName, Log::Level Level) + FileLogger(const std::string& rFileName, Log::Level Level, bool append) : Logger(Level), - mLogFile(rFileName, O_WRONLY | O_CREAT | O_APPEND) + mLogFile(rFileName, O_WRONLY | O_CREAT | (append ? O_APPEND : O_TRUNC)) { } - virtual bool Log(Log::Level Level, const std::string& rFile, - int Line, std::string& rMessage); + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); virtual const char* GetType() { return "FileLogger"; } virtual void SetProgramName(const std::string& rProgramName) { } @@ -451,6 +635,64 @@ class HideSpecificExceptionGuard static bool IsHidden(int type, int subtype); }; +class HideCategoryGuard : public Logger +{ + private: + std::list<Log::Category> mCategories; + HideCategoryGuard(const HideCategoryGuard& other); // no copying + HideCategoryGuard& operator=(const HideCategoryGuard& other); // no assignment + + public: + HideCategoryGuard(const Log::Category& category) + { + mCategories.push_back(category); + Logging::Add(this); + } + ~HideCategoryGuard() + { + Logging::Remove(this); + } + void Add(const Log::Category& category) + { + mCategories.push_back(category); + } + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); + virtual const char* GetType() { return "HideCategoryGuard"; } + virtual void SetProgramName(const std::string& rProgramName) { } +}; + +class HideFileGuard : public Logger +{ + private: + std::list<std::string> mFileNames; + HideFileGuard(const HideFileGuard& other); // no copying + HideFileGuard& operator=(const HideFileGuard& other); // no assignment + bool mHideAllButSelected; + + public: + HideFileGuard(const std::string& rFileName, bool HideAllButSelected = false) + : mHideAllButSelected(HideAllButSelected) + { + mFileNames.push_back(rFileName); + Logging::Add(this); + } + ~HideFileGuard() + { + Logging::Remove(this); + } + void Add(const std::string& rFileName) + { + mFileNames.push_back(rFileName); + } + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message); + virtual const char* GetType() { return "HideFileGuard"; } + virtual void SetProgramName(const std::string& rProgramName) { } +}; + std::string PrintEscapedBinaryData(const std::string& rInput); #endif // LOGGING__H diff --git a/lib/common/MainHelper.h b/lib/common/MainHelper.h index 3c6e9ff0..0303090e 100644 --- a/lib/common/MainHelper.h +++ b/lib/common/MainHelper.h @@ -19,18 +19,21 @@ #include "BoxException.h" #include "Logging.h" -#define MAINHELPER_START \ - if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \ - { printf(BOX_VERSION "\n"); return 0; } \ +#define MAINHELPER_START \ + if(argc == 2 && ::strcmp(argv[1], "--version") == 0) \ + { printf(BOX_VERSION "\n"); return 0; } \ MEMLEAKFINDER_INIT \ - MEMLEAKFINDER_START \ + MEMLEAKFINDER_START \ try { -#define MAINHELPER_END \ - } catch(std::exception &e) { \ +#define MAINHELPER_END \ + } catch(BoxException &e) { \ + BOX_FATAL(e.what() << ": " << e.GetMessage()); \ + return 1; \ + } catch(std::exception &e) { \ BOX_FATAL(e.what()); \ - return 1; \ - } catch(...) { \ + return 1; \ + } catch(...) { \ BOX_FATAL("UNKNOWN"); \ return 1; \ } diff --git a/lib/common/MemBlockStream.cpp b/lib/common/MemBlockStream.cpp index 3a43a304..f49ac96f 100644 --- a/lib/common/MemBlockStream.cpp +++ b/lib/common/MemBlockStream.cpp @@ -52,6 +52,26 @@ MemBlockStream::MemBlockStream(const void *pBuffer, int Size) // -------------------------------------------------------------------------- // // Function +// Name: MemBlockStream::MemBlockStream(const std::string& rMessage) +// Purpose: Convenience constructor for sending a simple string. +// Copies the string, so you can pass a temporary in. +// Created: 2014/01/20 +// +// -------------------------------------------------------------------------- +MemBlockStream::MemBlockStream(const std::string& rMessage) +: mReadPosition(0) +{ + mTempBuffer.Write(rMessage.c_str(), rMessage.size()); + mTempBuffer.SetForReading(); + mpBuffer = (const char *)(mTempBuffer.GetBuffer()); + mBytesInBuffer = rMessage.size(); + ASSERT(mpBuffer != 0); + ASSERT(mBytesInBuffer >= 0); +} + +// -------------------------------------------------------------------------- +// +// Function // Name: MemBlockStream::MemBlockStream(const StreamableMemBlock &) // Purpose: Constructor (doesn't copy block, careful with lifetimes) // Created: 2003/09/05 @@ -159,7 +179,7 @@ IOStream::pos_type MemBlockStream::BytesLeftToRead() // Created: 2003/09/05 // // -------------------------------------------------------------------------- -void MemBlockStream::Write(const void *pBuffer, int NBytes) +void MemBlockStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, MemBlockStreamNotSupported) } diff --git a/lib/common/MemBlockStream.h b/lib/common/MemBlockStream.h index 5234525b..1ba4b0a6 100644 --- a/lib/common/MemBlockStream.h +++ b/lib/common/MemBlockStream.h @@ -10,10 +10,10 @@ #ifndef MEMBLOCKSTREAM__H #define MEMBLOCKSTREAM__H +#include "CollectInBufferStream.h" #include "IOStream.h" class StreamableMemBlock; -class CollectInBufferStream; // -------------------------------------------------------------------------- // @@ -29,6 +29,7 @@ class MemBlockStream : public IOStream public: MemBlockStream(); MemBlockStream(const void *pBuffer, int Size); + MemBlockStream(const std::string& rMessage); MemBlockStream(const StreamableMemBlock &rBlock); MemBlockStream(const CollectInBufferStream &rBuffer); MemBlockStream(const MemBlockStream &rToCopy); @@ -37,7 +38,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual bool StreamDataLeft(); @@ -46,6 +48,9 @@ public: virtual int GetSize() const { return mBytesInBuffer; } private: + // Use mTempBuffer when we need to hold a copy of the memory block, + // and free it ourselves when done. + CollectInBufferStream mTempBuffer; const char *mpBuffer; int mBytesInBuffer; int mReadPosition; diff --git a/lib/common/MemLeakFindOn.h b/lib/common/MemLeakFindOn.h index c20fe25a..f1113184 100644 --- a/lib/common/MemLeakFindOn.h +++ b/lib/common/MemLeakFindOn.h @@ -15,6 +15,7 @@ #ifndef MEMLEAKFINDER_MALLOC_MONITORING_DEFINED #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) + #define calloc(X, Y) memleakfinder_calloc(X, Y, __FILE__, __LINE__) #define realloc memleakfinder_realloc #define free memleakfinder_free #define MEMLEAKFINDER_MALLOC_MONITORING_DEFINED diff --git a/lib/common/MemLeakFinder.h b/lib/common/MemLeakFinder.h index 1a2cf90c..07b52e26 100644 --- a/lib/common/MemLeakFinder.h +++ b/lib/common/MemLeakFinder.h @@ -43,7 +43,7 @@ void memleakfinder_reportleaks(); void memleakfinder_reportleaks_appendfile(const char *filename, const char *markertext); -void memleakfinder_setup_exit_report(const char *filename, const char *markertext); +void memleakfinder_setup_exit_report(const std::string& filename, const char *markertext); void memleakfinder_startsectionmonitor(); @@ -54,7 +54,8 @@ void memleakfinder_notaleak(void *ptr); void *operator new (size_t size, const char *file, int line); void *operator new[](size_t size, const char *file, int line); -// define the malloc functions now, if required +// Define the malloc functions now, if required. These should match the definitions +// in MemLeakFindOn.h. #ifdef MEMLEAKFINDER_FULL_MALLOC_MONITORING #define malloc(X) memleakfinder_malloc(X, __FILE__, __LINE__) #define calloc(X, Y) memleakfinder_calloc(X, Y, __FILE__, __LINE__) diff --git a/lib/common/NamedLock.cpp b/lib/common/NamedLock.cpp index f96f80b5..8e672ff5 100644 --- a/lib/common/NamedLock.cpp +++ b/lib/common/NamedLock.cpp @@ -21,8 +21,9 @@ #include <sys/file.h> #endif -#include "NamedLock.h" #include "CommonException.h" +#include "NamedLock.h" +#include "Utils.h" #include "MemLeakFindOn.h" @@ -35,7 +36,11 @@ // // -------------------------------------------------------------------------- NamedLock::NamedLock() - : mFileDescriptor(-1) +#ifdef WIN32 +: mFileDescriptor(INVALID_HANDLE_VALUE) +#else +: mFileDescriptor(-1) +#endif { } @@ -49,7 +54,11 @@ NamedLock::NamedLock() // -------------------------------------------------------------------------- NamedLock::~NamedLock() { +#ifdef WIN32 + if(mFileDescriptor != INVALID_HANDLE_VALUE) +#else if(mFileDescriptor != -1) +#endif { ReleaseLock(); } @@ -68,76 +77,151 @@ NamedLock::~NamedLock() bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) { // Check +#ifdef WIN32 + if(mFileDescriptor != INVALID_HANDLE_VALUE) +#else if(mFileDescriptor != -1) +#endif { THROW_EXCEPTION(CommonException, NamedLockAlreadyLockingSomething) } + mFileName = rFilename; + // See if the lock can be got + int flags = O_WRONLY | O_CREAT | O_TRUNC; + #if HAVE_DECL_O_EXLOCK - int fd = ::open(rFilename.c_str(), - O_WRONLY | O_NONBLOCK | O_CREAT | O_TRUNC | O_EXLOCK, mode); - if(fd != -1) - { - // Got a lock, lovely - mFileDescriptor = fd; - return true; - } - - // Failed. Why? - if(errno != EWOULDBLOCK) - { - // Not the expected error - THROW_EXCEPTION(CommonException, OSFileError) - } + flags |= O_NONBLOCK | O_EXLOCK; + BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXLOCK"); +#elif defined BOX_OPEN_LOCK + flags |= BOX_OPEN_LOCK; + BOX_TRACE("Trying to create lockfile " << rFilename << " using BOX_OPEN_LOCK"); +#elif !HAVE_DECL_F_SETLK && !defined HAVE_FLOCK + // We have no other way to get a lock, so all we can do is fail if + // the file already exists, and take the risk of stale locks. + flags |= O_EXCL; + BOX_TRACE("Trying to create lockfile " << rFilename << " using O_EXCL"); +#else + BOX_TRACE("Trying to create lockfile " << rFilename << " without special flags"); +#endif - return false; +#ifdef WIN32 + HANDLE fd = openfile(rFilename.c_str(), flags, mode); + if(fd == INVALID_HANDLE_VALUE) #else - int fd = ::open(rFilename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, mode); + int fd = ::open(rFilename.c_str(), flags, mode); if(fd == -1) - { - BOX_WARNING("Failed to open lockfile: " << rFilename); - THROW_EXCEPTION(CommonException, OSFileError) - } - -#ifdef HAVE_FLOCK - if(::flock(fd, LOCK_EX | LOCK_NB) != 0) - { - ::close(fd); +#endif +#if HAVE_DECL_O_EXLOCK + { // if() if(errno == EWOULDBLOCK) { + // Lockfile already exists, and we tried to open it + // exclusively, which means we failed to lock it. + BOX_NOTICE("Failed to lock lockfile with O_EXLOCK: " << rFilename + << ": already locked by another process?"); return false; } else { - THROW_EXCEPTION(CommonException, OSFileError) + THROW_SYS_FILE_ERROR("Failed to open lockfile with O_EXLOCK", + rFilename, CommonException, OSFileError); } } -#elif HAVE_DECL_F_SETLK - struct flock desc; - desc.l_type = F_WRLCK; - desc.l_whence = SEEK_SET; - desc.l_start = 0; - desc.l_len = 0; - if(::fcntl(fd, F_SETLK, &desc) != 0) - { - ::close(fd); - if(errno == EAGAIN) +#else // !HAVE_DECL_O_EXLOCK + { // if() +# if defined BOX_OPEN_LOCK + if(errno == EBUSY) +# else // !BOX_OPEN_LOCK + if(errno == EEXIST && (flags & O_EXCL)) +# endif { + // Lockfile already exists, and we tried to open it + // exclusively, which means we failed to lock it. + BOX_NOTICE("Failed to lock lockfile with O_EXCL: " << rFilename + << ": already locked by another process?"); return false; } else { - THROW_EXCEPTION(CommonException, OSFileError) + THROW_SYS_FILE_ERROR("Failed to open lockfile with O_EXCL", + rFilename, CommonException, OSFileError); } } -#endif + + try + { +# ifdef HAVE_FLOCK + BOX_TRACE("Trying to lock lockfile " << rFilename << " using flock()"); + if(::flock(fd, LOCK_EX | LOCK_NB) != 0) + { + if(errno == EWOULDBLOCK) + { + ::close(fd); + BOX_NOTICE("Failed to lock lockfile with flock(): " << rFilename + << ": already locked by another process"); + return false; + } + else + { + THROW_SYS_FILE_ERROR("Failed to lock lockfile with flock()", + rFilename, CommonException, OSFileError); + } + } +# elif HAVE_DECL_F_SETLK + struct flock desc; + desc.l_type = F_WRLCK; + desc.l_whence = SEEK_SET; + desc.l_start = 0; + desc.l_len = 0; + BOX_TRACE("Trying to lock lockfile " << rFilename << " using fcntl()"); + if(::fcntl(fd, F_SETLK, &desc) != 0) + { + if(errno == EAGAIN) + { + ::close(fd); + BOX_NOTICE("Failed to lock lockfile with fcntl(): " << rFilename + << ": already locked by another process"); + return false; + } + else + { + THROW_SYS_FILE_ERROR("Failed to lock lockfile with fcntl()", + rFilename, CommonException, OSFileError); + } + } +# endif + } + catch(BoxException &e) + { +# ifdef WIN32 + CloseHandle(fd); +# else + ::close(fd); +# endif + BOX_NOTICE("Failed to lock lockfile " << rFilename << ": " << e.what()); + throw; + } +#endif // HAVE_DECL_O_EXLOCK + + if(!FileExists(rFilename)) + { + BOX_ERROR("Locked lockfile " << rFilename << ", but lockfile no longer " + "exists, bailing out"); +# ifdef WIN32 + CloseHandle(fd); +# else + ::close(fd); +# endif + return false; + } // Success mFileDescriptor = fd; + BOX_TRACE("Successfully locked lockfile " << rFilename); return true; -#endif } // -------------------------------------------------------------------------- @@ -151,20 +235,65 @@ bool NamedLock::TryAndGetLock(const std::string& rFilename, int mode) void NamedLock::ReleaseLock() { // Got a lock? +#ifdef WIN32 + if(mFileDescriptor == INVALID_HANDLE_VALUE) +#else if(mFileDescriptor == -1) +#endif { THROW_EXCEPTION(CommonException, NamedLockNotHeld) } - + +#ifndef WIN32 + // Delete the file. We need to do this before closing the filehandle, + // if we used flock() or fcntl() to lock it, otherwise someone could + // acquire the lock, release and delete it between us closing (and + // hence releasing) and deleting it, and we'd fail when it came to + // deleting the file. This happens in tests much more often than + // you'd expect! + // + // This doesn't apply on systems using plain lockfile locking, such as + // Windows, and there we need to close the file before deleting it, + // otherwise the system won't let us delete it. + + if(::unlink(mFileName.c_str()) != 0) + { + THROW_EMU_ERROR( + BOX_FILE_MESSAGE(mFileName, "Failed to delete lockfile"), + CommonException, OSFileError); + } +#endif // !WIN32 + // Close the file +# ifdef WIN32 + if(!CloseHandle(mFileDescriptor)) +# else if(::close(mFileDescriptor) != 0) +# endif { - THROW_EXCEPTION(CommonException, OSFileError) + THROW_EMU_ERROR( + BOX_FILE_MESSAGE(mFileName, "Failed to close lockfile"), + CommonException, OSFileError); } - // Mark as unlocked - mFileDescriptor = -1; -} + // Mark as unlocked, so we don't try to close it again if the unlink() fails. +#ifdef WIN32 + mFileDescriptor = INVALID_HANDLE_VALUE; +#else + mFileDescriptor = -1; +#endif +#ifdef WIN32 + // On Windows we need to close the file before deleting it, otherwise + // the system won't let us delete it. + if(::unlink(mFileName.c_str()) != 0) + { + THROW_EMU_ERROR( + BOX_FILE_MESSAGE(mFileName, "Failed to delete lockfile"), + CommonException, OSFileError); + } +#endif // WIN32 + BOX_TRACE("Released lock and deleted lockfile " << mFileName); +} diff --git a/lib/common/NamedLock.h b/lib/common/NamedLock.h index 534115db..a7d0d778 100644 --- a/lib/common/NamedLock.h +++ b/lib/common/NamedLock.h @@ -29,12 +29,21 @@ private: public: bool TryAndGetLock(const std::string& rFilename, int mode = 0755); +# ifdef WIN32 + bool GotLock() {return mFileDescriptor != INVALID_HANDLE_VALUE;} +# else bool GotLock() {return mFileDescriptor != -1;} +# endif void ReleaseLock(); - private: +# ifdef WIN32 + HANDLE mFileDescriptor; +# else int mFileDescriptor; +# endif + + std::string mFileName; }; #endif // NAMEDLOCK__H diff --git a/lib/common/PartialReadStream.cpp b/lib/common/PartialReadStream.cpp index f2f79715..b5f99bb5 100644 --- a/lib/common/PartialReadStream.cpp +++ b/lib/common/PartialReadStream.cpp @@ -104,7 +104,7 @@ IOStream::pos_type PartialReadStream::BytesLeftToRead() // Created: 2003/08/26 // // -------------------------------------------------------------------------- -void PartialReadStream::Write(const void *pBuffer, int NBytes) +void PartialReadStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, CantWriteToPartialReadStream) } diff --git a/lib/common/PartialReadStream.h b/lib/common/PartialReadStream.h index 1b46b0bd..61bdd7d1 100644 --- a/lib/common/PartialReadStream.h +++ b/lib/common/PartialReadStream.h @@ -33,7 +33,8 @@ private: public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); diff --git a/lib/common/RateLimitingStream.h b/lib/common/RateLimitingStream.h index a322b99b..818c90af 100644 --- a/lib/common/RateLimitingStream.h +++ b/lib/common/RateLimitingStream.h @@ -30,9 +30,10 @@ public: int Timeout = IOStream::TimeOutInfinite); // Everything else is delegated to the sink - virtual void Write(const void *pBuffer, int NBytes) + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) { - Write(pBuffer, NBytes); + mrSink.Write(pBuffer, NBytes, Timeout); } virtual pos_type BytesLeftToRead() { diff --git a/lib/common/ReadGatherStream.cpp b/lib/common/ReadGatherStream.cpp index f50e6664..ae252832 100644 --- a/lib/common/ReadGatherStream.cpp +++ b/lib/common/ReadGatherStream.cpp @@ -213,7 +213,7 @@ IOStream::pos_type ReadGatherStream::BytesLeftToRead() // Created: 10/12/03 // // -------------------------------------------------------------------------- -void ReadGatherStream::Write(const void *pBuffer, int NBytes) +void ReadGatherStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, CannotWriteToReadGatherStream); } diff --git a/lib/common/ReadGatherStream.h b/lib/common/ReadGatherStream.h index 613ede3e..9a44480b 100644 --- a/lib/common/ReadGatherStream.h +++ b/lib/common/ReadGatherStream.h @@ -37,7 +37,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); virtual pos_type GetPosition() const; diff --git a/lib/common/ReadLoggingStream.cpp b/lib/common/ReadLoggingStream.cpp index 54c99c95..df493344 100644 --- a/lib/common/ReadLoggingStream.cpp +++ b/lib/common/ReadLoggingStream.cpp @@ -96,7 +96,7 @@ IOStream::pos_type ReadLoggingStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void ReadLoggingStream::Write(const void *pBuffer, int NBytes) +void ReadLoggingStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, NotSupported); } diff --git a/lib/common/ReadLoggingStream.h b/lib/common/ReadLoggingStream.h index b23b542c..bee7e1d6 100644 --- a/lib/common/ReadLoggingStream.h +++ b/lib/common/ReadLoggingStream.h @@ -39,7 +39,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); @@ -48,7 +49,7 @@ public: virtual bool StreamClosed(); private: - ReadLoggingStream(const ReadLoggingStream &rToCopy) + ReadLoggingStream(const ReadLoggingStream &rToCopy) : mrSource(rToCopy.mrSource), mrLogger(rToCopy.mrLogger) { /* do not call */ } }; diff --git a/lib/common/SelfFlushingStream.h b/lib/common/SelfFlushingStream.h index 36e9a4d3..b4efa294 100644 --- a/lib/common/SelfFlushingStream.h +++ b/lib/common/SelfFlushingStream.h @@ -33,6 +33,12 @@ public: ~SelfFlushingStream() { + if(StreamDataLeft()) + { + BOX_WARNING("Not all data was read from stream, " + "discarding the rest"); + } + Flush(); } @@ -50,9 +56,10 @@ public: { return mrSource.BytesLeftToRead(); } - virtual void Write(const void *pBuffer, int NBytes) + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) { - mrSource.Write(pBuffer, NBytes); + mrSource.Write(pBuffer, NBytes, Timeout); } virtual bool StreamDataLeft() { diff --git a/lib/common/StreamableMemBlock.cpp b/lib/common/StreamableMemBlock.cpp index b376f037..9abf78d3 100644 --- a/lib/common/StreamableMemBlock.cpp +++ b/lib/common/StreamableMemBlock.cpp @@ -125,7 +125,9 @@ void StreamableMemBlock::Set(IOStream &rStream, int Timeout) try { // Read in - if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */)) + if(!rStream.ReadFullBuffer(pblock, size, + 0 /* not interested in bytes read if this fails */, + Timeout)) { THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) } @@ -252,7 +254,9 @@ void StreamableMemBlock::ReadFromStream(IOStream &rStream, int Timeout) { // Get the size of the block int32_t size_s; - if(!rStream.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */)) + if(!rStream.ReadFullBuffer(&size_s, sizeof(size_s), + 0, /* not interested in bytes read if this fails */ + Timeout)) { THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) } @@ -270,7 +274,9 @@ void StreamableMemBlock::ReadFromStream(IOStream &rStream, int Timeout) try { // Read in - if(!rStream.ReadFullBuffer(pblock, size, 0 /* not interested in bytes read if this fails */)) + if(!rStream.ReadFullBuffer(pblock, size, + 0, /* not interested in bytes read if this fails */ + Timeout)) { THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead) } diff --git a/lib/common/Test.cpp b/lib/common/Test.cpp index de87c465..2c51cd61 100644 --- a/lib/common/Test.cpp +++ b/lib/common/Test.cpp @@ -22,7 +22,236 @@ #endif #include "BoxTime.h" +#include "FileStream.h" #include "Test.h" +#include "Utils.h" + +int num_tests_selected = 0; +int num_failures = 0; +int old_failure_count = 0; +int first_fail_line; +std::string original_working_dir; +std::string first_fail_file; +std::string current_test_name; +std::list<std::string> run_only_named_tests; +std::map<std::string, std::string> s_test_status; + +bool setUp(const char* function_name) +{ + current_test_name = function_name; + + if (!run_only_named_tests.empty()) + { + bool run_this_test = false; + + for (std::list<std::string>::iterator + i = run_only_named_tests.begin(); + i != run_only_named_tests.end(); i++) + { + if (*i == current_test_name) + { + run_this_test = true; + break; + } + } + + if (!run_this_test) + { + // not in the list, so don't run it. + return false; + } + } + + printf("\n\n== %s ==\n", function_name); + num_tests_selected++; + old_failure_count = num_failures; + + if (original_working_dir == "") + { + char buf[1024]; + if (getcwd(buf, sizeof(buf)) == NULL) + { + BOX_LOG_SYS_ERROR("getcwd"); + } + original_working_dir = buf; + } + else + { + if (chdir(original_working_dir.c_str()) != 0) + { + BOX_LOG_SYS_ERROR("chdir"); + } + } + +#ifdef _MSC_VER + DIR* pDir = opendir("testfiles"); + if(!pDir) + { + THROW_SYS_FILE_ERROR("Failed to open test temporary directory", + "testfiles", CommonException, Internal); + } + struct dirent* pEntry; + for(pEntry = readdir(pDir); pEntry; pEntry = readdir(pDir)) + { + std::string filename = pEntry->d_name; + if(StartsWith("TestDir", filename) || + StartsWith("0_", filename) || + filename == "accounts.txt" || + StartsWith("file", filename) || + StartsWith("notifyran", filename) || + StartsWith("notifyscript.tag", filename) || + StartsWith("restore", filename) || + filename == "bbackupd-data" || + filename == "syncallowscript.control" || + StartsWith("syncallowscript.notifyran.", filename) || + filename == "test2.downloaded" || + EndsWith("testfile", filename)) + { + std::string filepath = std::string("testfiles\\") + filename; + + int filetype = ObjectExists(filepath); + if(filetype == ObjectExists_File) + { + if(::unlink(filepath.c_str()) != 0) + { + TEST_FAIL_WITH_MESSAGE(BOX_SYS_ERROR_MESSAGE("Failed to delete " + "test fixture file: unlink(\"" << filepath << "\")")); + } + } + else if(filetype == ObjectExists_Dir) + { + std::string cmd = "cmd /c rd /s /q " + filepath; + WCHAR* wide_cmd = ConvertUtf8ToWideString(cmd.c_str()); + if(wide_cmd == NULL) + { + TEST_FAIL_WITH_MESSAGE("Failed to convert string " + "to wide string: " << cmd); + continue; + } + + STARTUPINFOW si; + PROCESS_INFORMATION pi; + + ZeroMemory( &si, sizeof(si) ); + si.cb = sizeof(si); + ZeroMemory( &pi, sizeof(pi) ); + + BOOL result = CreateProcessW( + NULL, // lpApplicationName + wide_cmd, // lpCommandLine + NULL, // lpProcessAttributes + NULL, // lpThreadAttributes + TRUE, // bInheritHandles + 0, // dwCreationFlags + NULL, // lpEnvironment + NULL, // lpCurrentDirectory + &si, // lpStartupInfo + &pi // lpProcessInformation + ); + delete [] wide_cmd; + + if(result == FALSE) + { + TEST_FAIL_WITH_MESSAGE("Failed to delete test " + "fixture file: failed to execute command " + "'" << cmd << "': " << + GetErrorMessage(GetLastError())); + continue; + } + + // Wait until child process exits. + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD exit_code; + result = GetExitCodeProcess(pi.hProcess, &exit_code); + + if(result == FALSE) + { + TEST_FAIL_WITH_MESSAGE("Failed to delete " + "test fixture file: failed to get " + "command exit status: '" << + cmd << "': " << + GetErrorMessage(GetLastError())); + } + else if(exit_code != 0) + { + TEST_FAIL_WITH_MESSAGE("Failed to delete test " + "fixture file: command '" << cmd << "' " + "exited with status " << exit_code); + } + + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + } + else + { + TEST_FAIL_WITH_MESSAGE("Don't know how to delete file " << filepath << + " of type " << filetype); + } + } + } + closedir(pDir); + FileStream touch("testfiles/accounts.txt", O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); +#else + TEST_THAT_THROWONFAIL(system( + "rm -rf testfiles/TestDir* testfiles/0_0 testfiles/0_1 " + "testfiles/0_2 testfiles/accounts.txt " // testfiles/test* .tgz! + "testfiles/file* testfiles/notifyran testfiles/notifyran.* " + "testfiles/notifyscript.tag* " + "testfiles/restore* testfiles/bbackupd-data " + "testfiles/syncallowscript.control " + "testfiles/syncallowscript.notifyran.* " + "testfiles/test2.downloaded" + ) == 0); + TEST_THAT_THROWONFAIL(system("touch testfiles/accounts.txt") == 0); +#endif + TEST_THAT_THROWONFAIL(mkdir("testfiles/0_0", 0755) == 0); + TEST_THAT_THROWONFAIL(mkdir("testfiles/0_1", 0755) == 0); + TEST_THAT_THROWONFAIL(mkdir("testfiles/0_2", 0755) == 0); + TEST_THAT_THROWONFAIL(mkdir("testfiles/bbackupd-data", 0755) == 0); + + return true; +} + +bool tearDown() +{ + if (num_failures == old_failure_count) + { + BOX_NOTICE(current_test_name << " passed"); + s_test_status[current_test_name] = "passed"; + return true; + } + else + { + BOX_NOTICE(current_test_name << " failed"); \ + s_test_status[current_test_name] = "FAILED"; + return false; + } +} + +bool fail() +{ + num_failures++; + return tearDown(); +} + +int finish_test_suite() +{ + printf("\n"); + printf("Test results:\n"); + + typedef std::map<std::string, std::string>::iterator s_test_status_iterator; + for(s_test_status_iterator i = s_test_status.begin(); + i != s_test_status.end(); i++) + { + BOX_NOTICE("test result: " << i->second << ": " << i->first); + } + + TEST_LINE(num_tests_selected > 0, "No tests matched the patterns " + "specified on the command line"); + + return (num_failures == 0 && num_tests_selected > 0) ? 0 : 1; +} bool TestFileExists(const char *Filename) { @@ -136,7 +365,7 @@ int ReadPidFile(const char *pidFile) if(!TestFileNotEmpty(pidFile)) { TEST_FAIL_WITH_MESSAGE("Server didn't save PID file " - "(perhaps one was already running?)"); + "(perhaps one was already running?)"); return -1; } @@ -145,7 +374,7 @@ int ReadPidFile(const char *pidFile) FILE *f = fopen(pidFile, "r"); if(f == NULL || fscanf(f, "%d", &pid) != 1) { - TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); + TEST_FAIL_WITH_MESSAGE("Couldn't read PID file"); return -1; } fclose(f); @@ -155,7 +384,7 @@ int ReadPidFile(const char *pidFile) int LaunchServer(const std::string& rCommandLine, const char *pidFile) { - ::fprintf(stdout, "Starting server: %s\n", rCommandLine.c_str()); + BOX_INFO("Starting server: " << rCommandLine); #ifdef WIN32 @@ -189,14 +418,10 @@ int LaunchServer(const std::string& rCommandLine, const char *pidFile) free(tempCmd); - if (result == 0) - { - DWORD err = GetLastError(); - printf("Launch failed: %s: error %d\n", rCommandLine.c_str(), - (int)err); - TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + TEST_THAT_OR(result != 0, + BOX_LOG_WIN_ERROR("Launch failed: " << rCommandLine); return -1; - } + ); CloseHandle(procInfo.hProcess); CloseHandle(procInfo.hThread); @@ -205,11 +430,10 @@ int LaunchServer(const std::string& rCommandLine, const char *pidFile) #else // !WIN32 - if(RunCommand(rCommandLine) != 0) - { - TEST_FAIL_WITH_MESSAGE("Couldn't start server"); + TEST_THAT_OR(RunCommand(rCommandLine) == 0, + TEST_FAIL_WITH_MESSAGE("Failed to start server: " << rCommandLine); return -1; - } + ) return WaitForServerStartup(pidFile, 0); @@ -230,18 +454,11 @@ int WaitForServerStartup(const char *pidFile, int pidIfKnown) #endif // time for it to start up - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_TRACE("Waiting for server to start"); - } - else - { - ::fprintf(stdout, "Waiting for server to start: "); - } + BOX_TRACE("Waiting for server to start"); for (int i = 0; i < 15; i++) { - if (TestFileNotEmpty(pidFile)) + if (TestFileNotEmpty(pidFile)) { break; } @@ -251,12 +468,6 @@ int WaitForServerStartup(const char *pidFile, int pidIfKnown) break; } - if (Logging::GetGlobalLevel() < Log::TRACE) - { - ::fprintf(stdout, "."); - ::fflush(stdout); - } - ::sleep(1); } @@ -265,42 +476,17 @@ int WaitForServerStartup(const char *pidFile, int pidIfKnown) if (pidIfKnown && !ServerIsAlive(pidIfKnown)) { - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_ERROR("server died!"); - } - else - { - ::fprintf(stdout, " server died!\n"); - } - - TEST_FAIL_WITH_MESSAGE("Server died!"); + TEST_FAIL_WITH_MESSAGE("Server died!"); return -1; } if (!TestFileNotEmpty(pidFile)) { - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_ERROR("timed out!"); - } - else - { - ::fprintf(stdout, " timed out!\n"); - } - - TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); + TEST_FAIL_WITH_MESSAGE("Server didn't save PID file"); return -1; } - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_TRACE("Server started"); - } - else - { - ::fprintf(stdout, " done.\n"); - } + BOX_TRACE("Server started"); // wait a second for the pid to be written to the file ::sleep(1); @@ -330,12 +516,12 @@ void TestRemoteProcessMemLeaksFunc(const char *filename, // Does the file exist? if(!TestFileExists(filename)) { - if (failures == 0) + if (num_failures == 0) { first_fail_file = file; first_fail_line = line; } - ++failures; + ++num_failures; printf("FAILURE: MemLeak report not available (file %s) " "at %s:%d\n", filename, file, line); } @@ -344,12 +530,12 @@ void TestRemoteProcessMemLeaksFunc(const char *filename, // Is it empty? if(TestGetFileSize(filename) > 0) { - if (failures == 0) + if (num_failures == 0) { first_fail_file = file; first_fail_line = line; } - ++failures; + ++num_failures; printf("FAILURE: Memory leaks found in other process " "(file %s) at %s:%d\n==========\n", filename, file, line); @@ -378,23 +564,29 @@ void force_sync() void wait_for_sync_start() { + BOX_TRACE("Waiting for sync to start..."); TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " "wait-for-sync") == 0); TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + BOX_TRACE("Backup daemon reported that sync has started."); } void wait_for_sync_end() { + BOX_TRACE("Waiting for sync to finish..."); TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " "wait-for-end") == 0); TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + BOX_TRACE("Backup daemon reported that sync has finished."); } void sync_and_wait() { + BOX_TRACE("Starting a sync and waiting for it to finish..."); TEST_THAT(::system(BBACKUPCTL " -q -c testfiles/bbackupd.conf " "sync-and-wait") == 0); TestRemoteProcessMemLeaks("bbackupctl.memleaks"); + BOX_TRACE("Backup daemon reported that sync has finished."); } void terminate_bbackupd(int pid) @@ -419,35 +611,14 @@ void terminate_bbackupd(int pid) // Wait a given number of seconds for something to complete void wait_for_operation(int seconds, const char* message) { - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_TRACE("Waiting " << seconds << " seconds for " << message); - } - else - { - printf("Waiting for %s: ", message); - fflush(stdout); - } + BOX_INFO("Waiting " << seconds << " seconds for " << message); for(int l = 0; l < seconds; ++l) { sleep(1); - if (Logging::GetGlobalLevel() < Log::TRACE) - { - printf("."); - fflush(stdout); - } } - if (Logging::GetGlobalLevel() >= Log::TRACE) - { - BOX_TRACE("Finished waiting for " << message); - } - else - { - printf(" done.\n"); - fflush(stdout); - } + BOX_TRACE("Finished waiting for " << message); } void safe_sleep(int seconds) @@ -455,3 +626,15 @@ void safe_sleep(int seconds) ShortSleep(SecondsToBoxTime(seconds), true); } +std::auto_ptr<Configuration> load_config_file(const std::string& config_file, + const ConfigurationVerify& verify) +{ + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify(config_file, &verify, errs)); + TEST_EQUAL_LINE(0, errs.size(), "Failed to load configuration file: " + config_file + + ": " + errs); + TEST_EQUAL_OR(0, errs.size(), config.reset()); + return config; +} + diff --git a/lib/common/Test.h b/lib/common/Test.h index f318c811..4b5cef61 100644 --- a/lib/common/Test.h +++ b/lib/common/Test.h @@ -11,6 +11,10 @@ #define TEST__H #include <cstring> +#include <list> +#include <map> + +#include "Configuration.h" #ifdef WIN32 #define BBACKUPCTL "..\\..\\bin\\bbackupctl\\bbackupctl.exe" @@ -28,30 +32,70 @@ #define TEST_RETURN(actual, expected) TEST_EQUAL((expected << 8), actual); #endif -extern int failures; +extern int num_failures; extern int first_fail_line; +extern int num_tests_selected; +extern int old_failure_count; extern std::string first_fail_file; extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; +extern std::list<std::string> run_only_named_tests; +extern std::string current_test_name; +extern std::map<std::string, std::string> s_test_status; + +//! Simplifies calling setUp() with the current function name in each test. +#define SETUP() \ + if (!setUp(__FUNCTION__)) return true; \ + try \ + { // left open for TEARDOWN() + +#define TEARDOWN() \ + return tearDown(); \ + } \ + catch (BoxException &e) \ + { \ + BOX_NOTICE(__FUNCTION__ << " errored: " << e.what()); \ + num_failures++; \ + tearDown(); \ + s_test_status[__FUNCTION__] = "ERRORED"; \ + return false; \ + } + +//! End the current test. Only use within a test function, because it just returns false! +#define FAIL { \ + std::ostringstream os; \ + os << "failed at " << __FUNCTION__ << ":" << __LINE__; \ + s_test_status[current_test_name] = os.str(); \ + return fail(); \ +} #define TEST_FAIL_WITH_MESSAGE(msg) \ { \ - if (failures == 0) \ + if (num_failures == 0) \ { \ first_fail_file = __FILE__; \ first_fail_line = __LINE__; \ } \ - failures++; \ + num_failures++; \ BOX_ERROR("**** TEST FAILURE: " << msg << " at " << __FILE__ << \ ":" << __LINE__); \ } #define TEST_ABORT_WITH_MESSAGE(msg) {TEST_FAIL_WITH_MESSAGE(msg); return 1;} -#define TEST_THAT(condition) {if(!(condition)) TEST_FAIL_WITH_MESSAGE("Condition [" #condition "] failed")} +#define TEST_THAT_OR(condition, or_command) \ + if(!(condition)) \ + { \ + TEST_FAIL_WITH_MESSAGE("Condition [" #condition "] failed"); \ + or_command; \ + } +#define TEST_THAT(condition) TEST_THAT_OR(condition,) #define TEST_THAT_ABORTONFAIL(condition) {if(!(condition)) TEST_ABORT_WITH_MESSAGE("Condition [" #condition "] failed")} +#define TEST_THAT_THROWONFAIL(condition) \ + TEST_THAT_OR(condition, THROW_EXCEPTION_MESSAGE(CommonException, \ + AssertFailed, "Condition [" #condition "] failed")); // NOTE: The 0- bit is to allow this to work with stuff which has negative constants for flags (eg ConnectionException) -#define TEST_CHECK_THROWS(statement, excepttype, subtype) \ +#define TEST_CHECK_THROWS_AND_OR(statement, excepttype, subtype, and_command, or_command) \ { \ bool didthrow = false; \ HideExceptionMessageGuard hide; \ @@ -69,6 +113,7 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; throw; \ } \ didthrow = true; \ + and_command; \ } \ catch(...) \ { \ @@ -76,12 +121,15 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; } \ if(!didthrow) \ { \ - TEST_FAIL_WITH_MESSAGE("Didn't throw exception " #excepttype "(" #subtype ")") \ + TEST_FAIL_WITH_MESSAGE("Didn't throw exception " #excepttype "(" #subtype ")"); \ + or_command; \ } \ } +#define TEST_CHECK_THROWS(statement, excepttype, subtype) \ + TEST_CHECK_THROWS_AND_OR(statement, excepttype, subtype,,) // utility macro for comparing two strings in a line -#define TEST_EQUAL(_expected, _found) \ +#define TEST_EQUAL_OR(_expected, _found, or_command) \ { \ std::ostringstream _oss1; \ _oss1 << _expected; \ @@ -100,8 +148,11 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; _oss3 << #_found << " != " << #_expected; \ \ TEST_FAIL_WITH_MESSAGE(_oss3.str().c_str()); \ + or_command; \ } \ } +#define TEST_EQUAL(_expected, _found) \ + TEST_EQUAL_OR(_expected, _found,) // utility macro for comparing two strings in a line #define TEST_EQUAL_LINE(_expected, _found, _line) \ @@ -129,7 +180,7 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; } \ } -// utility macro for testing a line +// utility macros for testing a string/output line #define TEST_LINE(_condition, _line) \ TEST_THAT(_condition); \ if (!(_condition)) \ @@ -140,6 +191,28 @@ extern std::string bbackupd_args, bbstored_args, bbackupquery_args, test_args; printf("Test failed on <%s>\n", _line_str.c_str()); \ } +#define TEST_LINE_OR(_condition, _line, _or_command) \ + TEST_LINE(_condition, _line); \ + if(!(_condition)) \ + { \ + _or_command; \ + } + +#define TEST_STARTSWITH(expected, actual) \ + TEST_EQUAL_LINE(expected, actual.substr(0, std::string(expected).size()), actual); + +//! Sets up (cleans up) test environment at the start of every test. +bool setUp(const char* function_name); + +//! Checks account for errors and shuts down daemons at end of every test. +bool tearDown(); + +//! Like tearDown() but returns false, because a test failure was detected. +bool fail(); + +//! Report final status of all tests, and return the correct value to test main(). +int finish_test_suite(); + bool TestFileExists(const char *Filename); bool TestDirExists(const char *Filename); @@ -167,5 +240,17 @@ void terminate_bbackupd(int pid); // Wait a given number of seconds for something to complete void wait_for_operation(int seconds, const char* message); void safe_sleep(int seconds); +std::auto_ptr<Configuration> load_config_file(const std::string& config_file, + const ConfigurationVerify& verify); + +#ifndef TEST_EXECUTABLE +# ifdef _MSC_VER + // Our CMakeFiles compile tests to different executable filenames, + // e.g. test_common.exe instead of _test.exe. + #define TEST_EXECUTABLE BOX_MODULE ".exe" +# else + #define TEST_EXECUTABLE "./_test" +# endif +#endif // TEST_EXECUTABLE #endif // TEST__H diff --git a/lib/common/Timer.cpp b/lib/common/Timer.cpp index ad6b5e8d..4f8c989e 100644 --- a/lib/common/Timer.cpp +++ b/lib/common/Timer.cpp @@ -55,8 +55,8 @@ void Timers::Init() sigemptyset(&newact.sa_mask); if (::sigaction(SIGALRM, &newact, &oldact) != 0) { - BOX_ERROR("Failed to install signal handler"); - THROW_EXCEPTION(CommonException, Internal); + THROW_SYS_ERROR("Failed to install signal handler", + CommonException, Internal); } ASSERT(oldact.sa_handler == 0); #endif // WIN32 && !PLATFORM_CYGWIN @@ -72,13 +72,23 @@ void Timers::Init() // Created: 6/11/2006 // // -------------------------------------------------------------------------- -void Timers::Cleanup() +void Timers::Cleanup(bool throw_exception_if_not_initialised) { - ASSERT(spTimers); - if (!spTimers) + if (throw_exception_if_not_initialised) { - BOX_ERROR("Tried to clean up timers when not initialised!"); - return; + ASSERT(spTimers); + if (!spTimers) + { + BOX_ERROR("Tried to clean up timers when not initialised!"); + return; + } + } + else + { + if (!spTimers) + { + return; + } } #if defined WIN32 && ! defined PLATFORM_CYGWIN @@ -87,8 +97,11 @@ void Timers::Cleanup() struct itimerval timeout; memset(&timeout, 0, sizeof(timeout)); - int result = ::setitimer(ITIMER_REAL, &timeout, NULL); - ASSERT(result == 0); + if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) + { + THROW_SYS_ERROR("Failed to set interval timer", + CommonException, Internal); + } struct sigaction newact, oldact; newact.sa_handler = SIG_DFL; @@ -96,8 +109,8 @@ void Timers::Cleanup() sigemptyset(&(newact.sa_mask)); if (::sigaction(SIGALRM, &newact, &oldact) != 0) { - BOX_ERROR("Failed to remove signal handler"); - THROW_EXCEPTION(CommonException, Internal); + THROW_SYS_ERROR("Failed to remove signal handler", + CommonException, Internal); } ASSERT(oldact.sa_handler == Timers::SignalHandler); #endif // WIN32 && !PLATFORM_CYGWIN @@ -118,7 +131,6 @@ void Timers::Cleanup() void Timers::Add(Timer& rTimer) { ASSERT(spTimers); - ASSERT(&rTimer); BOX_TRACE(TIMER_ID_OF(rTimer) " added to global queue, rescheduling"); spTimers->push_back(&rTimer); Reschedule(); @@ -135,8 +147,14 @@ void Timers::Add(Timer& rTimer) // -------------------------------------------------------------------------- void Timers::Remove(Timer& rTimer) { + if(!spTimers) + { + BOX_WARNING(TIMER_ID_OF(rTimer) " was still active after " + "timer subsystem was cleaned up, already removed."); + return; + } + ASSERT(spTimers); - ASSERT(&rTimer); BOX_TRACE(TIMER_ID_OF(rTimer) " removed from global queue, rescheduling"); bool restart = true; @@ -155,7 +173,7 @@ void Timers::Remove(Timer& rTimer) } } } - + Reschedule(); } @@ -185,26 +203,24 @@ void Timers::Reschedule() ASSERT(spTimers); if (spTimers == NULL) { - THROW_EXCEPTION(CommonException, Internal) + THROW_EXCEPTION(CommonException, TimersNotInitialised); } #ifndef WIN32 struct sigaction oldact; if (::sigaction(SIGALRM, NULL, &oldact) != 0) { - BOX_ERROR("Failed to check signal handler"); - THROW_EXCEPTION(CommonException, Internal) + THROW_SYS_ERROR("Failed to check signal handler", + CommonException, Internal); } ASSERT(oldact.sa_handler == Timers::SignalHandler); if (oldact.sa_handler != Timers::SignalHandler) { - BOX_ERROR("Signal handler was " << - (void *)oldact.sa_handler << - ", expected " << - (void *)Timers::SignalHandler); - THROW_EXCEPTION(CommonException, Internal) + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Signal handler was " << (void *)oldact.sa_handler << + ", expected " << (void *)Timers::SignalHandler); } #endif @@ -218,6 +234,8 @@ void Timers::Reschedule() // win32 timers need no management #else box_time_t timeNow = GetCurrentBoxTime(); + int64_t timeToNextEvent; + std::string nameOfNextEvent; // scan for, trigger and remove expired timers. Removal requires // us to restart the scan each time, due to std::vector semantics. @@ -225,6 +243,7 @@ void Timers::Reschedule() while (restart) { restart = false; + timeToNextEvent = 0; for (std::vector<Timer*>::iterator i = spTimers->begin(); i != spTimers->end(); i++) @@ -252,35 +271,14 @@ void Timers::Reschedule() " seconds"); */ } - } - } - - // Now the only remaining timers should all be in the future. - // Scan to find the next one to fire (earliest deadline). - - int64_t timeToNextEvent = 0; - std::string nameOfNextEvent; - - for (std::vector<Timer*>::iterator i = spTimers->begin(); - i != spTimers->end(); i++) - { - Timer& rTimer = **i; - int64_t timeToExpiry = rTimer.GetExpiryTime() - timeNow; - ASSERT(timeToExpiry > 0) - if (timeToExpiry <= 0) - { - timeToExpiry = 1; - } - - if (timeToNextEvent == 0 || timeToNextEvent > timeToExpiry) - { - timeToNextEvent = timeToExpiry; - nameOfNextEvent = rTimer.GetName(); + if (timeToNextEvent == 0 || timeToNextEvent > timeToExpiry) + { + timeToNextEvent = timeToExpiry; + nameOfNextEvent = rTimer.GetName(); + } } } - - ASSERT(timeToNextEvent >= 0); if (timeToNextEvent == 0) { @@ -302,8 +300,8 @@ void Timers::Reschedule() if(::setitimer(ITIMER_REAL, &timeout, NULL) != 0) { - BOX_ERROR("Failed to initialise system timer\n"); - THROW_EXCEPTION(CommonException, Internal) + THROW_SYS_ERROR("Failed to initialise system timer", + CommonException, Internal); } #endif } @@ -322,7 +320,6 @@ void Timers::Reschedule() // -------------------------------------------------------------------------- void Timers::SignalHandler(int unused) { - // ASSERT(spTimers); Timers::RequestReschedule(); } diff --git a/lib/common/Timer.h b/lib/common/Timer.h index 09be58fa..17233203 100644 --- a/lib/common/Timer.h +++ b/lib/common/Timer.h @@ -43,7 +43,7 @@ class Timers public: static void Init(); - static void Cleanup(); + static void Cleanup(bool throw_exception_if_not_initialised = true); static void Add (Timer& rTimer); static void Remove(Timer& rTimer); static void RequestReschedule(); diff --git a/lib/common/Utils.cpp b/lib/common/Utils.cpp index decc80e8..0915f29a 100644 --- a/lib/common/Utils.cpp +++ b/lib/common/Utils.cpp @@ -15,7 +15,7 @@ #include <cstdlib> -#ifdef SHOW_BACKTRACE_ON_EXCEPTION +#ifdef HAVE_EXECINFO_H #include <execinfo.h> #include <stdlib.h> #endif @@ -51,25 +51,40 @@ std::string GetBoxBackupVersion() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput) +void SplitString(std::string String, char SplitOn, std::vector<std::string> &rOutput) { // Split it up. - std::string::size_type b = 0; - std::string::size_type e = 0; - while(e = String.find_first_of(SplitOn, b), e != String.npos) + std::string::size_type begin = 0, end = 0, pos = 0; + + while(end = String.find_first_of(SplitOn, pos), end != String.npos) { - // Get this string - unsigned int len = e - b; - if(len >= 1) + // Is it preceded by the escape character? + if(end > 0 && String[end - 1] == '\\') + { + // Ignore this one, don't change begin, let the next + // match/fallback consume it instead. But remove the + // backslash from the string, and set pos to the + // current position, which no longer contains a + // separator character. + String.erase(end - 1, 1); + pos = end; + } + else { - rOutput.push_back(String.substr(b, len)); + // Extract the substring and move past it. + unsigned int len = end - begin; + if(len >= 1) + { + rOutput.push_back(String.substr(begin, len)); + } + begin = end + 1; + pos = begin; } - b = e + 1; } // Last string - if(b < String.size()) + if(begin < String.size()) { - rOutput.push_back(String.substr(b)); + rOutput.push_back(String.substr(begin)); } /*#ifndef BOX_RELEASE_BUILD BOX_TRACE("Splitting string '" << String << " on " << (char)SplitOn); @@ -80,71 +95,95 @@ void SplitString(const std::string &String, char SplitOn, std::vector<std::strin #endif*/ } -#ifdef SHOW_BACKTRACE_ON_EXCEPTION +bool StartsWith(const std::string& prefix, const std::string& haystack) +{ + return haystack.size() >= prefix.size() && + haystack.substr(0, prefix.size()) == prefix; +} + +bool EndsWith(const std::string& suffix, const std::string& haystack) +{ + return haystack.size() >= suffix.size() && + haystack.substr(haystack.size() - suffix.size()) == suffix; +} + +std::string RemovePrefix(const std::string& prefix, const std::string& haystack) +{ + if(StartsWith(prefix, haystack)) + { + return haystack.substr(prefix.size()); + } + else + { + return ""; + } +} + +std::string RemoveSuffix(const std::string& suffix, const std::string& haystack) +{ + if(EndsWith(suffix, haystack)) + { + return haystack.substr(0, haystack.size() - suffix.size()); + } + else + { + return ""; + } +} + static std::string demangle(const std::string& mangled_name) { + std::string demangled_name = mangled_name; + #ifdef HAVE_CXXABI_H + char buffer[1024]; int status; + size_t length = sizeof(buffer); -#include "MemLeakFindOff.h" char* result = abi::__cxa_demangle(mangled_name.c_str(), - NULL, NULL, &status); -#include "MemLeakFindOn.h" + buffer, &length, &status); - if (result == NULL) + if (status == 0) { - if (status == 0) - { - BOX_WARNING("Demangle failed but no error: " << - mangled_name); - } - else if (status == -1) - { - BOX_WARNING("Demangle failed with " - "memory allocation error: " << - mangled_name); - } - else if (status == -2) - { - // Probably non-C++ name, don't demangle - /* - BOX_WARNING("Demangle failed with " - "with invalid name: " << - mangled_name); - */ - } - else if (status == -3) - { - BOX_WARNING("Demangle failed with " - "with invalid argument: " << - mangled_name); - } - else - { - BOX_WARNING("Demangle failed with " - "with unknown error " << status << - ": " << mangled_name); - } - - return std::string(mangled_name); + demangled_name = result; + } + else if (status == -1) + { + BOX_WARNING("Demangle failed with " + "memory allocation error: " << + mangled_name); + } + else if (status == -2) + { + // Probably non-C++ name, don't demangle + /* + BOX_WARNING("Demangle failed with " + "with invalid name: " << + mangled_name); + */ + } + else if (status == -3) + { + BOX_WARNING("Demangle failed with " + "with invalid argument: " << + mangled_name); } else { - std::string output = result; -#include "MemLeakFindOff.h" - free(result); -#include "MemLeakFindOn.h" - return output; + BOX_WARNING("Demangle failed with " + "with unknown error " << status << + ": " << mangled_name); } - #else // !HAVE_CXXABI_H - return mangled_name; #endif // HAVE_CXXABI_H + + return demangled_name; } void DumpStackBacktrace() { - void *array[10]; - size_t size = backtrace(array, 10); +#ifdef HAVE_EXECINFO_H + void *array[20]; + size_t size = backtrace(array, 20); BOX_TRACE("Obtained " << size << " stack frames."); for(size_t i = 0; i < size; i++) @@ -179,8 +218,10 @@ void DumpStackBacktrace() BOX_TRACE(output.str()); } +#else // !HAVE_EXECINFO_H + BOX_TRACE("Backtrace support was not compiled in"); +#endif // HAVE_EXECINFO_H } -#endif // SHOW_BACKTRACE_ON_EXCEPTION @@ -340,30 +381,3 @@ std::string FormatUsageLineStart(const std::string& rName, return result.str(); } -std::string BoxGetTemporaryDirectoryName() -{ -#ifdef WIN32 - // http://msdn.microsoft.com/library/default.asp? - // url=/library/en-us/fileio/fs/creating_and_using_a_temporary_file.asp - - DWORD dwRetVal; - char lpPathBuffer[1024]; - DWORD dwBufSize = sizeof(lpPathBuffer); - - // Get the temp path. - dwRetVal = GetTempPath(dwBufSize, // length of the buffer - lpPathBuffer); // buffer for path - if (dwRetVal > dwBufSize) - { - THROW_EXCEPTION(CommonException, TempDirPathTooLong) - } - - return std::string(lpPathBuffer); -#elif defined TEMP_DIRECTORY_NAME - return std::string(TEMP_DIRECTORY_NAME); -#else - #error non-static temporary directory names not supported yet -#endif -} - - diff --git a/lib/common/Utils.h b/lib/common/Utils.h index 3134245a..d306ce1c 100644 --- a/lib/common/Utils.h +++ b/lib/common/Utils.h @@ -17,11 +17,13 @@ std::string GetBoxBackupVersion(); -void SplitString(const std::string &String, char SplitOn, std::vector<std::string> &rOutput); +void SplitString(std::string String, char SplitOn, std::vector<std::string> &rOutput); +bool StartsWith(const std::string& prefix, const std::string& haystack); +bool EndsWith(const std::string& prefix, const std::string& haystack); +std::string RemovePrefix(const std::string& prefix, const std::string& haystack); +std::string RemoveSuffix(const std::string& suffix, const std::string& haystack); -#ifdef SHOW_BACKTRACE_ON_EXCEPTION - void DumpStackBacktrace(); -#endif +void DumpStackBacktrace(); bool FileExists(const std::string& rFilename, int64_t *pFileSize = 0, bool TreatLinksAsNotExisting = false); diff --git a/lib/common/ZeroStream.cpp b/lib/common/ZeroStream.cpp index d11ed80c..e1342e6f 100644 --- a/lib/common/ZeroStream.cpp +++ b/lib/common/ZeroStream.cpp @@ -76,7 +76,7 @@ IOStream::pos_type ZeroStream::BytesLeftToRead() // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void ZeroStream::Write(const void *pBuffer, int NBytes) +void ZeroStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(CommonException, NotSupported); } diff --git a/lib/common/ZeroStream.h b/lib/common/ZeroStream.h index 0119045b..f91221b0 100644 --- a/lib/common/ZeroStream.h +++ b/lib/common/ZeroStream.h @@ -22,7 +22,8 @@ public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(IOStream::pos_type Offset, int SeekType); virtual void Close(); diff --git a/lib/common/makeexception.pl.in b/lib/common/makeexception.pl.in index b1b3a8ac..bddaa94a 100755 --- a/lib/common/makeexception.pl.in +++ b/lib/common/makeexception.pl.in @@ -73,12 +73,13 @@ class ${class}Exception : public BoxException public: ${class}Exception(unsigned int SubType, const std::string& rMessage = "") - : mSubType(SubType), mMessage(rMessage) + : mSubType(SubType), mMessage(rMessage), + mWhat(GetMessage(SubType) + std::string(rMessage.empty() ? "" : ": ") + rMessage) { } ${class}Exception(const ${class}Exception &rToCopy) - : mSubType(rToCopy.mSubType), mMessage(rToCopy.mMessage) + : mSubType(rToCopy.mSubType), mMessage(rToCopy.mMessage), mWhat(rToCopy.mWhat) { } @@ -113,10 +114,11 @@ print H <<__E; { return mMessage; } - + static const char* GetMessage(int SubType); private: unsigned int mSubType; std::string mMessage; + std::string mWhat; }; #endif // $guardname @@ -133,74 +135,39 @@ print CPP <<__E; #include "MemLeakFindOn.h" -#ifdef EXCEPTION_CODENAMES_EXTENDED - #ifdef EXCEPTION_CODENAMES_EXTENDED_WITH_DESCRIPTION -static const char *whats[] = { -__E - -my $last_seen = -1; -for(my $e = 0; $e <= $#exception; $e++) +unsigned int ${class}Exception::GetType() const throw() { - if($exception[$e] ne '') - { - for(my $s = $last_seen + 1; $s < $e; $s++) - { - print CPP "\t\"UNUSED\",\n" - } - my $ext = ($exception_desc[$e] ne '')?" ($exception_desc[$e])":''; - print CPP "\t\"${class} ".$exception[$e].$ext.'"'.(($e==$#exception)?'':',')."\n"; - $last_seen = $e; - } + return ${class}Exception::ExceptionType; } -print CPP <<__E; -}; - #else -static const char *whats[] = { -__E - -$last_seen = -1; -for(my $e = 0; $e <= $#exception; $e++) +unsigned int ${class}Exception::GetSubType() const throw() { - if($exception[$e] ne '') - { - for(my $s = $last_seen + 1; $s < $e; $s++) - { - print CPP "\t\"UNUSED\",\n" - } - print CPP "\t\"${class} ".$exception[$e].'"'.(($e==$#exception)?'':',')."\n"; - $last_seen = $e; - } + return mSubType; } -print CPP <<__E; -}; - #endif -#endif - -unsigned int ${class}Exception::GetType() const throw() +const char * ${class}Exception::what() const throw() { - return ${class}Exception::ExceptionType; + return mWhat.c_str(); } -unsigned int ${class}Exception::GetSubType() const throw() +const char * ${class}Exception::GetMessage(int SubType) { - return mSubType; -} + switch(SubType) + { +__E -const char *${class}Exception::what() const throw() +for(my $e = 0; $e <= $#exception; $e++) { -#ifdef EXCEPTION_CODENAMES_EXTENDED - if(mSubType > (sizeof(whats) / sizeof(whats[0]))) + if($exception[$e] ne '') { - return "${class}"; + print CPP "\t\tcase ".$exception[$e].': return "'.$exception[$e].'";'."\n"; } - return whats[mSubType]; -#else - return "${class}"; -#endif } +print CPP <<__E; + default: return "Unknown"; + } +} __E close H; diff --git a/lib/compress/CompressStream.cpp b/lib/compress/CompressStream.cpp index 9bb73e3d..f7728a21 100644 --- a/lib/compress/CompressStream.cpp +++ b/lib/compress/CompressStream.cpp @@ -177,12 +177,12 @@ int CompressStream::Read(void *pBuffer, int NBytes, int Timeout) // Created: 27/5/04 // // -------------------------------------------------------------------------- -void CompressStream::Write(const void *pBuffer, int NBytes) +void CompressStream::Write(const void *pBuffer, int NBytes, int Timeout) { USE_WRITE_COMPRESSOR if(pCompress == 0) { - mpStream->Write(pBuffer, NBytes); + mpStream->Write(pBuffer, NBytes, Timeout); return; } @@ -207,7 +207,7 @@ void CompressStream::Write(const void *pBuffer, int NBytes) // Created: 27/5/04 // // -------------------------------------------------------------------------- -void CompressStream::WriteAllBuffered() +void CompressStream::WriteAllBuffered(int Timeout) { if(mIsClosed) { @@ -215,7 +215,7 @@ void CompressStream::WriteAllBuffered() } // Just ask compressed data to be written out, but with the sync flag set - WriteCompressedData(true); + WriteCompressedData(true, Timeout); } @@ -238,7 +238,7 @@ void CompressStream::Close() pCompress->FinishInput(); WriteCompressedData(); - // Mark as definately closed + // Mark as definitely closed mIsClosed = true; } } @@ -257,7 +257,7 @@ void CompressStream::Close() // Created: 28/5/04 // // -------------------------------------------------------------------------- -void CompressStream::WriteCompressedData(bool SyncFlush) +void CompressStream::WriteCompressedData(bool SyncFlush, int Timeout) { USE_WRITE_COMPRESSOR if(pCompress == 0) {THROW_EXCEPTION(CompressException, Internal)} @@ -268,7 +268,7 @@ void CompressStream::WriteCompressedData(bool SyncFlush) s = pCompress->Output(mpBuffer, BUFFER_SIZE, SyncFlush); if(s > 0) { - mpStream->Write(mpBuffer, s); + mpStream->Write(mpBuffer, s, Timeout); } } while(s > 0); // Check assumption -- all input has been consumed diff --git a/lib/compress/CompressStream.h b/lib/compress/CompressStream.h index 7959e3dc..7d6b2501 100644 --- a/lib/compress/CompressStream.h +++ b/lib/compress/CompressStream.h @@ -33,8 +33,9 @@ private: public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); - virtual void WriteAllBuffered(); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + virtual void WriteAllBuffered(int Timeout = IOStream::TimeOutInfinite); virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); @@ -43,7 +44,8 @@ protected: void CheckRead(); void CheckWrite(); void CheckBuffer(); - void WriteCompressedData(bool SyncFlush = false); + void WriteCompressedData(bool SyncFlush = false, + int Timeout = IOStream::TimeOutInfinite); private: IOStream *mpStream; diff --git a/lib/crypto/CipherBlowfish.cpp b/lib/crypto/CipherBlowfish.cpp index e16cc6ed..4c75b1de 100644 --- a/lib/crypto/CipherBlowfish.cpp +++ b/lib/crypto/CipherBlowfish.cpp @@ -206,7 +206,7 @@ void CipherBlowfish::SetupParameters(EVP_CIPHER_CTX *pCipherContext) const } // Set key #ifndef HAVE_OLD_SSL - if(EVP_CipherInit_ex(pCipherContext, NULL, NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) + if(EVP_CipherInit_ex(pCipherContext, GetCipher(), NULL, (unsigned char*)mpKey, (unsigned char*)mpInitialisationVector, -1) != 1) #else if(EVP_CipherInit(pCipherContext, NULL, (unsigned char*)mKey.c_str(), (unsigned char*)mInitialisationVector, -1) != 1) #endif diff --git a/lib/crypto/CipherContext.cpp b/lib/crypto/CipherContext.cpp index fd149395..3de88c64 100644 --- a/lib/crypto/CipherContext.cpp +++ b/lib/crypto/CipherContext.cpp @@ -2,7 +2,7 @@ // // File // Name: CipherContext.cpp -// Purpose: Context for symmetric encryption / descryption +// Purpose: Context for symmetric encryption / decryption // Created: 1/12/03 // // -------------------------------------------------------------------------- @@ -50,7 +50,7 @@ CipherContext::~CipherContext() if(mInitialised) { // Clean up - EVP_CIPHER_CTX_cleanup(&ctx); + BOX_OPENSSL_CLEANUP_CTX(ctx); mInitialised = false; } #ifdef HAVE_OLD_SSL @@ -98,7 +98,7 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes // Check for bad usage if(mInitialised) { - THROW_EXCEPTION(CipherException, AlreadyInitialised) + THROW_EXCEPTION(CipherException, AlreadyInitialised); } if(Function != Decrypt && Function != Encrypt) { @@ -109,43 +109,45 @@ void CipherContext::Init(CipherContext::CipherFunction Function, const CipherDes mFunction = Function; // Initialise the cipher -#ifndef HAVE_OLD_SSL - EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does - - if(EVP_CipherInit_ex(&ctx, rDescription.GetCipher(), NULL, NULL, NULL, - (mFunction == Encrypt) ? 1 : 0) != 1) -#else +#ifdef HAVE_OLD_SSL // Use old version of init call if(EVP_CipherInit(&ctx, rDescription.GetCipher(), NULL, NULL, (mFunction == Encrypt) ? 1 : 0) != 1) +#else + BOX_OPENSSL_INIT_CTX(ctx); + + // Don't set key or IV yet, because we will modify the parameters: + if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), rDescription.GetCipher(), NULL, NULL, NULL, + (mFunction == Encrypt) ? 1 : 0) != 1) #endif { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, "Failed to initialise " << rDescription.GetFullName() - << "cipher: " << LogError("initialising cipher")); + << ": " << LogError("initialising cipher")); } + UsePadding(mPaddingOn); try { mCipherName = rDescription.GetFullName(); #ifndef HAVE_OLD_SSL // Let the description set up everything else - rDescription.SetupParameters(&ctx); + mpDescription = &rDescription; #else // With the old version, a copy needs to be taken first. mpDescription = rDescription.Clone(); // Mark it as not a leak, otherwise static cipher contexts // cause spurious memory leaks to be reported MEMLEAKFINDER_NOT_A_LEAK(mpDescription); - mpDescription->SetupParameters(&ctx); #endif + mpDescription->SetupParameters(BOX_OPENSSL_CTX(ctx)); } catch(...) { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, - "Failed to configure " << mCipherName << " cipher: " << + "Failed to configure " << mCipherName << ": " << LogError("configuring cipher")); - EVP_CIPHER_CTX_cleanup(&ctx); + BOX_OPENSSL_CLEANUP_CTX(ctx); throw; } @@ -166,7 +168,7 @@ void CipherContext::Reset() if(mInitialised) { // Clean up - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_cleanup(BOX_OPENSSL_CTX(ctx)); mInitialised = false; } #ifdef HAVE_OLD_SSL @@ -177,6 +179,7 @@ void CipherContext::Reset() } #endif mWithinTransform = false; + mIV.clear(); } @@ -192,24 +195,22 @@ void CipherContext::Begin() { if(!mInitialised) { - THROW_EXCEPTION(CipherException, NotInitialised) + THROW_EXCEPTION(CipherException, NotInitialised); } - // Warn if in a transformation (not an error, because a context might not have been finalised if an exception occured) if(mWithinTransform) { - BOX_WARNING("CipherContext::Begin called when context " - "flagged as within a transform"); + THROW_EXCEPTION(CipherException, AlreadyInTransform); } - // Initialise the cipher context again - if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) + if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), NULL, NULL, NULL, + (const unsigned char *)(mIV.size() > 0 ? mIV.c_str() : NULL), + -1) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, - "Failed to reset " << mCipherName << " cipher: " << - LogError("resetting cipher")); + "Failed to set IV for " << mCipherName << ": " << LogError(GetFunction())); } - + // Mark as being within a transform mWithinTransform = true; } @@ -251,18 +252,18 @@ int CipherContext::Transform(void *pOutBuffer, int OutLength, const void *pInBuf } // Check output buffer size - if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)))) { THROW_EXCEPTION(CipherException, OutputBufferTooSmall); } // Do the transform int outLength = OutLength; - if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + if(EVP_CipherUpdate(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &outLength, + (unsigned char*)pInBuffer, InLength) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure, - "Failed to " << GetFunction() << " (update) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to update " << mCipherName << ": " << LogError(GetFunction())); } return outLength; @@ -300,7 +301,7 @@ int CipherContext::Final(void *pOutBuffer, int OutLength) } // Check output buffer size - if(OutLength < (2 * EVP_CIPHER_CTX_block_size(&ctx))) + if(OutLength < (2 * EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)))) { THROW_EXCEPTION(CipherException, OutputBufferTooSmall); } @@ -308,12 +309,11 @@ int CipherContext::Final(void *pOutBuffer, int OutLength) // Do the transform int outLength = OutLength; #ifndef HAVE_OLD_SSL - if(EVP_CipherFinal(&ctx, (unsigned char*)pOutBuffer, &outLength) != 1) + if(EVP_CipherFinal(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &outLength) != 1) { mWithinTransform = false; THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure, - "Failed to " << GetFunction() << " (final) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to finalise " << mCipherName << ": " << LogError(GetFunction())); } #else OldOpenSSLFinal((unsigned char*)pOutBuffer, outLength); @@ -340,11 +340,11 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) // Old version needs to use a different form, and then set up the cipher again for next time around int outLength = rOutLengthOut; // Have to emulate padding off... - int blockSize = EVP_CIPHER_CTX_block_size(&ctx); + int blockSize = EVP_CIPHER_CTX_block_size(ctx); if(mPaddingOn) { // Just use normal final call - if(EVP_CipherFinal(&ctx, Buffer, &outLength) != 1) + if(EVP_CipherFinal(ctx, Buffer, &outLength) != 1) { THROW_EXCEPTION(CipherException, EVPFinalFailure) } @@ -357,13 +357,13 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) { // NASTY -- fiddling around with internals like this is bad. // But only way to get this working on old versions of OpenSSL. - if(!EVP_EncryptUpdate(&ctx,Buffer,&outLength,ctx.buf,0) + if(!EVP_EncryptUpdate(ctx,Buffer,&outLength,ctx.buf,0) || outLength != blockSize) { THROW_EXCEPTION(CipherException, EVPFinalFailure) } // Clean up - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_free(ctx); } else { @@ -391,12 +391,14 @@ void CipherContext::OldOpenSSLFinal(unsigned char *Buffer, int &rOutLengthOut) } } // Reinitialise the cipher for the next time around - if(EVP_CipherInit(&ctx, mpDescription->GetCipher(), NULL, NULL, + if(EVP_CipherInit_ex(&ctx, mpDescription->GetCipher(), NULL, NULL, + (const unsigned char *)(mIV.size() > 0 ? mIV.c_str() : NULL), (mFunction == Encrypt) ? 1 : 0) != 1) { THROW_EXCEPTION(CipherException, EVPInitFailure) } mpDescription->SetupParameters(&ctx); + UsePadding(mPaddingOn); // Update length for caller rOutLengthOut = outLength; @@ -421,7 +423,7 @@ int CipherContext::InSizeForOutBufferSize(int OutLength) // Strictly speaking, the *2 is unnecessary. However... // Final() is paranoid, and requires two input blocks of space to work. - return OutLength - (EVP_CIPHER_CTX_block_size(&ctx) * 2); + return OutLength - (EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)) * 2); } // -------------------------------------------------------------------------- @@ -442,7 +444,7 @@ int CipherContext::MaxOutSizeForInBufferSize(int InLength) // Final() is paranoid, and requires two input blocks of space to work, and so we need to add // three blocks on to be absolutely sure. - return InLength + (EVP_CIPHER_CTX_block_size(&ctx) * 3); + return InLength + (EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)) * 3); } @@ -456,20 +458,8 @@ int CipherContext::MaxOutSizeForInBufferSize(int InLength) // -------------------------------------------------------------------------- int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *pInBuffer, int InLength) { - if(!mInitialised) - { - THROW_EXCEPTION(CipherException, NotInitialised) - } - - // Warn if in a transformation - if(mWithinTransform) - { - BOX_WARNING("CipherContext::TransformBlock called when " - "context flagged as within a transform"); - } - // Check output buffer size - if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(&ctx))) + if(OutLength < (InLength + EVP_CIPHER_CTX_block_size(BOX_OPENSSL_CTX(ctx)))) { // Check if padding is off, in which case the buffer can be smaller if(!mPaddingOn && OutLength <= InLength) @@ -481,40 +471,36 @@ int CipherContext::TransformBlock(void *pOutBuffer, int OutLength, const void *p THROW_EXCEPTION(CipherException, OutputBufferTooSmall); } } - - // Initialise the cipher context again - if(EVP_CipherInit(&ctx, NULL, NULL, NULL, -1) != 1) - { - THROW_EXCEPTION(CipherException, EVPInitFailure) - } + + Begin(); // Do the entire block - int outLength = 0; + int output_space_used = OutLength; // Update - outLength = OutLength; - if(EVP_CipherUpdate(&ctx, (unsigned char*)pOutBuffer, &outLength, (unsigned char*)pInBuffer, InLength) != 1) + if(EVP_CipherUpdate(BOX_OPENSSL_CTX(ctx), (unsigned char*)pOutBuffer, &output_space_used, + (unsigned char*)pInBuffer, InLength) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPUpdateFailure, - "Failed to " << GetFunction() << " (update) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to update " << mCipherName << ": " << LogError(GetFunction())); } // Finalise - int outLength2 = OutLength - outLength; -#ifndef HAVE_OLD_SSL - if(EVP_CipherFinal(&ctx, ((unsigned char*)pOutBuffer) + outLength, &outLength2) != 1) + int output_space_remain = OutLength - output_space_used; + +#ifdef HAVE_OLD_SSL + OldOpenSSLFinal(((unsigned char*)pOutBuffer) + output_space_used, output_space_remain); +#else + if(EVP_CipherFinal(BOX_OPENSSL_CTX(ctx), ((unsigned char*)pOutBuffer) + output_space_used, + &output_space_remain) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPFinalFailure, - "Failed to " << GetFunction() << " (final) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to finalise " << mCipherName << ": " << LogError(GetFunction())); } -#else - OldOpenSSLFinal(((unsigned char*)pOutBuffer) + outLength, outLength2); #endif - outLength += outLength2; - return outLength; + mWithinTransform = false; + return output_space_used + output_space_remain; } @@ -533,7 +519,7 @@ int CipherContext::GetIVLength() THROW_EXCEPTION(CipherException, NotInitialised) } - return EVP_CIPHER_CTX_iv_length(&ctx); + return EVP_CIPHER_CTX_iv_length(BOX_OPENSSL_CTX(ctx)); } @@ -559,12 +545,14 @@ void CipherContext::SetIV(const void *pIV) "flagged as within a transform"); } + mIV = std::string((const char *)pIV, GetIVLength()); + // Set IV - if(EVP_CipherInit(&ctx, NULL, NULL, (unsigned char *)pIV, -1) != 1) + if(EVP_CipherInit_ex(BOX_OPENSSL_CTX(ctx), NULL, NULL, NULL, + (const unsigned char *)mIV.c_str(), -1) != 1) { THROW_EXCEPTION_MESSAGE(CipherException, EVPInitFailure, - "Failed to " << GetFunction() << " (set IV) " << - mCipherName << " cipher: " << LogError(GetFunction())); + "Failed to set IV for " << mCipherName << ": " << LogError(GetFunction())); } #ifdef HAVE_OLD_SSL @@ -601,19 +589,20 @@ const void *CipherContext::SetRandomIV(int &rLengthOut) } // Get length of IV - unsigned int ivLen = EVP_CIPHER_CTX_iv_length(&ctx); - if(ivLen > sizeof(mGeneratedIV)) + uint8_t generated_iv[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; + unsigned int ivLen = EVP_CIPHER_CTX_iv_length(BOX_OPENSSL_CTX(ctx)); + if(ivLen > sizeof(generated_iv)) { THROW_EXCEPTION(CipherException, IVSizeImplementationLimitExceeded) } // Generate some random data - Random::Generate(mGeneratedIV, ivLen); - SetIV(mGeneratedIV); + Random::Generate(generated_iv, ivLen); + SetIV(generated_iv); // Return the IV and it's length rLengthOut = ivLen; - return mGeneratedIV; + return mIV.c_str(); } @@ -628,9 +617,11 @@ const void *CipherContext::SetRandomIV(int &rLengthOut) void CipherContext::UsePadding(bool Padding) { #ifndef HAVE_OLD_SSL - if(EVP_CIPHER_CTX_set_padding(&ctx, Padding) != 1) + if(EVP_CIPHER_CTX_set_padding(BOX_OPENSSL_CTX(ctx), Padding) != 1) { - THROW_EXCEPTION(CipherException, EVPSetPaddingFailure) + THROW_EXCEPTION_MESSAGE(CipherException, EVPSetPaddingFailure, + "Failed to set padding for " << mCipherName << ": " << + LogError(GetFunction())); } #endif mPaddingOn = Padding; diff --git a/lib/crypto/CipherContext.h b/lib/crypto/CipherContext.h index 93c889d6..b6e97b4e 100644 --- a/lib/crypto/CipherContext.h +++ b/lib/crypto/CipherContext.h @@ -19,6 +19,22 @@ class CipherDescription; #define CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH 32 +// Macros to allow compatibility with OpenSSL 1.0 and 1.1 APIs. See +// https://github.com/charybdis-ircd/charybdis/blob/release/3.5/libratbox/src/openssl_ratbox.h +// for the gory details. +#if defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER >= 0x10100000L) // OpenSSL >= 1.1 +# define BOX_OPENSSL_INIT_CTX(ctx) ctx = EVP_CIPHER_CTX_new(); +# define BOX_OPENSSL_CTX(ctx) ctx +# define BOX_OPENSSL_CLEANUP_CTX(ctx) EVP_CIPHER_CTX_free(ctx) +typedef EVP_CIPHER_CTX* BOX_EVP_CIPHER_CTX; +#else // OpenSSL < 1.1 +# define BOX_OPENSSL_INIT_CTX(ctx) EVP_CIPHER_CTX_init(&ctx); // no error return code, even though the docs says it does +# define BOX_OPENSSL_CTX(ctx) &ctx +# define BOX_OPENSSL_CLEANUP_CTX(ctx) EVP_CIPHER_CTX_cleanup(&ctx) +typedef EVP_CIPHER_CTX BOX_EVP_CIPHER_CTX; +#endif + + // -------------------------------------------------------------------------- // // Class @@ -74,16 +90,14 @@ public: #endif private: - EVP_CIPHER_CTX ctx; + BOX_EVP_CIPHER_CTX ctx; bool mInitialised; bool mWithinTransform; bool mPaddingOn; - uint8_t mGeneratedIV[CIPHERCONTEXT_MAX_GENERATED_IV_LENGTH]; CipherFunction mFunction; std::string mCipherName; -#ifdef HAVE_OLD_SSL - CipherDescription *mpDescription; -#endif + const CipherDescription *mpDescription; + std::string mIV; }; diff --git a/lib/crypto/CipherException.txt b/lib/crypto/CipherException.txt index abdbac87..494ed3cc 100644 --- a/lib/crypto/CipherException.txt +++ b/lib/crypto/CipherException.txt @@ -16,3 +16,4 @@ PseudoRandNotAvailable 12 EVPSetPaddingFailure 13 RandomInitFailed 14 Failed to read from random device LengthRequestedTooLongForRandomHex 15 +AlreadyInTransform 16 Tried to initialise crypto when already in a transform diff --git a/lib/crypto/Random.cpp b/lib/crypto/Random.cpp index 1d6a07f0..c34a6eea 100644 --- a/lib/crypto/Random.cpp +++ b/lib/crypto/Random.cpp @@ -50,7 +50,7 @@ void Random::Initialise() // -------------------------------------------------------------------------- void Random::Generate(void *pOutput, int Length) { - if(RAND_pseudo_bytes((uint8_t*)pOutput, Length) == -1) + if(RAND_bytes((uint8_t*)pOutput, Length) == -1) { THROW_EXCEPTION(CipherException, PseudoRandNotAvailable) } diff --git a/lib/httpserver/HTTPException.txt b/lib/httpserver/HTTPException.txt index 52630cda..c9b3f940 100644 --- a/lib/httpserver/HTTPException.txt +++ b/lib/httpserver/HTTPException.txt @@ -1,16 +1,17 @@ EXCEPTION HTTP 10 -Internal 0 -RequestReadFailed 1 -RequestAlreadyBeenRead 2 -BadRequest 3 -UnknownResponseCodeUsed 4 -NoContentTypeSet 5 -POSTContentTooLong 6 -CannotSetRedirectIfReponseHasData 7 -CannotSetNotFoundIfReponseHasData 8 -NotImplemented 9 -RequestNotInitialised 10 -BadResponse 11 -ResponseReadFailed 12 -NoStreamConfigured 13 +Internal 0 +RequestReadFailed 1 +RequestAlreadyBeenRead 2 +BadRequest 3 +UnknownResponseCodeUsed 4 +NoContentTypeSet 5 +POSTContentTooLong 6 +CannotSetRedirectIfReponseHasData 7 +CannotSetNotFoundIfReponseHasData 8 +NotImplemented 9 +RequestNotInitialised 10 +BadResponse 11 +ResponseReadFailed 12 +NoStreamConfigured 13 +RequestFailedUnexpectedly 14 The request was expected to succeed, but it failed. diff --git a/lib/httpserver/HTTPRequest.cpp b/lib/httpserver/HTTPRequest.cpp index 4c5dc149..a94d96b0 100644 --- a/lib/httpserver/HTTPRequest.cpp +++ b/lib/httpserver/HTTPRequest.cpp @@ -10,10 +10,11 @@ #include "Box.h" #include <string.h> -#include <strings.h> #include <stdlib.h> #include <stdio.h> +#include <sstream> + #include "HTTPRequest.h" #include "HTTPResponse.h" #include "HTTPQueryDecoder.h" @@ -94,6 +95,32 @@ HTTPRequest::~HTTPRequest() } } +// -------------------------------------------------------------------------- +// +// Function +// Name: HTTPRequest::GetMethodName() +// Purpose: Returns the name of the request's HTTP method verb +// as a string. +// Created: 28/7/15 +// +// -------------------------------------------------------------------------- + +std::string HTTPRequest::GetMethodName() const +{ + switch(mMethod) + { + case Method_UNINITIALISED: return "uninitialised"; + case Method_UNKNOWN: return "unknown"; + case Method_GET: return "GET"; + case Method_HEAD: return "HEAD"; + case Method_POST: return "POST"; + case Method_PUT: return "PUT"; + default: + std::ostringstream oss; + oss << "unknown-" << mMethod; + return oss.str(); + }; +} // -------------------------------------------------------------------------- // @@ -111,7 +138,7 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) // Check caller's logic if(mMethod != Method_UNINITIALISED) { - THROW_EXCEPTION(HTTPException, RequestAlreadyBeenRead) + THROW_EXCEPTION(HTTPException, RequestAlreadyBeenRead); } // Read the first line, which is of a different format to the rest of the lines @@ -126,8 +153,8 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) // Check the method size_t p = 0; // current position in string p = requestLine.find(' '); // end of first word - - if (p == std::string::npos) + + if(p == std::string::npos) { // No terminating space, looks bad p = requestLine.size(); @@ -163,14 +190,14 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) { ++p; } - + // Check there's a URI following... if(requestLinePtr[p] == '\0') { // Didn't get the request line, probably end of connection which had been kept alive return false; } - + // Read the URI, unescaping any %XX hex codes while(requestLinePtr[p] != ' ' && requestLinePtr[p] != '\0') { @@ -201,10 +228,10 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) { code[1] = requestLinePtr[++p]; } - + // Convert into a char code long c = ::strtol(code, NULL, 16); - + // Accept it? if(c > 0 && c <= 255) { @@ -241,28 +268,32 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) int major, minor; if(::sscanf(requestLinePtr + p + 5, "%d.%d", &major, &minor) != 2) { - THROW_EXCEPTION(HTTPException, BadRequest) + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Unable to parse HTTP version number: " << + requestLinePtr); } - + // Store version mHTTPVersion = (major * HTTPVersion__MajorMultiplier) + minor; } else { // Not good -- wrong string found - THROW_EXCEPTION(HTTPException, BadRequest) + THROW_EXCEPTION_MESSAGE(HTTPException, BadRequest, + "Unable to parse HTTP request line: " << + requestLinePtr); } } - + BOX_TRACE("HTTPRequest: method=" << mMethod << ", uri=" << mRequestURI << ", version=" << mHTTPVersion); - + // If HTTP 1.1 or greater, assume keep-alive if(mHTTPVersion >= HTTPVersion_1_1) { mClientKeepAliveRequested = true; } - + // Decode query string? if((mMethod == Method_GET || mMethod == Method_HEAD) && !mQueryString.empty()) { @@ -270,28 +301,28 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) decoder.DecodeChunk(mQueryString.c_str(), mQueryString.size()); decoder.Finish(); } - + // Now parse the headers ParseHeaders(rGetLine, Timeout); - + std::string expected; - if (GetHeader("Expect", &expected)) + if(GetHeader("Expect", &expected)) { - if (expected == "100-continue") + if(expected == "100-continue") { mExpectContinue = true; } } - + // Parse form data? if(mMethod == Method_POST && mContentLength >= 0) { // Too long? Don't allow people to be nasty by sending lots of data if(mContentLength > MAX_CONTENT_SIZE) { - THROW_EXCEPTION(HTTPException, POSTContentTooLong) + THROW_EXCEPTION(HTTPException, POSTContentTooLong); } - + // Some data in the request to follow, parsing it bit by bit HTTPQueryDecoder decoder(mQuery); // Don't forget any data left in the GetLine object @@ -317,7 +348,8 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) if(r == 0) { // Timeout, just error - THROW_EXCEPTION(HTTPException, RequestReadFailed) + THROW_EXCEPTION_MESSAGE(HTTPException, RequestReadFailed, + "Failed to read complete request with the timeout"); } decoder.DecodeChunk(buf, r); bytesToGo -= r; @@ -336,7 +368,7 @@ bool HTTPRequest::Receive(IOStreamGetLine &rGetLine, int Timeout) SetForReading(); mpStreamToReadFrom = &(rGetLine.GetUnderlyingStream()); } - + return true; } @@ -346,7 +378,7 @@ void HTTPRequest::ReadContent(IOStream& rStreamToWriteTo) CopyStreamTo(rStreamToWriteTo); IOStream::pos_type bytesCopied = GetSize(); - + while (bytesCopied < mContentLength) { char buffer[1024]; @@ -397,7 +429,8 @@ bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) case HTTPVersion_1_0: rStream.Write("HTTP/1.0"); break; case HTTPVersion_1_1: rStream.Write("HTTP/1.1"); break; default: - THROW_EXCEPTION(HTTPException, NotImplemented); + THROW_EXCEPTION_MESSAGE(HTTPException, NotImplemented, + "Unsupported HTTP version: " << mHTTPVersion); } rStream.Write("\n"); @@ -428,7 +461,8 @@ bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) if (mpCookies) { - THROW_EXCEPTION(HTTPException, NotImplemented); + THROW_EXCEPTION_MESSAGE(HTTPException, NotImplemented, + "Cookie support not implemented yet"); } if (mClientKeepAliveRequested) @@ -445,7 +479,7 @@ bool HTTPRequest::Send(IOStream &rStream, int Timeout, bool ExpectContinue) { oss << i->first << ": " << i->second << "\n"; } - + if (ExpectContinue) { oss << "Expect: 100-continue\n"; @@ -461,23 +495,22 @@ void HTTPRequest::SendWithStream(IOStream &rStreamToSendTo, int Timeout, IOStream* pStreamToSend, HTTPResponse& rResponse) { IOStream::pos_type size = pStreamToSend->BytesLeftToRead(); - if (size != IOStream::SizeOfStreamUnknown) { mContentLength = size; } - + Send(rStreamToSendTo, Timeout, true); - + rResponse.Receive(rStreamToSendTo, Timeout); if (rResponse.GetResponseCode() != 100) { // bad response, abort now return; } - + pStreamToSend->CopyStreamTo(rStreamToSendTo, Timeout); - + // receive the final response rResponse.Receive(rStreamToSendTo, Timeout); } @@ -502,13 +535,13 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) THROW_EXCEPTION(HTTPException, BadRequest) } - std::string currentLine; + std::string currentLine; if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) { // Timeout THROW_EXCEPTION(HTTPException, RequestReadFailed) } - + // Is this a continuation of the previous line? bool processHeader = haveHeader; if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) @@ -517,7 +550,7 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) processHeader = false; } //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); - + // Parse the header -- this will actually process the header // from the previous run around the loop. if(processHeader) @@ -538,7 +571,7 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) std::string header_name(ToLowerCase(std::string(h, p))); - + if (header_name == "content-length") { // Decode number @@ -556,17 +589,17 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) { // Store host header mHostName = h + dataStart; - + // Is there a port number to split off? std::string::size_type colon = mHostName.find_first_of(':'); if(colon != std::string::npos) { // There's a port in the string... attempt to turn it into an int mHostPort = ::strtol(mHostName.c_str() + colon + 1, 0, 10); - + // Truncate the string to just the hostname mHostName = mHostName.substr(0, colon); - + BOX_TRACE("Host: header, hostname = " << "'" << mHostName << "', host " "port = " << mHostPort); @@ -596,7 +629,7 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) mExtraHeaders.push_back(Header(header_name, h + dataStart)); } - + // Unset have header flag, as it's now been processed haveHeader = false; } @@ -617,7 +650,7 @@ void HTTPRequest::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) { // All done! break; - } + } } } @@ -636,13 +669,13 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) const char *pos = data; const char *itemStart = pos; std::string name; - + enum { s_NAME, s_VALUE, s_VALUE_QUOTED, s_FIND_NEXT_NAME } state = s_NAME; - do + do { switch(state) { @@ -665,7 +698,7 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) } } break; - + case s_VALUE: { if(*pos == ';' || *pos == ',' || *pos == '\0') @@ -679,7 +712,7 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) } } break; - + case s_VALUE_QUOTED: { if(*pos == '"') @@ -693,7 +726,7 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) } } break; - + case s_FIND_NEXT_NAME: { // Skip over terminators and white space to get to the next name @@ -705,7 +738,7 @@ void HTTPRequest::ParseCookies(const std::string &rHeader, int DataStarts) } } break; - + default: // Ooops THROW_EXCEPTION(HTTPException, Internal) @@ -732,7 +765,7 @@ bool HTTPRequest::GetCookie(const char *CookieName, std::string &rValueOut) cons { return false; } - + // See if it's there CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); if(v != mpCookies->end()) @@ -741,7 +774,7 @@ bool HTTPRequest::GetCookie(const char *CookieName, std::string &rValueOut) cons rValueOut = v->second; return true; } - + return false; } @@ -764,7 +797,7 @@ const std::string &HTTPRequest::GetCookie(const char *CookieName) const { return noCookie; } - + // See if it's there CookieJar_t::const_iterator v(mpCookies->find(std::string(CookieName))); if(v != mpCookies->end()) @@ -772,7 +805,7 @@ const std::string &HTTPRequest::GetCookie(const char *CookieName) const // Return the value return v->second; } - + return noCookie; } diff --git a/lib/httpserver/HTTPRequest.h b/lib/httpserver/HTTPRequest.h index 25effb70..16c4d16c 100644 --- a/lib/httpserver/HTTPRequest.h +++ b/lib/httpserver/HTTPRequest.h @@ -3,7 +3,7 @@ // File // Name: HTTPRequest.h // Purpose: Request object for HTTP connections -// Created: 26/3/04 +// Created: 26/3/2004 // // -------------------------------------------------------------------------- @@ -23,8 +23,12 @@ class IOStreamGetLine; // // Class // Name: HTTPRequest -// Purpose: Request object for HTTP connections -// Created: 26/3/04 +// Purpose: Request object for HTTP connections. Although it +// inherits from CollectInBufferStream, not all of the +// request data is held in memory, only the beginning. +// Use ReadContent() to write it all (including the +// buffered beginning) to another stream, e.g. a file. +// Created: 26/3/2004 // // -------------------------------------------------------------------------- class HTTPRequest : public CollectInBufferStream @@ -77,6 +81,7 @@ public: // // -------------------------------------------------------------------------- enum Method GetMethod() const {return mMethod;} + std::string GetMethodName() const; const std::string &GetRequestURI() const {return mRequestURI;} // Note: the HTTPRequest generates and parses the Host: header diff --git a/lib/httpserver/HTTPResponse.cpp b/lib/httpserver/HTTPResponse.cpp index 1a8c8447..c56f286f 100644 --- a/lib/httpserver/HTTPResponse.cpp +++ b/lib/httpserver/HTTPResponse.cpp @@ -138,8 +138,7 @@ void HTTPResponse::SetContentType(const char *ContentType) // Function // Name: HTTPResponse::Send(IOStream &, bool) // Purpose: Build the response, and send via the stream. -// Optionally omitting the content. -// Created: 26/3/04 +// Created: 26/3/2004 // // -------------------------------------------------------------------------- void HTTPResponse::Send(bool OmitContent) @@ -148,7 +147,7 @@ void HTTPResponse::Send(bool OmitContent) { THROW_EXCEPTION(HTTPException, NoStreamConfigured); } - + if (GetSize() != 0 && mContentType.empty()) { THROW_EXCEPTION(HTTPException, NoContentTypeSet); @@ -184,6 +183,7 @@ void HTTPResponse::Send(bool OmitContent) // static is allowed to be cached for a day header += "\r\nCache-Control: max-age=86400"; } + if(mKeepAlive) { header += "\r\nConnection: keep-alive\r\n\r\n"; @@ -192,12 +192,13 @@ void HTTPResponse::Send(bool OmitContent) { header += "\r\nConnection: close\r\n\r\n"; } + // NOTE: header ends with blank line in all cases - + // Write to stream mpStreamToSendTo->Write(header.c_str(), header.size()); } - + // Send content if(!OmitContent) { @@ -227,16 +228,16 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) if(rGetLine.IsEOF()) { // Header terminates unexpectedly - THROW_EXCEPTION(HTTPException, BadRequest) + THROW_EXCEPTION(HTTPException, BadRequest) } - std::string currentLine; + std::string currentLine; if(!rGetLine.GetLine(currentLine, false /* no preprocess */, Timeout)) { // Timeout THROW_EXCEPTION(HTTPException, RequestReadFailed) } - + // Is this a continuation of the previous line? bool processHeader = haveHeader; if(!currentLine.empty() && (currentLine[0] == ' ' || currentLine[0] == '\t')) @@ -245,7 +246,7 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) processHeader = false; } //TRACE3("%d:%d:%s\n", processHeader, haveHeader, currentLine.c_str()); - + // Parse the header -- this will actually process the header // from the previous run around the loop. if(processHeader) @@ -263,7 +264,7 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) { ++dataStart; } - + if(p == sizeof("Content-Length")-1 && ::strncasecmp(h, "Content-Length", sizeof("Content-Length")-1) == 0) { @@ -308,7 +309,7 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) std::string headerName = header.substr(0, p); AddHeader(headerName, h + dataStart); } - + // Unset have header flag, as it's now been processed haveHeader = false; } @@ -329,7 +330,7 @@ void HTTPResponse::ParseHeaders(IOStreamGetLine &rGetLine, int Timeout) { // All done! break; - } + } } } @@ -340,22 +341,22 @@ void HTTPResponse::Receive(IOStream& rStream, int Timeout) if(rGetLine.IsEOF()) { // Connection terminated unexpectedly - THROW_EXCEPTION(HTTPException, BadResponse) + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "HTTP server closed the connection without sending a response"); } - std::string statusLine; + std::string statusLine; if(!rGetLine.GetLine(statusLine, false /* no preprocess */, Timeout)) { // Timeout - THROW_EXCEPTION(HTTPException, ResponseReadFailed) + THROW_EXCEPTION_MESSAGE(HTTPException, ResponseReadFailed, + "Failed to get a response from the HTTP server within the timeout"); } - if (statusLine.substr(0, 7) != "HTTP/1." || - statusLine[8] != ' ') + if (statusLine.substr(0, 7) != "HTTP/1." || statusLine[8] != ' ') { - // Status line terminated unexpectedly - BOX_ERROR("Bad response status line: " << statusLine); - THROW_EXCEPTION(HTTPException, BadResponse) + THROW_EXCEPTION_MESSAGE(HTTPException, BadResponse, + "HTTP server sent an invalid HTTP status line: " << statusLine); } if (statusLine[5] == '1' && statusLine[7] == '1') @@ -363,7 +364,7 @@ void HTTPResponse::Receive(IOStream& rStream, int Timeout) // HTTP/1.1 default is to keep alive mKeepAlive = true; } - + // Decode the status code long status = ::strtol(statusLine.substr(9, 3).c_str(), NULL, 10); // returns zero in error case, this is OK @@ -376,7 +377,7 @@ void HTTPResponse::Receive(IOStream& rStream, int Timeout) { return; } - + ParseHeaders(rGetLine, Timeout); // push back whatever bytes we have left @@ -549,7 +550,7 @@ void HTTPResponse::SetAsRedirect(const char *RedirectTo, bool IsLocalURI) if(IsLocalURI) header += msDefaultURIPrefix; header += RedirectTo; mExtraHeaders.push_back(Header("Location", header)); - + // Set up some default content mContentType = "text/html"; #define REDIRECT_HTML_1 "<html><head><title>Redirection</title></head>\n<body><p><a href=\"" @@ -622,7 +623,7 @@ void HTTPResponse::WriteStringDefang(const char *String, unsigned int StringLen) StringLen -= toWrite; String += toWrite; } - + // Is it a bad character next? while(StringLen > 0) { diff --git a/lib/httpserver/HTTPResponse.h b/lib/httpserver/HTTPResponse.h index 04051958..f39825d9 100644 --- a/lib/httpserver/HTTPResponse.h +++ b/lib/httpserver/HTTPResponse.h @@ -63,9 +63,10 @@ public: typedef std::pair<std::string, std::string> Header; void SetResponseCode(int Code); - int GetResponseCode() { return mResponseCode; } + int GetResponseCode() const { return mResponseCode; } void SetContentType(const char *ContentType); const std::string& GetContentType() { return mContentType; } + int64_t GetContentLength() { return mContentLength; } void SetAsRedirect(const char *RedirectTo, bool IsLocalURI = true); void SetAsNotFound(const char *URI); @@ -107,6 +108,7 @@ public: void SetResponseIsDynamicContent(bool IsDynamic) {mResponseIsDynamicContent = IsDynamic;} // Set keep alive control, default is to mark as to be closed void SetKeepAlive(bool KeepAlive) {mKeepAlive = KeepAlive;} + bool IsKeepAlive() {return mKeepAlive;} void SetCookie(const char *Name, const char *Value, const char *Path = "/", int ExpiresAt = 0); @@ -127,7 +129,7 @@ public: }; static const char *ResponseCodeToString(int ResponseCode); - + void WriteStringDefang(const char *String, unsigned int StringLen); void WriteStringDefang(const std::string &rString) {WriteStringDefang(rString.c_str(), rString.size());} @@ -163,9 +165,9 @@ private: bool mKeepAlive; std::string mContentType; std::vector<Header> mExtraHeaders; - int mContentLength; // only used when reading response from stream + int64_t mContentLength; // only used when reading response from stream IOStream* mpStreamToSendTo; // nonzero only when constructed with a stream - + static std::string msDefaultURIPrefix; void ParseHeaders(IOStreamGetLine &rGetLine, int Timeout); diff --git a/lib/httpserver/HTTPServer.cpp b/lib/httpserver/HTTPServer.cpp index be1db687..a2daed99 100644 --- a/lib/httpserver/HTTPServer.cpp +++ b/lib/httpserver/HTTPServer.cpp @@ -27,8 +27,8 @@ // Created: 26/3/04 // // -------------------------------------------------------------------------- -HTTPServer::HTTPServer() - : mTimeout(20000) // default timeout leaves a little while for clients to get the second request in. +HTTPServer::HTTPServer(int Timeout) +: mTimeout(Timeout) { } @@ -86,7 +86,7 @@ const ConfigurationVerify *HTTPServer::GetConfigVerify() const 0 } }; - + static ConfigurationVerifyKey verifyrootkeys[] = { HTTPSERVER_VERIFY_ROOT_KEYS @@ -132,10 +132,10 @@ void HTTPServer::Run() // Created: 26/3/04 // // -------------------------------------------------------------------------- -void HTTPServer::Connection(SocketStream &rStream) +void HTTPServer::Connection(std::auto_ptr<SocketStream> apConn) { // Create a get line object to use - IOStreamGetLine getLine(rStream); + IOStreamGetLine getLine(*apConn); // Notify dervived claases HTTPConnectionOpening(); @@ -150,10 +150,10 @@ void HTTPServer::Connection(SocketStream &rStream) // Didn't get request, connection probably closed. break; } - + // Generate a response - HTTPResponse response(&rStream); - + HTTPResponse response(apConn.get()); + try { Handle(request, response); @@ -169,9 +169,18 @@ void HTTPServer::Connection(SocketStream &rStream) { SendInternalErrorResponse("unknown", response); } - - // Keep alive? - if(request.GetClientKeepAliveRequested()) + + // Keep alive? response.GetSize() works for CollectInBufferStream, but + // when we make HTTPResponse stream the response instead, we'll need to + // figure out whether we can get the response length from the IOStream + // to be streamed, or not. Also, we don't currently support chunked + // encoding, and http://tools.ietf.org/html/rfc7230#section-3.3.1 says + // that "If any transfer coding other than chunked is applied to a + // response payload body, the sender MUST either apply chunked as the + // final transfer coding or terminate the message by closing the + // connection. So for now, keepalive stays off. + if(false && request.GetClientKeepAliveRequested() && + response.GetSize() >= 0) { // Mark the response to the client as supporting keepalive response.SetKeepAlive(true); diff --git a/lib/httpserver/HTTPServer.h b/lib/httpserver/HTTPServer.h index d9f74949..8ac1ff83 100644 --- a/lib/httpserver/HTTPServer.h +++ b/lib/httpserver/HTTPServer.h @@ -27,7 +27,8 @@ class HTTPResponse; class HTTPServer : public ServerStream<SocketStream, 80> { public: - HTTPServer(); + HTTPServer(int Timeout = 60000); + // default timeout leaves 1 minute for clients to get a second request in. ~HTTPServer(); private: // no copying @@ -62,7 +63,7 @@ private: const char *DaemonName() const; const ConfigurationVerify *GetConfigVerify() const; void Run(); - void Connection(SocketStream &rStream); + void Connection(std::auto_ptr<SocketStream> apStream); }; // Root level diff --git a/lib/httpserver/S3Client.cpp b/lib/httpserver/S3Client.cpp index cd5988d5..21814066 100644 --- a/lib/httpserver/S3Client.cpp +++ b/lib/httpserver/S3Client.cpp @@ -34,7 +34,7 @@ // Name: S3Client::GetObject(const std::string& rObjectURI) // Purpose: Retrieve the object with the specified URI (key) // from your S3 bucket. -// Created: 09/01/09 +// Created: 09/01/2009 // // -------------------------------------------------------------------------- @@ -46,12 +46,29 @@ HTTPResponse S3Client::GetObject(const std::string& rObjectURI) // -------------------------------------------------------------------------- // // Function +// Name: S3Client::HeadObject(const std::string& rObjectURI) +// Purpose: Retrieve the metadata for the object with the +// specified URI (key) from your S3 bucket. +// Created: 03/08/2015 +// +// -------------------------------------------------------------------------- + +HTTPResponse S3Client::HeadObject(const std::string& rObjectURI) +{ + return FinishAndSendRequest(HTTPRequest::Method_HEAD, rObjectURI); +} + + +HTTPResponse HeadObject(const std::string& rObjectURI); +// -------------------------------------------------------------------------- +// +// Function // Name: S3Client::PutObject(const std::string& rObjectURI, // IOStream& rStreamToSend, const char* pContentType) // Purpose: Upload the stream to S3, creating or overwriting the // object with the specified URI (key) in your S3 // bucket. -// Created: 09/01/09 +// Created: 09/01/2009 // // -------------------------------------------------------------------------- @@ -77,7 +94,7 @@ HTTPResponse S3Client::PutObject(const std::string& rObjectURI, // connection to the server if necessary, which may // throw a ConnectionException. Returns the HTTP // response returned by S3, which may be a 500 error. -// Created: 09/01/09 +// Created: 09/01/2009 // // -------------------------------------------------------------------------- @@ -113,7 +130,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, { request.AddHeader("Content-Type", pStreamContentType); } - + std::string s3suffix = ".s3.amazonaws.com"; std::string bucket; if (mHostName.size() > s3suffix.size()) @@ -126,18 +143,18 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, s3suffix.size()); } } - + std::ostringstream data; data << request.GetVerb() << "\n"; data << "\n"; /* Content-MD5 */ data << request.GetContentType() << "\n"; data << date.str() << "\n"; - + if (! bucket.empty()) { data << "/" << bucket; } - + data << request.GetRequestURI(); std::string data_string = data.str(); @@ -148,7 +165,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, (const unsigned char*)data_string.c_str(), data_string.size(), digest_buffer, &digest_size); std::string digest((const char *)digest_buffer, digest_size); - + base64::encoder encoder; std::string auth_code = "AWS " + mAccessKey + ":" + encoder.encode(digest); @@ -200,6 +217,7 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, } else { + BOX_TRACE("S3Client: " << mHostName << " ! " << ce.what()); throw; } } @@ -218,26 +236,57 @@ HTTPResponse S3Client::FinishAndSendRequest(HTTPRequest::Method Method, // necessary, which may throw a ConnectionException. // Returns the HTTP response returned by S3, which may // be a 500 error. -// Created: 09/01/09 +// Created: 09/01/2009 // // -------------------------------------------------------------------------- HTTPResponse S3Client::SendRequest(HTTPRequest& rRequest, IOStream* pStreamToSend, const char* pStreamContentType) -{ +{ HTTPResponse response; - + if (pStreamToSend) { - rRequest.SendWithStream(*mapClientSocket, - 30000 /* milliseconds */, + rRequest.SendWithStream(*mapClientSocket, mNetworkTimeout, pStreamToSend, response); } else { - rRequest.Send(*mapClientSocket, 30000 /* milliseconds */); - response.Receive(*mapClientSocket, 30000 /* milliseconds */); + rRequest.Send(*mapClientSocket, mNetworkTimeout); + response.Receive(*mapClientSocket, mNetworkTimeout); + } + + if(!response.IsKeepAlive()) + { + BOX_TRACE("Server will close the connection, closing our end too."); + mapClientSocket.reset(); + } + else + { + BOX_TRACE("Server will keep the connection open for more requests."); } - + return response; -} +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: S3Client::CheckResponse(HTTPResponse&, +// std::string& message) +// Purpose: Check the status code of an Amazon S3 response, and +// throw an exception with a useful message (including +// the supplied message) if it's not a 200 OK response. +// Created: 26/07/2015 +// +// -------------------------------------------------------------------------- + +void S3Client::CheckResponse(const HTTPResponse& response, const std::string& message) const +{ + if(response.GetResponseCode() != HTTPResponse::Code_OK) + { + THROW_EXCEPTION_MESSAGE(HTTPException, RequestFailedUnexpectedly, + message); + } +} + diff --git a/lib/httpserver/S3Client.h b/lib/httpserver/S3Client.h index 3c4126ac..4cbb4b96 100644 --- a/lib/httpserver/S3Client.h +++ b/lib/httpserver/S3Client.h @@ -36,7 +36,8 @@ class S3Client : mpSimulator(pSimulator), mHostName(rHostName), mAccessKey(rAccessKey), - mSecretKey(rSecretKey) + mSecretKey(rSecretKey), + mNetworkTimeout(30000) { } S3Client(std::string HostName, int Port, const std::string& rAccessKey, @@ -45,12 +46,16 @@ class S3Client mHostName(HostName), mPort(Port), mAccessKey(rAccessKey), - mSecretKey(rSecretKey) + mSecretKey(rSecretKey), + mNetworkTimeout(30000) { } HTTPResponse GetObject(const std::string& rObjectURI); + HTTPResponse HeadObject(const std::string& rObjectURI); HTTPResponse PutObject(const std::string& rObjectURI, IOStream& rStreamToSend, const char* pContentType = NULL); + void CheckResponse(const HTTPResponse& response, const std::string& message) const; + int GetNetworkTimeout() const { return mNetworkTimeout; } private: HTTPServer* mpSimulator; @@ -58,6 +63,7 @@ class S3Client int mPort; std::auto_ptr<SocketStream> mapClientSocket; std::string mAccessKey, mSecretKey; + int mNetworkTimeout; // milliseconds HTTPResponse FinishAndSendRequest(HTTPRequest::Method Method, const std::string& rRequestURI, diff --git a/lib/httpserver/S3Simulator.cpp b/lib/httpserver/S3Simulator.cpp index 4f6bb3e6..df8910d7 100644 --- a/lib/httpserver/S3Simulator.cpp +++ b/lib/httpserver/S3Simulator.cpp @@ -40,12 +40,12 @@ // -------------------------------------------------------------------------- const ConfigurationVerify* S3Simulator::GetConfigVerify() const { - static ConfigurationVerifyKey verifyserverkeys[] = + static ConfigurationVerifyKey verifyserverkeys[] = { HTTPSERVER_VERIFY_SERVER_KEYS(ConfigurationVerifyKey::NoDefaultValue) // no default addresses }; - static ConfigurationVerify verifyserver[] = + static ConfigurationVerify verifyserver[] = { { "Server", @@ -55,8 +55,8 @@ const ConfigurationVerify* S3Simulator::GetConfigVerify() const 0 } }; - - static ConfigurationVerifyKey verifyrootkeys[] = + + static ConfigurationVerifyKey verifyrootkeys[] = { ConfigurationVerifyKey("AccessKey", ConfigTest_Exists), ConfigurationVerifyKey("SecretKey", ConfigTest_Exists), @@ -99,11 +99,11 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) const Configuration& rConfig(GetConfiguration()); std::string access_key = rConfig.GetKeyValue("AccessKey"); std::string secret_key = rConfig.GetKeyValue("SecretKey"); - + std::string md5, date, bucket; rRequest.GetHeader("content-md5", &md5); rRequest.GetHeader("date", &date); - + std::string host = rRequest.GetHostName(); std::string s3suffix = ".s3.amazonaws.com"; if (host.size() > s3suffix.size()) @@ -116,7 +116,7 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) s3suffix.size()); } } - + std::ostringstream data; data << rRequest.GetVerb() << "\n"; data << md5 << "\n"; @@ -127,7 +127,7 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) std::vector<HTTPRequest::Header> headers = rRequest.GetHeaders(); std::sort(headers.begin(), headers.end()); - + for (std::vector<HTTPRequest::Header>::iterator i = headers.begin(); i != headers.end(); i++) { @@ -135,13 +135,13 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) { data << i->first << ":" << i->second << "\n"; } - } - + } + if (! bucket.empty()) { data << "/" << bucket; } - + data << rRequest.GetRequestURI(); std::string data_string = data.str(); @@ -152,17 +152,17 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) (const unsigned char*)data_string.c_str(), data_string.size(), digest_buffer, &digest_size); std::string digest((const char *)digest_buffer, digest_size); - + base64::encoder encoder; std::string expectedAuth = "AWS " + access_key + ":" + encoder.encode(digest); - + if (expectedAuth[expectedAuth.size() - 1] == '\n') { expectedAuth = expectedAuth.substr(0, expectedAuth.size() - 1); } - + std::string actualAuth; if (!rRequest.GetHeader("authorization", &actualAuth) || actualAuth != expectedAuth) @@ -170,7 +170,7 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) rResponse.SetResponseCode(HTTPResponse::Code_Unauthorized); SendInternalErrorResponse("Authentication Failed", rResponse); - } + } else if (rRequest.GetMethod() == HTTPRequest::Method_GET) { HandleGet(rRequest, rResponse); @@ -198,7 +198,7 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) { SendInternalErrorResponse("Unknown exception", rResponse); } - + if (rResponse.GetResponseCode() != 200 && rResponse.GetSize() == 0) { @@ -207,7 +207,10 @@ void S3Simulator::Handle(HTTPRequest &rRequest, HTTPResponse &rResponse) s << rResponse.GetResponseCode(); SendInternalErrorResponse(s.str().c_str(), rResponse); } - + + BOX_NOTICE(rResponse.GetResponseCode() << " " << rRequest.GetMethodName() << " " << + rRequest.GetRequestURI()); + return; } diff --git a/lib/httpserver/S3Simulator.h b/lib/httpserver/S3Simulator.h index f80770ee..eef4f400 100644 --- a/lib/httpserver/S3Simulator.h +++ b/lib/httpserver/S3Simulator.h @@ -27,13 +27,20 @@ class HTTPResponse; class S3Simulator : public HTTPServer { public: - S3Simulator() { } + // Increase timeout to 5 minutes, from HTTPServer default of 1 minute, + // to help with debugging. + S3Simulator() : HTTPServer(300000) { } ~S3Simulator() { } const ConfigurationVerify* GetConfigVerify() const; virtual void Handle(HTTPRequest &rRequest, HTTPResponse &rResponse); virtual void HandleGet(HTTPRequest &rRequest, HTTPResponse &rResponse); virtual void HandlePut(HTTPRequest &rRequest, HTTPResponse &rResponse); + + virtual const char *DaemonName() const + { + return "s3simulator"; + } }; #endif // S3SIMULATOR__H diff --git a/lib/httpserver/cdecode.cpp b/lib/httpserver/cdecode.cpp index e632f182..11c59d62 100644 --- a/lib/httpserver/cdecode.cpp +++ b/lib/httpserver/cdecode.cpp @@ -12,7 +12,7 @@ extern "C" int base64_decode_value(char value_in) { - static const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; + static signed const char decoding[] = {62,-1,-1,-1,63,52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-2,-1,-1,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1,-1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51}; static const char decoding_size = sizeof(decoding); value_in -= 43; if (value_in < 0 || value_in > decoding_size) return -1; diff --git a/lib/intercept/intercept.cpp b/lib/intercept/intercept.cpp index 7a33b610..72bd8d4e 100644 --- a/lib/intercept/intercept.cpp +++ b/lib/intercept/intercept.cpp @@ -15,7 +15,6 @@ #include <sys/syscall.h> #endif #include <sys/types.h> -#include <unistd.h> #ifdef HAVE_SYS_UIO_H #include <sys/uio.h> @@ -132,7 +131,7 @@ void intercept_setup_error(const char *filename, unsigned int errorafter, int er intercept_delay_ms = 0; } -void intercept_setup_delay(const char *filename, unsigned int delay_after, +void intercept_setup_delay(const char *filename, unsigned int delay_after, int delay_ms, int syscall_to_delay, int num_delays) { BOX_TRACE("Setup for delay: " << filename << @@ -243,6 +242,10 @@ extern "C" int open(const char *path, int flags, ...) #endif // DEFINE_ONLY_OPEN64 { + // Some newer architectures don't have an open() syscall, but use openat() instead. + // In these cases we will need to call sys_openat() instead of sys_open(). + // https://chromium.googlesource.com/linux-syscall-support/ + if(intercept_count > 0) { if(intercept_filename != NULL && @@ -265,6 +268,8 @@ extern "C" int #ifdef PLATFORM_NO_SYSCALL int r = TEST_open(path, flags, mode); +#elif HAVE_DECL_SYS_OPENAT && !HAVE_DECL_SYS_OPEN + int r = syscall(SYS_openat, AT_FDCWD, path, flags, mode); #else int r = syscall(SYS_open, path, flags, mode); #endif @@ -390,7 +395,7 @@ lseek(int fildes, off_t offset, int whence) #else #ifdef HAVE_LSEEK_DUMMY_PARAM off_t r = syscall(SYS_lseek, fildes, 0 /* extra 0 required here! */, offset, whence); - #elif defined(_FILE_OFFSET_BITS) + #elif defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS == 32 // Don't bother trying to call SYS__llseek on 32 bit since it is // fiddly and not needed for the tests off_t r = syscall(SYS_lseek, fildes, (uint32_t)offset, whence); @@ -514,7 +519,7 @@ DIR *opendir(const char *dirname) { if (opendir_real == NULL) { - opendir_real = (opendir_t*)find_function("opendir"); + opendir_real = (opendir_t*)find_function(FUNC_OPENDIR); } if (opendir_real == NULL) @@ -547,7 +552,7 @@ struct dirent *readdir(DIR *dir) if (readdir_real == NULL) { - readdir_real = (readdir_t*)find_function("readdir"); + readdir_real = (readdir_t*)find_function(FUNC_READDIR); } if (readdir_real == NULL) diff --git a/lib/intercept/intercept.h b/lib/intercept/intercept.h index 80a17d3f..4de5f9f2 100644 --- a/lib/intercept/intercept.h +++ b/lib/intercept/intercept.h @@ -13,6 +13,18 @@ #include <dirent.h> +#ifdef __NetBSD__ //__NetBSD_Version__ is defined in sys/param.h +#include <sys/param.h> +#endif + +#if defined __NetBSD_Version__ && __NetBSD_Version__ >= 399000800 //3.99.8 vers. +#define FUNC_OPENDIR "__opendir30" +#define FUNC_READDIR "__readdir30" +#else +#define FUNC_OPENDIR "opendir" +#define FUNC_READDIR "readdir" +#endif + #include <sys/types.h> #include <sys/stat.h> @@ -50,5 +62,14 @@ void intercept_setup_stat_post_hook (lstat_post_hook_t hookfn); void intercept_clear_setup(); +// Some newer architectures don't have an open() syscall, but use openat() instead. +// In these cases we define SYS_open (which is otherwise undefined) to equal SYS_openat +// (which is defined) so that everywhere else we can call intercept_setup_error(SYS_open) +// without caring about the difference. +// https://chromium.googlesource.com/linux-syscall-support/ +#if !HAVE_DECL_SYS_OPEN && HAVE_DECL_SYS_OPENAT +# define SYS_open SYS_openat +#endif + #endif // !PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE #endif // !INTERCEPT_H diff --git a/lib/raidfile/RaidFileRead.cpp b/lib/raidfile/RaidFileRead.cpp index bcff54c6..7b755395 100644 --- a/lib/raidfile/RaidFileRead.cpp +++ b/lib/raidfile/RaidFileRead.cpp @@ -44,8 +44,12 @@ #define READ_NUMBER_DISCS_REQUIRED 3 #define READV_MAX_BLOCKS 64 -// We want to use POSIX fstat() for now, not the emulated one -#undef fstat +// We want to use POSIX fstat() for now, not the emulated one, because it's +// difficult to rewrite all this code to use HANDLEs instead of ints. + +const RaidFileReadCategory RaidFileRead::OPEN_IN_RECOVERY("OpenInRecovery"); +const RaidFileReadCategory RaidFileRead::IO_ERROR("IoError"); +const RaidFileReadCategory RaidFileRead::RECOVERING_IO_ERROR("RecoverIoError"); // -------------------------------------------------------------------------- // @@ -548,7 +552,8 @@ void RaidFileRead_Raid::MoveDamagedFileAlertDaemon(int SetNumber, const std::str // -------------------------------------------------------------------------- void RaidFileRead_Raid::AttemptToRecoverFromIOError(bool Stripe1) { - BOX_WARNING("Attempting to recover from I/O error: " << mSetNumber << + BOX_LOG_CATEGORY(Log::WARNING, RaidFileRead::RECOVERING_IO_ERROR, + "Attempting to recover from I/O error: " << mSetNumber << " " << mFilename << ", on stripe " << (Stripe1?1:2)); // Close offending file @@ -740,7 +745,7 @@ int RaidFileRead_Raid::ReadRecovered(void *pBuffer, int NBytes) // Go XORing! unsigned int *b1 = (unsigned int*)mRecoveryBuffer; unsigned int *b2 = (unsigned int *)(mRecoveryBuffer + mBlockSize); - if((mStripe1Handle == -1)) + if(mStripe1Handle == -1) { b1 = b2; b2 = (unsigned int*)mRecoveryBuffer; @@ -861,10 +866,10 @@ void RaidFileRead_Raid::SetPosition(pos_type FilePosition) { if(errno == EIO) { - BOX_ERROR("I/O error when seeking in " << - mSetNumber << " " << mFilename << - " (to " << FilePosition << "), " << - "stripe 1"); + BOX_LOG_CATEGORY(Log::ERROR, RaidFileRead::IO_ERROR, + "I/O error when seeking in set " << mSetNumber << + ": " << mFilename << " (to " << FilePosition << + "), " << "stripe 1"); // Attempt to recover AttemptToRecoverFromIOError(true /* is stripe 1 */); ASSERT(mStripe1Handle == -1); @@ -881,10 +886,10 @@ void RaidFileRead_Raid::SetPosition(pos_type FilePosition) { if(errno == EIO) { - BOX_ERROR("I/O error when seeking in " << - mSetNumber << " " << mFilename << - " (to " << FilePosition << "), " << - "stripe 2"); + BOX_LOG_CATEGORY(Log::ERROR, RaidFileRead::IO_ERROR, + "I/O error when seeking in set " << mSetNumber << + ": " << mFilename << " (to " << FilePosition << + "), " << "stripe 2"); // Attempt to recover AttemptToRecoverFromIOError(false /* is stripe 2 */); ASSERT(mStripe2Handle == -1); @@ -1155,8 +1160,9 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string bool oktotryagain = true; if(stripe1errno == EIO) { - BOX_ERROR("I/O error on opening " << - SetNumber << " " << Filename << + BOX_LOG_CATEGORY(Log::ERROR, + RaidFileRead::RECOVERING_IO_ERROR, "I/O error " + "on opening " << SetNumber << " " << Filename << " stripe 1, trying recovery mode"); RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, true /* is stripe 1 */); @@ -1172,8 +1178,9 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string if(stripe2errno == EIO) { - BOX_ERROR("I/O error on opening " << - SetNumber << " " << Filename << + BOX_LOG_CATEGORY(Log::ERROR, + RaidFileRead::RECOVERING_IO_ERROR, "I/O error " + "on opening " << SetNumber << " " << Filename << " stripe 2, trying recovery mode"); RaidFileRead_Raid::MoveDamagedFileAlertDaemon(SetNumber, Filename, false /* is stripe 2 */); @@ -1196,7 +1203,8 @@ std::auto_ptr<RaidFileRead> RaidFileRead::Open(int SetNumber, const std::string if(existance == RaidFileUtil::AsRaidWithMissingReadable) { - BOX_ERROR("Attempting to open RAID file " << SetNumber << + BOX_LOG_CATEGORY(Log::ERROR, RaidFileRead::OPEN_IN_RECOVERY, + "Attempting to open RAID file " << SetNumber << " " << Filename << " in recovery mode (stripe " << ((existingFiles & RaidFileUtil::Stripe1Exists)?1:2) << " present)"); @@ -1498,8 +1506,8 @@ bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::strin std::string dn(rSet[l] + DIRECTORY_SEPARATOR + rDirName); // check for existence - struct stat st; - if(::stat(dn.c_str(), &st) == 0) + EMU_STRUCT_STAT st; + if(EMU_STAT(dn.c_str(), &st) == 0) { // Directory? if(st.st_mode & S_IFDIR) @@ -1510,7 +1518,10 @@ bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::strin else { // No. It's a file. Bad! - THROW_EXCEPTION(RaidFileException, UnexpectedFileInDirPlace) + THROW_FILE_ERROR("Expected a directory, " + "found something else", dn, + RaidFileException, + UnexpectedFileInDirPlace); } } else @@ -1519,7 +1530,9 @@ bool RaidFileRead::DirectoryExists(const RaidFileDiscSet &rSet, const std::strin if(errno != ENOENT) { // No. Bad things. - THROW_EXCEPTION(RaidFileException, OSError) + THROW_SYS_FILE_ERROR("Failed to check for " + "existing RaidFile directory", dn, + RaidFileException, OSError); } } } @@ -1617,12 +1630,15 @@ bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirN #ifdef HAVE_VALID_DIRENT_D_TYPE if(DirReadType == DirReadType_FilesOnly && en->d_type == DT_REG) #else - struct stat st; + EMU_STRUCT_STAT st; std::string fullName(dn + DIRECTORY_SEPARATOR + en->d_name); - if(::lstat(fullName.c_str(), &st) != 0) + if(EMU_LSTAT(fullName.c_str(), &st) != 0) { - THROW_EXCEPTION(RaidFileException, OSError) + THROW_SYS_FILE_ERROR("Failed to stat", + fullName, RaidFileException, + OSError); } + if(DirReadType == DirReadType_FilesOnly && (st.st_mode & S_IFDIR) == 0) #endif { @@ -1721,7 +1737,7 @@ bool RaidFileRead::ReadDirectoryContents(int SetNumber, const std::string &rDirN // Created: 2003/08/21 // // -------------------------------------------------------------------------- -void RaidFileRead::Write(const void *pBuffer, int NBytes) +void RaidFileRead::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(RaidFileException, UnsupportedReadWriteOrClose) } @@ -1767,6 +1783,7 @@ IOStream::pos_type RaidFileRead::GetDiscUsageInBlocks() return RaidFileUtil::DiscUsageInBlocks(GetFileSize(), rdiscSet); } - - - +std::string RaidFileRead::ToString() const +{ + return std::string("RaidFile ") + mFilename; +} diff --git a/lib/raidfile/RaidFileRead.h b/lib/raidfile/RaidFileRead.h index 8a04409d..a3c792d0 100644 --- a/lib/raidfile/RaidFileRead.h +++ b/lib/raidfile/RaidFileRead.h @@ -16,9 +16,17 @@ #include <vector> #include "IOStream.h" +#include "Logging.h" class RaidFileDiscSet; +class RaidFileReadCategory : public Log::Category +{ + public: + RaidFileReadCategory(const std::string& name) + : Log::Category(std::string("RaidFileRead/") + name) + { } +}; // -------------------------------------------------------------------------- // @@ -56,14 +64,20 @@ public: static bool ReadDirectoryContents(int SetNumber, const std::string &rDirName, int DirReadType, std::vector<std::string> &rOutput); // Common IOStream interface implementation - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamClosed(); virtual pos_type BytesLeftToRead(); pos_type GetDiscUsageInBlocks(); + std::string ToString() const; typedef int64_t FileSizeType; + static const RaidFileReadCategory OPEN_IN_RECOVERY; + static const RaidFileReadCategory IO_ERROR; + static const RaidFileReadCategory RECOVERING_IO_ERROR; + protected: int mSetNumber; std::string mFilename; diff --git a/lib/raidfile/RaidFileUtil.cpp b/lib/raidfile/RaidFileUtil.cpp index 7c6299ec..48625997 100644 --- a/lib/raidfile/RaidFileUtil.cpp +++ b/lib/raidfile/RaidFileUtil.cpp @@ -14,10 +14,30 @@ #include "RaidFileUtil.h" #include "FileModificationTime.h" -#include "RaidFileRead.h" // for type definition +#include "RaidFileRead.h" // for type definition #include "MemLeakFindOn.h" +int64_t adjust_timestamp(int64_t timestamp, size_t file_size) +{ +#ifndef BOX_RELEASE_BUILD + // Remove the microseconds part of the timestamp, + // to simulate filesystem with 1-second timestamp + // resolution, e.g. MacOS X HFS, old Linuxes. + // Otherwise it's easy to write tests that rely + // on more accurate timestamps, and pass on + // platforms that have them, and fail on others. + timestamp -= (timestamp % MICRO_SEC_IN_SEC); +#endif + + // The resolution of timestamps may be very + // low, e.g. 1 second. So add the size to it + // to give a bit more chance of it changing. + // TODO: Make this better. + timestamp += file_size; + + return timestamp; +} // -------------------------------------------------------------------------- // @@ -39,8 +59,7 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, *pExistingFiles = 0; } - // For stat call, although the results are not examined - struct stat st; + EMU_STRUCT_STAT st; // check various files int startDisc = 0; @@ -50,29 +69,15 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, { *pStartDisc = startDisc; } - if(::stat(writeFile.c_str(), &st) == 0) + if(EMU_STAT(writeFile.c_str(), &st) == 0) { // write file exists, use that // Get unique ID if(pRevisionID != 0) { - #ifdef WIN32 - *pRevisionID = st.st_mtime; - #else - *pRevisionID = FileModificationTime(st); - #endif - -#ifdef BOX_RELEASE_BUILD - // The resolution of timestamps may be very - // low, e.g. 1 second. So add the size to it - // to give a bit more chance of it changing. - // TODO: Make this better. - // Disabled in debug mode, to simulate - // filesystem with 1-second timestamp - // resolution, e.g. MacOS X HFS, Linux. - (*pRevisionID) += st.st_size; -#endif + *pRevisionID = FileModificationTime(st); + *pRevisionID = adjust_timestamp(*pRevisionID, st.st_size); } // return non-raid file @@ -91,7 +96,7 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, for(int f = 0; f < setSize; ++f) { std::string componentFile(RaidFileUtil::MakeRaidComponentName(rDiscSet, rFilename, (f + startDisc) % setSize)); - if(::stat(componentFile.c_str(), &st) == 0) + if(EMU_STAT(componentFile.c_str(), &st) == 0) { // Component file exists, add to count rfCount++; @@ -103,12 +108,7 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, // Revision ID if(pRevisionID != 0) { - #ifdef WIN32 - int64_t rid = st.st_mtime; - #else - int64_t rid = FileModificationTime(st); - #endif - + int64_t rid = FileModificationTime(st); if(rid > revisionID) revisionID = rid; revisionIDplus += st.st_size; } @@ -116,16 +116,8 @@ RaidFileUtil::ExistType RaidFileUtil::RaidFileExists(RaidFileDiscSet &rDiscSet, } if(pRevisionID != 0) { + revisionID = adjust_timestamp(revisionID, revisionIDplus); (*pRevisionID) = revisionID; -#ifdef BOX_RELEASE_BUILD - // The resolution of timestamps may be very low, e.g. - // 1 second. So add the size to it to give a bit more - // chance of it changing. - // TODO: Make this better. - // Disabled in debug mode, to simulate filesystem with - // 1-second timestamp resolution, e.g. MacOS X HFS, Linux. - (*pRevisionID) += revisionIDplus; -#endif } // Return a status based on how many parts are available diff --git a/lib/raidfile/RaidFileWrite.cpp b/lib/raidfile/RaidFileWrite.cpp index 82aeef3d..8f95ba65 100644 --- a/lib/raidfile/RaidFileWrite.cpp +++ b/lib/raidfile/RaidFileWrite.cpp @@ -42,8 +42,8 @@ // Must have this number of discs in the set #define TRANSFORM_NUMBER_DISCS_REQUIRED 3 -// we want to use POSIX fstat() for now, not the emulated one -#undef fstat +// We want to use POSIX fstat() for now, not the emulated one, because it's +// difficult to rewrite all this code to use HANDLEs instead of ints. // -------------------------------------------------------------------------- // @@ -243,7 +243,7 @@ void RaidFileWrite::Open(bool AllowOverwrite) // Created: 2003/07/10 // // -------------------------------------------------------------------------- -void RaidFileWrite::Write(const void *pBuffer, int Length) +void RaidFileWrite::Write(const void *pBuffer, int Length, int Timeout) { // open? if(mOSFileHandle == -1) @@ -672,7 +672,9 @@ void RaidFileWrite::TransformToRaidStorage() { \ if (::unlink(file) != 0 && errno != ENOENT) \ { \ - THROW_EXCEPTION(RaidFileException, OSError); \ + THROW_EMU_ERROR("Failed to unlink raidfile " \ + "stripe: " << file, RaidFileException, \ + OSError); \ } \ } CHECK_UNLINK(stripe1Filename.c_str()); diff --git a/lib/raidfile/RaidFileWrite.h b/lib/raidfile/RaidFileWrite.h index e2887167..ab9b399a 100644 --- a/lib/raidfile/RaidFileWrite.h +++ b/lib/raidfile/RaidFileWrite.h @@ -41,13 +41,15 @@ public: RaidFileWrite(int SetNumber, const std::string &Filename, int refcount); ~RaidFileWrite(); + private: RaidFileWrite(const RaidFileWrite &rToCopy); public: // IOStream interface virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); // will exception - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual pos_type GetPosition() const; virtual void Seek(pos_type Offset, int SeekType); virtual void Close(); // will discard the file! Use commit instead. @@ -65,8 +67,6 @@ public: static void CreateDirectory(int SetNumber, const std::string &rDirName, bool Recursive = false, int mode = 0777); static void CreateDirectory(const RaidFileDiscSet &rSet, const std::string &rDirName, bool Recursive = false, int mode = 0777); - -private: private: int mSetNumber; diff --git a/lib/server/ConnectionException.txt b/lib/server/ConnectionException.txt index c3429116..7dcaadeb 100644 --- a/lib/server/ConnectionException.txt +++ b/lib/server/ConnectionException.txt @@ -25,3 +25,4 @@ Protocol_HandshakeFailed 48 Protocol_StreamWhenObjExpected 49 Protocol_ObjWhenStreamExpected 50 Protocol_TimeOutWhenSendingStream 52 Probably a network issue between client and server. +Protocol_StreamsNotConsumed 53 The server command handler did not consume all streams that were sent. diff --git a/lib/server/Daemon.cpp b/lib/server/Daemon.cpp index 7419f973..d3c8441f 100644 --- a/lib/server/Daemon.cpp +++ b/lib/server/Daemon.cpp @@ -9,23 +9,27 @@ #include "Box.h" -#ifdef HAVE_UNISTD_H - #include <unistd.h> -#endif - #include <errno.h> #include <stdio.h> #include <signal.h> #include <string.h> #include <stdarg.h> +#ifdef HAVE_PROCESS_H +# include <process.h> +#endif + +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + #ifdef HAVE_BSD_UNISTD_H #include <bsd/unistd.h> #endif #ifdef WIN32 + #include <Strsafe.h> #include <ws2tcpip.h> - #include <process.h> #endif #include "depot.h" @@ -36,12 +40,13 @@ # include "BoxVersion.h" #endif +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "Configuration.h" #include "Daemon.h" #include "FileModificationTime.h" #include "Guards.h" #include "Logging.h" -#include "ServerException.h" #include "UnixUser.h" #include "Utils.h" @@ -106,11 +111,11 @@ Daemon::~Daemon() // -------------------------------------------------------------------------- std::string Daemon::GetOptionString() { - return "c:" + return std::string("c:" #ifndef WIN32 "DF" #endif - "hkKo:O:PqQt:TUvVW:"; + "hkKo:O:") + Logging::OptionParser::GetOptionString(); } void Daemon::Usage() @@ -133,16 +138,7 @@ void Daemon::Usage() " -K Stop writing log messages to console while daemon is running\n" " -o <file> Log to a file, defaults to maximum verbosity\n" " -O <level> Set file log verbosity to error/warning/notice/info/trace/everything\n" - " -P Show process ID (PID) in console output\n" - " -q Run more quietly, reduce verbosity level by one, can repeat\n" - " -Q Run at minimum verbosity, log nothing to console and system\n" - " -t <tag> Tag console output with specified marker\n" - " -T Timestamp console output\n" - " -U Timestamp console output with microseconds\n" - " -v Run more verbosely, increase verbosity level by one, can repeat\n" - " -V Run at maximum verbosity, log everything to console and system\n" - " -W <level> Set verbosity to error/warning/notice/info/trace/everything\n" - ; + << Logging::OptionParser::GetUsageString(); } // -------------------------------------------------------------------------- @@ -218,94 +214,9 @@ int Daemon::ProcessOption(signed int option) } break; - case 'P': - { - Console::SetShowPID(true); - } - break; - - case 'q': - { - if(mLogLevel == Log::NOTHING) - { - BOX_FATAL("Too many '-q': " - "Cannot reduce logging " - "level any more"); - return 2; - } - mLogLevel--; - } - break; - - case 'Q': - { - mLogLevel = Log::NOTHING; - } - break; - - case 't': - { - Logging::SetProgramName(optarg); - Console::SetShowTag(true); - } - break; - - case 'T': - { - Console::SetShowTime(true); - } - break; - - case 'U': - { - Console::SetShowTime(true); - Console::SetShowTimeMicros(true); - } - break; - - case 'v': - { - if(mLogLevel == Log::EVERYTHING) - { - BOX_FATAL("Too many '-v': " - "Cannot increase logging " - "level any more"); - return 2; - } - mLogLevel++; - } - break; - - case 'V': - { - mLogLevel = Log::EVERYTHING; - } - break; - - case 'W': - { - mLogLevel = Logging::GetNamedLevel(optarg); - if (mLogLevel == Log::INVALID) - { - BOX_FATAL("Invalid logging level: " << optarg); - return 2; - } - } - break; - - case '?': - { - BOX_FATAL("Unknown option on command line: " - << "'" << (char)optopt << "'"); - return 2; - } - break; - default: { - BOX_FATAL("Unknown error in getopt: returned " - << "'" << option << "'"); - return 1; + return mLogLevel.ProcessOption(option); } } @@ -351,12 +262,6 @@ int Daemon::Main(const std::string& rDefaultConfigFile, int argc, int Daemon::ProcessOptions(int argc, const char *argv[]) { - #ifdef BOX_RELEASE_BUILD - mLogLevel = Log::NOTICE; - #else - mLogLevel = Log::INFO; - #endif - if (argc == 2 && strcmp(argv[1], "/?") == 0) { Usage(); @@ -368,7 +273,7 @@ int Daemon::ProcessOptions(int argc, const char *argv[]) // reset getopt, just in case anybody used it before. // unfortunately glibc and BSD differ on this point! // http://www.ussg.iu.edu/hypermail/linux/kernel/0305.3/0262.html - #if HAVE_DECL_OPTRESET == 1 || defined WIN32 + #if HAVE_DECL_OPTRESET == 1 || defined BOX_BSD_GETOPT optind = 1; optreset = 1; #elif defined __GLIBC__ @@ -406,13 +311,14 @@ int Daemon::ProcessOptions(int argc, const char *argv[]) return 2; } - Logging::FilterConsole((Log::Level)mLogLevel); - Logging::FilterSyslog ((Log::Level)mLogLevel); + Logging::FilterConsole(mLogLevel.GetCurrentLevel()); + Logging::FilterSyslog (mLogLevel.GetCurrentLevel()); if (mLogFileLevel != Log::INVALID) { mapLogFileLogger.reset( - new FileLogger(mLogFile, mLogFileLevel)); + new FileLogger(mLogFile, mLogFileLevel, + !mLogLevel.mTruncateLogFile)); } return 0; @@ -473,17 +379,17 @@ bool Daemon::Configure(const std::string& rConfigFileName) BOX_ERROR("Failed to load or verify configuration file"); return false; } - + if(!Configure(*apConfig)) { BOX_ERROR("Failed to verify configuration file"); - return false; + return false; } - + // Store configuration mConfigFileName = rConfigFileName; mLoadedConfigModifiedTime = GetConfigFileModifiedTime(); - + return true; } @@ -513,14 +419,14 @@ bool Daemon::Configure(const Configuration& rConfig) BOX_ERROR("Configuration errors: " << errors); return false; } - + // Store configuration mapConfiguration = apConf; - + // Let the derived class have a go at setting up stuff // in the initial process SetupInInitialProcess(); - + return true; } @@ -664,7 +570,7 @@ int Daemon::Main(const std::string &rConfigFileName) // Write PID to file char pid[32]; - int pidsize = sprintf(pid, "%d", (int)getpid()); + int pidsize = snprintf(pid, sizeof(pid), "%d", (int)getpid()); if(::write(pidFile, pid, pidsize) != pidsize) { @@ -676,9 +582,8 @@ int Daemon::Main(const std::string &rConfigFileName) // Set up memory leak reporting #ifdef BOX_MEMORY_LEAK_TESTING { - char filename[256]; - sprintf(filename, "%s.memleaks", DaemonName()); - memleakfinder_setup_exit_report(filename, DaemonName()); + memleakfinder_setup_exit_report(std::string(DaemonName()) + + ".memleaks", DaemonName()); } #endif // BOX_MEMORY_LEAK_TESTING @@ -986,7 +891,9 @@ const Configuration &Daemon::GetConfiguration() const if(mapConfiguration.get() == 0) { // Shouldn't get anywhere near this if a configuration file can't be loaded - THROW_EXCEPTION(ServerException, Internal) + THROW_EXCEPTION_MESSAGE(ServerException, Internal, + "The daemon has not been configured; no config file " + "has been loaded."); } return *mapConfiguration; diff --git a/lib/server/Daemon.h b/lib/server/Daemon.h index 2718c288..b5384918 100644 --- a/lib/server/Daemon.h +++ b/lib/server/Daemon.h @@ -85,7 +85,16 @@ protected: bool IsSingleProcess() { return mSingleProcess; } virtual std::string GetOptionString(); virtual int ProcessOption(signed int option); - + void ResetLogFile() + { + if(mapLogFileLogger.get()) + { + mapLogFileLogger.reset( + new FileLogger(mLogFile, mLogFileLevel, + !mLogLevel.mTruncateLogFile)); + } + } + private: static void SignalHandler(int sigraised); box_time_t GetConfigFileModifiedTime() const; @@ -99,7 +108,7 @@ private: bool mRunInForeground; bool mKeepConsoleOpenAfterFork; bool mHaveConfigFile; - int mLogLevel; // need an int to do math with + Logging::OptionParser mLogLevel; std::string mLogFile; Log::Level mLogFileLevel; std::auto_ptr<FileLogger> mapLogFileLogger; diff --git a/lib/server/Message.h b/lib/server/Message.h index 0d073d49..9f2245ec 100644 --- a/lib/server/Message.h +++ b/lib/server/Message.h @@ -37,10 +37,11 @@ public: // reading and writing with Protocol objects virtual void SetPropertiesFromStreamData(Protocol &rProtocol); - virtual void WritePropertiesToStreamData(Protocol &rProtocol) const; + virtual void WritePropertiesToStreamData(Protocol &rProtocol) const; virtual void LogSysLog(const char *Action) const { } virtual void LogFile(const char *Action, FILE *file) const { } + virtual std::string ToString() const = 0; }; /* diff --git a/lib/server/Protocol.cpp b/lib/server/Protocol.cpp index 382f1c37..0adf9543 100644 --- a/lib/server/Protocol.cpp +++ b/lib/server/Protocol.cpp @@ -17,10 +17,11 @@ #include <new> +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "Protocol.h" #include "ProtocolWire.h" -#include "IOStream.h" -#include "ServerException.h" +#include "SocketStream.h" #include "PartialReadStream.h" #include "ProtocolUncertainStream.h" #include "Logging.h" @@ -44,8 +45,8 @@ // Created: 2003/08/19 // // -------------------------------------------------------------------------- -Protocol::Protocol(IOStream &rStream) -: mrStream(rStream), +Protocol::Protocol(std::auto_ptr<SocketStream> apConn) +: mapConn(apConn), mHandshakeDone(false), mMaxObjectSize(PROTOCOL_DEFAULT_MAXOBJSIZE), mTimeout(PROTOCOL_DEFAULT_TIMEOUT), @@ -103,8 +104,8 @@ void Protocol::Handshake() ::strncpy(hsSend.mIdent, GetProtocolIdentString(), sizeof(hsSend.mIdent)); // Send it - mrStream.Write(&hsSend, sizeof(hsSend)); - mrStream.WriteAllBuffered(); + mapConn->Write(&hsSend, sizeof(hsSend), GetTimeout()); + mapConn->WriteAllBuffered(); // Receive a handshake from the peer PW_Handshake hsReceive; @@ -114,10 +115,10 @@ void Protocol::Handshake() while(bytesToRead > 0) { // Get some data from the stream - int bytesRead = mrStream.Read(readInto, bytesToRead, mTimeout); + int bytesRead = mapConn->Read(readInto, bytesToRead, GetTimeout()); if(bytesRead == 0) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + THROW_EXCEPTION(ConnectionException, Protocol_Timeout) } readInto += bytesRead; bytesToRead -= bytesRead; @@ -127,7 +128,7 @@ void Protocol::Handshake() // Are they the same? if(::memcmp(&hsSend, &hsReceive, sizeof(hsSend)) != 0) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_HandshakeFailed) + THROW_EXCEPTION(ConnectionException, Protocol_HandshakeFailed) } // Mark as done @@ -158,9 +159,10 @@ void Protocol::CheckAndReadHdr(void *hdr) } // Get some data into this header - if(!mrStream.ReadFullBuffer(hdr, sizeof(PW_ObjectHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) + if(!mapConn->ReadFullBuffer(hdr, sizeof(PW_ObjectHeader), + 0 /* not interested in bytes read if this fails */, mTimeout)) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + THROW_EXCEPTION(ConnectionException, Protocol_Timeout) } } @@ -168,8 +170,9 @@ void Protocol::CheckAndReadHdr(void *hdr) // -------------------------------------------------------------------------- // // Function -// Name: Protocol::Recieve() -// Purpose: Recieves an object from the stream, creating it from the factory object type +// Name: Protocol::ReceiveInternal() +// Purpose: Receives an object from the stream, creating it +// from the factory object type // Created: 2003/08/19 // // -------------------------------------------------------------------------- @@ -182,14 +185,14 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() // Hope it's not a stream if(ntohl(objHeader.mObjType) == SPECIAL_STREAM_OBJECT_TYPE) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_StreamWhenObjExpected) + THROW_EXCEPTION(ConnectionException, Protocol_StreamWhenObjExpected) } // Check the object size - u_int32_t objSize = ntohl(objHeader.mObjSize); + uint32_t objSize = ntohl(objHeader.mObjSize); if(objSize < sizeof(objHeader) || objSize > mMaxObjectSize) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjTooBig) + THROW_EXCEPTION(ConnectionException, Protocol_ObjTooBig) } // Create a blank object @@ -199,9 +202,10 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() EnsureBufferAllocated(objSize); // Read data - if(!mrStream.ReadFullBuffer(mpBuffer, objSize - sizeof(objHeader), 0 /* not interested in bytes read if this fails */, mTimeout)) + if(!mapConn->ReadFullBuffer(mpBuffer, objSize - sizeof(objHeader), + 0 /* not interested in bytes read if this fails */, mTimeout)) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_Timeout) + THROW_EXCEPTION(ConnectionException, Protocol_Timeout) } // Setup ready to read out data from the buffer @@ -231,7 +235,7 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() // Exception if not all the data was consumed if(dataLeftOver) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved) + THROW_EXCEPTION(ConnectionException, Protocol_BadCommandRecieved) } return obj; @@ -240,7 +244,7 @@ std::auto_ptr<Message> Protocol::ReceiveInternal() // -------------------------------------------------------------------------- // // Function -// Name: Protocol::Send() +// Name: Protocol::SendInternal() // Purpose: Send an object to the other side of the connection. // Created: 2003/08/19 // @@ -292,8 +296,8 @@ void Protocol::SendInternal(const Message &rObject) pobjHeader->mObjType = htonl(rObject.GetType()); // Write data - mrStream.Write(mpBuffer, writtenSize); - mrStream.WriteAllBuffered(); + mapConn->Write(mpBuffer, writtenSize, GetTimeout()); + mapConn->WriteAllBuffered(); } // -------------------------------------------------------------------------- @@ -346,7 +350,7 @@ void Protocol::EnsureBufferAllocated(int Size) #define READ_CHECK_BYTES_AVAILABLE(bytesRequired) \ if((mReadOffset + (int)(bytesRequired)) > mValidDataSize) \ { \ - THROW_EXCEPTION(ConnectionException, Conn_Protocol_BadCommandRecieved) \ + THROW_EXCEPTION(ConnectionException, Protocol_BadCommandRecieved) \ } // -------------------------------------------------------------------------- @@ -619,7 +623,7 @@ void Protocol::Write(const std::string &rValue) // -------------------------------------------------------------------------- // // Function -// Name: Protocol::ReceieveStream() +// Name: Protocol::ReceiveStream() // Purpose: Receive a stream from the remote side // Created: 2003/08/26 // @@ -633,11 +637,11 @@ std::auto_ptr<IOStream> Protocol::ReceiveStream() // Hope it's not an object if(ntohl(objHeader.mObjType) != SPECIAL_STREAM_OBJECT_TYPE) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_ObjWhenStreamExpected) + THROW_EXCEPTION(ConnectionException, Protocol_ObjWhenStreamExpected) } // Get the stream size - u_int32_t streamSize = ntohl(objHeader.mObjSize); + uint32_t streamSize = ntohl(objHeader.mObjSize); // Inform sub class InformStreamReceiving(streamSize); @@ -647,13 +651,13 @@ std::auto_ptr<IOStream> Protocol::ReceiveStream() { BOX_TRACE("Receiving stream, size uncertain"); return std::auto_ptr<IOStream>( - new ProtocolUncertainStream(mrStream)); + new ProtocolUncertainStream(*mapConn)); } else { BOX_TRACE("Receiving stream, size " << streamSize << " bytes"); return std::auto_ptr<IOStream>( - new PartialReadStream(mrStream, streamSize)); + new PartialReadStream(*mapConn, streamSize)); } } @@ -709,7 +713,7 @@ void Protocol::SendStream(IOStream &rStream) objHeader.mObjType = htonl(SPECIAL_STREAM_OBJECT_TYPE); // Write header - mrStream.Write(&objHeader, sizeof(objHeader)); + mapConn->Write(&objHeader, sizeof(objHeader), GetTimeout()); // Could be sent in one of two ways if(uncertainSize) { @@ -744,7 +748,7 @@ void Protocol::SendStream(IOStream &rStream) // Send final byte to finish the stream BOX_TRACE("Sending end of stream byte"); uint8_t endOfStream = ProtocolStreamHeader_EndOfStream; - mrStream.Write(&endOfStream, 1); + mapConn->Write(&endOfStream, 1, GetTimeout()); BOX_TRACE("Sent end of stream byte"); } catch(...) @@ -759,13 +763,14 @@ void Protocol::SendStream(IOStream &rStream) else { // Fixed size stream, send it all in one go - if(!rStream.CopyStreamTo(mrStream, mTimeout, 4096 /* slightly larger buffer */)) + if(!rStream.CopyStreamTo(*mapConn, GetTimeout(), + 4096 /* slightly larger buffer */)) { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_TimeOutWhenSendingStream) + THROW_EXCEPTION(ConnectionException, Protocol_TimeOutWhenSendingStream) } } // Make sure everything is written - mrStream.WriteAllBuffered(); + mapConn->WriteAllBuffered(); } @@ -816,7 +821,7 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) Block[-1] = header; // Write everything out - mrStream.Write(Block - 1, writeSize + 1); + mapConn->Write(Block - 1, writeSize + 1, GetTimeout()); BOX_TRACE("Sent " << (writeSize+1) << " bytes to stream"); // move the remainer to the beginning of the block for the next time round @@ -831,12 +836,12 @@ int Protocol::SendStreamSendBlock(uint8_t *Block, int BytesInBlock) // -------------------------------------------------------------------------- // // Function -// Name: Protocol::InformStreamReceiving(u_int32_t) +// Name: Protocol::InformStreamReceiving(uint32_t) // Purpose: Informs sub classes about streams being received // Created: 2003/10/27 // // -------------------------------------------------------------------------- -void Protocol::InformStreamReceiving(u_int32_t Size) +void Protocol::InformStreamReceiving(uint32_t Size) { if(GetLogToSysLog()) { @@ -863,12 +868,12 @@ void Protocol::InformStreamReceiving(u_int32_t Size) // -------------------------------------------------------------------------- // // Function -// Name: Protocol::InformStreamSending(u_int32_t) +// Name: Protocol::InformStreamSending(uint32_t) // Purpose: Informs sub classes about streams being sent // Created: 2003/10/27 // // -------------------------------------------------------------------------- -void Protocol::InformStreamSending(u_int32_t Size) +void Protocol::InformStreamSending(uint32_t Size) { if(GetLogToSysLog()) { @@ -1177,6 +1182,12 @@ const uint16_t Protocol::sProtocolStreamHeaderLengths[256] = 0 // 255 = special (reserved) }; +int64_t Protocol::GetBytesRead() const +{ + return mapConn->GetBytesRead(); +} - - +int64_t Protocol::GetBytesWritten() const +{ + return mapConn->GetBytesWritten(); +} diff --git a/lib/server/Protocol.h b/lib/server/Protocol.h index 42cb0ff8..fbe6461c 100644 --- a/lib/server/Protocol.h +++ b/lib/server/Protocol.h @@ -19,6 +19,7 @@ #include "Message.h" class IOStream; +class SocketStream; // default timeout is 15 minutes #define PROTOCOL_DEFAULT_TIMEOUT (15*60*1000) @@ -36,7 +37,7 @@ class IOStream; class Protocol { public: - Protocol(IOStream &rStream); + Protocol(std::auto_ptr<SocketStream> apConn); virtual ~Protocol(); private: @@ -66,14 +67,14 @@ public: // Purpose: Sets the timeout for sending and reciving // Created: 2003/08/19 // - // -------------------------------------------------------------------------- + // -------------------------------------------------------------------------- void SetTimeout(int NewTimeout) {mTimeout = NewTimeout;} // -------------------------------------------------------------------------- // // Function - // Name: Protocol::GetTimeout() + // Name: Protocol::GetTimeout() // Purpose: Get current timeout for sending and receiving // Created: 2003/09/06 // @@ -175,6 +176,8 @@ public: FILE *GetLogToFile() { return mLogToFile; } void SetLogToSysLog(bool Log = false) {mLogToSysLog = Log;} void SetLogToFile(FILE *File = 0) {mLogToFile = File;} + int64_t GetBytesRead() const; + int64_t GetBytesWritten() const; protected: virtual std::auto_ptr<Message> MakeMessage(int ObjType) = 0; @@ -183,14 +186,14 @@ protected: void CheckAndReadHdr(void *hdr); // don't use type here to avoid dependency // Will be used for logging - virtual void InformStreamReceiving(u_int32_t Size); - virtual void InformStreamSending(u_int32_t Size); + virtual void InformStreamReceiving(uint32_t Size); + virtual void InformStreamSending(uint32_t Size); private: void EnsureBufferAllocated(int Size); int SendStreamSendBlock(uint8_t *Block, int BytesInBlock); - IOStream &mrStream; + std::auto_ptr<SocketStream> mapConn; bool mHandshakeDone; unsigned int mMaxObjectSize; int mTimeout; @@ -208,4 +211,3 @@ class ProtocolContext }; #endif // PROTOCOL__H - diff --git a/lib/server/ProtocolUncertainStream.cpp b/lib/server/ProtocolUncertainStream.cpp index 84a213a8..aeb15816 100644 --- a/lib/server/ProtocolUncertainStream.cpp +++ b/lib/server/ProtocolUncertainStream.cpp @@ -8,8 +8,9 @@ // -------------------------------------------------------------------------- #include "Box.h" +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "ProtocolUncertainStream.h" -#include "ServerException.h" #include "Protocol.h" #include "MemLeakFindOn.h" @@ -172,7 +173,7 @@ IOStream::pos_type ProtocolUncertainStream::BytesLeftToRead() // Created: 2003/12/05 // // -------------------------------------------------------------------------- -void ProtocolUncertainStream::Write(const void *pBuffer, int NBytes) +void ProtocolUncertainStream::Write(const void *pBuffer, int NBytes, int Timeout) { THROW_EXCEPTION(ServerException, CantWriteToProtocolUncertainStream) } diff --git a/lib/server/ProtocolUncertainStream.h b/lib/server/ProtocolUncertainStream.h index 4954cf88..2e97ba6a 100644 --- a/lib/server/ProtocolUncertainStream.h +++ b/lib/server/ProtocolUncertainStream.h @@ -33,7 +33,8 @@ private: public: virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); virtual pos_type BytesLeftToRead(); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual bool StreamDataLeft(); virtual bool StreamClosed(); diff --git a/lib/server/ProtocolWire.h b/lib/server/ProtocolWire.h index ff62b66e..6dee445b 100644 --- a/lib/server/ProtocolWire.h +++ b/lib/server/ProtocolWire.h @@ -26,8 +26,8 @@ typedef struct typedef struct { - u_int32_t mObjSize; - u_int32_t mObjType; + uint32_t mObjSize; + uint32_t mObjType; } PW_ObjectHeader; #define SPECIAL_STREAM_OBJECT_TYPE 0xffffffff diff --git a/lib/server/SSLLib.cpp b/lib/server/SSLLib.cpp index 004d2d98..1bcadb0d 100644 --- a/lib/server/SSLLib.cpp +++ b/lib/server/SSLLib.cpp @@ -18,9 +18,10 @@ #include <wincrypt.h> #endif +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "CryptoUtils.h" #include "SSLLib.h" -#include "ServerException.h" #include "MemLeakFindOn.h" @@ -79,7 +80,7 @@ void SSLLib::Initialise() BOX_LOG_WIN_ERROR("Failed to release crypto context"); } } -#elif HAVE_RANDOM_DEVICE +#elif defined HAVE_RANDOM_DEVICE if(::RAND_load_file(RANDOM_DEVICE, 1024) != 1024) { THROW_EXCEPTION(ServerException, SSLRandomInitFailed) diff --git a/lib/server/ServerControl.cpp b/lib/server/ServerControl.cpp index b9650cee..f1a718df 100644 --- a/lib/server/ServerControl.cpp +++ b/lib/server/ServerControl.cpp @@ -15,13 +15,14 @@ #include <signal.h> #endif +#include "BoxTime.h" +#include "IOStreamGetLine.h" #include "ServerControl.h" #include "Test.h" #ifdef WIN32 #include "WinNamedPipeStream.h" -#include "IOStreamGetLine.h" #include "BoxPortsAndFiles.h" static std::string sPipeName; @@ -197,18 +198,18 @@ bool KillServer(int pid, bool WaitForProcess) } #endif - for (int i = 0; i < 30; i++) + printf("Waiting for server to die (pid %d): ", pid); + + for (int i = 0; i < 300; i++) { - if (i == 0) + if (i % 10 == 0) { - printf("Waiting for server to die (pid %d): ", pid); + printf("."); + fflush(stdout); } - printf("."); - fflush(stdout); - if (!ServerIsAlive(pid)) break; - ::sleep(1); + ShortSleep(MilliSecondsToBoxTime(100), false); if (!ServerIsAlive(pid)) break; } @@ -226,3 +227,64 @@ bool KillServer(int pid, bool WaitForProcess) return !ServerIsAlive(pid); } +bool KillServer(std::string pid_file, bool WaitForProcess) +{ + FileStream fs(pid_file); + IOStreamGetLine getline(fs); + std::string line = getline.GetLine(); + int pid = atoi(line.c_str()); + bool status = KillServer(pid, WaitForProcess); + TEST_EQUAL_LINE(true, status, std::string("kill(") + pid_file + ")"); + +#ifdef WIN32 + if(WaitForProcess) + { + int unlink_result = unlink(pid_file.c_str()); + TEST_EQUAL_LINE(0, unlink_result, std::string("unlink ") + pid_file); + if(unlink_result != 0) + { + return false; + } + } +#endif + + return status; +} + +int StartDaemon(int current_pid, const std::string& cmd_line, const char* pid_file) +{ + TEST_THAT_OR(current_pid == 0, return 0); + + int new_pid = LaunchServer(cmd_line, pid_file); + TEST_THAT_OR(new_pid != -1 && new_pid != 0, return 0); + + ::sleep(1); + TEST_THAT_OR(ServerIsAlive(new_pid), return 0); + return new_pid; +} + +bool StopDaemon(int current_pid, const std::string& pid_file, + const std::string& memleaks_file, bool wait_for_process) +{ + TEST_THAT_OR(current_pid != 0, return false); + TEST_THAT_OR(ServerIsAlive(current_pid), return false); + TEST_THAT_OR(KillServer(current_pid, wait_for_process), return false); + ::sleep(1); + + TEST_THAT_OR(!ServerIsAlive(current_pid), return false); + + #ifdef WIN32 + int unlink_result = unlink(pid_file.c_str()); + TEST_EQUAL_LINE(0, unlink_result, std::string("unlink ") + pid_file); + if(unlink_result != 0) + { + return false; + } + #else + TestRemoteProcessMemLeaks(memleaks_file.c_str()); + #endif + + return true; +} + + diff --git a/lib/server/ServerControl.h b/lib/server/ServerControl.h index b2e51864..be2464c1 100644 --- a/lib/server/ServerControl.h +++ b/lib/server/ServerControl.h @@ -5,6 +5,10 @@ bool HUPServer(int pid); bool KillServer(int pid, bool WaitForProcess = false); +bool KillServer(std::string pid_file, bool WaitForProcess = false); +int StartDaemon(int current_pid, const std::string& cmd_line, const char* pid_file); +bool StopDaemon(int current_pid, const std::string& pid_file, + const std::string& memleaks_file, bool wait_for_process); #ifdef WIN32 #include "WinNamedPipeStream.h" diff --git a/lib/server/ServerException.h b/lib/server/ServerException.h deleted file mode 100644 index 8851b90a..00000000 --- a/lib/server/ServerException.h +++ /dev/null @@ -1,46 +0,0 @@ -// -------------------------------------------------------------------------- -// -// File -// Name: ServerException.h -// Purpose: Exception -// Created: 2003/07/08 -// -// -------------------------------------------------------------------------- - -#ifndef SERVEREXCEPTION__H -#define SERVEREXCEPTION__H - -// Compatibility header -#include "autogen_ServerException.h" -#include "autogen_ConnectionException.h" - -// Rename old connection exception names to new names without Conn_ prefix -// This is all because ConnectionException used to be derived from ServerException -// with some funky magic with subtypes. Perhaps a little unreliable, and the -// usefulness of it never really was used. -#define Conn_SocketWriteError SocketWriteError -#define Conn_SocketReadError SocketReadError -#define Conn_SocketNameLookupError SocketNameLookupError -#define Conn_SocketShutdownError SocketShutdownError -#define Conn_SocketConnectError SocketConnectError -#define Conn_TLSHandshakeFailed TLSHandshakeFailed -#define Conn_TLSShutdownFailed TLSShutdownFailed -#define Conn_TLSWriteFailed TLSWriteFailed -#define Conn_TLSReadFailed TLSReadFailed -#define Conn_TLSNoPeerCertificate TLSNoPeerCertificate -#define Conn_TLSPeerCertificateInvalid TLSPeerCertificateInvalid -#define Conn_TLSClosedWhenWriting TLSClosedWhenWriting -#define Conn_TLSHandshakeTimedOut TLSHandshakeTimedOut -#define Conn_Protocol_Timeout Protocol_Timeout -#define Conn_Protocol_ObjTooBig Protocol_ObjTooBig -#define Conn_Protocol_BadCommandRecieved Protocol_BadCommandRecieved -#define Conn_Protocol_UnknownCommandRecieved Protocol_UnknownCommandRecieved -#define Conn_Protocol_TriedToExecuteReplyCommand Protocol_TriedToExecuteReplyCommand -#define Conn_Protocol_UnexpectedReply Protocol_UnexpectedReply -#define Conn_Protocol_HandshakeFailed Protocol_HandshakeFailed -#define Conn_Protocol_StreamWhenObjExpected Protocol_StreamWhenObjExpected -#define Conn_Protocol_ObjWhenStreamExpected Protocol_ObjWhenStreamExpected -#define Conn_Protocol_TimeOutWhenSendingStream Protocol_TimeOutWhenSendingStream - -#endif // SERVEREXCEPTION__H - diff --git a/lib/server/ServerStream.h b/lib/server/ServerStream.h index a9b56169..3f6eed7e 100644 --- a/lib/server/ServerStream.h +++ b/lib/server/ServerStream.h @@ -17,6 +17,7 @@ #include <sys/wait.h> #endif +#include "autogen_ServerException.h" #include "Daemon.h" #include "SocketListen.h" #include "Utils.h" @@ -286,7 +287,7 @@ public: #endif // The derived class does some server magic with the connection - HandleConnection(*connection); + HandleConnection(connection); // Since rChildExit == true, the forked process will call _exit() on return from this fn return; @@ -305,7 +306,7 @@ public: #endif // !WIN32 // Just handle in this process SetProcessTitle("handling"); - HandleConnection(*connection); + HandleConnection(connection); SetProcessTitle("idle"); #ifndef WIN32 } @@ -344,10 +345,20 @@ public: p = ::waitpid(0 /* any child in process group */, &status, WNOHANG); - if(p == -1 && errno != ECHILD && errno != EINTR) + if(p == -1) { - THROW_EXCEPTION(ServerException, - ServerWaitOnChildError) + if (errno == ECHILD || errno == EINTR) + { + // Nothing actually happened, so there's no reason + // to wait again. + break; + } + else + { + THROW_SYS_ERROR("Failed to wait for daemon child " + "process", ServerException, + ServerWaitOnChildError); + } } else if(p == 0) { @@ -377,12 +388,12 @@ public: } #endif - virtual void HandleConnection(StreamType &rStream) + virtual void HandleConnection(std::auto_ptr<StreamType> apStream) { - Connection(rStream); + Connection(apStream); } - virtual void Connection(StreamType &rStream) = 0; + virtual void Connection(std::auto_ptr<StreamType> apStream) = 0; protected: // For checking code in derived classes -- use if you have an algorithm which diff --git a/lib/server/ServerTLS.h b/lib/server/ServerTLS.h index a74a671e..f748f4b2 100644 --- a/lib/server/ServerTLS.h +++ b/lib/server/ServerTLS.h @@ -52,18 +52,19 @@ public: std::string certFile(serverconf.GetKeyValue("CertificateFile")); std::string keyFile(serverconf.GetKeyValue("PrivateKeyFile")); std::string caFile(serverconf.GetKeyValue("TrustedCAsFile")); - mContext.Initialise(true /* as server */, certFile.c_str(), keyFile.c_str(), caFile.c_str()); + mContext.Initialise(true /* as server */, certFile.c_str(), + keyFile.c_str(), caFile.c_str()); // Then do normal stream server stuff ServerStream<SocketStreamTLS, Port, ListenBacklog, ForkToHandleRequests>::Run2(rChildExit); } - virtual void HandleConnection(SocketStreamTLS &rStream) + virtual void HandleConnection(std::auto_ptr<SocketStreamTLS> apStream) { - rStream.Handshake(mContext, true /* is server */); + apStream->Handshake(mContext, true /* is server */); // this-> in next line required to build under some gcc versions - this->Connection(rStream); + this->Connection(apStream); } private: diff --git a/lib/server/Socket.cpp b/lib/server/Socket.cpp index f2a4996b..c9c1773d 100644 --- a/lib/server/Socket.cpp +++ b/lib/server/Socket.cpp @@ -24,8 +24,9 @@ #include <string.h> #include <stdio.h> +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "Socket.h" -#include "ServerException.h" #include "MemLeakFindOn.h" @@ -69,12 +70,12 @@ void Socket::NameLookupToSockAddr(SocketAllAddr &addr, int &sockDomain, } else { - THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError); + THROW_EXCEPTION(ConnectionException, SocketNameLookupError); } } else { - THROW_EXCEPTION(ConnectionException, Conn_SocketNameLookupError); + THROW_EXCEPTION(ConnectionException, SocketNameLookupError); } } break; diff --git a/lib/server/SocketListen.h b/lib/server/SocketListen.h index 39c60ba6..39fe7e24 100644 --- a/lib/server/SocketListen.h +++ b/lib/server/SocketListen.h @@ -29,8 +29,9 @@ #include <memory> #include <string> +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "Socket.h" -#include "ServerException.h" #include "MemLeakFindOn.h" @@ -73,7 +74,8 @@ private: // Created: 2003/07/31 // // -------------------------------------------------------------------------- -template<typename SocketType, int ListenBacklog = 128, typename SocketLockingType = _NoSocketLocking, int MaxMultiListenSockets = 16> +template<typename SocketType, int ListenBacklog = 128, + typename SocketLockingType = _NoSocketLocking, int MaxMultiListenSockets = 16> class SocketListen { public: @@ -112,10 +114,9 @@ public: if(::close(mSocketHandle) == -1) #endif { - BOX_LOG_SOCKET_ERROR(mType, mName, mPort, - "Failed to close network socket"); - THROW_EXCEPTION(ServerException, - SocketCloseError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketCloseError, + BOX_SOCKET_ERROR_MESSAGE(mType, mName, mPort, + "Failed to close network socket")); } } mSocketHandle = -1; @@ -152,9 +153,9 @@ public: 0 /* let OS choose protocol */); if(mSocketHandle == -1) { - BOX_LOG_SOCKET_ERROR(Type, Name, Port, - "Failed to create a network socket"); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, Name, Port, + "Failed to create a network socket")); } // Set an option to allow reuse (useful for -HUP situations!) @@ -167,28 +168,28 @@ public: &option, sizeof(option)) == -1) #endif { - BOX_LOG_SOCKET_ERROR(Type, Name, Port, - "Failed to set socket options"); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, Name, Port, + "Failed to set socket options")); } // Bind it to the right port, and start listening if(::bind(mSocketHandle, &addr.sa_generic, addrLen) == -1 || ::listen(mSocketHandle, ListenBacklog) == -1) { - int err_number = errno; - - BOX_LOG_SOCKET_ERROR(Type, Name, Port, - "Failed to bind socket"); - - // Dispose of the socket - ::close(mSocketHandle); - mSocketHandle = -1; - - THROW_SYS_FILE_ERRNO("Failed to bind or listen " - "on socket", Name, err_number, - ServerException, SocketBindError); - } + try + { + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, Name, Port, + "Failed to bind socket to name/port")); + } + catch(ServerException &e) // finally + { + // Dispose of the socket + Close(); + throw; + } + } } // ------------------------------------------------------------------ @@ -248,10 +249,10 @@ public: } else { - BOX_LOG_SOCKET_ERROR(mType, mName, mPort, - "Failed to poll connection"); - THROW_EXCEPTION(ServerException, - SocketPollError) + THROW_EXCEPTION_MESSAGE(ServerException, + SocketPollError, + BOX_SOCKET_ERROR_MESSAGE(mType, mName, + mPort, "Failed to poll connection")); } break; case 0: // timed out @@ -268,9 +269,9 @@ public: // Got socket (or error), unlock (implicit in destruction) if(sock == -1) { - BOX_LOG_SOCKET_ERROR(mType, mName, mPort, - "Failed to accept connection"); - THROW_EXCEPTION(ServerException, SocketAcceptError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketAcceptError, + BOX_SOCKET_ERROR_MESSAGE(mType, mName, + mPort, "Failed to accept connection")); } // Log it diff --git a/lib/server/SocketStream.cpp b/lib/server/SocketStream.cpp index 6ef4b8d1..edb5e5b8 100644 --- a/lib/server/SocketStream.cpp +++ b/lib/server/SocketStream.cpp @@ -25,10 +25,24 @@ #include <ucred.h> #endif +#ifdef HAVE_BSD_UNISTD_H + #include <bsd/unistd.h> +#endif + +#ifdef HAVE_SYS_PARAM_H + #include <sys/param.h> +#endif + +#ifdef HAVE_SYS_UCRED_H + #include <sys/ucred.h> +#endif + +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "SocketStream.h" -#include "ServerException.h" #include "CommonException.h" #include "Socket.h" +#include "Utils.h" #include "MemLeakFindOn.h" @@ -162,25 +176,31 @@ void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) 0 /* let OS choose protocol */); if(mSocketHandle == INVALID_SOCKET_VALUE) { - BOX_LOG_SOCKET_ERROR(Type, rName, Port, - "Failed to create a network socket"); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, rName, Port, + "Failed to create a network socket")); } // Connect it if(::connect(mSocketHandle, &addr.sa_generic, addrLen) == -1) { // Dispose of the socket - BOX_LOG_SOCKET_ERROR(Type, rName, Port, - "Failed to connect to socket"); + try + { + THROW_EXCEPTION_MESSAGE(ServerException, SocketOpenError, + BOX_SOCKET_ERROR_MESSAGE(Type, rName, Port, + "Failed to connect to socket")); + } + catch(ServerException &e) + { #ifdef WIN32 - ::closesocket(mSocketHandle); + ::closesocket(mSocketHandle); #else // !WIN32 - ::close(mSocketHandle); + ::close(mSocketHandle); #endif // WIN32 - - mSocketHandle = INVALID_SOCKET_VALUE; - THROW_EXCEPTION(ConnectionException, Conn_SocketConnectError) + mSocketHandle = INVALID_SOCKET_VALUE; + throw; + } } ResetCounters(); @@ -199,7 +219,9 @@ void SocketStream::Open(Socket::Type Type, const std::string& rName, int Port) // -------------------------------------------------------------------------- int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) { - if(mSocketHandle == INVALID_SOCKET_VALUE) + CheckForMissingTimeout(Timeout); + + if(mSocketHandle == INVALID_SOCKET_VALUE) { THROW_EXCEPTION(ServerException, BadSocketHandle) } @@ -210,7 +232,7 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) p.fd = mSocketHandle; p.events = POLLIN; p.revents = 0; - switch(::poll(&p, 1, (Timeout == IOStream::TimeOutInfinite)?INFTIM:Timeout)) + switch(::poll(&p, 1, PollTimeout(Timeout, 0))) { case -1: // error @@ -256,7 +278,7 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) // Other error BOX_LOG_SYS_ERROR("Failed to read from socket"); THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError); + SocketReadError); } } @@ -270,6 +292,41 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) return r; } +bool SocketStream::Poll(short Events, int Timeout) +{ + // Wait for data to send. + struct pollfd p; + p.fd = GetSocketHandle(); + p.events = Events; + p.revents = 0; + + box_time_t start = GetCurrentBoxTime(); + int result; + + do + { + result = ::poll(&p, 1, PollTimeout(Timeout, start)); + } + while(result == -1 && errno == EINTR); + + switch(result) + { + case -1: + // error - Bad! + THROW_SYS_ERROR("Failed to poll socket", ServerException, + SocketPollError); + break; + + case 0: + // Condition not met, timed out + return false; + + default: + // good to go! + return true; + } +} + // -------------------------------------------------------------------------- // // Function @@ -278,20 +335,21 @@ int SocketStream::Read(void *pBuffer, int NBytes, int Timeout) // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void SocketStream::Write(const void *pBuffer, int NBytes) +void SocketStream::Write(const void *pBuffer, int NBytes, int Timeout) { - if(mSocketHandle == INVALID_SOCKET_VALUE) + if(mSocketHandle == INVALID_SOCKET_VALUE) { THROW_EXCEPTION(ServerException, BadSocketHandle) } - + // Buffer in byte sized type. ASSERT(sizeof(char) == 1); const char *buffer = (char *)pBuffer; - + // Bytes left to send int bytesLeft = NBytes; - + box_time_t start = GetCurrentBoxTime(); + while(bytesLeft > 0) { // Try to send. @@ -304,41 +362,30 @@ void SocketStream::Write(const void *pBuffer, int NBytes) { // Error. mWriteClosed = true; // assume can't write again - BOX_LOG_SYS_ERROR("Failed to write to socket"); - THROW_EXCEPTION(ConnectionException, - Conn_SocketWriteError); + THROW_SYS_ERROR("Failed to write to socket", + ConnectionException, SocketWriteError); } - + // Knock off bytes sent bytesLeft -= sent; // Move buffer pointer buffer += sent; mBytesWritten += sent; - + // Need to wait until it can send again? if(bytesLeft > 0) { - BOX_TRACE("Waiting to send data on socket " << + BOX_TRACE("Waiting to send data on socket " << mSocketHandle << " (" << bytesLeft << " of " << NBytes << " bytes left)"); - - // Wait for data to send. - struct pollfd p; - p.fd = mSocketHandle; - p.events = POLLOUT; - p.revents = 0; - - if(::poll(&p, 1, 16000 /* 16 seconds */) == -1) + + if(!Poll(POLLOUT, PollTimeout(Timeout, start))) { - // Don't exception if it's just a signal - if(errno != EINTR) - { - BOX_LOG_SYS_ERROR("Failed to poll " - "socket"); - THROW_EXCEPTION(ServerException, - SocketPollError) - } + THROW_EXCEPTION_MESSAGE(ConnectionException, + Protocol_Timeout, "Timed out waiting " + "to send " << bytesLeft << " of " << + NBytes << " bytes"); } } } @@ -354,7 +401,7 @@ void SocketStream::Write(const void *pBuffer, int NBytes) // -------------------------------------------------------------------------- void SocketStream::Close() { - if(mSocketHandle == INVALID_SOCKET_VALUE) + if(mSocketHandle == INVALID_SOCKET_VALUE) { THROW_EXCEPTION(ServerException, BadSocketHandle) } @@ -385,19 +432,19 @@ void SocketStream::Shutdown(bool Read, bool Write) { THROW_EXCEPTION(ServerException, BadSocketHandle) } - + // Do anything? if(!Read && !Write) return; - + int how = SHUT_RDWR; if(Read && !Write) how = SHUT_RD; if(!Read && Write) how = SHUT_WR; - + // Shut it down! if(::shutdown(mSocketHandle, how) == -1) { BOX_LOG_SYS_ERROR("Failed to shutdown socket"); - THROW_EXCEPTION(ConnectionException, Conn_SocketShutdownError) + THROW_EXCEPTION(ConnectionException, SocketShutdownError) } } @@ -478,8 +525,13 @@ bool SocketStream::GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut) if(::getsockopt(mSocketHandle, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == 0) { +#ifdef HAVE_STRUCT_UCRED_UID rUidOut = cred.uid; rGidOut = cred.gid; +#else // HAVE_STRUCT_UCRED_CR_UID + rUidOut = cred.cr_uid; + rGidOut = cred.cr_gid; +#endif return true; } @@ -509,3 +561,11 @@ bool SocketStream::GetPeerCredentials(uid_t &rUidOut, gid_t &rGidOut) return false; } +void SocketStream::CheckForMissingTimeout(int Timeout) +{ + if (Timeout == IOStream::TimeOutInfinite) + { + BOX_WARNING("Network operation started with no timeout!"); + DumpStackBacktrace(); + } +} diff --git a/lib/server/SocketStream.h b/lib/server/SocketStream.h index 2fb5e391..fd57af8f 100644 --- a/lib/server/SocketStream.h +++ b/lib/server/SocketStream.h @@ -10,6 +10,13 @@ #ifndef SOCKETSTREAM__H #define SOCKETSTREAM__H +#include <climits> + +#ifdef HAVE_SYS_POLL_H +# include <sys/poll.h> +#endif + +#include "BoxTime.h" #include "IOStream.h" #include "Socket.h" @@ -41,7 +48,16 @@ public: void Attach(int socket); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); + + // Why not inherited from IOStream? Never mind, we want to enforce + // supplying a timeout for network operations anyway. + virtual void Write(const std::string& rBuffer, int Timeout) + { + IOStream::Write(rBuffer, Timeout); + } + virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); @@ -53,6 +69,42 @@ public: protected: void MarkAsReadClosed() {mReadClosed = true;} void MarkAsWriteClosed() {mWriteClosed = true;} + void CheckForMissingTimeout(int Timeout); + + // Converts a timeout in milliseconds (or IOStream::TimeOutInfinite) + // into one that can be passed to poll() (also in milliseconds), also + // compensating for time elapsed since the wait should have started, + // if known. + int PollTimeout(int timeout, box_time_t start_time) + { + if (timeout == IOStream::TimeOutInfinite) + { + return INFTIM; + } + + if (start_time == 0) + { + return timeout; // no adjustment possible + } + + box_time_t end_time = start_time + MilliSecondsToBoxTime(timeout); + box_time_t now = GetCurrentBoxTime(); + box_time_t remaining = end_time - now; + + if (remaining < 0) + { + return 0; // no delay + } + else if (BoxTimeToMilliSeconds(remaining) > INT_MAX) + { + return INT_MAX; + } + else + { + return (int) BoxTimeToMilliSeconds(remaining); + } + } + bool Poll(short Events, int Timeout); private: tOSSocketHandle mSocketHandle; diff --git a/lib/server/SocketStreamTLS.cpp b/lib/server/SocketStreamTLS.cpp index 576b53a2..e6299bfa 100644 --- a/lib/server/SocketStreamTLS.cpp +++ b/lib/server/SocketStreamTLS.cpp @@ -19,9 +19,10 @@ #include <poll.h> #endif +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "BoxTime.h" #include "CryptoUtils.h" -#include "ServerException.h" #include "SocketStreamTLS.h" #include "SSLLib.h" #include "TLSContext.h" @@ -131,7 +132,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) tOSSocketHandle socket = GetSocketHandle(); BIO_set_fd(mpBIO, socket, BIO_NOCLOSE); - + // Then the SSL object mpSSL = ::SSL_new(rContext.GetRawContext()); if(mpSSL == 0) @@ -154,7 +155,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) THROW_EXCEPTION(ServerException, SocketSetNonBlockingFailed) } #endif - + // FIXME: This is less portable than the above. However, it MAY be needed // for cygwin, which has/had bugs with fcntl // @@ -196,7 +197,7 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) if(WaitWhenRetryRequired(se, TLS_HANDSHAKE_TIMEOUT) == false) { // timed out - THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeTimedOut) + THROW_EXCEPTION(ConnectionException, TLSHandshakeTimedOut) } break; @@ -205,12 +206,12 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) if(IsServer) { CryptoUtils::LogError("accepting connection"); - THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) + THROW_EXCEPTION(ConnectionException, TLSHandshakeFailed) } else { CryptoUtils::LogError("connecting"); - THROW_EXCEPTION(ConnectionException, Conn_TLSHandshakeFailed) + THROW_EXCEPTION(ConnectionException, TLSHandshakeFailed) } } } @@ -222,23 +223,25 @@ void SocketStreamTLS::Handshake(const TLSContext &rContext, bool IsServer) // // Function // Name: WaitWhenRetryRequired(int, int) -// Purpose: Waits until the condition required by the TLS layer is met. -// Returns true if the condition is met, false if timed out. +// Purpose: Waits until the condition required by the TLS layer +// is met. Returns true if the condition is met, false +// if timed out. // Created: 2003/08/15 // // -------------------------------------------------------------------------- bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout) { - struct pollfd p; - p.fd = GetSocketHandle(); + CheckForMissingTimeout(Timeout); + + short events; switch(SSLErrorCode) { case SSL_ERROR_WANT_READ: - p.events = POLLIN; + events = POLLIN; break; case SSL_ERROR_WANT_WRITE: - p.events = POLLOUT; + events = POLLOUT; break; default: @@ -246,45 +249,8 @@ bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout) THROW_EXCEPTION(ServerException, Internal) break; } - p.revents = 0; - - int64_t start, end; - start = BoxTimeToMilliSeconds(GetCurrentBoxTime()); - end = start + Timeout; - int result; - - do - { - int64_t now = BoxTimeToMilliSeconds(GetCurrentBoxTime()); - int poll_timeout = (int)(end - now); - if (poll_timeout < 0) poll_timeout = 0; - if (Timeout == IOStream::TimeOutInfinite) - { - poll_timeout = INFTIM; - } - result = ::poll(&p, 1, poll_timeout); - } - while(result == -1 && errno == EINTR); - - switch(result) - { - case -1: - // error - Bad! - THROW_EXCEPTION(ServerException, SocketPollError) - break; - - case 0: - // Condition not met, timed out - return false; - break; - - default: - // good to go! - return true; - break; - } - return true; + return Poll(events, Timeout); } // -------------------------------------------------------------------------- @@ -297,6 +263,7 @@ bool SocketStreamTLS::WaitWhenRetryRequired(int SSLErrorCode, int Timeout) // -------------------------------------------------------------------------- int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) { + CheckForMissingTimeout(Timeout); if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} // Make sure zero byte reads work as expected @@ -304,7 +271,7 @@ int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) { return 0; } - + while(true) { int r = ::SSL_read(mpSSL, pBuffer, NBytes); @@ -337,7 +304,7 @@ int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) default: CryptoUtils::LogError("reading"); - THROW_EXCEPTION(ConnectionException, Conn_TLSReadFailed) + THROW_EXCEPTION(ConnectionException, TLSReadFailed) break; } } @@ -351,23 +318,23 @@ int SocketStreamTLS::Read(void *pBuffer, int NBytes, int Timeout) // Created: 2003/08/06 // // -------------------------------------------------------------------------- -void SocketStreamTLS::Write(const void *pBuffer, int NBytes) +void SocketStreamTLS::Write(const void *pBuffer, int NBytes, int Timeout) { if(!mpSSL) {THROW_EXCEPTION(ServerException, TLSNoSSLObject)} - + // Make sure zero byte writes work as expected if(NBytes == 0) { return; } - + // from man SSL_write // // SSL_write() will only return with success, when the // complete contents of buf of length num has been written. // // So no worries about partial writes and moving the buffer around - + while(true) { // try the write @@ -385,24 +352,24 @@ void SocketStreamTLS::Write(const void *pBuffer, int NBytes) case SSL_ERROR_ZERO_RETURN: // Connection closed MarkAsWriteClosed(); - THROW_EXCEPTION(ConnectionException, Conn_TLSClosedWhenWriting) + THROW_EXCEPTION(ConnectionException, TLSClosedWhenWriting) break; case SSL_ERROR_WANT_READ: case SSL_ERROR_WANT_WRITE: - // wait for the requried data + // wait for the required data { #ifndef BOX_RELEASE_BUILD - bool conditionmet = + bool conditionmet = #endif - WaitWhenRetryRequired(se, IOStream::TimeOutInfinite); + WaitWhenRetryRequired(se, Timeout); ASSERT(conditionmet); } break; default: CryptoUtils::LogError("writing"); - THROW_EXCEPTION(ConnectionException, Conn_TLSWriteFailed) + THROW_EXCEPTION(ConnectionException, TLSWriteFailed) break; } } @@ -444,7 +411,7 @@ void SocketStreamTLS::Shutdown(bool Read, bool Write) if(::SSL_shutdown(mpSSL) < 0) { CryptoUtils::LogError("shutting down"); - THROW_EXCEPTION(ConnectionException, Conn_TLSShutdownFailed) + THROW_EXCEPTION(ConnectionException, TLSShutdownFailed) } // Don't ask the base class to shutdown -- BIO does this, apparently. @@ -467,15 +434,15 @@ std::string SocketStreamTLS::GetPeerCommonName() if(cert == 0) { ::X509_free(cert); - THROW_EXCEPTION(ConnectionException, Conn_TLSNoPeerCertificate) + THROW_EXCEPTION(ConnectionException, TLSNoPeerCertificate) } - // Subject details - X509_NAME *subject = ::X509_get_subject_name(cert); + // Subject details + X509_NAME *subject = ::X509_get_subject_name(cert); if(subject == 0) { ::X509_free(cert); - THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid) + THROW_EXCEPTION(ConnectionException, TLSPeerCertificateInvalid) } // Common name @@ -483,7 +450,7 @@ std::string SocketStreamTLS::GetPeerCommonName() if(::X509_NAME_get_text_by_NID(subject, NID_commonName, commonName, sizeof(commonName)) <= 0) { ::X509_free(cert); - THROW_EXCEPTION(ConnectionException, Conn_TLSPeerCertificateInvalid) + THROW_EXCEPTION(ConnectionException, TLSPeerCertificateInvalid) } // Terminate just in case commonName[sizeof(commonName)-1] = '\0'; diff --git a/lib/server/SocketStreamTLS.h b/lib/server/SocketStreamTLS.h index bb40ed10..3fda98c1 100644 --- a/lib/server/SocketStreamTLS.h +++ b/lib/server/SocketStreamTLS.h @@ -43,7 +43,8 @@ public: void Handshake(const TLSContext &rContext, bool IsServer = false); virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual void Close(); virtual void Shutdown(bool Read = true, bool Write = true); diff --git a/lib/server/TLSContext.cpp b/lib/server/TLSContext.cpp index 341043e9..1a6d4a53 100644 --- a/lib/server/TLSContext.cpp +++ b/lib/server/TLSContext.cpp @@ -12,8 +12,9 @@ #define TLS_CLASS_IMPLEMENTATION_CPP #include <openssl/ssl.h> +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" #include "CryptoUtils.h" -#include "ServerException.h" #include "SSLLib.h" #include "TLSContext.h" @@ -22,6 +23,17 @@ #define MAX_VERIFICATION_DEPTH 2 #define CIPHER_LIST "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH" +// Macros to allow compatibility with OpenSSL 1.0 and 1.1 APIs. See +// https://github.com/charybdis-ircd/charybdis/blob/release/3.5/libratbox/src/openssl_ratbox.h +// for the gory details. +#if defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER >= 0x10100000L) // OpenSSL >= 1.1 +# define BOX_TLS_SERVER_METHOD TLS_server_method +# define BOX_TLS_CLIENT_METHOD TLS_client_method +#else // OpenSSL < 1.1 +# define BOX_TLS_SERVER_METHOD TLSv1_server_method +# define BOX_TLS_CLIENT_METHOD TLSv1_client_method +#endif + // -------------------------------------------------------------------------- // // Function @@ -66,7 +78,7 @@ void TLSContext::Initialise(bool AsServer, const char *CertificatesFile, const c ::SSL_CTX_free(mpContext); } - mpContext = ::SSL_CTX_new(AsServer?TLSv1_server_method():TLSv1_client_method()); + mpContext = ::SSL_CTX_new(AsServer ? BOX_TLS_SERVER_METHOD() : BOX_TLS_CLIENT_METHOD()); if(mpContext == NULL) { THROW_EXCEPTION(ServerException, TLSAllocationFailed) diff --git a/lib/server/TcpNice.cpp b/lib/server/TcpNice.cpp index 20619e49..79e91eeb 100644 --- a/lib/server/TcpNice.cpp +++ b/lib/server/TcpNice.cpp @@ -146,7 +146,7 @@ void NiceSocketStream::Write(const void *pBuffer, int NBytes) int socket = mapSocket->GetSocketHandle(); int rtt = 50; // WAG -# if HAVE_DECL_SOL_TCP && HAVE_DECL_TCP_INFO && HAVE_STRUCT_TCP_INFO_TCPI_RTT +# if HAVE_DECL_SOL_TCP && defined HAVE_STRUCT_TCP_INFO_TCPI_RTT struct tcp_info info; socklen_t optlen = sizeof(info); if(getsockopt(socket, SOL_TCP, TCP_INFO, &info, &optlen) == -1) @@ -154,7 +154,7 @@ void NiceSocketStream::Write(const void *pBuffer, int NBytes) BOX_LOG_SYS_WARNING("getsockopt(" << socket << ", SOL_TCP, " "TCP_INFO) failed"); } - else if(optlen != sizeof(info)) + else if(optlen < sizeof(info)) { BOX_WARNING("getsockopt(" << socket << ", SOL_TCP, " "TCP_INFO) return structure size " << optlen << ", " @@ -164,7 +164,7 @@ void NiceSocketStream::Write(const void *pBuffer, int NBytes) { rtt = info.tcpi_rtt; } -# endif +# endif // HAVE_DECL_SOL_TCP && defined HAVE_STRUCT_TCP_INFO_TCPI_RTT int newWindow = mTcpNice.GetNextWindowSize(mBytesWrittenThisPeriod, elapsed, rtt); diff --git a/lib/server/TcpNice.h b/lib/server/TcpNice.h index e2027749..4381df42 100644 --- a/lib/server/TcpNice.h +++ b/lib/server/TcpNice.h @@ -83,7 +83,7 @@ private: // // -------------------------------------------------------------------------- -class NiceSocketStream : public IOStream +class NiceSocketStream : public SocketStream { private: std::auto_ptr<SocketStream> mapSocket; @@ -166,6 +166,10 @@ public: } virtual void SetEnabled(bool enabled); + off_t GetBytesRead() const { return mapSocket->GetBytesRead(); } + off_t GetBytesWritten() const { return mapSocket->GetBytesWritten(); } + void ResetCounters() { mapSocket->ResetCounters(); } + private: NiceSocketStream(const NiceSocketStream &rToCopy) { /* do not call */ } diff --git a/lib/server/WinNamedPipeListener.h b/lib/server/WinNamedPipeListener.h index 26e76e3d..956a7b5a 100644 --- a/lib/server/WinNamedPipeListener.h +++ b/lib/server/WinNamedPipeListener.h @@ -11,10 +11,10 @@ #ifndef WINNAMEDPIPELISTENER__H #define WINNAMEDPIPELISTENER__H -#include <OverlappedIO.h> -#include <WinNamedPipeStream.h> - -#include "ServerException.h" +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" +#include "OverlappedIO.h" +#include "WinNamedPipeStream.h" #include "MemLeakFindOn.h" @@ -53,8 +53,8 @@ private: socket.c_str(), // pipe name PIPE_ACCESS_DUPLEX | // read/write access FILE_FLAG_OVERLAPPED, // enabled overlapped I/O - PIPE_TYPE_BYTE | // message type pipe - PIPE_READMODE_BYTE | // message-read mode + PIPE_TYPE_BYTE | + PIPE_READMODE_BYTE | PIPE_WAIT, // blocking mode ListenBacklog + 1, // max. instances 4096, // output buffer size @@ -64,9 +64,9 @@ private: if (handle == INVALID_HANDLE_VALUE) { - BOX_LOG_WIN_ERROR("Failed to create named pipe " << - socket); - THROW_EXCEPTION(ServerException, SocketOpenError) + THROW_WIN_FILE_ERRNO("Failed to create named pipe", + socket, GetLastError(), ServerException, + SocketOpenError); } return handle; diff --git a/lib/server/WinNamedPipeStream.cpp b/lib/server/WinNamedPipeStream.cpp index 1179516e..448a3c9d 100644 --- a/lib/server/WinNamedPipeStream.cpp +++ b/lib/server/WinNamedPipeStream.cpp @@ -19,10 +19,12 @@ #include <errno.h> #include <windows.h> -#include "WinNamedPipeStream.h" -#include "ServerException.h" +#include "autogen_ConnectionException.h" +#include "autogen_ServerException.h" +#include "BoxTime.h" #include "CommonException.h" #include "Socket.h" +#include "WinNamedPipeStream.h" #include "MemLeakFindOn.h" @@ -37,13 +39,14 @@ std::string WinNamedPipeStream::sPipeNamePrefix = "\\\\.\\pipe\\"; // // -------------------------------------------------------------------------- WinNamedPipeStream::WinNamedPipeStream() - : mSocketHandle(INVALID_HANDLE_VALUE), - mReadableEvent(INVALID_HANDLE_VALUE), - mBytesInBuffer(0), - mReadClosed(false), - mWriteClosed(false), - mIsServer(false), - mIsConnected(false) +: mSocketHandle(INVALID_HANDLE_VALUE), + mReadableEvent(INVALID_HANDLE_VALUE), + mBytesInBuffer(0), + mReadClosed(false), + mWriteClosed(false), + mIsServer(false), + mIsConnected(false), + mNeedAnotherRead(false) { } // -------------------------------------------------------------------------- @@ -55,14 +58,21 @@ WinNamedPipeStream::WinNamedPipeStream() // // -------------------------------------------------------------------------- WinNamedPipeStream::WinNamedPipeStream(HANDLE hNamedPipe) - : mSocketHandle(hNamedPipe), - mReadableEvent(INVALID_HANDLE_VALUE), - mBytesInBuffer(0), - mReadClosed(false), - mWriteClosed(false), - mIsServer(true), - mIsConnected(true) +: mSocketHandle(hNamedPipe), + mReadableEvent(INVALID_HANDLE_VALUE), + mBytesInBuffer(0), + mReadClosed(false), + mWriteClosed(false), + mIsServer(true), + mIsConnected(true), + mNeedAnotherRead(false) { + StartFirstRead(); +} + +// Start the first overlapped read +void WinNamedPipeStream::StartFirstRead() +{ // create the Readable event mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); @@ -74,23 +84,50 @@ WinNamedPipeStream::WinNamedPipeStream(HANDLE hNamedPipe) THROW_EXCEPTION(CommonException, Internal) } - // initialise the OVERLAPPED structure + StartOverlappedRead(); +} + +void WinNamedPipeStream::StartOverlappedRead() +{ + // We should only do this when the buffer is empty. We don't want + // to start an overlapped read anywhere else than the start of the + // buffer, because it could complete at any time and we don't want + // to mess about with interrupting the read already in progress. + ASSERT(mBytesInBuffer == 0); + + // Initialise the OVERLAPPED structure memset(&mReadOverlap, 0, sizeof(mReadOverlap)); mReadOverlap.hEvent = mReadableEvent; - // start the first overlapped read if (!ReadFile(mSocketHandle, mReadBuffer, sizeof(mReadBuffer), NULL, &mReadOverlap)) { DWORD err = GetLastError(); - - if (err != ERROR_IO_PENDING) + if (err == ERROR_IO_PENDING) + { + // Don't reset yet, there might be data + // in the buffer waiting to be read, + // will check below. + // ResetEvent(mReadableEvent); + } + else if (err == ERROR_HANDLE_EOF) + { + BOX_INFO("Control client disconnected"); + mReadClosed = true; + } + else if (err == ERROR_BROKEN_PIPE || + err == ERROR_PIPE_NOT_CONNECTED) + { + BOX_NOTICE("Control client disconnected"); + mReadClosed = true; + mIsConnected = false; + } + else { - BOX_ERROR("Failed to start overlapped read: " << - GetErrorMessage(err)); Close(); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) + THROW_WIN_ERROR_NUMBER("Failed to start overlapped " + "read", err, ConnectionException, + SocketReadError) } } } @@ -105,6 +142,12 @@ WinNamedPipeStream::WinNamedPipeStream(HANDLE hNamedPipe) // -------------------------------------------------------------------------- WinNamedPipeStream::~WinNamedPipeStream() { + for(std::list<WriteInProgress*>::iterator i = mWritesInProgress.begin(); + i != mWritesInProgress.end(); i++) + { + delete *i; + } + if (mSocketHandle != INVALID_HANDLE_VALUE) { try @@ -157,36 +200,7 @@ void WinNamedPipeStream::Accept() mIsServer = true; // must flush and disconnect before closing mIsConnected = true; - // create the Readable event - mReadableEvent = CreateEvent(NULL, TRUE, FALSE, NULL); - - if (mReadableEvent == INVALID_HANDLE_VALUE) - { - BOX_ERROR("Failed to create the Readable event: " << - GetErrorMessage(GetLastError())); - Close(); - THROW_EXCEPTION(CommonException, Internal) - } - - // initialise the OVERLAPPED structure - memset(&mReadOverlap, 0, sizeof(mReadOverlap)); - mReadOverlap.hEvent = mReadableEvent; - - // start the first overlapped read - if (!ReadFile(mSocketHandle, mReadBuffer, sizeof(mReadBuffer), - NULL, &mReadOverlap)) - { - DWORD err = GetLastError(); - - if (err != ERROR_IO_PENDING) - { - BOX_ERROR("Failed to start overlapped read: " << - GetErrorMessage(err)); - Close(); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) - } - } + StartFirstRead(); } */ @@ -214,7 +228,7 @@ void WinNamedPipeStream::Connect(const std::string& rName) 0, // no sharing NULL, // default security attributes OPEN_EXISTING, - 0, // default attributes + 0, // FILE_FLAG_OVERLAPPED, // dwFlagsAndAttributes NULL); // no template file if (mSocketHandle == INVALID_HANDLE_VALUE) @@ -237,6 +251,86 @@ void WinNamedPipeStream::Connect(const std::string& rName) mWriteClosed = false; mIsServer = false; // just close the socket mIsConnected = true; + + StartFirstRead(); +} + +// Returns true if the operation is complete (and you will need to start +// another one), or false otherwise (you can wait again). +bool WinNamedPipeStream::WaitForOverlappedOperation(OVERLAPPED& Overlapped, + int Timeout, int64_t* pBytesTransferred) +{ + if (Timeout == IOStream::TimeOutInfinite) + { + Timeout = INFINITE; + } + + // overlapped I/O completed successfully? (wait if needed) + DWORD waitResult = WaitForSingleObject(Overlapped.hEvent, Timeout); + DWORD NumBytesTransferred = -1; + + if (waitResult == WAIT_FAILED) + { + THROW_WIN_ERROR_NUMBER("Failed to wait for overlapped I/O", + GetLastError(), ServerException, Internal); + } + + if (waitResult == WAIT_ABANDONED) + { + THROW_EXCEPTION_MESSAGE(ServerException, Internal, + "Wait for overlapped I/O abandoned by system"); + } + + if (waitResult == WAIT_TIMEOUT) + { + // wait timed out, nothing to read + *pBytesTransferred = 0; + return false; + } + + if (waitResult != WAIT_OBJECT_0) + { + THROW_EXCEPTION_MESSAGE(ServerException, BadSocketHandle, + "Failed to wait for overlapped I/O: unknown " + "result code: " << waitResult); + } + + // Overlapped operation completed successfully. Return the number + // of bytes transferred. + if (GetOverlappedResult(mSocketHandle, &Overlapped, + &NumBytesTransferred, TRUE)) + { + *pBytesTransferred = NumBytesTransferred; + return true; + } + + // We are here because GetOverlappedResult() informed us that the + // overlapped operation encountered an error, so what was it? + DWORD err = GetLastError(); + + if (err == ERROR_HANDLE_EOF) + { + Close(); + *pBytesTransferred = 0; + return true; + } + + // ERROR_NO_DATA is a strange name for + // "The pipe is being closed". No exception wanted. + + if (err == ERROR_NO_DATA || + err == ERROR_PIPE_NOT_CONNECTED || + err == ERROR_BROKEN_PIPE) + { + BOX_INFO(BOX_WIN_ERRNO_MESSAGE(err, + "Named pipe peer disconnected")); + Close(); + *pBytesTransferred = 0; + return true; + } + + THROW_WIN_ERROR_NUMBER("Failed to wait for overlapped I/O " + "to complete", err, ConnectionException, SocketReadError); } // -------------------------------------------------------------------------- @@ -249,192 +343,61 @@ void WinNamedPipeStream::Connect(const std::string& rName) // -------------------------------------------------------------------------- int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) { - // TODO no support for timeouts yet - if (!mIsServer && Timeout != IOStream::TimeOutInfinite) - { - THROW_EXCEPTION(CommonException, AssertFailed) - } - if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) { - THROW_EXCEPTION(ServerException, BadSocketHandle) + THROW_EXCEPTION_MESSAGE(ServerException, BadSocketHandle, + "Tried to read from closed pipe"); } if (mReadClosed) { - THROW_EXCEPTION(ConnectionException, SocketShutdownError) + THROW_EXCEPTION_MESSAGE(ConnectionException, + SocketShutdownError, "Tried to read from closing pipe"); } // ensure safe to cast NBytes to unsigned if (NBytes < 0) { - THROW_EXCEPTION(CommonException, AssertFailed) + THROW_EXCEPTION(CommonException, AssertFailed); } - DWORD NumBytesRead; + int64_t NumBytesRead; - if (mIsServer) + // Satisfy from buffer if possible, to avoid blocking on read. + if (mBytesInBuffer == 0) { - // satisfy from buffer if possible, to avoid - // blocking on read. - bool needAnotherRead = false; - if (mBytesInBuffer == 0) - { - // overlapped I/O completed successfully? - // (wait if needed) - DWORD waitResult = WaitForSingleObject( - mReadOverlap.hEvent, Timeout); - - if (waitResult == WAIT_ABANDONED) - { - BOX_ERROR("Wait for command socket read " - "abandoned by system"); - THROW_EXCEPTION(ServerException, - BadSocketHandle); - } - else if (waitResult == WAIT_TIMEOUT) - { - // wait timed out, nothing to read - NumBytesRead = 0; - } - else if (waitResult != WAIT_OBJECT_0) - { - BOX_ERROR("Failed to wait for command " - "socket read: unknown result " << - waitResult); - } - // object is ready to read from - else if (GetOverlappedResult(mSocketHandle, - &mReadOverlap, &NumBytesRead, TRUE)) - { - needAnotherRead = true; - } - else - { - DWORD err = GetLastError(); - - if (err == ERROR_HANDLE_EOF) - { - mReadClosed = true; - } - else - { - if (err == ERROR_BROKEN_PIPE) - { - BOX_NOTICE("Control client " - "disconnected"); - } - else - { - BOX_ERROR("Failed to wait for " - "ReadFile to complete: " - << GetErrorMessage(err)); - } - - Close(); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) - } - } - } - else - { - NumBytesRead = 0; - } - - size_t BytesToCopy = NumBytesRead + mBytesInBuffer; - size_t BytesRemaining = 0; - - if (BytesToCopy > (size_t)NBytes) - { - BytesRemaining = BytesToCopy - NBytes; - BytesToCopy = NBytes; - } - - memcpy(pBuffer, mReadBuffer, BytesToCopy); - memmove(mReadBuffer, mReadBuffer + BytesToCopy, BytesRemaining); - - mBytesInBuffer = BytesRemaining; - NumBytesRead = BytesToCopy; - - if (needAnotherRead) + if (mNeedAnotherRead) { - // reinitialise the OVERLAPPED structure - memset(&mReadOverlap, 0, sizeof(mReadOverlap)); - mReadOverlap.hEvent = mReadableEvent; + // Start the next overlapped read + StartOverlappedRead(); } - // start the next overlapped read - if (needAnotherRead && !ReadFile(mSocketHandle, - mReadBuffer + mBytesInBuffer, - sizeof(mReadBuffer) - mBytesInBuffer, - NULL, &mReadOverlap)) - { - DWORD err = GetLastError(); - if (err == ERROR_IO_PENDING) - { - // Don't reset yet, there might be data - // in the buffer waiting to be read, - // will check below. - // ResetEvent(mReadableEvent); - } - else if (err == ERROR_HANDLE_EOF) - { - mReadClosed = true; - } - else if (err == ERROR_BROKEN_PIPE) - { - BOX_ERROR("Control client disconnected"); - mReadClosed = true; - } - else - { - BOX_ERROR("Failed to start overlapped read: " - << GetErrorMessage(err)); - Close(); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) - } - } + mNeedAnotherRead = WaitForOverlappedOperation(mReadOverlap, + Timeout, &NumBytesRead); } else { - if (!ReadFile( - mSocketHandle, // pipe handle - pBuffer, // buffer to receive reply - NBytes, // size of buffer - &NumBytesRead, // number of bytes read - NULL)) // not overlapped - { - DWORD err = GetLastError(); - - Close(); + // Just return the existing data from the buffer + // this time around. The caller should call again, + // and then the buffer will be empty. + NumBytesRead = 0; + } - // ERROR_NO_DATA is a strange name for - // "The pipe is being closed". No exception wanted. - - if (err == ERROR_NO_DATA || - err == ERROR_PIPE_NOT_CONNECTED) - { - NumBytesRead = 0; - } - else - { - BOX_ERROR("Failed to read from control socket: " - << GetErrorMessage(err)); - THROW_EXCEPTION(ConnectionException, - Conn_SocketReadError) - } - } - - // Closed for reading at EOF? - if (NumBytesRead == 0) - { - mReadClosed = true; - } + int BytesToCopy = NumBytesRead + mBytesInBuffer; + + if (NBytes < BytesToCopy) + { + BytesToCopy = NBytes; } - - return NumBytesRead; + + memcpy(pBuffer, mReadBuffer, BytesToCopy); + + size_t BytesRemaining = mBytesInBuffer + NumBytesRead - BytesToCopy; + ASSERT(BytesToCopy + BytesRemaining <= sizeof(mReadBuffer)); + memmove(mReadBuffer, mReadBuffer + BytesToCopy, BytesRemaining); + mBytesInBuffer = BytesRemaining; + + return BytesToCopy; } // -------------------------------------------------------------------------- @@ -445,8 +408,15 @@ int WinNamedPipeStream::Read(void *pBuffer, int NBytes, int Timeout) // Created: 2003/07/31 // // -------------------------------------------------------------------------- -void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) +void WinNamedPipeStream::Write(const void *pBuffer, int NBytes, int Timeout) { + // Calculate the deadline at the beginning. Not valid if Timeout is + // IOStream::TimeOutInfinite! + ASSERT(Timeout != IOStream::TimeOutInfinite); + + box_time_t deadline = GetCurrentBoxTime() + + MilliSecondsToBoxTime(Timeout); + if (mSocketHandle == INVALID_HANDLE_VALUE || !mIsConnected) { THROW_EXCEPTION(ServerException, BadSocketHandle) @@ -454,41 +424,62 @@ void WinNamedPipeStream::Write(const void *pBuffer, int NBytes) // Buffer in byte sized type. ASSERT(sizeof(char) == 1); - const char *pByteBuffer = (char *)pBuffer; - - int NumBytesWrittenTotal = 0; - - while (NumBytesWrittenTotal < NBytes) + WriteInProgress* new_write = new WriteInProgress( + std::string((char *)pBuffer, NBytes)); + + // Start the WriteFile operation, and add to queue if pending. + BOOL Success = WriteFile( + mSocketHandle, // pipe handle + new_write->mBuffer.c_str(), // message + NBytes, // message length + NULL, // bytes written this time + &(new_write->mOverlap)); + + if (Success == TRUE) { - DWORD NumBytesWrittenThisTime = 0; - - bool Success = WriteFile( - mSocketHandle, // pipe handle - pByteBuffer + NumBytesWrittenTotal, // message - NBytes - NumBytesWrittenTotal, // message length - &NumBytesWrittenThisTime, // bytes written this time - NULL); // not overlapped + // Unfortunately this does happen. We should still call + // GetOverlappedResult() to get the number of bytes written, + // so we can treat it just the same. + // BOX_NOTICE("Write claimed success while overlapped?"); + mWritesInProgress.push_back(new_write); + } + else + { + DWORD err = GetLastError(); - if (!Success) + if (err == ERROR_IO_PENDING) { - // ERROR_NO_DATA is a strange name for - // "The pipe is being closed". - - DWORD err = GetLastError(); - - if (err != ERROR_NO_DATA) - { - BOX_ERROR("Failed to write to control " - "socket: " << GetErrorMessage(err)); - } - + BOX_TRACE("WriteFile is pending, adding to queue"); + mWritesInProgress.push_back(new_write); + } + else + { + // Not in progress any more, pop it Close(); - - THROW_EXCEPTION(ConnectionException, - Conn_SocketWriteError) + THROW_WIN_ERROR_NUMBER("Failed to start overlapped " + "write", err, ConnectionException, + SocketWriteError); } + } + + // Wait for previous WriteFile operations to complete, one at a time, + // until the deadline expires or the pipe becomes disconnected. + for(box_time_t remaining = deadline - GetCurrentBoxTime(); + remaining > 0 && !mWritesInProgress.empty() && mIsConnected; + remaining = deadline - GetCurrentBoxTime()) + { + int new_timeout = BoxTimeToMilliSeconds(remaining); + WriteInProgress* oldest_write = + *(mWritesInProgress.begin()); - NumBytesWrittenTotal += NumBytesWrittenThisTime; + int64_t bytes_written = 0; + if(WaitForOverlappedOperation(oldest_write->mOverlap, + new_timeout, &bytes_written)) + { + // This one is complete, pop it and start a new one + delete oldest_write; + mWritesInProgress.pop_front(); + } } } @@ -513,59 +504,47 @@ void WinNamedPipeStream::Close() THROW_EXCEPTION(ServerException, BadSocketHandle) } - if (mIsServer) + if (!CancelIo(mSocketHandle)) { - if (!CancelIo(mSocketHandle)) - { - BOX_ERROR("Failed to cancel outstanding I/O: " << - GetErrorMessage(GetLastError())); - } + BOX_LOG_WIN_ERROR("Failed to cancel outstanding I/O"); + } - if (mReadableEvent == INVALID_HANDLE_VALUE) - { - BOX_ERROR("Failed to destroy Readable event: " - "invalid handle"); - } - else if (!CloseHandle(mReadableEvent)) - { - BOX_ERROR("Failed to destroy Readable event: " << - GetErrorMessage(GetLastError())); - } + if (mReadableEvent == INVALID_HANDLE_VALUE) + { + BOX_ERROR("Failed to destroy Readable event: " + "invalid handle"); + } + else if (!CloseHandle(mReadableEvent)) + { + BOX_LOG_WIN_ERROR("Failed to destroy Readable event"); + } - mReadableEvent = INVALID_HANDLE_VALUE; + mReadableEvent = INVALID_HANDLE_VALUE; - if (!FlushFileBuffers(mSocketHandle)) - { - BOX_ERROR("Failed to FlushFileBuffers: " << - GetErrorMessage(GetLastError())); - } - - if (!DisconnectNamedPipe(mSocketHandle)) + if (mIsConnected && !FlushFileBuffers(mSocketHandle)) + { + BOX_LOG_WIN_ERROR("Failed to FlushFileBuffers"); + } + + if (mIsServer && mIsConnected && !DisconnectNamedPipe(mSocketHandle)) + { + DWORD err = GetLastError(); + if (err != ERROR_PIPE_NOT_CONNECTED) { - DWORD err = GetLastError(); - if (err != ERROR_PIPE_NOT_CONNECTED) - { - BOX_ERROR("Failed to DisconnectNamedPipe: " << - GetErrorMessage(err)); - } + BOX_LOG_WIN_ERROR("Failed to DisconnectNamedPipe"); } - - mIsServer = false; } - bool result = CloseHandle(mSocketHandle); + if (!CloseHandle(mSocketHandle)) + { + THROW_WIN_ERROR_NUMBER("Failed to CloseHandle", + GetLastError(), ServerException, SocketCloseError); + } mSocketHandle = INVALID_HANDLE_VALUE; mIsConnected = false; mReadClosed = true; mWriteClosed = true; - - if (!result) - { - BOX_ERROR("Failed to CloseHandle: " << - GetErrorMessage(GetLastError())); - THROW_EXCEPTION(ServerException, SocketCloseError) - } } // -------------------------------------------------------------------------- diff --git a/lib/server/WinNamedPipeStream.h b/lib/server/WinNamedPipeStream.h index 386ff7e3..5473c690 100644 --- a/lib/server/WinNamedPipeStream.h +++ b/lib/server/WinNamedPipeStream.h @@ -10,6 +10,8 @@ #if ! defined WINNAMEDPIPESTREAM__H && defined WIN32 #define WINNAMEDPIPESTREAM__H +#include <list> + #include "IOStream.h" // -------------------------------------------------------------------------- @@ -36,15 +38,27 @@ public: // both sides virtual int Read(void *pBuffer, int NBytes, int Timeout = IOStream::TimeOutInfinite); - virtual void Write(const void *pBuffer, int NBytes); + virtual void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite); virtual void WriteAllBuffered(); virtual void Close(); virtual bool StreamDataLeft(); virtual bool StreamClosed(); + // Why not inherited from IOStream? Never mind, we want to enforce + // supplying a timeout for network operations anyway. + virtual void Write(const std::string& rBuffer, int Timeout) + { + IOStream::Write(rBuffer, Timeout); + } + protected: void MarkAsReadClosed() {mReadClosed = true;} void MarkAsWriteClosed() {mWriteClosed = true;} + bool WaitForOverlappedOperation(OVERLAPPED& Overlapped, + int Timeout, int64_t* pBytesTransferred); + void StartFirstRead(); + void StartOverlappedRead(); private: WinNamedPipeStream(const WinNamedPipeStream &rToCopy) @@ -59,6 +73,37 @@ private: bool mWriteClosed; bool mIsServer; bool mIsConnected; + bool mNeedAnotherRead; + + class WriteInProgress { + private: + friend class WinNamedPipeStream; + std::string mBuffer; + OVERLAPPED mOverlap; + WriteInProgress(const WriteInProgress& other); // do not call + public: + WriteInProgress(const std::string& dataToWrite) + : mBuffer(dataToWrite) + { + // create the Writable event + HANDLE writable_event = CreateEvent(NULL, TRUE, FALSE, + NULL); + if (writable_event == INVALID_HANDLE_VALUE) + { + BOX_LOG_WIN_ERROR("Failed to create the " + "Writable event"); + THROW_EXCEPTION(CommonException, Internal) + } + + memset(&mOverlap, 0, sizeof(mOverlap)); + mOverlap.hEvent = writable_event; + } + ~WriteInProgress() + { + CloseHandle(mOverlap.hEvent); + } + }; + std::list<WriteInProgress*> mWritesInProgress; public: static std::string sPipeNamePrefix; diff --git a/lib/server/makeprotocol.pl.in b/lib/server/makeprotocol.pl.in index a074b435..d6c0e216 100755 --- a/lib/server/makeprotocol.pl.in +++ b/lib/server/makeprotocol.pl.in @@ -78,7 +78,7 @@ sub add_type my ($protocol_name, $cpp_name, $header_file) = split /\s+/,$_[0]; $translate_type_info{$protocol_name} = [0, $cpp_name]; - push @extra_header_files, $header_file; + push @extra_header_files, $header_file if $header_file; } # check attributes @@ -158,7 +158,10 @@ print CPP <<__E; #include <sstream> #include "$filename_base.h" -#include "IOStream.h" +#include "CollectInBufferStream.h" +#include "MemBlockStream.h" +#include "SelfFlushingStream.h" +#include "SocketStream.h" __E print H <<__E; @@ -174,12 +177,10 @@ print H <<__E; #include <syslog.h> #endif +#include "autogen_ConnectionException.h" #include "Protocol.h" #include "Message.h" -#include "ServerException.h" - -class IOStream; - +#include "SocketStream.h" __E @@ -210,19 +211,26 @@ __E my $request_base_class = "${protocol_name}ProtocolRequest"; my $reply_base_class = "${protocol_name}ProtocolReply"; # the abstract protocol interface -my $protocol_base_class = $protocol_name."ProtocolBase"; +my $custom_protocol_subclass = $protocol_name."Protocol"; +my $client_server_base_class = $protocol_name."ProtocolClientServer"; my $replyable_base_class = $protocol_name."ProtocolReplyable"; +my $callable_base_class = $protocol_name."ProtocolCallable"; +my $send_receive_class = $protocol_name."ProtocolSendReceive"; print H <<__E; -class $protocol_base_class; +class $custom_protocol_subclass; +class $client_server_base_class; +class $callable_base_class; class $replyable_base_class; -class $reply_base_class; class $message_base_class : public Message { public: virtual std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, $context_class &rContext) const; + virtual std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, + $context_class &rContext, IOStream& rDataStream) const; + virtual bool HasStreamWithCommand() const = 0; }; class $reply_base_class @@ -233,17 +241,44 @@ class $request_base_class { }; +class $send_receive_class { +public: + virtual void Send(const $message_base_class &rObject) = 0; + virtual std::auto_ptr<$message_base_class> Receive() = 0; +}; + +class $custom_protocol_subclass : public Protocol +{ +public: + $custom_protocol_subclass(std::auto_ptr<SocketStream> apConn) + : Protocol(apConn) + { } + virtual ~$custom_protocol_subclass() { } + virtual std::auto_ptr<Message> MakeMessage(int ObjType); + virtual const char *GetProtocolIdentString(); + +private: + $custom_protocol_subclass(const $custom_protocol_subclass &rToCopy); +}; + __E print CPP <<__E; std::auto_ptr<$message_base_class> $message_base_class\::DoCommand($replyable_base_class &rProtocol, $context_class &rContext) const { - THROW_EXCEPTION(ConnectionException, Conn_Protocol_TriedToExecuteReplyCommand) + THROW_EXCEPTION(ConnectionException, Protocol_TriedToExecuteReplyCommand) +} + +std::auto_ptr<$message_base_class> $message_base_class\::DoCommand($replyable_base_class &rProtocol, + $context_class &rContext, IOStream& rDataStream) const +{ + THROW_EXCEPTION(ConnectionException, Protocol_TriedToExecuteReplyCommand) } __E -my %cmd_class; +my %cmd_classes; +my $error_message = undef; # output the classes foreach my $cmd (@cmd_list) @@ -262,7 +297,7 @@ foreach my $cmd (@cmd_list) my $cmd_base_class = join(", ", map {"public $_"} @cmd_base_classes); my $cmd_class = $protocol_name."Protocol".$cmd; - $cmd_class{$cmd} = $cmd_class; + $cmd_classes{$cmd} = $cmd_class; print H <<__E; class $cmd_class : $cmd_base_class @@ -294,18 +329,50 @@ __E if(obj_is_type($cmd,'IsError')) { - print H "\tbool IsError(int &rTypeOut, int &rSubTypeOut) const;\n"; - print H "\tstd::string GetMessage() const;\n"; + $error_message = $cmd; + my ($mem_type,$mem_subtype) = split /,/,obj_get_type_params($cmd,'IsError'); + my $error_type = $cmd_constants{"ErrorType"}; + print H <<__E; + $cmd_class(int SubType) : m$mem_type($error_type), m$mem_subtype(SubType) { } + bool IsError(int &rTypeOut, int &rSubTypeOut) const; + std::string GetMessage() const { return GetMessage(m$mem_subtype); }; + static std::string GetMessage(int subtype); +__E } - if(obj_is_type($cmd, 'Command')) + my $has_stream = obj_is_type($cmd, 'StreamWithCommand'); + + if(obj_is_type($cmd, 'Command') && $has_stream) + { + print H <<__E; + std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, + $context_class &rContext, IOStream& rDataStream) const; // IMPLEMENT THIS\n + std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, + $context_class &rContext) const + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "This command requires a stream parameter"); + } +__E + } + elsif(obj_is_type($cmd, 'Command') && !$has_stream) { print H <<__E; std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, $context_class &rContext) const; // IMPLEMENT THIS\n + std::auto_ptr<$message_base_class> DoCommand($replyable_base_class &rProtocol, + $context_class &rContext, IOStream& rDataStream) const + { + THROW_EXCEPTION_MESSAGE(CommonException, NotSupported, + "This command requires no stream parameter"); + } __E } + print H <<__E; + bool HasStreamWithCommand() const { return $has_stream; } +__E + # want to be able to read from streams? print H "\tvoid SetPropertiesFromStreamData(Protocol &rProtocol);\n"; @@ -442,9 +509,9 @@ bool $cmd_class\::IsError(int &rTypeOut, int &rSubTypeOut) const rSubTypeOut = m$mem_subtype; return true; } -std::string $cmd_class\::GetMessage() const +std::string $cmd_class\::GetMessage(int subtype) { - switch(m$mem_subtype) + switch(subtype) { __E foreach my $const (@{$cmd_constants{$cmd}}) @@ -459,7 +526,7 @@ __E print CPP <<__E; default: std::ostringstream out; - out << "Unknown subtype " << m$mem_subtype; + out << "Unknown subtype " << subtype; return out.str(); } } @@ -505,47 +572,44 @@ my $error_class = $protocol_name."ProtocolError"; # the abstract protocol interface print H <<__E; -class $protocol_base_class + +class $client_server_base_class { public: - $protocol_base_class(); - virtual ~$protocol_base_class(); - virtual const char *GetIdentString(); + $client_server_base_class(); + virtual ~$client_server_base_class(); + virtual std::auto_ptr<IOStream> ReceiveStream() = 0; bool GetLastError(int &rTypeOut, int &rSubTypeOut); + int GetLastErrorType() { return mLastErrorSubType; } protected: - void CheckReply(const std::string& requestCommand, - const $message_base_class &rReply, int expectedType); void SetLastError(int Type, int SubType) { mLastErrorType = Type; mLastErrorSubType = SubType; } + std::string mPreviousCommand; + std::string mPreviousReply; private: - $protocol_base_class(const $protocol_base_class &rToCopy); /* do not call */ + $client_server_base_class(const $client_server_base_class &rToCopy); /* do not call */ int mLastErrorType; int mLastErrorSubType; }; -class $replyable_base_class : public virtual $protocol_base_class +class $replyable_base_class : public virtual $client_server_base_class { public: - $replyable_base_class(); + $replyable_base_class() { } virtual ~$replyable_base_class(); - /* - virtual std::auto_ptr<$message_base_class> Receive() = 0; - virtual void Send(const ${message_base_class} &rObject) = 0; - */ - - virtual std::auto_ptr<IOStream> ReceiveStream() = 0; virtual int GetTimeout() = 0; void SendStreamAfterCommand(std::auto_ptr<IOStream> apStream); - + protected: std::list<IOStream*> mStreamsToSend; void DeleteStreamsToSend(); + virtual std::auto_ptr<$message_base_class> HandleException(BoxException& e) const; private: $replyable_base_class(const $replyable_base_class &rToCopy); /* do not call */ @@ -554,24 +618,47 @@ private: __E print CPP <<__E; -$protocol_base_class\::$protocol_base_class() +$client_server_base_class\::$client_server_base_class() : mLastErrorType(Protocol::NoError), mLastErrorSubType(Protocol::NoError) { } -$protocol_base_class\::~$protocol_base_class() +$client_server_base_class\::~$client_server_base_class() { } -const char *$protocol_base_class\::GetIdentString() +const char *$custom_protocol_subclass\::GetProtocolIdentString() { return "$ident_string"; } -$replyable_base_class\::$replyable_base_class() -{ } +std::auto_ptr<Message> $custom_protocol_subclass\::MakeMessage(int ObjType) +{ + switch(ObjType) + { +__E + +# do objects within this +for my $cmd (@cmd_list) +{ + print CPP <<__E; + case $cmd_id{$cmd}: + return std::auto_ptr<Message>(new $cmd_classes{$cmd}()); + break; +__E +} + +print CPP <<__E; + default: + THROW_EXCEPTION(ConnectionException, Protocol_UnknownCommandRecieved) + } +} $replyable_base_class\::~$replyable_base_class() -{ } +{ + // If there were any streams left over, there's no longer any way to + // access them, and we're responsible for them, so we'd better delete them. + DeleteStreamsToSend(); +} void $replyable_base_class\::SendStreamAfterCommand(std::auto_ptr<IOStream> apStream) { @@ -589,12 +676,14 @@ void $replyable_base_class\::DeleteStreamsToSend() mStreamsToSend.clear(); } -void $protocol_base_class\::CheckReply(const std::string& requestCommand, - const $message_base_class &rReply, int expectedType) +void $callable_base_class\::CheckReply(const std::string& requestCommandName, + const $message_base_class &rCommand, const $message_base_class &rReply, + int expectedType) { if(rReply.GetType() == expectedType) { // Correct response, do nothing + SetLastError(Protocol::NoError, Protocol::NoError); } else { @@ -605,8 +694,8 @@ void $protocol_base_class\::CheckReply(const std::string& requestCommand, { SetLastError(type, subType); THROW_EXCEPTION_MESSAGE(ConnectionException, - Conn_Protocol_UnexpectedReply, - requestCommand << " command failed: " + Protocol_UnexpectedReply, + requestCommandName << " command failed: " "received error " << (($error_class&)rReply).GetMessage()); } @@ -614,12 +703,18 @@ void $protocol_base_class\::CheckReply(const std::string& requestCommand, { SetLastError(Protocol::UnknownError, Protocol::UnknownError); THROW_EXCEPTION_MESSAGE(ConnectionException, - Conn_Protocol_UnexpectedReply, - requestCommand << " command failed: " + Protocol_UnexpectedReply, + requestCommandName << " command failed: " "received unexpected response type " << rReply.GetType()); } } + + // As a client, if we get an unexpected reply later, we'll want to know + // the last command that we executed, and the reply, to help debug the + // server. + mPreviousCommand = rCommand.ToString(); + mPreviousReply = rReply.ToString(); } // -------------------------------------------------------------------------- @@ -630,7 +725,7 @@ void $protocol_base_class\::CheckReply(const std::string& requestCommand, // Created: 2003/08/19 // // -------------------------------------------------------------------------- -bool $protocol_base_class\::GetLastError(int &rTypeOut, int &rSubTypeOut) +bool $client_server_base_class\::GetLastError(int &rTypeOut, int &rSubTypeOut) { if(mLastErrorType == Protocol::NoError) { @@ -653,13 +748,19 @@ __E # the callable protocol interface (implemented by Client and Local classes) # with Query methods that don't take a context parameter -my $callable_base_class = $protocol_name."ProtocolCallable"; print H <<__E; -class $callable_base_class : public virtual $protocol_base_class +class $callable_base_class : public virtual $client_server_base_class, + public $send_receive_class { public: - virtual std::auto_ptr<IOStream> ReceiveStream() = 0; virtual int GetTimeout() = 0; + +protected: + void CheckReply(const std::string& requestCommandName, + const $message_base_class &rCommand, + const $message_base_class &rReply, int expectedType); + +public: __E # add plain object taking query functions @@ -671,8 +772,8 @@ for my $cmd (@cmd_list) my $has_stream = obj_is_type($cmd,'StreamWithCommand'); my $argextra = $has_stream?', std::auto_ptr<IOStream> apStream':''; my $queryextra = $has_stream?', apStream':''; - my $request_class = $cmd_class{$cmd}; - my $reply_class = $cmd_class{obj_get_type_params($cmd,'Command')}; + my $request_class = $cmd_classes{$cmd}; + my $reply_class = $cmd_classes{obj_get_type_params($cmd,'Command')}; print H "\tvirtual std::auto_ptr<$reply_class> Query(const $request_class &rQuery$argextra) = 0;\n"; my @a; @@ -720,13 +821,15 @@ foreach my $type ('Client', 'Server', 'Local') { push @base_classes, $replyable_base_class; } + if (not $writing_server) { push @base_classes, $callable_base_class; } + if (not $writing_local) { - push @base_classes, "Protocol"; + push @base_classes, $custom_protocol_subclass; } my $base_classes_str = join(", ", map {"public $_"} @base_classes); @@ -735,6 +838,7 @@ foreach my $type ('Client', 'Server', 'Local') class $server_or_client_class : $base_classes_str { public: + virtual ~$server_or_client_class(); __E if($writing_local) @@ -743,18 +847,12 @@ __E $server_or_client_class($context_class &rContext); __E } - else - { - print H <<__E; - $server_or_client_class(IOStream &rStream); + + print H <<__E; + $server_or_client_class(std::auto_ptr<SocketStream> apConn); std::auto_ptr<$message_base_class> Receive(); void Send(const $message_base_class &rObject); __E - } - - print H <<__E; - virtual ~$server_or_client_class(); -__E if($writing_server) { @@ -775,37 +873,29 @@ __E my $has_stream = obj_is_type($cmd,'StreamWithCommand'); my $argextra = $has_stream?', std::auto_ptr<IOStream> apStream':''; my $queryextra = $has_stream?', apStream':''; - my $request_class = $cmd_class{$cmd}; - my $reply_class = $cmd_class{obj_get_type_params($cmd,'Command')}; + my $request_class = $cmd_classes{$cmd}; + my $reply_class = $cmd_classes{obj_get_type_params($cmd,'Command')}; print H "\tstd::auto_ptr<$reply_class> Query(const $request_class &rQuery$argextra);\n"; } } } - + if($writing_local) { print H <<__E; private: $context_class &mrContext; -__E - } - - print H <<__E; - -protected: - virtual std::auto_ptr<Message> MakeMessage(int ObjType); - -__E - - if($writing_local) - { - print H <<__E; - virtual void InformStreamReceiving(u_int32_t Size) { } - virtual void InformStreamSending(u_int32_t Size) { } - + std::auto_ptr<$message_base_class> mapLastReply; public: virtual std::auto_ptr<IOStream> ReceiveStream() { + if(mStreamsToSend.empty()) + { + THROW_EXCEPTION_MESSAGE(CommonException, Internal, + "Tried to ReceiveStream when none was sent or " + "made available"); + } + std::auto_ptr<IOStream> apStream(mStreamsToSend.front()); mStreamsToSend.pop_front(); return apStream; @@ -815,29 +905,33 @@ __E else { print H <<__E; - virtual void InformStreamReceiving(u_int32_t Size) - { - this->Protocol::InformStreamReceiving(Size); - } - virtual void InformStreamSending(u_int32_t Size) - { - this->Protocol::InformStreamSending(Size); - } + virtual std::auto_ptr<IOStream> ReceiveStream(); +__E -public: - virtual std::auto_ptr<IOStream> ReceiveStream() + print CPP <<__E; +std::auto_ptr<IOStream> $server_or_client_class\::ReceiveStream() +{ + try { - return this->Protocol::ReceiveStream(); - } -__E + return $custom_protocol_subclass\::ReceiveStream(); } - - print H <<__E; - virtual const char *GetProtocolIdentString() + catch(ConnectionException &e) { - return GetIdentString(); + if(e.GetSubType() == ConnectionException::Protocol_ObjWhenStreamExpected) + { + THROW_EXCEPTION_MESSAGE(ConnectionException, + Protocol_ObjWhenStreamExpected, + "Last exchange was " << mPreviousCommand << + " => " << mPreviousReply); + } + else + { + throw; + } } +} __E + } if($writing_local) { @@ -853,23 +947,13 @@ __E print H <<__E; virtual int GetTimeout() { - return this->Protocol::GetTimeout(); + return $custom_protocol_subclass\::GetTimeout(); } __E } - + print H <<__E; - /* - virtual void Handshake() - { - this->Protocol::Handshake(); - } - virtual bool GetLastError(int &rTypeOut, int &rSubTypeOut) - { - return this->Protocol::GetLastError(rTypeOut, rSubTypeOut); - } - */ - + private: $server_or_client_class(const $server_or_client_class &rToCopy); /* no copies */ }; @@ -890,8 +974,8 @@ __E else { print CPP <<__E; -$server_or_client_class\::$server_or_client_class(IOStream &rStream) -: Protocol(rStream) +$server_or_client_class\::$server_or_client_class(std::auto_ptr<SocketStream> apConn) +: $custom_protocol_subclass(apConn) { } __E } @@ -903,49 +987,58 @@ $server_or_client_class\::~$server_or_client_class() __E # write receive and send functions - print CPP <<__E; -std::auto_ptr<Message> $server_or_client_class\::MakeMessage(int ObjType) -{ - switch(ObjType) - { -__E - - # do objects within this - for my $cmd (@cmd_list) + if($writing_local) { print CPP <<__E; - case $cmd_id{$cmd}: - return std::auto_ptr<Message>(new $cmd_class{$cmd}()); - break; -__E - } - - print CPP <<__E; - default: - THROW_EXCEPTION(ConnectionException, Conn_Protocol_UnknownCommandRecieved) - } +std::auto_ptr<$message_base_class> $server_or_client_class\::Receive() +{ + return mapLastReply; +} +void $server_or_client_class\::Send(const $message_base_class &rObject) +{ + mapLastReply = rObject.DoCommand(*this, mrContext); } __E - - if(not $writing_local) + } + else { print CPP <<__E; std::auto_ptr<$message_base_class> $server_or_client_class\::Receive() { - std::auto_ptr<$message_base_class> preply(($message_base_class *) - Protocol::ReceiveInternal().release()); + std::auto_ptr<$message_base_class> apReply; + + try + { + apReply = std::auto_ptr<$message_base_class>( + static_cast<$message_base_class *> + ($custom_protocol_subclass\::ReceiveInternal().release())); + } + catch(ConnectionException &e) + { + if(e.GetSubType() == ConnectionException::Protocol_StreamWhenObjExpected) + { + THROW_EXCEPTION_MESSAGE(ConnectionException, + Protocol_StreamWhenObjExpected, + "Last exchange was " << mPreviousCommand << + " => " << mPreviousReply); + } + else + { + throw; + } + } if(GetLogToSysLog()) { - preply->LogSysLog("Receive"); + apReply->LogSysLog("Receive"); } if(GetLogToFile() != 0) { - preply->LogFile("Receive", GetLogToFile()); + apReply->LogFile("Receive", GetLogToFile()); } - return preply; + return apReply; } void $server_or_client_class\::Send(const $message_base_class &rObject) @@ -981,10 +1074,40 @@ void $server_or_client_class\::DoServer($context_class &rContext) { // Get an object from the conversation std::auto_ptr<$message_base_class> pobj = Receive(); + std::auto_ptr<$message_base_class> preply; // Run the command - std::auto_ptr<$message_base_class> preply = pobj->DoCommand(*this, rContext); - + try + { + try + { + if(pobj->HasStreamWithCommand()) + { + std::auto_ptr<IOStream> apDataStream = ReceiveStream(); + SelfFlushingStream autoflush(*apDataStream); + preply = pobj->DoCommand(*this, rContext, *apDataStream); + } + else + { + preply = pobj->DoCommand(*this, rContext); + } + } + catch(BoxException &e) + { + // First try a the built-in exception handler + preply = HandleException(e); + } + } + catch (...) + { + // Fallback in case the exception isn't a BoxException + // or the exception handler fails as well. This path + // throws the exception upwards, killing the process + // that handles the current client. + Send($cmd_classes{$error_message}(-1)); + throw; + } + // Send the reply Send(*preply); @@ -995,7 +1118,16 @@ void $server_or_client_class\::DoServer($context_class &rContext) { SendStream(**i); } - + + // As a server, if we get an unexpected message later, we'll + // want to know the last command that we received, and the + // reply, to help debug our response to it. + mPreviousCommand = pobj->ToString(); + std::ostringstream reply; + reply << preply->ToString() << " and " << + mStreamsToSend.size() << " streams"; + mPreviousReply = reply.str(); + // Delete these streams DeleteStreamsToSend(); @@ -1004,7 +1136,7 @@ void $server_or_client_class\::DoServer($context_class &rContext) { inProgress = false; } - } + } } __E @@ -1017,67 +1149,86 @@ __E { if(obj_is_type($cmd,'Command')) { - my $request_class = $cmd_class{$cmd}; + my $request_class = $cmd_classes{$cmd}; my $reply_msg = obj_get_type_params($cmd,'Command'); - my $reply_class = $cmd_class{$reply_msg}; + my $reply_class = $cmd_classes{$reply_msg}; my $reply_id = $cmd_id{$reply_msg}; my $has_stream = obj_is_type($cmd,'StreamWithCommand'); - my $argextra = $has_stream?', std::auto_ptr<IOStream> apStream':''; + my $argextra = $has_stream?', std::auto_ptr<IOStream> apDataStream':''; my $send_stream_extra = ''; - my $send_stream_method = $writing_client ? "SendStream" - : "SendStreamAfterCommand"; - + + print CPP <<__E; +std::auto_ptr<$reply_class> $server_or_client_class\::Query(const $request_class &rQuery$argextra) +{ +__E + if($writing_client) { if($has_stream) { $send_stream_extra = <<__E; // Send stream after the command - SendStream(*apStream); + try + { + SendStream(*apDataStream); + } + catch (BoxException &e) + { + BOX_WARNING("Failed to send stream after command: " << + rQuery.ToString() << ": " << e.what()); + throw; + } __E } print CPP <<__E; -std::auto_ptr<$reply_class> $server_or_client_class\::Query(const $request_class &rQuery$argextra) -{ // Send query Send(rQuery); - $send_stream_extra +$send_stream_extra // Wait for the reply - std::auto_ptr<$message_base_class> preply = Receive(); - - CheckReply("$cmd", *preply, $reply_id); - - // Correct response, if no exception thrown by CheckReply - return std::auto_ptr<$reply_class>(($reply_class *)preply.release()); -} + std::auto_ptr<$message_base_class> apReply = Receive(); __E } elsif($writing_local) { + print CPP <<__E; + std::auto_ptr<$message_base_class> apReply; + try + { +__E if($has_stream) { - $send_stream_extra = <<__E; - // Send stream after the command - SendStreamAfterCommand(apStream); + print CPP <<__E; + apReply = rQuery.DoCommand(*this, mrContext, *apDataStream); +__E + } + else + { + print CPP <<__E; + apReply = rQuery.DoCommand(*this, mrContext); __E } print CPP <<__E; -std::auto_ptr<$reply_class> $server_or_client_class\::Query(const $request_class &rQuery$argextra) -{ - // Send query - $send_stream_extra - std::auto_ptr<$message_base_class> preply = rQuery.DoCommand(*this, mrContext); - - CheckReply("$cmd", *preply, $reply_id); + } + catch(BoxException &e) + { + // First try a the built-in exception handler + apReply = HandleException(e); + } +__E + } + + # Common to both client and local + print CPP <<__E; + CheckReply("$cmd", rQuery, *apReply, $reply_id); // Correct response, if no exception thrown by CheckReply - return std::auto_ptr<$reply_class>(($reply_class *)preply.release()); + return std::auto_ptr<$reply_class>( + static_cast<$reply_class *>(apReply.release())); } __E - } } } } @@ -1110,7 +1261,7 @@ sub obj_get_type_params { return $1 if $_ =~ m/\A$ty\((.+?)\)\Z/; } - die "Can't find attribute $ty\n" + die "Can't find attribute $ty on command $c\n" } # returns (is basic type, typename) diff --git a/lib/win32/box_getopt.h b/lib/win32/box_getopt.h new file mode 100644 index 00000000..f18446d4 --- /dev/null +++ b/lib/win32/box_getopt.h @@ -0,0 +1,14 @@ +#if defined _MSC_VER || defined __MINGW32__ +#define REPLACE_GETOPT 1 /* use this getopt as the system getopt(3) */ +#else +#define REPLACE_GETOPT 0 // force a conflict if included multiple times +#endif + +#if REPLACE_GETOPT +# include "bsd_getopt.h" +# define BOX_BSD_GETOPT +#else +# include <getopt.h> +# undef BOX_BSD_GETOPT +#endif + diff --git a/lib/win32/getopt.h b/lib/win32/bsd_getopt.h index 7c290343..3e2441ca 100755 --- a/lib/win32/getopt.h +++ b/lib/win32/bsd_getopt.h @@ -1,98 +1,105 @@ -/* $OpenBSD: getopt.h,v 1.1 2002/12/03 20:24:29 millert Exp $ */
-/* $NetBSD: getopt.h,v 1.4 2000/07/07 10:43:54 ad Exp $ */
-
-/*-
- * Copyright (c) 2000 The NetBSD Foundation, Inc.
- * All rights reserved.
- *
- * This code is derived from software contributed to The NetBSD Foundation
- * by Dieter Baron and Thomas Klausner.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- * must display the following acknowledgement:
- * This product includes software developed by the NetBSD
- * Foundation, Inc. and its contributors.
- * 4. Neither the name of The NetBSD Foundation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _GETOPT_H_
-#define _GETOPT_H_
-
-// copied from: http://www.la.utexas.edu/lab/software/devtool/gnu/libtool/C_header_files.html
-
-/* __BEGIN_DECLS should be used at the beginning of your declarations,
- so that C++ compilers don't mangle their names. Use __END_DECLS at
- the end of C declarations. */
-#undef __BEGIN_DECLS
-#undef __END_DECLS
-#ifdef __cplusplus
-# define __BEGIN_DECLS extern "C" {
-# define __END_DECLS }
-#else
-# define __BEGIN_DECLS /* empty */
-# define __END_DECLS /* empty */
-#endif
-
-/*
- * GNU-like getopt_long() and 4.4BSD getsubopt()/optreset extensions
- */
-#define no_argument 0
-#define required_argument 1
-#define optional_argument 2
-
-struct option {
- /* name of long option */
- const char *name;
- /*
- * one of no_argument, required_argument, and optional_argument:
- * whether option takes an argument
- */
- int has_arg;
- /* if not NULL, set *flag to val when option found */
- int *flag;
- /* if flag not NULL, value to set *flag to; else return value */
- int val;
-};
-
-__BEGIN_DECLS
-int getopt_long(int, char * const *, const char *,
- const struct option *, int *);
-int getopt_long_only(int, char * const *, const char *,
- const struct option *, int *);
-#ifndef _GETOPT_DEFINED_
-#define _GETOPT_DEFINED_
-int getopt(int, char * const *, const char *);
-int getsubopt(char **, char * const *, char **);
-
-extern char *optarg; /* getopt(3) external variables */
-extern int opterr;
-extern int optind;
-extern int optopt;
-extern int optreset;
-extern char *suboptarg; /* getsubopt(3) external variable */
-#endif
-__END_DECLS
-
-#endif /* !_GETOPT_H_ */
+/* $OpenBSD: getopt.h,v 1.1 2002/12/03 20:24:29 millert Exp $ */ +/* $NetBSD: getopt.h,v 1.4 2000/07/07 10:43:54 ad Exp $ */ + +/*- + * Copyright (c) 2000 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Dieter Baron and Thomas Klausner. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the NetBSD + * Foundation, Inc. and its contributors. + * 4. Neither the name of The NetBSD Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef REPLACE_GETOPT +#error You must include box_getopt.h, not bsd_getopt.h +#endif + +#if REPLACE_GETOPT // defined in box_getopt.h; until end of file + +#ifndef _GETOPT_H_ +#define _GETOPT_H_ + +// copied from: http://www.la.utexas.edu/lab/software/devtool/gnu/libtool/C_header_files.html + +/* __BEGIN_DECLS should be used at the beginning of your declarations, + so that C++ compilers don't mangle their names. Use __END_DECLS at + the end of C declarations. */ +#undef __BEGIN_DECLS +#undef __END_DECLS +#ifdef __cplusplus +# define __BEGIN_DECLS extern "C" { +# define __END_DECLS } +#else +# define __BEGIN_DECLS /* empty */ +# define __END_DECLS /* empty */ +#endif + +/* + * GNU-like getopt_long() and 4.4BSD getsubopt()/optreset extensions + */ +#define no_argument 0 +#define required_argument 1 +#define optional_argument 2 + +struct option { + /* name of long option */ + const char *name; + /* + * one of no_argument, required_argument, and optional_argument: + * whether option takes an argument + */ + int has_arg; + /* if not NULL, set *flag to val when option found */ + int *flag; + /* if flag not NULL, value to set *flag to; else return value */ + int val; +}; + +__BEGIN_DECLS +int getopt_long(int, char * const *, const char *, + const struct option *, int *); +int getopt_long_only(int, char * const *, const char *, + const struct option *, int *); +#ifndef _GETOPT_DEFINED_ +#define _GETOPT_DEFINED_ +int getopt(int, char * const *, const char *); +int getsubopt(char **, char * const *, char **); + +extern char *optarg; /* getopt(3) external variables */ +extern int opterr; +extern int optind; +extern int optopt; +extern int optreset; +extern char *suboptarg; /* getsubopt(3) external variable */ +#endif +__END_DECLS + +#endif /* !_GETOPT_H_ */ +#endif // REPLACE_GETOPT diff --git a/lib/win32/emu.cpp b/lib/win32/emu.cpp index ef237671..1f6392d5 100644 --- a/lib/win32/emu.cpp +++ b/lib/win32/emu.cpp @@ -32,8 +32,9 @@ bool EnableBackupRights() if(!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &hToken)) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to open process token: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return false; } @@ -45,8 +46,9 @@ bool EnableBackupRights() SE_BACKUP_NAME, //the name of the privilege &( token_priv.Privileges[0].Luid ))) //result { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to lookup backup privilege: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); CloseHandle(hToken); return false; } @@ -68,8 +70,9 @@ bool EnableBackupRights() //this function is a little tricky - if we were adjusting //more than one privilege, it could return success but not //adjust them all - in the general case, you need to trap this + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to enable backup privilege: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); CloseHandle(hToken); return false; @@ -238,9 +241,10 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EINVAL; return NULL; } @@ -270,9 +274,10 @@ char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage) if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EACCES; delete [] buffer; return NULL; @@ -299,9 +304,10 @@ bool ConvertFromWideString(const std::wstring& rInput, if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EINVAL; return false; } @@ -331,9 +337,10 @@ bool ConvertFromWideString(const std::wstring& rInput, if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to convert wide string to narrow: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EACCES; delete [] buffer; return false; @@ -363,10 +370,11 @@ bool ConvertEncoding(const std::string& rSource, int sourceCodePage, true); if (pWide == NULL) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to convert string '%s' from " "current code page %d to wide string: %s", rSource.c_str(), sourceCodePage, - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return false; } @@ -528,7 +536,10 @@ std::string GetErrorMessage(DWORD errorCode) if (chars == 0 || pMsgBuf == NULL) { - return std::string("failed to get error message"); + std::ostringstream oss; + oss << "Failed to get error message for error code " << errorCode << ": error " << + GetLastError(); + return oss.str(); } // remove embedded newline @@ -605,7 +616,7 @@ HANDLE openfile(const char *pFileName, int flags, int mode) createDisposition = CREATE_NEW; } - if (flags & O_LOCK) + if (flags & BOX_OPEN_LOCK) { shareMode = 0; } @@ -641,7 +652,7 @@ HANDLE openfile(const char *pFileName, int flags, int mode) ::syslog(LOG_WARNING, "Failed to open file '%s': " "%s", pFileName, - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return INVALID_HANDLE_VALUE; } @@ -684,16 +695,18 @@ int emu_fstat(HANDLE hdir, struct emu_stat * st) BY_HANDLE_FILE_INFORMATION fi; if (!GetFileInformationByHandle(hdir, &fi)) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to read file information: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EACCES; return -1; } if (INVALID_FILE_ATTRIBUTES == fi.dwFileAttributes) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to get file attributes: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); errno = EACCES; return -1; } @@ -826,10 +839,10 @@ HANDLE OpenFileByNameUtf8(const char* pFileName, DWORD flags) if (handle == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); + winerrno = GetLastError(); - if (err == ERROR_FILE_NOT_FOUND || - err == ERROR_PATH_NOT_FOUND) + if (winerrno == ERROR_FILE_NOT_FOUND || + winerrno == ERROR_PATH_NOT_FOUND) { errno = ENOENT; } @@ -837,7 +850,7 @@ HANDLE OpenFileByNameUtf8(const char* pFileName, DWORD flags) { ::syslog(LOG_WARNING, "Failed to open '%s': " "%s", pFileName, - GetErrorMessage(err).c_str()); + GetErrorMessage(winerrno).c_str()); errno = EACCES; } @@ -906,9 +919,10 @@ int statfs(const char * pName, struct statfs * s) BY_HANDLE_FILE_INFORMATION fi; if (!GetFileInformationByHandle(handle, &fi)) { + winerrno = GetLastError(); ::syslog(LOG_WARNING, "Failed to get file information " "for '%s': %s", pName, - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); CloseHandle(handle); errno = EACCES; return -1; @@ -961,8 +975,9 @@ int emu_utimes(const char * pName, const struct timeval times[]) if (!SetFileTime(handle, &creationTime, NULL, &modificationTime)) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to set times on '%s': %s", pName, - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); CloseHandle(handle); return 1; } @@ -1004,8 +1019,9 @@ int emu_chmod(const char * pName, mode_t mode) DWORD attribs = GetFileAttributesW(pBuffer); if (attribs == INVALID_FILE_ATTRIBUTES) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to get file attributes of '%s': %s", - pName, GetErrorMessage(GetLastError()).c_str()); + pName, GetErrorMessage(winerrno).c_str()); errno = EACCES; free(pBuffer); return -1; @@ -1022,8 +1038,9 @@ int emu_chmod(const char * pName, mode_t mode) if (!SetFileAttributesW(pBuffer, attribs)) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to set file attributes of '%s': %s", - pName, GetErrorMessage(GetLastError()).c_str()); + pName, GetErrorMessage(winerrno).c_str()); errno = EACCES; free(pBuffer); return -1; @@ -1078,7 +1095,6 @@ DIR *opendir(const char *name) } pDir->fd = FindFirstFileW(pDir->name, &pDir->info); - DWORD tmp = GetLastError(); if (pDir->fd == INVALID_HANDLE_VALUE) { @@ -1297,7 +1313,7 @@ int poll (struct pollfd *ufds, unsigned long nfds, int timeout) BOOL AddEventSource ( - LPTSTR pszSrcName, // event source name + const std::string& name, // event source name DWORD dwNum // number of categories ) { @@ -1309,8 +1325,9 @@ BOOL AddEventSource if (len == 0) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to get the program file name: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return FALSE; } @@ -1318,31 +1335,39 @@ BOOL AddEventSource std::string regkey("SYSTEM\\CurrentControlSet\\Services\\EventLog\\" "Application\\"); - regkey += pszSrcName; + regkey += name; HKEY hk; DWORD dwDisp; - if (RegCreateKeyEx(HKEY_LOCAL_MACHINE, regkey.c_str(), - 0, NULL, REG_OPTION_NON_VOLATILE, - KEY_WRITE, NULL, &hk, &dwDisp)) + winerrno = RegCreateKeyEx(HKEY_LOCAL_MACHINE, regkey.c_str(), + 0, NULL, REG_OPTION_NON_VOLATILE, + KEY_WRITE, NULL, &hk, &dwDisp); + if (winerrno == ERROR_ACCESS_DENIED) { - ::syslog(LOG_ERR, "Failed to create the registry key: %s", - GetErrorMessage(GetLastError()).c_str()); + ::syslog(LOG_ERR, "Failed to create the registry key: access denied. You must " + "be an Administrator to register new event sources in %s", regkey.c_str()); + return FALSE; + } + else if (winerrno != ERROR_SUCCESS) + { + ::syslog(LOG_ERR, "Failed to create the registry key: %s: %s", + GetErrorMessage(winerrno).c_str(), regkey.c_str()); return FALSE; } // Set the name of the message file. - if (RegSetValueExW(hk, // subkey handle - L"EventMessageFile", // value name - 0, // must be zero - REG_EXPAND_SZ, // value type - (LPBYTE)cmd, // pointer to value data - len*sizeof(WCHAR))) // data size + winerrno = RegSetValueExW(hk, // subkey handle + L"EventMessageFile", // value name + 0, // must be zero + REG_EXPAND_SZ, // value type + (LPBYTE)cmd, // pointer to value data + len*sizeof(WCHAR)); // data size + if (winerrno != ERROR_SUCCESS) { ::syslog(LOG_ERR, "Failed to set the event message file: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); RegCloseKey(hk); return FALSE; } @@ -1352,43 +1377,46 @@ BOOL AddEventSource DWORD dwData = EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | EVENTLOG_INFORMATION_TYPE; - if (RegSetValueEx(hk, // subkey handle - "TypesSupported", // value name - 0, // must be zero - REG_DWORD, // value type - (LPBYTE) &dwData, // pointer to value data - sizeof(DWORD))) // length of value data + winerrno = RegSetValueEx(hk, // subkey handle + "TypesSupported", // value name + 0, // must be zero + REG_DWORD, // value type + (LPBYTE) &dwData, // pointer to value data + sizeof(DWORD)); // length of value data + if (winerrno != ERROR_SUCCESS) { ::syslog(LOG_ERR, "Failed to set the supported types: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); RegCloseKey(hk); return FALSE; } // Set the category message file and number of categories. - if (RegSetValueExW(hk, // subkey handle - L"CategoryMessageFile", // value name - 0, // must be zero - REG_EXPAND_SZ, // value type - (LPBYTE)cmd, // pointer to value data - len*sizeof(WCHAR))) // data size + winerrno = RegSetValueExW(hk, // subkey handle + L"CategoryMessageFile", // value name + 0, // must be zero + REG_EXPAND_SZ, // value type + (LPBYTE)cmd, // pointer to value data + len*sizeof(WCHAR)); // data size + if (winerrno != ERROR_SUCCESS) { ::syslog(LOG_ERR, "Failed to set the category message file: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); RegCloseKey(hk); return FALSE; } - if (RegSetValueEx(hk, // subkey handle + winerrno = RegSetValueEx(hk, // subkey handle "CategoryCount", // value name 0, // must be zero REG_DWORD, // value type (LPBYTE) &dwNum, // pointer to value data - sizeof(DWORD))) // length of value data + sizeof(DWORD)); // length of value data + if (winerrno != ERROR_SUCCESS) { ::syslog(LOG_ERR, "Failed to set the category count: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); RegCloseKey(hk); return FALSE; } @@ -1397,7 +1425,7 @@ BOOL AddEventSource return TRUE; } -static HANDLE gSyslogH = 0; +static HANDLE gSyslogH = INVALID_HANDLE_VALUE; static bool sHaveWarnedEventLogFull = false; void openlog(const char * daemonName, int, int) @@ -1406,19 +1434,21 @@ void openlog(const char * daemonName, int, int) nameStr += daemonName; nameStr += ")"; - // register a default event source, so that we can - // log errors with the process of adding or registering our own. + // Don't try to open a new handle when one is already open. It will leak handles. + assert(gSyslogH == INVALID_HANDLE_VALUE); + + // Register a default event source, so that we can log errors with the process of + // adding or registering our own, which follows. If this fails, there's not much we + // can do about it, certainly not send anything to the event log! gSyslogH = RegisterEventSource( NULL, // uses local computer nameStr.c_str()); // source name if (gSyslogH == NULL) { + gSyslogH = INVALID_HANDLE_VALUE; } - char* name = strdup(nameStr.c_str()); - BOOL success = AddEventSource(name, 0); - free(name); - + BOOL success = AddEventSource(nameStr, 0); if (!success) { ::syslog(LOG_ERR, "Failed to add our own event source"); @@ -1428,8 +1458,9 @@ void openlog(const char * daemonName, int, int) HANDLE newSyslogH = RegisterEventSource(NULL, nameStr.c_str()); if (newSyslogH == NULL) { + winerrno = GetLastError(); ::syslog(LOG_ERR, "Failed to register our own event source: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); return; } @@ -1439,7 +1470,11 @@ void openlog(const char * daemonName, int, int) void closelog(void) { - DeregisterEventSource(gSyslogH); + if(gSyslogH != INVALID_HANDLE_VALUE) + { + DeregisterEventSource(gSyslogH); + gSyslogH = INVALID_HANDLE_VALUE; + } } void syslog(int loglevel, const char *frmt, ...) @@ -1494,7 +1529,7 @@ void syslog(int loglevel, const char *frmt, ...) va_end(args); - if (gSyslogH == 0) + if (gSyslogH == INVALID_HANDLE_VALUE) { printf("%s\r\n", buffer); fflush(stdout); @@ -1541,22 +1576,20 @@ void syslog(int loglevel, const char *frmt, ...) if (result == 0) { - DWORD err = GetLastError(); - if (err == ERROR_LOG_FILE_FULL) + winerrno = GetLastError(); + if (winerrno == ERROR_LOG_FILE_FULL) { if (!sHaveWarnedEventLogFull) { printf("Unable to send message to Event Log " - "(Event Log is full):\r\n"); - fflush(stdout); + "(Event Log is full): %s\r\n", buffer); sHaveWarnedEventLogFull = TRUE; } } else { - printf("Unable to send message to Event Log: %s:\r\n", - GetErrorMessage(err).c_str()); - fflush(stdout); + printf("Unable to send message to Event Log: %s: %s\r\n", + GetErrorMessage(winerrno).c_str(), buffer); } } else @@ -1589,8 +1622,9 @@ int emu_chdir(const char* pDirName) if (result != 0) return 0; errno = EACCES; + winerrno = GetLastError(); fprintf(stderr, "Failed to change directory to '%s': %s\n", - pDirName, GetErrorMessage(GetLastError()).c_str()); + pDirName, GetErrorMessage(winerrno).c_str()); return -1; } @@ -1668,6 +1702,74 @@ int emu_mkdir(const char* pPathName) return 0; } +int emu_link(const char* pOldPath, const char* pNewPath) +{ + std::string AbsOldPathWithUnicode = + ConvertPathToAbsoluteUnicode(pOldPath); + + if (AbsOldPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + std::string AbsNewPathWithUnicode = + ConvertPathToAbsoluteUnicode(pNewPath); + + if (AbsNewPathWithUnicode.size() == 0) + { + // error already logged by ConvertPathToAbsoluteUnicode() + return -1; + } + + WCHAR* pOldBuffer = ConvertUtf8ToWideString(AbsOldPathWithUnicode.c_str()); + if (!pOldBuffer) + { + return -1; + } + + WCHAR* pNewBuffer = ConvertUtf8ToWideString(AbsNewPathWithUnicode.c_str()); + if (!pNewBuffer) + { + delete [] pOldBuffer; + return -1; + } + + BOOL result = CreateHardLinkW(pNewBuffer, pOldBuffer, NULL); + winerrno = GetLastError(); + delete [] pOldBuffer; + delete [] pNewBuffer; + + if (!result) + { + if (winerrno == ERROR_FILE_NOT_FOUND || + winerrno == ERROR_PATH_NOT_FOUND) + { + errno = ENOENT; + } + else if (winerrno == ERROR_SHARING_VIOLATION) + { + errno = EBUSY; + } + else if (winerrno == ERROR_ACCESS_DENIED) + { + errno = EACCES; + } + else + { + ::syslog(LOG_WARNING, "Failed to hardlink file " + "'%s' to '%s': %s", pOldPath, pNewPath, + GetErrorMessage(winerrno).c_str()); + errno = ENOSYS; + } + + return -1; + } + + return 0; + +} + int emu_unlink(const char* pFileName) { std::string AbsPathWithUnicode = @@ -1686,20 +1788,21 @@ int emu_unlink(const char* pFileName) } BOOL result = DeleteFileW(pBuffer); - DWORD err = GetLastError(); + winerrno = GetLastError(); delete [] pBuffer; if (!result) { - if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + if (winerrno == ERROR_FILE_NOT_FOUND || + winerrno == ERROR_PATH_NOT_FOUND) { errno = ENOENT; } - else if (err == ERROR_SHARING_VIOLATION) + else if (winerrno == ERROR_SHARING_VIOLATION) { errno = EBUSY; } - else if (err == ERROR_ACCESS_DENIED) + else if (winerrno == ERROR_ACCESS_DENIED) { errno = EACCES; } @@ -1707,9 +1810,10 @@ int emu_unlink(const char* pFileName) { ::syslog(LOG_WARNING, "Failed to delete file " "'%s': %s", pFileName, - GetErrorMessage(err).c_str()); + GetErrorMessage(winerrno).c_str()); errno = ENOSYS; } + return -1; } @@ -1751,21 +1855,22 @@ int emu_rename(const char* pOldFileName, const char* pNewFileName) } BOOL result = MoveFileW(pOldBuffer, pNewBuffer); - DWORD err = GetLastError(); + winerrno = GetLastError(); delete [] pOldBuffer; delete [] pNewBuffer; if (!result) { - if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) + if (winerrno == ERROR_FILE_NOT_FOUND || + winerrno == ERROR_PATH_NOT_FOUND) { errno = ENOENT; } - else if (err == ERROR_SHARING_VIOLATION) + else if (winerrno == ERROR_SHARING_VIOLATION) { errno = EBUSY; } - else if (err == ERROR_ACCESS_DENIED) + else if (winerrno == ERROR_ACCESS_DENIED) { errno = EACCES; } @@ -1773,7 +1878,7 @@ int emu_rename(const char* pOldFileName, const char* pNewFileName) { ::syslog(LOG_WARNING, "Failed to rename file " "'%s' to '%s': %s", pOldFileName, pNewFileName, - GetErrorMessage(err).c_str()); + GetErrorMessage(winerrno).c_str()); errno = ENOSYS; } return -1; @@ -1788,8 +1893,9 @@ int console_read(char* pBuffer, size_t BufferSize) if (hConsole == INVALID_HANDLE_VALUE) { + winerrno = GetLastError(); ::fprintf(stderr, "Failed to get a handle on standard input: " - "%s", GetErrorMessage(GetLastError()).c_str()); + "%s", GetErrorMessage(winerrno).c_str()); return -1; } @@ -1812,8 +1918,9 @@ int console_read(char* pBuffer, size_t BufferSize) NULL // reserved )) { + winerrno = GetLastError(); ::fprintf(stderr, "Failed to read from console: %s\n", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return -1; } @@ -1864,7 +1971,7 @@ int writev(int filedes, const struct iovec *vector, size_t count) return bytes; } -// need this for conversions +// Need this for conversions. Works in UTC. time_t ConvertFileTimeToTime_t(FILETIME *fileTime) { SYSTEMTIME stUTC; @@ -1883,18 +1990,17 @@ time_t ConvertFileTimeToTime_t(FILETIME *fileTime) // timeinfo.tm_yday = ...; timeinfo.tm_year = stUTC.wYear - 1900; - time_t retVal = mktime(&timeinfo) - _timezone; + time_t retVal = _mkgmtime(&timeinfo); return retVal; } bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo) { - time_t adjusted = from + _timezone; - struct tm *time_breakdown = gmtime(&adjusted); + struct tm *time_breakdown = gmtime(&from); if (time_breakdown == NULL) { ::syslog(LOG_ERR, "Error: failed to convert time format: " - "%d is not a valid time\n", adjusted); + "%d is not a valid time\n", from); return false; } @@ -1911,8 +2017,9 @@ bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo) // Convert the last-write time to local time. if (!SystemTimeToFileTime(&stUTC, pTo)) { + winerrno = GetLastError(); syslog(LOG_ERR, "Failed to convert between time formats: %s", - GetErrorMessage(GetLastError()).c_str()); + GetErrorMessage(winerrno).c_str()); return false; } diff --git a/lib/win32/emu.h b/lib/win32/emu.h index bf408050..91793004 100644 --- a/lib/win32/emu.h +++ b/lib/win32/emu.h @@ -18,7 +18,14 @@ #define EMU_INCLUDE // Need feature detection macros below -#include "../common/BoxConfig.h" +#if defined BOX_CMAKE +# include "../common/BoxConfig.cmake.h" +#elif defined _MSC_VER +# include "../common/BoxConfig-MSVC.h" +# define NEED_BOX_VERSION_H +#else +# include "../common/BoxConfig.h" +#endif // Shut up stupid new warnings. Thanks MinGW! Ever heard of "compatibility"? #ifdef __MINGW32__ @@ -27,31 +34,21 @@ // basic types, may be required by other headers since we // don't include sys/types.h - -#ifdef __MINGW32__ - #include <stdint.h> -#else // MSVC - typedef unsigned __int64 u_int64_t; - typedef unsigned __int64 uint64_t; - typedef __int64 int64_t; - typedef unsigned __int32 uint32_t; - typedef unsigned __int32 u_int32_t; - typedef __int32 int32_t; - typedef unsigned __int16 uint16_t; - typedef __int16 int16_t; - typedef unsigned __int8 uint8_t; - typedef __int8 int8_t; -#endif +#include <stdint.h> // emulated types, present on MinGW but not MSVC or vice versa -#ifdef __MINGW32__ - typedef uint32_t u_int32_t; -#else +#ifndef __MINGW32__ typedef unsigned int mode_t; typedef unsigned int pid_t; + typedef unsigned int uid_t; + typedef unsigned int gid_t; #endif +// Disable Windows' non-standard implementation of min() and max(): +// http://stackoverflow.com/a/5004874/648162 +#define NOMINMAX + // Windows headers #include <winsock2.h> @@ -76,17 +73,6 @@ #define ITIMER_REAL 0 -#ifdef _MSC_VER -// Microsoft decided to deprecate the standard POSIX functions. Great! -#define open(file,flags,mode) _open(file,flags,mode) -#define close(fd) _close(fd) -#define dup(fd) _dup(fd) -#define read(fd,buf,count) _read(fd,buf,count) -#define write(fd,buf,count) _write(fd,buf,count) -#define lseek(fd,off,whence) _lseek(fd,off,whence) -#define fileno(struct_file) _fileno(struct_file) -#endif - struct passwd { char *pw_name; char *pw_passwd; @@ -110,26 +96,34 @@ inline struct passwd * getpwnam(const char * name) return &gTempPasswd; } -#define S_IRWXG 1 -#define S_IRWXO 2 -#define S_ISUID 4 -#define S_ISGID 8 -#define S_ISVTX 16 - -#ifndef __MINGW32__ +#ifndef S_IRGRP + // these constants are only defined in MinGW64, not the original MinGW headers, + // nor MSVC, so use poor man's feature detection to define them only if needed. //not sure if these are correct //S_IWRITE - writing permitted //_S_IREAD - reading permitted //_S_IREAD | _S_IWRITE - - #define S_IRUSR S_IWRITE - #define S_IWUSR S_IREAD - #define S_IRWXU (S_IREAD|S_IWRITE|S_IEXEC) +# define S_IRUSR S_IWRITE +# define S_IWUSR S_IREAD +# define S_IRGRP S_IWRITE +# define S_IWGRP S_IREAD +# define S_IROTH S_IWRITE | S_IREAD +# define S_IWOTH S_IREAD | S_IREAD +# define S_IRWXU (S_IREAD|S_IWRITE|S_IEXEC) +# define S_IRWXG 1 +# define S_IRWXO 2 +#endif +#define S_ISUID 4 +#define S_ISGID 8 +#define S_ISVTX 16 + +#ifndef __MINGW32__ #define S_ISREG(x) (S_IFREG & x) #define S_ISDIR(x) (S_IFDIR & x) #endif -inline int chown(const char * Filename, u_int32_t uid, u_int32_t gid) +inline int chown(const char * Filename, uint32_t uid, uint32_t gid) { //important - this needs implementing //If a large restore is required then @@ -183,7 +177,7 @@ inline int geteuid(void) // MinGW provides a getopt implementation #ifndef __MINGW32__ -#include "getopt.h" +#include "box_getopt.h" #endif // !__MINGW32__ #define timespec timeval @@ -195,11 +189,6 @@ inline int geteuid(void) typedef int socklen_t; #endif -#define S_IRGRP S_IWRITE -#define S_IWGRP S_IREAD -#define S_IROTH S_IWRITE | S_IREAD -#define S_IWOTH S_IREAD | S_IREAD - //again need to verify these #define S_IFLNK 1 #define S_IFSOCK 0 @@ -209,9 +198,14 @@ inline int geteuid(void) #define vsnprintf _vsnprintf #ifndef __MINGW32__ +#define snprintf _snprintf inline int strcasecmp(const char *s1, const char *s2) { - return _stricmp(s1,s2); + return _stricmp(s1, s2); +} +inline int strncasecmp(const char *s1, const char *s2, size_t count) +{ + return _strnicmp(s1, s2, count); } #endif @@ -238,7 +232,7 @@ struct dirent *readdir(DIR *dp); int closedir(DIR *dp); // local constant to open file exclusively without shared access -#define O_LOCK 0x10000 +#define BOX_OPEN_LOCK 0x10000 extern DWORD winerrno; /* used to report errors from openfile() */ HANDLE openfile(const char *filename, int flags, int mode); @@ -279,7 +273,7 @@ void syslog (int loglevel, const char *fmt, ...); #define strtoll _strtoi64 #endif -inline unsigned int sleep(unsigned int secs) +extern "C" inline unsigned int sleep(unsigned int secs) { Sleep(secs*1000); return(ERROR_SUCCESS); @@ -347,6 +341,7 @@ bool ConvertTime_tToFileTime(const time_t from, FILETIME *pTo); int emu_chdir (const char* pDirName); int emu_mkdir (const char* pPathName); +int emu_link (const char* pOldPath, const char* pNewPath); int emu_unlink (const char* pFileName); int emu_fstat (HANDLE file, struct emu_stat* st); int emu_stat (const char* pName, struct emu_stat* st); @@ -357,6 +352,7 @@ int emu_rename (const char* pOldName, const char* pNewName); #define chdir(directory) emu_chdir (directory) #define mkdir(path, mode) emu_mkdir (path) +#define link(oldpath, newpath) emu_link (oldpath, newpath) #define unlink(file) emu_unlink (file) #define utimes(buffer, times) emu_utimes (buffer, times) #define chmod(file, mode) emu_chmod (file, mode) @@ -403,6 +399,7 @@ bool ConvertConsoleToUtf8(const std::string& rSource, std::string& rDest); char* ConvertFromWideString(const WCHAR* pString, unsigned int codepage); bool ConvertFromWideString(const std::wstring& rInput, std::string* pOutput, unsigned int codepage); +WCHAR* ConvertUtf8ToWideString(const char* pString); std::string ConvertPathToAbsoluteUnicode(const char *pFileName); // Utility function which returns a default config file name, diff --git a/lib/win32/getopt_long.cpp b/lib/win32/getopt_long.cpp index 31695aa0..af2833a1 100755 --- a/lib/win32/getopt_long.cpp +++ b/lib/win32/getopt_long.cpp @@ -66,18 +66,15 @@ #include <stdio.h>
#include <string.h>
-#include "getopt.h"
+#include "box_getopt.h"
-#if defined _MSC_VER || defined __MINGW32__
-#define REPLACE_GETOPT /* use this getopt as the system getopt(3) */
+#ifdef REPLACE_GETOPT // until end of file
-#ifdef REPLACE_GETOPT
int opterr = 1; /* if error message should be printed */
int optind = 1; /* index into parent argv vector */
int optopt = '?'; /* character checked for validity */
int optreset; /* reset getopt */
char *optarg; /* argument associated with option */
-#endif
#define PRINT_ERROR ((opterr) && (*options != ':'))
@@ -499,7 +496,6 @@ start: return (optchar);
}
-#ifdef REPLACE_GETOPT
/*
* getopt --
* Parse argc/argv argument vector.
@@ -520,7 +516,6 @@ getopt(int nargc, char * const *nargv, const char *options) */
return (getopt_internal(nargc, nargv, options, NULL, NULL, 0));
}
-#endif /* REPLACE_GETOPT */
/*
* getopt_long --
@@ -548,4 +543,4 @@ getopt_long_only(int nargc, char * const *nargv, const char *options, FLAG_PERMUTE|FLAG_LONGONLY));
}
-#endif // defined _MSC_VER || defined __MINGW32__
+#endif // REPLACE_GETOPT
diff --git a/lib/win32/messages.h b/lib/win32/messages.h index 6959591b..22290226 100755 --- a/lib/win32/messages.h +++ b/lib/win32/messages.h @@ -1,57 +1,57 @@ - // Message source file, to be compiled to a resource file with
- // Microsoft Message Compiler (MC), to an object file with a Resource
- // Compiler, and linked into the application.
-
- // The main reason for this file is to work around Windows' stupid
- // messages in the Event Log, which say:
-
- // The description for Event ID ( 4 ) in Source ( Box Backup (bbackupd) )
- // cannot be found. The local computer may not have the necessary
- // registry information or message DLL files to display messages from a
- // remote computer. The following information is part of the event:
- // Message definitions follow
-//
-// Values are 32 bit values layed out as follows:
-//
-// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
-// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
-// +---+-+-+-----------------------+-------------------------------+
-// |Sev|C|R| Facility | Code |
-// +---+-+-+-----------------------+-------------------------------+
-//
-// where
-//
-// Sev - is the severity code
-//
-// 00 - Success
-// 01 - Informational
-// 10 - Warning
-// 11 - Error
-//
-// C - is the Customer code flag
-//
-// R - is a reserved bit
-//
-// Facility - is the facility code
-//
-// Code - is the facility's status code
-//
-//
-// Define the facility codes
-//
-
-
-//
-// Define the severity codes
-//
-
-
-//
-// MessageId: MSG_ERR
-//
-// MessageText:
-//
-// %1
-//
-#define MSG_ERR ((DWORD)0x40000001L)
-
+ // Message source file, to be compiled to a resource file with + // Microsoft Message Compiler (MC), to an object file with a Resource + // Compiler, and linked into the application. + + // The main reason for this file is to work around Windows' stupid + // messages in the Event Log, which say: + + // The description for Event ID ( 4 ) in Source ( Box Backup (bbackupd) ) + // cannot be found. The local computer may not have the necessary + // registry information or message DLL files to display messages from a + // remote computer. The following information is part of the event: + // Message definitions follow +// +// Values are 32 bit values layed out as follows: +// +// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 +// +---+-+-+-----------------------+-------------------------------+ +// |Sev|C|R| Facility | Code | +// +---+-+-+-----------------------+-------------------------------+ +// +// where +// +// Sev - is the severity code +// +// 00 - Success +// 01 - Informational +// 10 - Warning +// 11 - Error +// +// C - is the Customer code flag +// +// R - is a reserved bit +// +// Facility - is the facility code +// +// Code - is the facility's status code +// +// +// Define the facility codes +// + + +// +// Define the severity codes +// + + +// +// MessageId: MSG_ERR +// +// MessageText: +// +// %1 +// +#define MSG_ERR ((DWORD)0x40000001L) + diff --git a/modules.txt b/modules.txt index 670f4ac4..4fd27a2d 100644 --- a/modules.txt +++ b/modules.txt @@ -7,44 +7,53 @@ # -l libaries must be in the order they should appear on the command line. # Note that order is important on platforms which do not have shared libraries. +# QDBM is currently built from a copy distributed in-tree, which we need to fix. +qdbm + # Generic support code and modules -lib/raidfile -lib/crypto +lib/win32 +lib/common lib/win32 +lib/raidfile lib/common +lib/crypto lib/common lib/server qdbm lib/crypto -lib/compress -lib/intercept +lib/compress lib/common +lib/intercept lib/common -test/common -test/crypto lib/crypto -test/compress lib/compress -test/raidfile lib/raidfile lib/intercept -test/basicserver lib/server +test/common qdbm lib/common +test/crypto qdbm lib/crypto +test/compress qdbm lib/compress +test/raidfile qdbm lib/raidfile lib/intercept +test/basicserver qdbm lib/server # IF_DISTRIBUTION(boxbackup) # Backup system -lib/backupstore lib/server lib/raidfile lib/crypto lib/compress +lib/backupstore lib/server lib/raidfile lib/crypto lib/compress lib/httpserver lib/backupclient lib/backupstore - -bin/bbackupobjdump lib/backupclient lib/backupstore -bin/bbstored lib/raidfile lib/server lib/backupstore -bin/bbstoreaccounts lib/raidfile lib/backupstore -bin/bbackupd lib/server lib/backupclient qdbm -bin/bbackupquery lib/server lib/backupclient -bin/bbackupctl lib/server lib/backupclient - -test/backupstore bin/bbstored bin/bbstoreaccounts lib/server lib/backupstore lib/backupclient lib/raidfile -test/backupstorefix bin/bbstored bin/bbstoreaccounts lib/backupstore lib/raidfile bin/bbackupquery bin/bbackupd bin/bbackupctl +lib/bbackupd lib/backupclient qdbm +lib/bbackupquery lib/backupclient +lib/bbstored lib/backupstore + +bin/bbackupobjdump lib/backupclient +bin/bbstored lib/bbstored +bin/bbstoreaccounts lib/backupclient +bin/bbackupd lib/bbackupd +bin/bbackupquery lib/bbackupquery +bin/bbackupctl lib/backupclient qdbm lib/bbackupd + +test/backupstore bin/bbstored bin/bbstoreaccounts lib/backupclient lib/raidfile +test/backupstorefix bin/bbstored bin/bbstoreaccounts lib/backupclient bin/bbackupquery bin/bbackupd bin/bbackupctl test/backupstorepatch bin/bbstored bin/bbstoreaccounts lib/backupclient test/backupdiff lib/backupclient -test/bbackupd bin/bbackupd bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupctl lib/server lib/backupstore lib/backupclient lib/intercept +test/bbackupd bin/bbackupd bin/bbstored bin/bbstoreaccounts bin/bbackupquery bin/bbackupctl lib/bbackupquery lib/bbackupd lib/bbstored lib/server lib/intercept +bin/s3simulator lib/httpserver +test/s3store lib/backupclient lib/httpserver bin/s3simulator bin/bbstoreaccounts # HTTP server system -lib/httpserver lib/server -test/httpserver lib/httpserver -bin/s3simulator lib/httpserver +lib/httpserver lib/server +test/httpserver lib/httpserver # END_IF_DISTRIBUTION diff --git a/parcels.txt b/parcels.txt index cbb6f7c7..7f961b10 100644 --- a/parcels.txt +++ b/parcels.txt @@ -18,8 +18,6 @@ backup-client html bbackupd-config html bbackupd.conf - subdir qdbm libqdbm.a - EXCEPT:mingw32,mingw32msvc man bbackupd.8 man bbackupquery.8 @@ -35,9 +33,12 @@ ONLY:mingw32,mingw32msvc END-ONLY ONLY:mingw32 - script /usr/i686-pc-mingw32/sys-root/mingw/bin/libz-1.dll - script /usr/i686-pc-mingw32/sys-root/mingw/bin/mingwm10.dll - script /usr/i686-pc-mingw32/sys-root/mingw/bin/libgcc_s_dw2-1.dll + script /usr/$ac_target/sys-root/mingw/bin/zlib1.dll + script /usr/$ac_target/sys-root/mingw/bin/libgcc_s_seh-1.dll +END-ONLY + +ONLY:i686-w64-mingw32 + script /usr/$ac_target/sys-root/mingw/bin/libgcc_s_sjlj-1.dll END-ONLY ONLY:SunOS diff --git a/qdbm/Makefile.in b/qdbm/Makefile.in index f87d7e46..c6d69b93 100644 --- a/qdbm/Makefile.in +++ b/qdbm/Makefile.in @@ -56,7 +56,7 @@ CPPFLAGS = @CPPFLAGS@ -I$(srcdir) -I$(MYHEADDIR) \ -D_XOPEN_SOURCE_EXTENDED=1 -D_GNU_SOURCE=1 -D__EXTENSIONS__=1 -D_HPUX_SOURCE=1 \ -D_POSIX_MAPPED_FILES=1 -D_POSIX_SYNCHRONIZED_IO=1 \ -DPIC=1 -D_THREAD_SAFE=1 -D_REENTRANT=1 -DNDEBUG -CFLAGS = @CFLAGS@ -Wall -pedantic -fPIC -fsigned-char -O0 -fforce-addr @MYOPTS@ +CFLAGS = @CFLAGS@ -Wall -pedantic -fPIC -fsigned-char -O0 @MYOPTS@ LD = @LD@ LIBS = -lqdbm @LIBS@ LIBLDFLAGS = @LDFLAGS@ -L. -L$(MYLIBDIR) -L$(HOME)/lib -L/usr/local/lib @LIBS@ diff --git a/qdbm/configure b/qdbm/configure deleted file mode 100755 index 60e3d2fa..00000000 --- a/qdbm/configure +++ /dev/null @@ -1,3913 +0,0 @@ -#! /bin/sh -# Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.59 for qdbm 1.8.77. -# -# Copyright (C) 2003 Free Software Foundation, Inc. -# This configure script is free software; the Free Software Foundation -# gives unlimited permission to copy, distribute and modify it. -## --------------------- ## -## M4sh Initialization. ## -## --------------------- ## - -# Be Bourne compatible -if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then - emulate sh - NULLCMD=: - # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which - # is contrary to our usage. Disable this feature. - alias -g '${1+"$@"}'='"$@"' -elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then - set -o posix -fi -DUALCASE=1; export DUALCASE # for MKS sh - -# Support unset when possible. -if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then - as_unset=unset -else - as_unset=false -fi - - -# Work around bugs in pre-3.0 UWIN ksh. -$as_unset ENV MAIL MAILPATH -PS1='$ ' -PS2='> ' -PS4='+ ' - -# NLS nuisances. -for as_var in \ - LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \ - LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \ - LC_TELEPHONE LC_TIME -do - if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then - eval $as_var=C; export $as_var - else - $as_unset $as_var - fi -done - -# Required to use basename. -if expr a : '\(a\)' >/dev/null 2>&1; then - as_expr=expr -else - as_expr=false -fi - -if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then - as_basename=basename -else - as_basename=false -fi - - -# Name of the executable. -as_me=`$as_basename "$0" || -$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ - X"$0" : 'X\(//\)$' \| \ - X"$0" : 'X\(/\)$' \| \ - . : '\(.\)' 2>/dev/null || -echo X/"$0" | - sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; } - /^X\/\(\/\/\)$/{ s//\1/; q; } - /^X\/\(\/\).*/{ s//\1/; q; } - s/.*/./; q'` - - -# PATH needs CR, and LINENO needs CR and PATH. -# Avoid depending upon Character Ranges. -as_cr_letters='abcdefghijklmnopqrstuvwxyz' -as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' -as_cr_Letters=$as_cr_letters$as_cr_LETTERS -as_cr_digits='0123456789' -as_cr_alnum=$as_cr_Letters$as_cr_digits - -# The user is always right. -if test "${PATH_SEPARATOR+set}" != set; then - echo "#! /bin/sh" >conf$$.sh - echo "exit 0" >>conf$$.sh - chmod +x conf$$.sh - if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then - PATH_SEPARATOR=';' - else - PATH_SEPARATOR=: - fi - rm -f conf$$.sh -fi - - - as_lineno_1=$LINENO - as_lineno_2=$LINENO - as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` - test "x$as_lineno_1" != "x$as_lineno_2" && - test "x$as_lineno_3" = "x$as_lineno_2" || { - # Find who we are. Look in the path if we contain no path at all - # relative or not. - case $0 in - *[\\/]* ) as_myself=$0 ;; - *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break -done - - ;; - esac - # We did not find ourselves, most probably we were run as `sh COMMAND' - # in which case we are not to be found in the path. - if test "x$as_myself" = x; then - as_myself=$0 - fi - if test ! -f "$as_myself"; then - { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2 - { (exit 1); exit 1; }; } - fi - case $CONFIG_SHELL in - '') - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for as_base in sh bash ksh sh5; do - case $as_dir in - /*) - if ("$as_dir/$as_base" -c ' - as_lineno_1=$LINENO - as_lineno_2=$LINENO - as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` - test "x$as_lineno_1" != "x$as_lineno_2" && - test "x$as_lineno_3" = "x$as_lineno_2" ') 2>/dev/null; then - $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; } - $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; } - CONFIG_SHELL=$as_dir/$as_base - export CONFIG_SHELL - exec "$CONFIG_SHELL" "$0" ${1+"$@"} - fi;; - esac - done -done -;; - esac - - # Create $as_me.lineno as a copy of $as_myself, but with $LINENO - # uniformly replaced by the line number. The first 'sed' inserts a - # line-number line before each line; the second 'sed' does the real - # work. The second script uses 'N' to pair each line-number line - # with the numbered line, and appends trailing '-' during - # substitution so that $LINENO is not a special case at line end. - # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the - # second 'sed' script. Blame Lee E. McMahon for sed's syntax. :-) - sed '=' <$as_myself | - sed ' - N - s,$,-, - : loop - s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3, - t loop - s,-$,, - s,^['$as_cr_digits']*\n,, - ' >$as_me.lineno && - chmod +x $as_me.lineno || - { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2 - { (exit 1); exit 1; }; } - - # Don't try to exec as it changes $[0], causing all sort of problems - # (the dirname of $[0] is not the place where we might find the - # original and so on. Autoconf is especially sensible to this). - . ./$as_me.lineno - # Exit status is that of the last command. - exit -} - - -case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in - *c*,-n*) ECHO_N= ECHO_C=' -' ECHO_T=' ' ;; - *c*,* ) ECHO_N=-n ECHO_C= ECHO_T= ;; - *) ECHO_N= ECHO_C='\c' ECHO_T= ;; -esac - -if expr a : '\(a\)' >/dev/null 2>&1; then - as_expr=expr -else - as_expr=false -fi - -rm -f conf$$ conf$$.exe conf$$.file -echo >conf$$.file -if ln -s conf$$.file conf$$ 2>/dev/null; then - # We could just check for DJGPP; but this test a) works b) is more generic - # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04). - if test -f conf$$.exe; then - # Don't use ln at all; we don't have any links - as_ln_s='cp -p' - else - as_ln_s='ln -s' - fi -elif ln conf$$.file conf$$ 2>/dev/null; then - as_ln_s=ln -else - as_ln_s='cp -p' -fi -rm -f conf$$ conf$$.exe conf$$.file - -if mkdir -p . 2>/dev/null; then - as_mkdir_p=: -else - test -d ./-p && rmdir ./-p - as_mkdir_p=false -fi - -as_executable_p="test -f" - -# Sed expression to map a string onto a valid CPP name. -as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" - -# Sed expression to map a string onto a valid variable name. -as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" - - -# IFS -# We need space, tab and new line, in precisely that order. -as_nl=' -' -IFS=" $as_nl" - -# CDPATH. -$as_unset CDPATH - - -# Name of the host. -# hostname on some systems (SVR3.2, Linux) returns a bogus exit status, -# so uname gets run too. -ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` - -exec 6>&1 - -# -# Initializations. -# -ac_default_prefix=/usr/local -ac_config_libobj_dir=. -cross_compiling=no -subdirs= -MFLAGS= -MAKEFLAGS= -SHELL=${CONFIG_SHELL-/bin/sh} - -# Maximum number of lines to put in a shell here document. -# This variable seems obsolete. It should probably be removed, and -# only ac_max_sed_lines should be used. -: ${ac_max_here_lines=38} - -# Identity of this package. -PACKAGE_NAME='qdbm' -PACKAGE_TARNAME='qdbm' -PACKAGE_VERSION='1.8.77' -PACKAGE_STRING='qdbm 1.8.77' -PACKAGE_BUGREPORT='' - -ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS CC CFLAGS LDFLAGS CPPFLAGS ac_ct_CC EXEEXT OBJEXT LIBVER LIBREV TARGETS MYDEFS MYOPTS MGWLIBS LD AR LIBOBJS LTLIBOBJS' -ac_subst_files='' - -# Initialize some variables set by options. -ac_init_help= -ac_init_version=false -# The variables have the same names as the options, with -# dashes changed to underlines. -cache_file=/dev/null -exec_prefix=NONE -no_create= -no_recursion= -prefix=NONE -program_prefix=NONE -program_suffix=NONE -program_transform_name=s,x,x, -silent= -site= -srcdir= -verbose= -x_includes=NONE -x_libraries=NONE - -# Installation directory options. -# These are left unexpanded so users can "make install exec_prefix=/foo" -# and all the variables that are supposed to be based on exec_prefix -# by default will actually change. -# Use braces instead of parens because sh, perl, etc. also accept them. -bindir='${exec_prefix}/bin' -sbindir='${exec_prefix}/sbin' -libexecdir='${exec_prefix}/libexec' -datadir='${prefix}/share' -sysconfdir='${prefix}/etc' -sharedstatedir='${prefix}/com' -localstatedir='${prefix}/var' -libdir='${exec_prefix}/lib' -includedir='${prefix}/include' -oldincludedir='/usr/include' -infodir='${prefix}/info' -mandir='${prefix}/man' - -ac_prev= -for ac_option -do - # If the previous option needs an argument, assign it. - if test -n "$ac_prev"; then - eval "$ac_prev=\$ac_option" - ac_prev= - continue - fi - - ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'` - - # Accept the important Cygnus configure options, so we can diagnose typos. - - case $ac_option in - - -bindir | --bindir | --bindi | --bind | --bin | --bi) - ac_prev=bindir ;; - -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*) - bindir=$ac_optarg ;; - - -build | --build | --buil | --bui | --bu) - ac_prev=build_alias ;; - -build=* | --build=* | --buil=* | --bui=* | --bu=*) - build_alias=$ac_optarg ;; - - -cache-file | --cache-file | --cache-fil | --cache-fi \ - | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) - ac_prev=cache_file ;; - -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ - | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*) - cache_file=$ac_optarg ;; - - --config-cache | -C) - cache_file=config.cache ;; - - -datadir | --datadir | --datadi | --datad | --data | --dat | --da) - ac_prev=datadir ;; - -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \ - | --da=*) - datadir=$ac_optarg ;; - - -disable-* | --disable-*) - ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'` - # Reject names that are not valid shell variable names. - expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null && - { echo "$as_me: error: invalid feature name: $ac_feature" >&2 - { (exit 1); exit 1; }; } - ac_feature=`echo $ac_feature | sed 's/-/_/g'` - eval "enable_$ac_feature=no" ;; - - -enable-* | --enable-*) - ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` - # Reject names that are not valid shell variable names. - expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null && - { echo "$as_me: error: invalid feature name: $ac_feature" >&2 - { (exit 1); exit 1; }; } - ac_feature=`echo $ac_feature | sed 's/-/_/g'` - case $ac_option in - *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;; - *) ac_optarg=yes ;; - esac - eval "enable_$ac_feature='$ac_optarg'" ;; - - -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \ - | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \ - | --exec | --exe | --ex) - ac_prev=exec_prefix ;; - -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \ - | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \ - | --exec=* | --exe=* | --ex=*) - exec_prefix=$ac_optarg ;; - - -gas | --gas | --ga | --g) - # Obsolete; use --with-gas. - with_gas=yes ;; - - -help | --help | --hel | --he | -h) - ac_init_help=long ;; - -help=r* | --help=r* | --hel=r* | --he=r* | -hr*) - ac_init_help=recursive ;; - -help=s* | --help=s* | --hel=s* | --he=s* | -hs*) - ac_init_help=short ;; - - -host | --host | --hos | --ho) - ac_prev=host_alias ;; - -host=* | --host=* | --hos=* | --ho=*) - host_alias=$ac_optarg ;; - - -includedir | --includedir | --includedi | --included | --include \ - | --includ | --inclu | --incl | --inc) - ac_prev=includedir ;; - -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \ - | --includ=* | --inclu=* | --incl=* | --inc=*) - includedir=$ac_optarg ;; - - -infodir | --infodir | --infodi | --infod | --info | --inf) - ac_prev=infodir ;; - -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*) - infodir=$ac_optarg ;; - - -libdir | --libdir | --libdi | --libd) - ac_prev=libdir ;; - -libdir=* | --libdir=* | --libdi=* | --libd=*) - libdir=$ac_optarg ;; - - -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \ - | --libexe | --libex | --libe) - ac_prev=libexecdir ;; - -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \ - | --libexe=* | --libex=* | --libe=*) - libexecdir=$ac_optarg ;; - - -localstatedir | --localstatedir | --localstatedi | --localstated \ - | --localstate | --localstat | --localsta | --localst \ - | --locals | --local | --loca | --loc | --lo) - ac_prev=localstatedir ;; - -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \ - | --localstate=* | --localstat=* | --localsta=* | --localst=* \ - | --locals=* | --local=* | --loca=* | --loc=* | --lo=*) - localstatedir=$ac_optarg ;; - - -mandir | --mandir | --mandi | --mand | --man | --ma | --m) - ac_prev=mandir ;; - -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*) - mandir=$ac_optarg ;; - - -nfp | --nfp | --nf) - # Obsolete; use --without-fp. - with_fp=no ;; - - -no-create | --no-create | --no-creat | --no-crea | --no-cre \ - | --no-cr | --no-c | -n) - no_create=yes ;; - - -no-recursion | --no-recursion | --no-recursio | --no-recursi \ - | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) - no_recursion=yes ;; - - -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \ - | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \ - | --oldin | --oldi | --old | --ol | --o) - ac_prev=oldincludedir ;; - -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \ - | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \ - | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*) - oldincludedir=$ac_optarg ;; - - -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) - ac_prev=prefix ;; - -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) - prefix=$ac_optarg ;; - - -program-prefix | --program-prefix | --program-prefi | --program-pref \ - | --program-pre | --program-pr | --program-p) - ac_prev=program_prefix ;; - -program-prefix=* | --program-prefix=* | --program-prefi=* \ - | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*) - program_prefix=$ac_optarg ;; - - -program-suffix | --program-suffix | --program-suffi | --program-suff \ - | --program-suf | --program-su | --program-s) - ac_prev=program_suffix ;; - -program-suffix=* | --program-suffix=* | --program-suffi=* \ - | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*) - program_suffix=$ac_optarg ;; - - -program-transform-name | --program-transform-name \ - | --program-transform-nam | --program-transform-na \ - | --program-transform-n | --program-transform- \ - | --program-transform | --program-transfor \ - | --program-transfo | --program-transf \ - | --program-trans | --program-tran \ - | --progr-tra | --program-tr | --program-t) - ac_prev=program_transform_name ;; - -program-transform-name=* | --program-transform-name=* \ - | --program-transform-nam=* | --program-transform-na=* \ - | --program-transform-n=* | --program-transform-=* \ - | --program-transform=* | --program-transfor=* \ - | --program-transfo=* | --program-transf=* \ - | --program-trans=* | --program-tran=* \ - | --progr-tra=* | --program-tr=* | --program-t=*) - program_transform_name=$ac_optarg ;; - - -q | -quiet | --quiet | --quie | --qui | --qu | --q \ - | -silent | --silent | --silen | --sile | --sil) - silent=yes ;; - - -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) - ac_prev=sbindir ;; - -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ - | --sbi=* | --sb=*) - sbindir=$ac_optarg ;; - - -sharedstatedir | --sharedstatedir | --sharedstatedi \ - | --sharedstated | --sharedstate | --sharedstat | --sharedsta \ - | --sharedst | --shareds | --shared | --share | --shar \ - | --sha | --sh) - ac_prev=sharedstatedir ;; - -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \ - | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \ - | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \ - | --sha=* | --sh=*) - sharedstatedir=$ac_optarg ;; - - -site | --site | --sit) - ac_prev=site ;; - -site=* | --site=* | --sit=*) - site=$ac_optarg ;; - - -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) - ac_prev=srcdir ;; - -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) - srcdir=$ac_optarg ;; - - -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \ - | --syscon | --sysco | --sysc | --sys | --sy) - ac_prev=sysconfdir ;; - -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \ - | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*) - sysconfdir=$ac_optarg ;; - - -target | --target | --targe | --targ | --tar | --ta | --t) - ac_prev=target_alias ;; - -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*) - target_alias=$ac_optarg ;; - - -v | -verbose | --verbose | --verbos | --verbo | --verb) - verbose=yes ;; - - -version | --version | --versio | --versi | --vers | -V) - ac_init_version=: ;; - - -with-* | --with-*) - ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` - # Reject names that are not valid shell variable names. - expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null && - { echo "$as_me: error: invalid package name: $ac_package" >&2 - { (exit 1); exit 1; }; } - ac_package=`echo $ac_package| sed 's/-/_/g'` - case $ac_option in - *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;; - *) ac_optarg=yes ;; - esac - eval "with_$ac_package='$ac_optarg'" ;; - - -without-* | --without-*) - ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'` - # Reject names that are not valid shell variable names. - expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null && - { echo "$as_me: error: invalid package name: $ac_package" >&2 - { (exit 1); exit 1; }; } - ac_package=`echo $ac_package | sed 's/-/_/g'` - eval "with_$ac_package=no" ;; - - --x) - # Obsolete; use --with-x. - with_x=yes ;; - - -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \ - | --x-incl | --x-inc | --x-in | --x-i) - ac_prev=x_includes ;; - -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \ - | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*) - x_includes=$ac_optarg ;; - - -x-libraries | --x-libraries | --x-librarie | --x-librari \ - | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l) - ac_prev=x_libraries ;; - -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \ - | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) - x_libraries=$ac_optarg ;; - - -*) { echo "$as_me: error: unrecognized option: $ac_option -Try \`$0 --help' for more information." >&2 - { (exit 1); exit 1; }; } - ;; - - *=*) - ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='` - # Reject names that are not valid shell variable names. - expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null && - { echo "$as_me: error: invalid variable name: $ac_envvar" >&2 - { (exit 1); exit 1; }; } - ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` - eval "$ac_envvar='$ac_optarg'" - export $ac_envvar ;; - - *) - # FIXME: should be removed in autoconf 3.0. - echo "$as_me: WARNING: you should use --build, --host, --target" >&2 - expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && - echo "$as_me: WARNING: invalid host type: $ac_option" >&2 - : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option} - ;; - - esac -done - -if test -n "$ac_prev"; then - ac_option=--`echo $ac_prev | sed 's/_/-/g'` - { echo "$as_me: error: missing argument to $ac_option" >&2 - { (exit 1); exit 1; }; } -fi - -# Be sure to have absolute paths. -for ac_var in exec_prefix prefix -do - eval ac_val=$`echo $ac_var` - case $ac_val in - [\\/$]* | ?:[\\/]* | NONE | '' ) ;; - *) { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2 - { (exit 1); exit 1; }; };; - esac -done - -# Be sure to have absolute paths. -for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \ - localstatedir libdir includedir oldincludedir infodir mandir -do - eval ac_val=$`echo $ac_var` - case $ac_val in - [\\/$]* | ?:[\\/]* ) ;; - *) { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2 - { (exit 1); exit 1; }; };; - esac -done - -# There might be people who depend on the old broken behavior: `$host' -# used to hold the argument of --host etc. -# FIXME: To remove some day. -build=$build_alias -host=$host_alias -target=$target_alias - -# FIXME: To remove some day. -if test "x$host_alias" != x; then - if test "x$build_alias" = x; then - cross_compiling=maybe - echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host. - If a cross compiler is detected then cross compile mode will be used." >&2 - elif test "x$build_alias" != "x$host_alias"; then - cross_compiling=yes - fi -fi - -ac_tool_prefix= -test -n "$host_alias" && ac_tool_prefix=$host_alias- - -test "$silent" = yes && exec 6>/dev/null - - -# Find the source files, if location was not specified. -if test -z "$srcdir"; then - ac_srcdir_defaulted=yes - # Try the directory containing this script, then its parent. - ac_confdir=`(dirname "$0") 2>/dev/null || -$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ - X"$0" : 'X\(//\)[^/]' \| \ - X"$0" : 'X\(//\)$' \| \ - X"$0" : 'X\(/\)' \| \ - . : '\(.\)' 2>/dev/null || -echo X"$0" | - sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } - /^X\(\/\/\)[^/].*/{ s//\1/; q; } - /^X\(\/\/\)$/{ s//\1/; q; } - /^X\(\/\).*/{ s//\1/; q; } - s/.*/./; q'` - srcdir=$ac_confdir - if test ! -r $srcdir/$ac_unique_file; then - srcdir=.. - fi -else - ac_srcdir_defaulted=no -fi -if test ! -r $srcdir/$ac_unique_file; then - if test "$ac_srcdir_defaulted" = yes; then - { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2 - { (exit 1); exit 1; }; } - else - { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2 - { (exit 1); exit 1; }; } - fi -fi -(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null || - { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2 - { (exit 1); exit 1; }; } -srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'` -ac_env_build_alias_set=${build_alias+set} -ac_env_build_alias_value=$build_alias -ac_cv_env_build_alias_set=${build_alias+set} -ac_cv_env_build_alias_value=$build_alias -ac_env_host_alias_set=${host_alias+set} -ac_env_host_alias_value=$host_alias -ac_cv_env_host_alias_set=${host_alias+set} -ac_cv_env_host_alias_value=$host_alias -ac_env_target_alias_set=${target_alias+set} -ac_env_target_alias_value=$target_alias -ac_cv_env_target_alias_set=${target_alias+set} -ac_cv_env_target_alias_value=$target_alias -ac_env_CC_set=${CC+set} -ac_env_CC_value=$CC -ac_cv_env_CC_set=${CC+set} -ac_cv_env_CC_value=$CC -ac_env_CFLAGS_set=${CFLAGS+set} -ac_env_CFLAGS_value=$CFLAGS -ac_cv_env_CFLAGS_set=${CFLAGS+set} -ac_cv_env_CFLAGS_value=$CFLAGS -ac_env_LDFLAGS_set=${LDFLAGS+set} -ac_env_LDFLAGS_value=$LDFLAGS -ac_cv_env_LDFLAGS_set=${LDFLAGS+set} -ac_cv_env_LDFLAGS_value=$LDFLAGS -ac_env_CPPFLAGS_set=${CPPFLAGS+set} -ac_env_CPPFLAGS_value=$CPPFLAGS -ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set} -ac_cv_env_CPPFLAGS_value=$CPPFLAGS - -# -# Report the --help message. -# -if test "$ac_init_help" = "long"; then - # Omit some internal or obsolete options to make the list less imposing. - # This message is too long to be a string in the A/UX 3.1 sh. - cat <<_ACEOF -\`configure' configures qdbm 1.8.77 to adapt to many kinds of systems. - -Usage: $0 [OPTION]... [VAR=VALUE]... - -To assign environment variables (e.g., CC, CFLAGS...), specify them as -VAR=VALUE. See below for descriptions of some of the useful variables. - -Defaults for the options are specified in brackets. - -Configuration: - -h, --help display this help and exit - --help=short display options specific to this package - --help=recursive display the short help of all the included packages - -V, --version display version information and exit - -q, --quiet, --silent do not print \`checking...' messages - --cache-file=FILE cache test results in FILE [disabled] - -C, --config-cache alias for \`--cache-file=config.cache' - -n, --no-create do not create output files - --srcdir=DIR find the sources in DIR [configure dir or \`..'] - -_ACEOF - - cat <<_ACEOF -Installation directories: - --prefix=PREFIX install architecture-independent files in PREFIX - [$ac_default_prefix] - --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX - [PREFIX] - -By default, \`make install' will install all the files in -\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify -an installation prefix other than \`$ac_default_prefix' using \`--prefix', -for instance \`--prefix=\$HOME'. - -For better control, use the options below. - -Fine tuning of the installation directories: - --bindir=DIR user executables [EPREFIX/bin] - --sbindir=DIR system admin executables [EPREFIX/sbin] - --libexecdir=DIR program executables [EPREFIX/libexec] - --datadir=DIR read-only architecture-independent data [PREFIX/share] - --sysconfdir=DIR read-only single-machine data [PREFIX/etc] - --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] - --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --libdir=DIR object code libraries [EPREFIX/lib] - --includedir=DIR C header files [PREFIX/include] - --oldincludedir=DIR C header files for non-gcc [/usr/include] - --infodir=DIR info documentation [PREFIX/info] - --mandir=DIR man documentation [PREFIX/man] -_ACEOF - - cat <<\_ACEOF -_ACEOF -fi - -if test -n "$ac_init_help"; then - case $ac_init_help in - short | recursive ) echo "Configuration of qdbm 1.8.77:";; - esac - cat <<\_ACEOF - -Optional Features: - --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) - --enable-FEATURE[=ARG] include FEATURE [ARG=yes] - --enable-debug build for debugging - --enable-devel build for development - --enable-stable build for stable release - --enable-pthread use POSIX thread and make APIs thread-safe - --disable-lock build for environments without file locking - --disable-mmap build for environments without memory mapping - --enable-zlib feature ZLIB for B+ tree and inverted index - --enable-lzo feature LZO for B+ tree and inverted index - --enable-bzip feature BZIP2 for B+ tree and inverted index - --enable-iconv feature ICONV utilities - --disable-warn hide warnings in the configuration - -Some influential environment variables: - CC C compiler command - CFLAGS C compiler flags - LDFLAGS linker flags, e.g. -L<lib dir> if you have libraries in a - nonstandard directory <lib dir> - CPPFLAGS C/C++ preprocessor flags, e.g. -I<include dir> if you have - headers in a nonstandard directory <include dir> - -Use these variables to override the choices made by `configure' or to help -it to find libraries and programs with nonstandard names/locations. - -_ACEOF -fi - -if test "$ac_init_help" = "recursive"; then - # If there are subdirs, report their specific --help. - ac_popdir=`pwd` - for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue - test -d $ac_dir || continue - ac_builddir=. - -if test "$ac_dir" != .; then - ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'` - # A "../" for each directory in $ac_dir_suffix. - ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'` -else - ac_dir_suffix= ac_top_builddir= -fi - -case $srcdir in - .) # No --srcdir option. We are building in place. - ac_srcdir=. - if test -z "$ac_top_builddir"; then - ac_top_srcdir=. - else - ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'` - fi ;; - [\\/]* | ?:[\\/]* ) # Absolute path. - ac_srcdir=$srcdir$ac_dir_suffix; - ac_top_srcdir=$srcdir ;; - *) # Relative path. - ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix - ac_top_srcdir=$ac_top_builddir$srcdir ;; -esac - -# Do not use `cd foo && pwd` to compute absolute paths, because -# the directories may not exist. -case `pwd` in -.) ac_abs_builddir="$ac_dir";; -*) - case "$ac_dir" in - .) ac_abs_builddir=`pwd`;; - [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";; - *) ac_abs_builddir=`pwd`/"$ac_dir";; - esac;; -esac -case $ac_abs_builddir in -.) ac_abs_top_builddir=${ac_top_builddir}.;; -*) - case ${ac_top_builddir}. in - .) ac_abs_top_builddir=$ac_abs_builddir;; - [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;; - *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;; - esac;; -esac -case $ac_abs_builddir in -.) ac_abs_srcdir=$ac_srcdir;; -*) - case $ac_srcdir in - .) ac_abs_srcdir=$ac_abs_builddir;; - [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;; - *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;; - esac;; -esac -case $ac_abs_builddir in -.) ac_abs_top_srcdir=$ac_top_srcdir;; -*) - case $ac_top_srcdir in - .) ac_abs_top_srcdir=$ac_abs_builddir;; - [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;; - *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;; - esac;; -esac - - cd $ac_dir - # Check for guested configure; otherwise get Cygnus style configure. - if test -f $ac_srcdir/configure.gnu; then - echo - $SHELL $ac_srcdir/configure.gnu --help=recursive - elif test -f $ac_srcdir/configure; then - echo - $SHELL $ac_srcdir/configure --help=recursive - elif test -f $ac_srcdir/configure.ac || - test -f $ac_srcdir/configure.in; then - echo - $ac_configure --help - else - echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2 - fi - cd $ac_popdir - done -fi - -test -n "$ac_init_help" && exit 0 -if $ac_init_version; then - cat <<\_ACEOF -qdbm configure 1.8.77 -generated by GNU Autoconf 2.59 - -Copyright (C) 2003 Free Software Foundation, Inc. -This configure script is free software; the Free Software Foundation -gives unlimited permission to copy, distribute and modify it. -_ACEOF - exit 0 -fi -exec 5>config.log -cat >&5 <<_ACEOF -This file contains any messages produced by compilers while -running configure, to aid debugging if configure makes a mistake. - -It was created by qdbm $as_me 1.8.77, which was -generated by GNU Autoconf 2.59. Invocation command line was - - $ $0 $@ - -_ACEOF -{ -cat <<_ASUNAME -## --------- ## -## Platform. ## -## --------- ## - -hostname = `(hostname || uname -n) 2>/dev/null | sed 1q` -uname -m = `(uname -m) 2>/dev/null || echo unknown` -uname -r = `(uname -r) 2>/dev/null || echo unknown` -uname -s = `(uname -s) 2>/dev/null || echo unknown` -uname -v = `(uname -v) 2>/dev/null || echo unknown` - -/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown` -/bin/uname -X = `(/bin/uname -X) 2>/dev/null || echo unknown` - -/bin/arch = `(/bin/arch) 2>/dev/null || echo unknown` -/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null || echo unknown` -/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown` -hostinfo = `(hostinfo) 2>/dev/null || echo unknown` -/bin/machine = `(/bin/machine) 2>/dev/null || echo unknown` -/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null || echo unknown` -/bin/universe = `(/bin/universe) 2>/dev/null || echo unknown` - -_ASUNAME - -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - echo "PATH: $as_dir" -done - -} >&5 - -cat >&5 <<_ACEOF - - -## ----------- ## -## Core tests. ## -## ----------- ## - -_ACEOF - - -# Keep a trace of the command line. -# Strip out --no-create and --no-recursion so they do not pile up. -# Strip out --silent because we don't want to record it for future runs. -# Also quote any args containing shell meta-characters. -# Make two passes to allow for proper duplicate-argument suppression. -ac_configure_args= -ac_configure_args0= -ac_configure_args1= -ac_sep= -ac_must_keep_next=false -for ac_pass in 1 2 -do - for ac_arg - do - case $ac_arg in - -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;; - -q | -quiet | --quiet | --quie | --qui | --qu | --q \ - | -silent | --silent | --silen | --sile | --sil) - continue ;; - *" "*|*" "*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*) - ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; - esac - case $ac_pass in - 1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;; - 2) - ac_configure_args1="$ac_configure_args1 '$ac_arg'" - if test $ac_must_keep_next = true; then - ac_must_keep_next=false # Got value, back to normal. - else - case $ac_arg in - *=* | --config-cache | -C | -disable-* | --disable-* \ - | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \ - | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \ - | -with-* | --with-* | -without-* | --without-* | --x) - case "$ac_configure_args0 " in - "$ac_configure_args1"*" '$ac_arg' "* ) continue ;; - esac - ;; - -* ) ac_must_keep_next=true ;; - esac - fi - ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'" - # Get rid of the leading space. - ac_sep=" " - ;; - esac - done -done -$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; } -$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; } - -# When interrupted or exit'd, cleanup temporary files, and complete -# config.log. We remove comments because anyway the quotes in there -# would cause problems or look ugly. -# WARNING: Be sure not to use single quotes in there, as some shells, -# such as our DU 5.0 friend, will then `close' the trap. -trap 'exit_status=$? - # Save into config.log some information that might help in debugging. - { - echo - - cat <<\_ASBOX -## ---------------- ## -## Cache variables. ## -## ---------------- ## -_ASBOX - echo - # The following way of writing the cache mishandles newlines in values, -{ - (set) 2>&1 | - case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in - *ac_space=\ *) - sed -n \ - "s/'"'"'/'"'"'\\\\'"'"''"'"'/g; - s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p" - ;; - *) - sed -n \ - "s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p" - ;; - esac; -} - echo - - cat <<\_ASBOX -## ----------------- ## -## Output variables. ## -## ----------------- ## -_ASBOX - echo - for ac_var in $ac_subst_vars - do - eval ac_val=$`echo $ac_var` - echo "$ac_var='"'"'$ac_val'"'"'" - done | sort - echo - - if test -n "$ac_subst_files"; then - cat <<\_ASBOX -## ------------- ## -## Output files. ## -## ------------- ## -_ASBOX - echo - for ac_var in $ac_subst_files - do - eval ac_val=$`echo $ac_var` - echo "$ac_var='"'"'$ac_val'"'"'" - done | sort - echo - fi - - if test -s confdefs.h; then - cat <<\_ASBOX -## ----------- ## -## confdefs.h. ## -## ----------- ## -_ASBOX - echo - sed "/^$/d" confdefs.h | sort - echo - fi - test "$ac_signal" != 0 && - echo "$as_me: caught signal $ac_signal" - echo "$as_me: exit $exit_status" - } >&5 - rm -f core *.core && - rm -rf conftest* confdefs* conf$$* $ac_clean_files && - exit $exit_status - ' 0 -for ac_signal in 1 2 13 15; do - trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal -done -ac_signal=0 - -# confdefs.h avoids OS command line length limits that DEFS can exceed. -rm -rf conftest* confdefs.h -# AIX cpp loses on an empty file, so make sure it contains at least a newline. -echo >confdefs.h - -# Predefined preprocessor variables. - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_NAME "$PACKAGE_NAME" -_ACEOF - - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_TARNAME "$PACKAGE_TARNAME" -_ACEOF - - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_VERSION "$PACKAGE_VERSION" -_ACEOF - - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_STRING "$PACKAGE_STRING" -_ACEOF - - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT" -_ACEOF - - -# Let the site file select an alternate cache file if it wants to. -# Prefer explicitly selected file to automatically selected ones. -if test -z "$CONFIG_SITE"; then - if test "x$prefix" != xNONE; then - CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site" - else - CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site" - fi -fi -for ac_site_file in $CONFIG_SITE; do - if test -r "$ac_site_file"; then - { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5 -echo "$as_me: loading site script $ac_site_file" >&6;} - sed 's/^/| /' "$ac_site_file" >&5 - . "$ac_site_file" - fi -done - -if test -r "$cache_file"; then - # Some versions of bash will fail to source /dev/null (special - # files actually), so we avoid doing that. - if test -f "$cache_file"; then - { echo "$as_me:$LINENO: loading cache $cache_file" >&5 -echo "$as_me: loading cache $cache_file" >&6;} - case $cache_file in - [\\/]* | ?:[\\/]* ) . $cache_file;; - *) . ./$cache_file;; - esac - fi -else - { echo "$as_me:$LINENO: creating cache $cache_file" >&5 -echo "$as_me: creating cache $cache_file" >&6;} - >$cache_file -fi - -# Check that the precious variables saved in the cache have kept the same -# value. -ac_cache_corrupted=false -for ac_var in `(set) 2>&1 | - sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do - eval ac_old_set=\$ac_cv_env_${ac_var}_set - eval ac_new_set=\$ac_env_${ac_var}_set - eval ac_old_val="\$ac_cv_env_${ac_var}_value" - eval ac_new_val="\$ac_env_${ac_var}_value" - case $ac_old_set,$ac_new_set in - set,) - { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 -echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} - ac_cache_corrupted=: ;; - ,set) - { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5 -echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} - ac_cache_corrupted=: ;; - ,);; - *) - if test "x$ac_old_val" != "x$ac_new_val"; then - { echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5 -echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} - { echo "$as_me:$LINENO: former value: $ac_old_val" >&5 -echo "$as_me: former value: $ac_old_val" >&2;} - { echo "$as_me:$LINENO: current value: $ac_new_val" >&5 -echo "$as_me: current value: $ac_new_val" >&2;} - ac_cache_corrupted=: - fi;; - esac - # Pass precious variables to config.status. - if test "$ac_new_set" = set; then - case $ac_new_val in - *" "*|*" "*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*) - ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; - *) ac_arg=$ac_var=$ac_new_val ;; - esac - case " $ac_configure_args " in - *" '$ac_arg' "*) ;; # Avoid dups. Use of quotes ensures accuracy. - *) ac_configure_args="$ac_configure_args '$ac_arg'" ;; - esac - fi -done -if $ac_cache_corrupted; then - { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5 -echo "$as_me: error: changes in the environment can compromise the build" >&2;} - { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5 -echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;} - { (exit 1); exit 1; }; } -fi - -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Export variables -LIBVER=14 -LIBREV=13 -TARGETS="all" -MYDEFS="" -MYOPTS="" -MGWLIBS="" -LD="ld" -AR="ar" - -# Building paths -pathtmp="$PATH" -PATH="$HOME/bin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin" -PATH="$PATH:/usr/ccs/bin:/usr/ucb:/usr/xpg4/bin:/usr/xpg6/bin:$pathtmp" -LIBRARY_PATH="$HOME/lib:/usr/local/lib:$LIBRARY_PATH" -LD_LIBRARY_PATH="$HOME/lib:/usr/local/lib:$LD_LIBRARY_PATH" -CPATH="$HOME/include:/usr/local/include:$CPATH" -export PATH LIBRARY_PATH LD_LIBRARY_PATH CPATH - - - -#================================================================ -# Options -#================================================================ - - -# Internal variables -enables="" -ispthread="" -iszlib="" -isiconv="" -isnowarn="" - -# Debug mode -# Check whether --enable-debug or --disable-debug was given. -if test "${enable_debug+set}" = set; then - enableval="$enable_debug" - -fi; -if test "$enable_debug" = "yes" -then - TARGETS="debug" - enables="$enables (debug)" -fi - -# Developping mode -# Check whether --enable-devel or --disable-devel was given. -if test "${enable_devel+set}" = set; then - enableval="$enable_devel" - -fi; -if test "$enable_devel" = "yes" -then - TARGETS="devel" - enables="$enables (devel)" -fi - -# Stable mode -# Check whether --enable-stable or --disable-stable was given. -if test "${enable_stable+set}" = set; then - enableval="$enable_stable" - -fi; -if test "$enable_stable" = "yes" -then - TARGETS="stable" - enables="$enables (stable)" -fi - -# Enable POSIX thread -# Check whether --enable-pthread or --disable-pthread was given. -if test "${enable_pthread+set}" = set; then - enableval="$enable_pthread" - -fi; -if test "$enable_pthread" = "yes" -then - MYDEFS="$MYDEFS -DMYPTHREAD" - enables="$enables (pthread)" - ispthread="yes" -fi - -# Disable file locking -# Check whether --enable-lock or --disable-lock was given. -if test "${enable_lock+set}" = set; then - enableval="$enable_lock" - -fi; -if test "$enable_lock" = "no" -then - MYDEFS="$MYDEFS -DMYNOLOCK" - enables="$enables (no-lock)" -fi - -# Disable memory mapping -# Check whether --enable-mmap or --disable-mmap was given. -if test "${enable_mmap+set}" = set; then - enableval="$enable_mmap" - -fi; -if test "$enable_mmap" = "no" -then - MYDEFS="$MYDEFS -DMYNOMMAP" - enables="$enables (no-mmap)" -fi - -# Enable ZLIB compression -# Check whether --enable-zlib or --disable-zlib was given. -if test "${enable_zlib+set}" = set; then - enableval="$enable_zlib" - -fi; -if test "$enable_zlib" = "yes" -then - MYDEFS="$MYDEFS -DMYZLIB" - MGWLIBS="-lz $MGWLIBS" - enables="$enables (zlib)" - iszlib="yes" -fi - -# Enable LZO compression -# Check whether --enable-lzo or --disable-lzo was given. -if test "${enable_lzo+set}" = set; then - enableval="$enable_lzo" - -fi; -if test "$enable_lzo" = "yes" -then - MYDEFS="$MYDEFS -DMYLZO" - MGWLIBS="-llzo2 $MGWLIBS" - enables="$enables (lzo)" - islzo="yes" -fi - -# Enable BZIP2 compression -# Check whether --enable-bzip or --disable-bzip was given. -if test "${enable_bzip+set}" = set; then - enableval="$enable_bzip" - -fi; -if test "$enable_bzip" = "yes" -then - MYDEFS="$MYDEFS -DMYBZIP" - MGWLIBS="-lbz2 $MGWLIBS" - enables="$enables (bzip)" - isbzip="yes" -fi - -# Enable ICONV utilities -# Check whether --enable-iconv or --disable-iconv was given. -if test "${enable_iconv+set}" = set; then - enableval="$enable_iconv" - -fi; -if test "$enable_iconv" = "yes" -then - MYDEFS="$MYDEFS -DMYICONV" - MGWLIBS="-liconv $MGWLIBS" - enables="$enables (iconv)" - isiconv="yes" -fi - -# No warning configuration -# Check whether --enable-warn or --disable-warn was given. -if test "${enable_warn+set}" = set; then - enableval="$enable_warn" - -fi; -if test "$enable_warn" = "no" -then - isnowarn="yes" -fi - -# Messages -printf '#================================================================\n' -printf '# Configuring QDBM version %s%s.\n' "$PACKAGE_VERSION" "$enables" -printf '#================================================================\n' - - - -#================================================================ -# Checking Commands to Build with -#================================================================ - - -# C compiler -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu -if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args. -set dummy ${ac_tool_prefix}gcc; ac_word=$2 -echo "$as_me:$LINENO: checking for $ac_word" >&5 -echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 -if test "${ac_cv_prog_CC+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="${ac_tool_prefix}gcc" - echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done -done - -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - echo "$as_me:$LINENO: result: $CC" >&5 -echo "${ECHO_T}$CC" >&6 -else - echo "$as_me:$LINENO: result: no" >&5 -echo "${ECHO_T}no" >&6 -fi - -fi -if test -z "$ac_cv_prog_CC"; then - ac_ct_CC=$CC - # Extract the first word of "gcc", so it can be a program name with args. -set dummy gcc; ac_word=$2 -echo "$as_me:$LINENO: checking for $ac_word" >&5 -echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 -if test "${ac_cv_prog_ac_ct_CC+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - if test -n "$ac_ct_CC"; then - ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="gcc" - echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done -done - -fi -fi -ac_ct_CC=$ac_cv_prog_ac_ct_CC -if test -n "$ac_ct_CC"; then - echo "$as_me:$LINENO: result: $ac_ct_CC" >&5 -echo "${ECHO_T}$ac_ct_CC" >&6 -else - echo "$as_me:$LINENO: result: no" >&5 -echo "${ECHO_T}no" >&6 -fi - - CC=$ac_ct_CC -else - CC="$ac_cv_prog_CC" -fi - -if test -z "$CC"; then - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. -set dummy ${ac_tool_prefix}cc; ac_word=$2 -echo "$as_me:$LINENO: checking for $ac_word" >&5 -echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 -if test "${ac_cv_prog_CC+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="${ac_tool_prefix}cc" - echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done -done - -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - echo "$as_me:$LINENO: result: $CC" >&5 -echo "${ECHO_T}$CC" >&6 -else - echo "$as_me:$LINENO: result: no" >&5 -echo "${ECHO_T}no" >&6 -fi - -fi -if test -z "$ac_cv_prog_CC"; then - ac_ct_CC=$CC - # Extract the first word of "cc", so it can be a program name with args. -set dummy cc; ac_word=$2 -echo "$as_me:$LINENO: checking for $ac_word" >&5 -echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 -if test "${ac_cv_prog_ac_ct_CC+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - if test -n "$ac_ct_CC"; then - ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="cc" - echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done -done - -fi -fi -ac_ct_CC=$ac_cv_prog_ac_ct_CC -if test -n "$ac_ct_CC"; then - echo "$as_me:$LINENO: result: $ac_ct_CC" >&5 -echo "${ECHO_T}$ac_ct_CC" >&6 -else - echo "$as_me:$LINENO: result: no" >&5 -echo "${ECHO_T}no" >&6 -fi - - CC=$ac_ct_CC -else - CC="$ac_cv_prog_CC" -fi - -fi -if test -z "$CC"; then - # Extract the first word of "cc", so it can be a program name with args. -set dummy cc; ac_word=$2 -echo "$as_me:$LINENO: checking for $ac_word" >&5 -echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 -if test "${ac_cv_prog_CC+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else - ac_prog_rejected=no -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then - ac_prog_rejected=yes - continue - fi - ac_cv_prog_CC="cc" - echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done -done - -if test $ac_prog_rejected = yes; then - # We found a bogon in the path, so make sure we never use it. - set dummy $ac_cv_prog_CC - shift - if test $# != 0; then - # We chose a different compiler from the bogus one. - # However, it has the same basename, so the bogon will be chosen - # first if we set CC to just the basename; use the full file name. - shift - ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@" - fi -fi -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - echo "$as_me:$LINENO: result: $CC" >&5 -echo "${ECHO_T}$CC" >&6 -else - echo "$as_me:$LINENO: result: no" >&5 -echo "${ECHO_T}no" >&6 -fi - -fi -if test -z "$CC"; then - if test -n "$ac_tool_prefix"; then - for ac_prog in cl - do - # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. -set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -echo "$as_me:$LINENO: checking for $ac_word" >&5 -echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 -if test "${ac_cv_prog_CC+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="$ac_tool_prefix$ac_prog" - echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done -done - -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - echo "$as_me:$LINENO: result: $CC" >&5 -echo "${ECHO_T}$CC" >&6 -else - echo "$as_me:$LINENO: result: no" >&5 -echo "${ECHO_T}no" >&6 -fi - - test -n "$CC" && break - done -fi -if test -z "$CC"; then - ac_ct_CC=$CC - for ac_prog in cl -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -echo "$as_me:$LINENO: checking for $ac_word" >&5 -echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6 -if test "${ac_cv_prog_ac_ct_CC+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - if test -n "$ac_ct_CC"; then - ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="$ac_prog" - echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done -done - -fi -fi -ac_ct_CC=$ac_cv_prog_ac_ct_CC -if test -n "$ac_ct_CC"; then - echo "$as_me:$LINENO: result: $ac_ct_CC" >&5 -echo "${ECHO_T}$ac_ct_CC" >&6 -else - echo "$as_me:$LINENO: result: no" >&5 -echo "${ECHO_T}no" >&6 -fi - - test -n "$ac_ct_CC" && break -done - - CC=$ac_ct_CC -fi - -fi - - -test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH -See \`config.log' for more details." >&5 -echo "$as_me: error: no acceptable C compiler found in \$PATH -See \`config.log' for more details." >&2;} - { (exit 1); exit 1; }; } - -# Provide some information about the compiler. -echo "$as_me:$LINENO:" \ - "checking for C compiler version" >&5 -ac_compiler=`set X $ac_compile; echo $2` -{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5 - (eval $ac_compiler --version </dev/null >&5) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } -{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5 - (eval $ac_compiler -v </dev/null >&5) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } -{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5 - (eval $ac_compiler -V </dev/null >&5) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } - -cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -ac_clean_files_save=$ac_clean_files -ac_clean_files="$ac_clean_files a.out a.exe b.out" -# Try to create an executable without -o first, disregard a.out. -# It will help us diagnose broken compilers, and finding out an intuition -# of exeext. -echo "$as_me:$LINENO: checking for C compiler default output file name" >&5 -echo $ECHO_N "checking for C compiler default output file name... $ECHO_C" >&6 -ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'` -if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5 - (eval $ac_link_default) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; then - # Find the output, starting from the most likely. This scheme is -# not robust to junk in `.', hence go to wildcards (a.*) only as a last -# resort. - -# Be careful to initialize this variable, since it used to be cached. -# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile. -ac_cv_exeext= -# b.out is created by i960 compilers. -for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out -do - test -f "$ac_file" || continue - case $ac_file in - *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) - ;; - conftest.$ac_ext ) - # This is the source file. - ;; - [ab].out ) - # We found the default executable, but exeext='' is most - # certainly right. - break;; - *.* ) - ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` - # FIXME: I believe we export ac_cv_exeext for Libtool, - # but it would be cool to find out if it's true. Does anybody - # maintain Libtool? --akim. - export ac_cv_exeext - break;; - * ) - break;; - esac -done -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -{ { echo "$as_me:$LINENO: error: C compiler cannot create executables -See \`config.log' for more details." >&5 -echo "$as_me: error: C compiler cannot create executables -See \`config.log' for more details." >&2;} - { (exit 77); exit 77; }; } -fi - -ac_exeext=$ac_cv_exeext -echo "$as_me:$LINENO: result: $ac_file" >&5 -echo "${ECHO_T}$ac_file" >&6 - -# Check the compiler produces executables we can run. If not, either -# the compiler is broken, or we cross compile. -echo "$as_me:$LINENO: checking whether the C compiler works" >&5 -echo $ECHO_N "checking whether the C compiler works... $ECHO_C" >&6 -# FIXME: These cross compiler hacks should be removed for Autoconf 3.0 -# If not cross compiling, check that we can run a simple program. -if test "$cross_compiling" != yes; then - if { ac_try='./$ac_file' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - cross_compiling=no - else - if test "$cross_compiling" = maybe; then - cross_compiling=yes - else - { { echo "$as_me:$LINENO: error: cannot run C compiled programs. -If you meant to cross compile, use \`--host'. -See \`config.log' for more details." >&5 -echo "$as_me: error: cannot run C compiled programs. -If you meant to cross compile, use \`--host'. -See \`config.log' for more details." >&2;} - { (exit 1); exit 1; }; } - fi - fi -fi -echo "$as_me:$LINENO: result: yes" >&5 -echo "${ECHO_T}yes" >&6 - -rm -f a.out a.exe conftest$ac_cv_exeext b.out -ac_clean_files=$ac_clean_files_save -# Check the compiler produces executables we can run. If not, either -# the compiler is broken, or we cross compile. -echo "$as_me:$LINENO: checking whether we are cross compiling" >&5 -echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6 -echo "$as_me:$LINENO: result: $cross_compiling" >&5 -echo "${ECHO_T}$cross_compiling" >&6 - -echo "$as_me:$LINENO: checking for suffix of executables" >&5 -echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6 -if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 - (eval $ac_link) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; then - # If both `conftest.exe' and `conftest' are `present' (well, observable) -# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will -# work properly (i.e., refer to `conftest.exe'), while it won't with -# `rm'. -for ac_file in conftest.exe conftest conftest.*; do - test -f "$ac_file" || continue - case $ac_file in - *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;; - *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` - export ac_cv_exeext - break;; - * ) break;; - esac -done -else - { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link -See \`config.log' for more details." >&5 -echo "$as_me: error: cannot compute suffix of executables: cannot compile and link -See \`config.log' for more details." >&2;} - { (exit 1); exit 1; }; } -fi - -rm -f conftest$ac_cv_exeext -echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5 -echo "${ECHO_T}$ac_cv_exeext" >&6 - -rm -f conftest.$ac_ext -EXEEXT=$ac_cv_exeext -ac_exeext=$EXEEXT -echo "$as_me:$LINENO: checking for suffix of object files" >&5 -echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6 -if test "${ac_cv_objext+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -rm -f conftest.o conftest.obj -if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 - (eval $ac_compile) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; then - for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do - case $ac_file in - *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;; - *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'` - break;; - esac -done -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile -See \`config.log' for more details." >&5 -echo "$as_me: error: cannot compute suffix of object files: cannot compile -See \`config.log' for more details." >&2;} - { (exit 1); exit 1; }; } -fi - -rm -f conftest.$ac_cv_objext conftest.$ac_ext -fi -echo "$as_me:$LINENO: result: $ac_cv_objext" >&5 -echo "${ECHO_T}$ac_cv_objext" >&6 -OBJEXT=$ac_cv_objext -ac_objext=$OBJEXT -echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5 -echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6 -if test "${ac_cv_c_compiler_gnu+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - -int -main () -{ -#ifndef __GNUC__ - choke me -#endif - - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext -if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 - (eval $ac_compile) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest.$ac_objext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_compiler_gnu=yes -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -ac_compiler_gnu=no -fi -rm -f conftest.err conftest.$ac_objext conftest.$ac_ext -ac_cv_c_compiler_gnu=$ac_compiler_gnu - -fi -echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5 -echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6 -GCC=`test $ac_compiler_gnu = yes && echo yes` -ac_test_CFLAGS=${CFLAGS+set} -ac_save_CFLAGS=$CFLAGS -CFLAGS="-g" -echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5 -echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6 -if test "${ac_cv_prog_cc_g+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext -if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 - (eval $ac_compile) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest.$ac_objext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_cv_prog_cc_g=yes -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -ac_cv_prog_cc_g=no -fi -rm -f conftest.err conftest.$ac_objext conftest.$ac_ext -fi -echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5 -echo "${ECHO_T}$ac_cv_prog_cc_g" >&6 -if test "$ac_test_CFLAGS" = set; then - CFLAGS=$ac_save_CFLAGS -elif test $ac_cv_prog_cc_g = yes; then - if test "$GCC" = yes; then - CFLAGS="-g -O2" - else - CFLAGS="-g" - fi -else - if test "$GCC" = yes; then - CFLAGS="-O2" - else - CFLAGS= - fi -fi -echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5 -echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6 -if test "${ac_cv_prog_cc_stdc+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - ac_cv_prog_cc_stdc=no -ac_save_CC=$CC -cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ -#include <stdarg.h> -#include <stdio.h> -#include <sys/types.h> -#include <sys/stat.h> -/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ -struct buf { int x; }; -FILE * (*rcsopen) (struct buf *, struct stat *, int); -static char *e (p, i) - char **p; - int i; -{ - return p[i]; -} -static char *f (char * (*g) (char **, int), char **p, ...) -{ - char *s; - va_list v; - va_start (v,p); - s = g (p, va_arg (v,int)); - va_end (v); - return s; -} - -/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has - function prototypes and stuff, but not '\xHH' hex character constants. - These don't provoke an error unfortunately, instead are silently treated - as 'x'. The following induces an error, until -std1 is added to get - proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an - array size at least. It's necessary to write '\x00'==0 to get something - that's true only with -std1. */ -int osf4_cc_array ['\x00' == 0 ? 1 : -1]; - -int test (int i, double x); -struct s1 {int (*f) (int a);}; -struct s2 {int (*f) (double a);}; -int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int); -int argc; -char **argv; -int -main () -{ -return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]; - ; - return 0; -} -_ACEOF -# Don't try gcc -ansi; that turns off useful extensions and -# breaks some systems' header files. -# AIX -qlanglvl=ansi -# Ultrix and OSF/1 -std1 -# HP-UX 10.20 and later -Ae -# HP-UX older versions -Aa -D_HPUX_SOURCE -# SVR4 -Xc -D__EXTENSIONS__ -for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" -do - CC="$ac_save_CC $ac_arg" - rm -f conftest.$ac_objext -if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 - (eval $ac_compile) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest.$ac_objext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_cv_prog_cc_stdc=$ac_arg -break -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -fi -rm -f conftest.err conftest.$ac_objext -done -rm -f conftest.$ac_ext conftest.$ac_objext -CC=$ac_save_CC - -fi - -case "x$ac_cv_prog_cc_stdc" in - x|xno) - echo "$as_me:$LINENO: result: none needed" >&5 -echo "${ECHO_T}none needed" >&6 ;; - *) - echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5 -echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6 - CC="$CC $ac_cv_prog_cc_stdc" ;; -esac - -# Some people use a C++ compiler to compile C. Since we use `exit', -# in C++ we need to declare it. In case someone uses the same compiler -# for both compiling C and C++ we need to have the C++ compiler decide -# the declaration of exit, since it's the most demanding environment. -cat >conftest.$ac_ext <<_ACEOF -#ifndef __cplusplus - choke me -#endif -_ACEOF -rm -f conftest.$ac_objext -if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 - (eval $ac_compile) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest.$ac_objext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - for ac_declaration in \ - '' \ - 'extern "C" void std::exit (int) throw (); using std::exit;' \ - 'extern "C" void std::exit (int); using std::exit;' \ - 'extern "C" void exit (int) throw ();' \ - 'extern "C" void exit (int);' \ - 'void exit (int);' -do - cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ -$ac_declaration -#include <stdlib.h> -int -main () -{ -exit (42); - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext -if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 - (eval $ac_compile) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest.$ac_objext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - : -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -continue -fi -rm -f conftest.err conftest.$ac_objext conftest.$ac_ext - cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ -$ac_declaration -int -main () -{ -exit (42); - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext -if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 - (eval $ac_compile) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest.$ac_objext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - break -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -fi -rm -f conftest.err conftest.$ac_objext conftest.$ac_ext -done -rm -f conftest* -if test -n "$ac_declaration"; then - echo '#ifdef __cplusplus' >>confdefs.h - echo $ac_declaration >>confdefs.h - echo '#endif' >>confdefs.h -fi - -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -fi -rm -f conftest.err conftest.$ac_objext conftest.$ac_ext -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - -if test "$GCC" != "yes" -then - if test "$isnowarn" != "yes" - then - printf '#================================================================\n' 1>&2 - printf '# WARNING: GCC is required to build this package.\n' 1>&2 - printf '#================================================================\n' 1>&2 - fi -fi -if uname | egrep -i 'Linux' > /dev/null 2>&1 && - uname -m | egrep '(x|i)(3|4|5|6|7|8|9)?86' > /dev/null 2>&1 -then - MYOPTS="-minline-all-stringops" -fi -if uname | egrep -i 'SunOS' > /dev/null 2>&1 -then - MYOPTS="-O1 -fno-omit-frame-pointer -fno-force-addr" -fi -if uname | egrep -i 'BSD' > /dev/null 2>&1 -then - MYOPTS="-O1 -fno-omit-frame-pointer -fno-force-addr" -fi -if gcc --version | egrep -i '^2\.(8|9)' > /dev/null 2>&1 -then - MYOPTS="-O1 -fno-omit-frame-pointer -fno-force-addr" -fi - -# Linker -printf 'checking for ld... ' -if which ld | grep '/ld$' > /dev/null 2>&1 -then - LD=`which ld` - printf '%s\n' "$LD" -else - printf 'no\n' - if test "$isnowarn" != "yes" - then - printf '#================================================================\n' 1>&2 - printf '# WARNING: ld is not found in PATH.\n' 1>&2 - printf '#================================================================\n' 1>&2 - fi -fi - -# Archiver -printf 'checking for ar... ' -if which ar | grep '/ar$' > /dev/null 2>&1 -then - AR=`which ar` - printf '%s\n' "$AR" -else - printf 'no\n' - if test "$isnowarn" != "yes" - then - printf '#================================================================\n' 1>&2 - printf '# WARNING: ar is not found in PATH.\n' 1>&2 - printf '#================================================================\n' 1>&2 - fi -fi - - - -#================================================================ -# Checking Libraries -#================================================================ - - -# Underlying libraries - - -echo "$as_me:$LINENO: checking for main in -lc" >&5 -echo $ECHO_N "checking for main in -lc... $ECHO_C" >&6 -if test "${ac_cv_lib_c_main+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lc $LIBS" -cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - - -int -main () -{ -main (); - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext conftest$ac_exeext -if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 - (eval $ac_link) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest$ac_exeext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_cv_lib_c_main=yes -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -ac_cv_lib_c_main=no -fi -rm -f conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -echo "$as_me:$LINENO: result: $ac_cv_lib_c_main" >&5 -echo "${ECHO_T}$ac_cv_lib_c_main" >&6 -if test $ac_cv_lib_c_main = yes; then - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBC 1 -_ACEOF - - LIBS="-lc $LIBS" - -fi - - -# for pthread -if test "$ispthread" = "yes" -then - -echo "$as_me:$LINENO: checking for main in -lpthread" >&5 -echo $ECHO_N "checking for main in -lpthread... $ECHO_C" >&6 -if test "${ac_cv_lib_pthread_main+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lpthread $LIBS" -cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - - -int -main () -{ -main (); - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext conftest$ac_exeext -if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 - (eval $ac_link) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest$ac_exeext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_cv_lib_pthread_main=yes -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -ac_cv_lib_pthread_main=no -fi -rm -f conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -echo "$as_me:$LINENO: result: $ac_cv_lib_pthread_main" >&5 -echo "${ECHO_T}$ac_cv_lib_pthread_main" >&6 -if test $ac_cv_lib_pthread_main = yes; then - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBPTHREAD 1 -_ACEOF - - LIBS="-lpthread $LIBS" - -fi - -fi - -# for zlib -if test "$iszlib" = "yes" -then - -echo "$as_me:$LINENO: checking for main in -lz" >&5 -echo $ECHO_N "checking for main in -lz... $ECHO_C" >&6 -if test "${ac_cv_lib_z_main+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lz $LIBS" -cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - - -int -main () -{ -main (); - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext conftest$ac_exeext -if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 - (eval $ac_link) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest$ac_exeext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_cv_lib_z_main=yes -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -ac_cv_lib_z_main=no -fi -rm -f conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -echo "$as_me:$LINENO: result: $ac_cv_lib_z_main" >&5 -echo "${ECHO_T}$ac_cv_lib_z_main" >&6 -if test $ac_cv_lib_z_main = yes; then - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBZ 1 -_ACEOF - - LIBS="-lz $LIBS" - -fi - -fi - -# for lzo -if test "$islzo" = "yes" -then - -echo "$as_me:$LINENO: checking for main in -llzo2" >&5 -echo $ECHO_N "checking for main in -llzo2... $ECHO_C" >&6 -if test "${ac_cv_lib_lzo2_main+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-llzo2 $LIBS" -cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - - -int -main () -{ -main (); - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext conftest$ac_exeext -if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 - (eval $ac_link) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest$ac_exeext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_cv_lib_lzo2_main=yes -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -ac_cv_lib_lzo2_main=no -fi -rm -f conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -echo "$as_me:$LINENO: result: $ac_cv_lib_lzo2_main" >&5 -echo "${ECHO_T}$ac_cv_lib_lzo2_main" >&6 -if test $ac_cv_lib_lzo2_main = yes; then - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBLZO2 1 -_ACEOF - - LIBS="-llzo2 $LIBS" - -fi - -fi - -# for bzip -if test "$isbzip" = "yes" -then - -echo "$as_me:$LINENO: checking for main in -lbz2" >&5 -echo $ECHO_N "checking for main in -lbz2... $ECHO_C" >&6 -if test "${ac_cv_lib_bz2_main+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lbz2 $LIBS" -cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - - -int -main () -{ -main (); - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext conftest$ac_exeext -if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 - (eval $ac_link) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest$ac_exeext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_cv_lib_bz2_main=yes -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -ac_cv_lib_bz2_main=no -fi -rm -f conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -echo "$as_me:$LINENO: result: $ac_cv_lib_bz2_main" >&5 -echo "${ECHO_T}$ac_cv_lib_bz2_main" >&6 -if test $ac_cv_lib_bz2_main = yes; then - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBBZ2 1 -_ACEOF - - LIBS="-lbz2 $LIBS" - -fi - -fi - -# for iconv -if test "$isiconv" = "yes" -then - -echo "$as_me:$LINENO: checking for main in -liconv" >&5 -echo $ECHO_N "checking for main in -liconv... $ECHO_C" >&6 -if test "${ac_cv_lib_iconv_main+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-liconv $LIBS" -cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - - -int -main () -{ -main (); - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext conftest$ac_exeext -if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 - (eval $ac_link) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest$ac_exeext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_cv_lib_iconv_main=yes -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -ac_cv_lib_iconv_main=no -fi -rm -f conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -echo "$as_me:$LINENO: result: $ac_cv_lib_iconv_main" >&5 -echo "${ECHO_T}$ac_cv_lib_iconv_main" >&6 -if test $ac_cv_lib_iconv_main = yes; then - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBICONV 1 -_ACEOF - - LIBS="-liconv $LIBS" - -fi - -fi - -# For old BSDs -if test "$ispthread" = "yes" && uname -a | grep BSD > /dev/null && - test -f /usr/lib/libc_r.a && test ! -f /usr/lib/libpthread.a -then - LIBS=`printf '%s' "$LIBS" | sed 's/-lc/-lc_r/g'` -fi - -# Duplication of QDBM for C -echo "$as_me:$LINENO: checking for main in -lqdbm" >&5 -echo $ECHO_N "checking for main in -lqdbm... $ECHO_C" >&6 -if test "${ac_cv_lib_qdbm_main+set}" = set; then - echo $ECHO_N "(cached) $ECHO_C" >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lqdbm $LIBS" -cat >conftest.$ac_ext <<_ACEOF -/* confdefs.h. */ -_ACEOF -cat confdefs.h >>conftest.$ac_ext -cat >>conftest.$ac_ext <<_ACEOF -/* end confdefs.h. */ - - -int -main () -{ -main (); - ; - return 0; -} -_ACEOF -rm -f conftest.$ac_objext conftest$ac_exeext -if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 - (eval $ac_link) 2>conftest.er1 - ac_status=$? - grep -v '^ *+' conftest.er1 >conftest.err - rm -f conftest.er1 - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } && - { ac_try='test -z "$ac_c_werror_flag" - || test ! -s conftest.err' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; } && - { ac_try='test -s conftest$ac_exeext' - { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 - (eval $ac_try) 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); }; }; then - ac_cv_lib_qdbm_main=yes -else - echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -ac_cv_lib_qdbm_main=no -fi -rm -f conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -echo "$as_me:$LINENO: result: $ac_cv_lib_qdbm_main" >&5 -echo "${ECHO_T}$ac_cv_lib_qdbm_main" >&6 -if test $ac_cv_lib_qdbm_main = yes; then - if test "$isnowarn" != "yes" - then - printf '#================================================================\n' 1>&2 - printf '# WARNING: The existing library was detected.\n' 1>&2 - printf '#================================================================\n' 1>&2 - fi - -fi - - - - -#================================================================ -# Generic Settings -#================================================================ - - -# Export variables - - - - - - - - - - - -# Targets - ac_config_files="$ac_config_files Makefile LTmakefile qdbm.spec qdbm.pc" -cat >confcache <<\_ACEOF -# This file is a shell script that caches the results of configure -# tests run on this system so they can be shared between configure -# scripts and configure runs, see configure's option --config-cache. -# It is not useful on other systems. If it contains results you don't -# want to keep, you may remove or edit it. -# -# config.status only pays attention to the cache file if you give it -# the --recheck option to rerun configure. -# -# `ac_cv_env_foo' variables (set or unset) will be overridden when -# loading this file, other *unset* `ac_cv_foo' will be assigned the -# following values. - -_ACEOF - -# The following way of writing the cache mishandles newlines in values, -# but we know of no workaround that is simple, portable, and efficient. -# So, don't put newlines in cache variables' values. -# Ultrix sh set writes to stderr and can't be redirected directly, -# and sets the high bit in the cache file unless we assign to the vars. -{ - (set) 2>&1 | - case `(ac_space=' '; set | grep ac_space) 2>&1` in - *ac_space=\ *) - # `set' does not quote correctly, so add quotes (double-quote - # substitution turns \\\\ into \\, and sed turns \\ into \). - sed -n \ - "s/'/'\\\\''/g; - s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" - ;; - *) - # `set' quotes correctly as required by POSIX, so do not add quotes. - sed -n \ - "s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p" - ;; - esac; -} | - sed ' - t clear - : clear - s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/ - t end - /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/ - : end' >>confcache -if diff $cache_file confcache >/dev/null 2>&1; then :; else - if test -w $cache_file; then - test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file" - cat confcache >$cache_file - else - echo "not updating unwritable cache $cache_file" - fi -fi -rm -f confcache - -test "x$prefix" = xNONE && prefix=$ac_default_prefix -# Let make expand exec_prefix. -test "x$exec_prefix" = xNONE && exec_prefix='${prefix}' - -# VPATH may cause trouble with some makes, so we remove $(srcdir), -# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and -# trailing colons and then remove the whole line if VPATH becomes empty -# (actually we leave an empty line to preserve line numbers). -if test "x$srcdir" = x.; then - ac_vpsub='/^[ ]*VPATH[ ]*=/{ -s/:*\$(srcdir):*/:/; -s/:*\${srcdir}:*/:/; -s/:*@srcdir@:*/:/; -s/^\([^=]*=[ ]*\):*/\1/; -s/:*$//; -s/^[^=]*=[ ]*$//; -}' -fi - -# Transform confdefs.h into DEFS. -# Protect against shell expansion while executing Makefile rules. -# Protect against Makefile macro expansion. -# -# If the first sed substitution is executed (which looks for macros that -# take arguments), then we branch to the quote section. Otherwise, -# look for a macro that doesn't take arguments. -cat >confdef2opt.sed <<\_ACEOF -t clear -: clear -s,^[ ]*#[ ]*define[ ][ ]*\([^ (][^ (]*([^)]*)\)[ ]*\(.*\),-D\1=\2,g -t quote -s,^[ ]*#[ ]*define[ ][ ]*\([^ ][^ ]*\)[ ]*\(.*\),-D\1=\2,g -t quote -d -: quote -s,[ `~#$^&*(){}\\|;'"<>?],\\&,g -s,\[,\\&,g -s,\],\\&,g -s,\$,$$,g -p -_ACEOF -# We use echo to avoid assuming a particular line-breaking character. -# The extra dot is to prevent the shell from consuming trailing -# line-breaks from the sub-command output. A line-break within -# single-quotes doesn't work because, if this script is created in a -# platform that uses two characters for line-breaks (e.g., DOS), tr -# would break. -ac_LF_and_DOT=`echo; echo .` -DEFS=`sed -n -f confdef2opt.sed confdefs.h | tr "$ac_LF_and_DOT" ' .'` -rm -f confdef2opt.sed - - -ac_libobjs= -ac_ltlibobjs= -for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue - # 1. Remove the extension, and $U if already installed. - ac_i=`echo "$ac_i" | - sed 's/\$U\././;s/\.o$//;s/\.obj$//'` - # 2. Add them. - ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext" - ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo' -done -LIBOBJS=$ac_libobjs - -LTLIBOBJS=$ac_ltlibobjs - - - -: ${CONFIG_STATUS=./config.status} -ac_clean_files_save=$ac_clean_files -ac_clean_files="$ac_clean_files $CONFIG_STATUS" -{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5 -echo "$as_me: creating $CONFIG_STATUS" >&6;} -cat >$CONFIG_STATUS <<_ACEOF -#! $SHELL -# Generated by $as_me. -# Run this file to recreate the current configuration. -# Compiler output produced by configure, useful for debugging -# configure, is in config.log if it exists. - -debug=false -ac_cs_recheck=false -ac_cs_silent=false -SHELL=\${CONFIG_SHELL-$SHELL} -_ACEOF - -cat >>$CONFIG_STATUS <<\_ACEOF -## --------------------- ## -## M4sh Initialization. ## -## --------------------- ## - -# Be Bourne compatible -if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then - emulate sh - NULLCMD=: - # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which - # is contrary to our usage. Disable this feature. - alias -g '${1+"$@"}'='"$@"' -elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then - set -o posix -fi -DUALCASE=1; export DUALCASE # for MKS sh - -# Support unset when possible. -if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then - as_unset=unset -else - as_unset=false -fi - - -# Work around bugs in pre-3.0 UWIN ksh. -$as_unset ENV MAIL MAILPATH -PS1='$ ' -PS2='> ' -PS4='+ ' - -# NLS nuisances. -for as_var in \ - LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \ - LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \ - LC_TELEPHONE LC_TIME -do - if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then - eval $as_var=C; export $as_var - else - $as_unset $as_var - fi -done - -# Required to use basename. -if expr a : '\(a\)' >/dev/null 2>&1; then - as_expr=expr -else - as_expr=false -fi - -if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then - as_basename=basename -else - as_basename=false -fi - - -# Name of the executable. -as_me=`$as_basename "$0" || -$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ - X"$0" : 'X\(//\)$' \| \ - X"$0" : 'X\(/\)$' \| \ - . : '\(.\)' 2>/dev/null || -echo X/"$0" | - sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; } - /^X\/\(\/\/\)$/{ s//\1/; q; } - /^X\/\(\/\).*/{ s//\1/; q; } - s/.*/./; q'` - - -# PATH needs CR, and LINENO needs CR and PATH. -# Avoid depending upon Character Ranges. -as_cr_letters='abcdefghijklmnopqrstuvwxyz' -as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' -as_cr_Letters=$as_cr_letters$as_cr_LETTERS -as_cr_digits='0123456789' -as_cr_alnum=$as_cr_Letters$as_cr_digits - -# The user is always right. -if test "${PATH_SEPARATOR+set}" != set; then - echo "#! /bin/sh" >conf$$.sh - echo "exit 0" >>conf$$.sh - chmod +x conf$$.sh - if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then - PATH_SEPARATOR=';' - else - PATH_SEPARATOR=: - fi - rm -f conf$$.sh -fi - - - as_lineno_1=$LINENO - as_lineno_2=$LINENO - as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` - test "x$as_lineno_1" != "x$as_lineno_2" && - test "x$as_lineno_3" = "x$as_lineno_2" || { - # Find who we are. Look in the path if we contain no path at all - # relative or not. - case $0 in - *[\\/]* ) as_myself=$0 ;; - *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break -done - - ;; - esac - # We did not find ourselves, most probably we were run as `sh COMMAND' - # in which case we are not to be found in the path. - if test "x$as_myself" = x; then - as_myself=$0 - fi - if test ! -f "$as_myself"; then - { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5 -echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;} - { (exit 1); exit 1; }; } - fi - case $CONFIG_SHELL in - '') - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for as_base in sh bash ksh sh5; do - case $as_dir in - /*) - if ("$as_dir/$as_base" -c ' - as_lineno_1=$LINENO - as_lineno_2=$LINENO - as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null` - test "x$as_lineno_1" != "x$as_lineno_2" && - test "x$as_lineno_3" = "x$as_lineno_2" ') 2>/dev/null; then - $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; } - $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; } - CONFIG_SHELL=$as_dir/$as_base - export CONFIG_SHELL - exec "$CONFIG_SHELL" "$0" ${1+"$@"} - fi;; - esac - done -done -;; - esac - - # Create $as_me.lineno as a copy of $as_myself, but with $LINENO - # uniformly replaced by the line number. The first 'sed' inserts a - # line-number line before each line; the second 'sed' does the real - # work. The second script uses 'N' to pair each line-number line - # with the numbered line, and appends trailing '-' during - # substitution so that $LINENO is not a special case at line end. - # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the - # second 'sed' script. Blame Lee E. McMahon for sed's syntax. :-) - sed '=' <$as_myself | - sed ' - N - s,$,-, - : loop - s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3, - t loop - s,-$,, - s,^['$as_cr_digits']*\n,, - ' >$as_me.lineno && - chmod +x $as_me.lineno || - { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5 -echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;} - { (exit 1); exit 1; }; } - - # Don't try to exec as it changes $[0], causing all sort of problems - # (the dirname of $[0] is not the place where we might find the - # original and so on. Autoconf is especially sensible to this). - . ./$as_me.lineno - # Exit status is that of the last command. - exit -} - - -case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in - *c*,-n*) ECHO_N= ECHO_C=' -' ECHO_T=' ' ;; - *c*,* ) ECHO_N=-n ECHO_C= ECHO_T= ;; - *) ECHO_N= ECHO_C='\c' ECHO_T= ;; -esac - -if expr a : '\(a\)' >/dev/null 2>&1; then - as_expr=expr -else - as_expr=false -fi - -rm -f conf$$ conf$$.exe conf$$.file -echo >conf$$.file -if ln -s conf$$.file conf$$ 2>/dev/null; then - # We could just check for DJGPP; but this test a) works b) is more generic - # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04). - if test -f conf$$.exe; then - # Don't use ln at all; we don't have any links - as_ln_s='cp -p' - else - as_ln_s='ln -s' - fi -elif ln conf$$.file conf$$ 2>/dev/null; then - as_ln_s=ln -else - as_ln_s='cp -p' -fi -rm -f conf$$ conf$$.exe conf$$.file - -if mkdir -p . 2>/dev/null; then - as_mkdir_p=: -else - test -d ./-p && rmdir ./-p - as_mkdir_p=false -fi - -as_executable_p="test -f" - -# Sed expression to map a string onto a valid CPP name. -as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" - -# Sed expression to map a string onto a valid variable name. -as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" - - -# IFS -# We need space, tab and new line, in precisely that order. -as_nl=' -' -IFS=" $as_nl" - -# CDPATH. -$as_unset CDPATH - -exec 6>&1 - -# Open the log real soon, to keep \$[0] and so on meaningful, and to -# report actual input values of CONFIG_FILES etc. instead of their -# values after options handling. Logging --version etc. is OK. -exec 5>>config.log -{ - echo - sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX -## Running $as_me. ## -_ASBOX -} >&5 -cat >&5 <<_CSEOF - -This file was extended by qdbm $as_me 1.8.77, which was -generated by GNU Autoconf 2.59. Invocation command line was - - CONFIG_FILES = $CONFIG_FILES - CONFIG_HEADERS = $CONFIG_HEADERS - CONFIG_LINKS = $CONFIG_LINKS - CONFIG_COMMANDS = $CONFIG_COMMANDS - $ $0 $@ - -_CSEOF -echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5 -echo >&5 -_ACEOF - -# Files that config.status was made for. -if test -n "$ac_config_files"; then - echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS -fi - -if test -n "$ac_config_headers"; then - echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS -fi - -if test -n "$ac_config_links"; then - echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS -fi - -if test -n "$ac_config_commands"; then - echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS -fi - -cat >>$CONFIG_STATUS <<\_ACEOF - -ac_cs_usage="\ -\`$as_me' instantiates files from templates according to the -current configuration. - -Usage: $0 [OPTIONS] [FILE]... - - -h, --help print this help, then exit - -V, --version print version number, then exit - -q, --quiet do not print progress messages - -d, --debug don't remove temporary files - --recheck update $as_me by reconfiguring in the same conditions - --file=FILE[:TEMPLATE] - instantiate the configuration file FILE - -Configuration files: -$config_files - -Report bugs to <bug-autoconf@gnu.org>." -_ACEOF - -cat >>$CONFIG_STATUS <<_ACEOF -ac_cs_version="\\ -qdbm config.status 1.8.77 -configured by $0, generated by GNU Autoconf 2.59, - with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\" - -Copyright (C) 2003 Free Software Foundation, Inc. -This config.status script is free software; the Free Software Foundation -gives unlimited permission to copy, distribute and modify it." -srcdir=$srcdir -_ACEOF - -cat >>$CONFIG_STATUS <<\_ACEOF -# If no file are specified by the user, then we need to provide default -# value. By we need to know if files were specified by the user. -ac_need_defaults=: -while test $# != 0 -do - case $1 in - --*=*) - ac_option=`expr "x$1" : 'x\([^=]*\)='` - ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'` - ac_shift=: - ;; - -*) - ac_option=$1 - ac_optarg=$2 - ac_shift=shift - ;; - *) # This is not an option, so the user has probably given explicit - # arguments. - ac_option=$1 - ac_need_defaults=false;; - esac - - case $ac_option in - # Handling of the options. -_ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF - -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) - ac_cs_recheck=: ;; - --version | --vers* | -V ) - echo "$ac_cs_version"; exit 0 ;; - --he | --h) - # Conflict between --help and --header - { { echo "$as_me:$LINENO: error: ambiguous option: $1 -Try \`$0 --help' for more information." >&5 -echo "$as_me: error: ambiguous option: $1 -Try \`$0 --help' for more information." >&2;} - { (exit 1); exit 1; }; };; - --help | --hel | -h ) - echo "$ac_cs_usage"; exit 0 ;; - --debug | --d* | -d ) - debug=: ;; - --file | --fil | --fi | --f ) - $ac_shift - CONFIG_FILES="$CONFIG_FILES $ac_optarg" - ac_need_defaults=false;; - --header | --heade | --head | --hea ) - $ac_shift - CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg" - ac_need_defaults=false;; - -q | -quiet | --quiet | --quie | --qui | --qu | --q \ - | -silent | --silent | --silen | --sile | --sil | --si | --s) - ac_cs_silent=: ;; - - # This is an error. - -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1 -Try \`$0 --help' for more information." >&5 -echo "$as_me: error: unrecognized option: $1 -Try \`$0 --help' for more information." >&2;} - { (exit 1); exit 1; }; } ;; - - *) ac_config_targets="$ac_config_targets $1" ;; - - esac - shift -done - -ac_configure_extra_args= - -if $ac_cs_silent; then - exec 6>/dev/null - ac_configure_extra_args="$ac_configure_extra_args --silent" -fi - -_ACEOF -cat >>$CONFIG_STATUS <<_ACEOF -if \$ac_cs_recheck; then - echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6 - exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion -fi - -_ACEOF - - - - - -cat >>$CONFIG_STATUS <<\_ACEOF -for ac_config_target in $ac_config_targets -do - case "$ac_config_target" in - # Handling of arguments. - "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;; - "LTmakefile" ) CONFIG_FILES="$CONFIG_FILES LTmakefile" ;; - "qdbm.spec" ) CONFIG_FILES="$CONFIG_FILES qdbm.spec" ;; - "qdbm.pc" ) CONFIG_FILES="$CONFIG_FILES qdbm.pc" ;; - *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5 -echo "$as_me: error: invalid argument: $ac_config_target" >&2;} - { (exit 1); exit 1; }; };; - esac -done - -# If the user did not use the arguments to specify the items to instantiate, -# then the envvar interface is used. Set only those that are not. -# We use the long form for the default assignment because of an extremely -# bizarre bug on SunOS 4.1.3. -if $ac_need_defaults; then - test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files -fi - -# Have a temporary directory for convenience. Make it in the build tree -# simply because there is no reason to put it here, and in addition, -# creating and moving files from /tmp can sometimes cause problems. -# Create a temporary directory, and hook for its removal unless debugging. -$debug || -{ - trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0 - trap '{ (exit 1); exit 1; }' 1 2 13 15 -} - -# Create a (secure) tmp directory for tmp files. - -{ - tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` && - test -n "$tmp" && test -d "$tmp" -} || -{ - tmp=./confstat$$-$RANDOM - (umask 077 && mkdir $tmp) -} || -{ - echo "$me: cannot create a temporary directory in ." >&2 - { (exit 1); exit 1; } -} - -_ACEOF - -cat >>$CONFIG_STATUS <<_ACEOF - -# -# CONFIG_FILES section. -# - -# No need to generate the scripts if there are no CONFIG_FILES. -# This happens for instance when ./config.status config.h -if test -n "\$CONFIG_FILES"; then - # Protect against being on the right side of a sed subst in config.status. - sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g; - s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF -s,@SHELL@,$SHELL,;t t -s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t -s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t -s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t -s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t -s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t -s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t -s,@exec_prefix@,$exec_prefix,;t t -s,@prefix@,$prefix,;t t -s,@program_transform_name@,$program_transform_name,;t t -s,@bindir@,$bindir,;t t -s,@sbindir@,$sbindir,;t t -s,@libexecdir@,$libexecdir,;t t -s,@datadir@,$datadir,;t t -s,@sysconfdir@,$sysconfdir,;t t -s,@sharedstatedir@,$sharedstatedir,;t t -s,@localstatedir@,$localstatedir,;t t -s,@libdir@,$libdir,;t t -s,@includedir@,$includedir,;t t -s,@oldincludedir@,$oldincludedir,;t t -s,@infodir@,$infodir,;t t -s,@mandir@,$mandir,;t t -s,@build_alias@,$build_alias,;t t -s,@host_alias@,$host_alias,;t t -s,@target_alias@,$target_alias,;t t -s,@DEFS@,$DEFS,;t t -s,@ECHO_C@,$ECHO_C,;t t -s,@ECHO_N@,$ECHO_N,;t t -s,@ECHO_T@,$ECHO_T,;t t -s,@LIBS@,$LIBS,;t t -s,@CC@,$CC,;t t -s,@CFLAGS@,$CFLAGS,;t t -s,@LDFLAGS@,$LDFLAGS,;t t -s,@CPPFLAGS@,$CPPFLAGS,;t t -s,@ac_ct_CC@,$ac_ct_CC,;t t -s,@EXEEXT@,$EXEEXT,;t t -s,@OBJEXT@,$OBJEXT,;t t -s,@LIBVER@,$LIBVER,;t t -s,@LIBREV@,$LIBREV,;t t -s,@TARGETS@,$TARGETS,;t t -s,@MYDEFS@,$MYDEFS,;t t -s,@MYOPTS@,$MYOPTS,;t t -s,@MGWLIBS@,$MGWLIBS,;t t -s,@LD@,$LD,;t t -s,@AR@,$AR,;t t -s,@LIBOBJS@,$LIBOBJS,;t t -s,@LTLIBOBJS@,$LTLIBOBJS,;t t -CEOF - -_ACEOF - - cat >>$CONFIG_STATUS <<\_ACEOF - # Split the substitutions into bite-sized pieces for seds with - # small command number limits, like on Digital OSF/1 and HP-UX. - ac_max_sed_lines=48 - ac_sed_frag=1 # Number of current file. - ac_beg=1 # First line for current file. - ac_end=$ac_max_sed_lines # Line after last line for current file. - ac_more_lines=: - ac_sed_cmds= - while $ac_more_lines; do - if test $ac_beg -gt 1; then - sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag - else - sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag - fi - if test ! -s $tmp/subs.frag; then - ac_more_lines=false - else - # The purpose of the label and of the branching condition is to - # speed up the sed processing (if there are no `@' at all, there - # is no need to browse any of the substitutions). - # These are the two extra sed commands mentioned above. - (echo ':t - /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed - if test -z "$ac_sed_cmds"; then - ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed" - else - ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed" - fi - ac_sed_frag=`expr $ac_sed_frag + 1` - ac_beg=$ac_end - ac_end=`expr $ac_end + $ac_max_sed_lines` - fi - done - if test -z "$ac_sed_cmds"; then - ac_sed_cmds=cat - fi -fi # test -n "$CONFIG_FILES" - -_ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF -for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue - # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in". - case $ac_file in - - | *:- | *:-:* ) # input from stdin - cat >$tmp/stdin - ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` - ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; - *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'` - ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;; - * ) ac_file_in=$ac_file.in ;; - esac - - # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories. - ac_dir=`(dirname "$ac_file") 2>/dev/null || -$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ - X"$ac_file" : 'X\(//\)[^/]' \| \ - X"$ac_file" : 'X\(//\)$' \| \ - X"$ac_file" : 'X\(/\)' \| \ - . : '\(.\)' 2>/dev/null || -echo X"$ac_file" | - sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } - /^X\(\/\/\)[^/].*/{ s//\1/; q; } - /^X\(\/\/\)$/{ s//\1/; q; } - /^X\(\/\).*/{ s//\1/; q; } - s/.*/./; q'` - { if $as_mkdir_p; then - mkdir -p "$ac_dir" - else - as_dir="$ac_dir" - as_dirs= - while test ! -d "$as_dir"; do - as_dirs="$as_dir $as_dirs" - as_dir=`(dirname "$as_dir") 2>/dev/null || -$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ - X"$as_dir" : 'X\(//\)[^/]' \| \ - X"$as_dir" : 'X\(//\)$' \| \ - X"$as_dir" : 'X\(/\)' \| \ - . : '\(.\)' 2>/dev/null || -echo X"$as_dir" | - sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; } - /^X\(\/\/\)[^/].*/{ s//\1/; q; } - /^X\(\/\/\)$/{ s//\1/; q; } - /^X\(\/\).*/{ s//\1/; q; } - s/.*/./; q'` - done - test ! -n "$as_dirs" || mkdir $as_dirs - fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5 -echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;} - { (exit 1); exit 1; }; }; } - - ac_builddir=. - -if test "$ac_dir" != .; then - ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'` - # A "../" for each directory in $ac_dir_suffix. - ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'` -else - ac_dir_suffix= ac_top_builddir= -fi - -case $srcdir in - .) # No --srcdir option. We are building in place. - ac_srcdir=. - if test -z "$ac_top_builddir"; then - ac_top_srcdir=. - else - ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'` - fi ;; - [\\/]* | ?:[\\/]* ) # Absolute path. - ac_srcdir=$srcdir$ac_dir_suffix; - ac_top_srcdir=$srcdir ;; - *) # Relative path. - ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix - ac_top_srcdir=$ac_top_builddir$srcdir ;; -esac - -# Do not use `cd foo && pwd` to compute absolute paths, because -# the directories may not exist. -case `pwd` in -.) ac_abs_builddir="$ac_dir";; -*) - case "$ac_dir" in - .) ac_abs_builddir=`pwd`;; - [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";; - *) ac_abs_builddir=`pwd`/"$ac_dir";; - esac;; -esac -case $ac_abs_builddir in -.) ac_abs_top_builddir=${ac_top_builddir}.;; -*) - case ${ac_top_builddir}. in - .) ac_abs_top_builddir=$ac_abs_builddir;; - [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;; - *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;; - esac;; -esac -case $ac_abs_builddir in -.) ac_abs_srcdir=$ac_srcdir;; -*) - case $ac_srcdir in - .) ac_abs_srcdir=$ac_abs_builddir;; - [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;; - *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;; - esac;; -esac -case $ac_abs_builddir in -.) ac_abs_top_srcdir=$ac_top_srcdir;; -*) - case $ac_top_srcdir in - .) ac_abs_top_srcdir=$ac_abs_builddir;; - [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;; - *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;; - esac;; -esac - - - - if test x"$ac_file" != x-; then - { echo "$as_me:$LINENO: creating $ac_file" >&5 -echo "$as_me: creating $ac_file" >&6;} - rm -f "$ac_file" - fi - # Let's still pretend it is `configure' which instantiates (i.e., don't - # use $as_me), people would be surprised to read: - # /* config.h. Generated by config.status. */ - if test x"$ac_file" = x-; then - configure_input= - else - configure_input="$ac_file. " - fi - configure_input=$configure_input"Generated from `echo $ac_file_in | - sed 's,.*/,,'` by configure." - - # First look for the input files in the build tree, otherwise in the - # src tree. - ac_file_inputs=`IFS=: - for f in $ac_file_in; do - case $f in - -) echo $tmp/stdin ;; - [\\/$]*) - # Absolute (can't be DOS-style, as IFS=:) - test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 -echo "$as_me: error: cannot find input file: $f" >&2;} - { (exit 1); exit 1; }; } - echo "$f";; - *) # Relative - if test -f "$f"; then - # Build tree - echo "$f" - elif test -f "$srcdir/$f"; then - # Source tree - echo "$srcdir/$f" - else - # /dev/null tree - { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5 -echo "$as_me: error: cannot find input file: $f" >&2;} - { (exit 1); exit 1; }; } - fi;; - esac - done` || { (exit 1); exit 1; } -_ACEOF -cat >>$CONFIG_STATUS <<_ACEOF - sed "$ac_vpsub -$extrasub -_ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF -:t -/@[a-zA-Z_][a-zA-Z_0-9]*@/!b -s,@configure_input@,$configure_input,;t t -s,@srcdir@,$ac_srcdir,;t t -s,@abs_srcdir@,$ac_abs_srcdir,;t t -s,@top_srcdir@,$ac_top_srcdir,;t t -s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t -s,@builddir@,$ac_builddir,;t t -s,@abs_builddir@,$ac_abs_builddir,;t t -s,@top_builddir@,$ac_top_builddir,;t t -s,@abs_top_builddir@,$ac_abs_top_builddir,;t t -" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out - rm -f $tmp/stdin - if test x"$ac_file" != x-; then - mv $tmp/out $ac_file - else - cat $tmp/out - rm -f $tmp/out - fi - -done -_ACEOF - -cat >>$CONFIG_STATUS <<\_ACEOF - -{ (exit 0); exit 0; } -_ACEOF -chmod +x $CONFIG_STATUS -ac_clean_files=$ac_clean_files_save - - -# configure is writing to config.log, and then calls config.status. -# config.status does its own redirection, appending to config.log. -# Unfortunately, on DOS this fails, as config.log is still kept open -# by configure, so config.status won't be able to write to it; its -# output is simply discarded. So we exec the FD to /dev/null, -# effectively closing config.log, so it can be properly (re)opened and -# appended to by config.status. When coming back to configure, we -# need to make the FD available again. -if test "$no_create" != yes; then - ac_cs_success=: - ac_config_status_args= - test "$silent" = yes && - ac_config_status_args="$ac_config_status_args --quiet" - exec 5>/dev/null - $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false - exec 5>>config.log - # Use ||, not &&, to avoid exiting from the if with $? = 1, which - # would make configure fail if this is the last instruction. - $ac_cs_success || { (exit 1); exit 1; } -fi - - -# Messages -printf '#================================================================\n' -printf '# Ready to make.\n' -printf '#================================================================\n' - - - -# END OF FILE diff --git a/qdbm/configure.in b/qdbm/configure.in index 9a6f4c40..95349aec 100644 --- a/qdbm/configure.in +++ b/qdbm/configure.in @@ -178,17 +178,18 @@ if uname | egrep -i 'Linux' > /dev/null 2>&1 && then MYOPTS="-minline-all-stringops" fi -if uname | egrep -i 'SunOS' > /dev/null 2>&1 -then - MYOPTS="-O1 -fno-omit-frame-pointer -fno-force-addr" -fi -if uname | egrep -i 'BSD' > /dev/null 2>&1 -then - MYOPTS="-O1 -fno-omit-frame-pointer -fno-force-addr" -fi -if gcc --version | egrep -i '^2\.(8|9)' > /dev/null 2>&1 + +echo "Compiler version:" `$CC -v` + +if $CC -v 2>&1 | egrep -iq '(^| )clang '; then + MYOPTS="-O1 -fno-omit-frame-pointer" +elif uname | egrep -iq 'SunOS' || \ + uname | egrep -iq 'BSD' || \ + gcc --version | egrep -iq '^2\.(8|9)' then MYOPTS="-O1 -fno-omit-frame-pointer -fno-force-addr" +else + MYOPTS="-O1 -fno-omit-frame-pointer -fforce-addr" fi # Linker diff --git a/runtest.pl.in b/runtest.pl.in index b4f59c1f..a864336b 100755 --- a/runtest.pl.in +++ b/runtest.pl.in @@ -3,12 +3,46 @@ use strict; use warnings; -use lib 'infrastructure'; +use Cwd; +use File::Basename; +use Getopt::Std; + +chdir(dirname($0)); +my $base_dir = getcwd(); +use lib dirname($0)."/infrastructure"; + use BoxPlatform; -my ($test_name,$test_mode) = @ARGV; +my %opts; +getopts('acnv', \%opts); + +# Don't actually run the test, just prepare for it. +my $cmake_build = $opts{'c'}; +my $prepare_only = $opts{'n'}; +my $verbose_build = $opts{'v'}; +my $appveyor_mode = $opts{'a'}; + +my $test_name = shift @ARGV; +my $test_mode = shift @ARGV; +my $test_src_exe; +my $test_dst_exe; + +if($cmake_build) +{ + # To support different build environments (Windows/MSVC and Linux/Makefile) which + # place compiled executables in different locations, we need to accept the name of + # the compiled test executable as an additional command-line parameter. + die "test name is required in cmake mode" unless $test_name; + die "test mode is required in cmake mode" unless $test_mode; + die "only a single test name is supported in cmake mode" if $test_name =~ /,/; + $test_src_exe = shift @ARGV; + die "test project source executable path is required in cmake mode" unless $test_src_exe; + $test_dst_exe = shift @ARGV; + die "test project destination executable name is required in cmake mode" unless $test_dst_exe; +} $test_mode = 'debug' if not defined $test_mode or $test_mode eq ''; +$test_mode = lc($test_mode); if($test_name eq '' || ($test_mode ne 'debug' && $test_mode ne 'release')) { @@ -38,7 +72,7 @@ if($test_name ne 'ALL') } else { - runtest($test_name); + runtest($test_name, $test_src_exe, $test_dst_exe); } } else @@ -52,7 +86,7 @@ else next if m/\AEND-OMIT/; if(m/\AOMIT:(.+)/) { - if($1 eq $build_os or $1 eq $target_os) + if($1 eq $build_os or $1 eq $ac_target_os) { while(<MODULES>) { @@ -93,25 +127,159 @@ __E exit $exit_code; +sub appveyor_test_status +{ + my ($test_name, $status, $duration, $message, $stdout) = @_; + + if(!$appveyor_mode) + { + return; + } + + # Assume that the test was already "Added" by CMakeLists.txt. + my $cmdline = "appveyor UpdateTest -Name $test_name -Framework Custom ". + "-FileName \"\" -Outcome $status"; + if(defined $duration) + { + $cmdline .= " -Duration ".($duration * 1000); + } + + if($message) + { + $cmdline .= " -ErrorMessage \"$message\""; + } + + if(system($cmdline)) + { + warn "AppVeyor test command failed: $cmdline"; + } + else + { + print "Notified: $cmdline\n"; + } +} + sub runtest { - my ($t) = @_; + my ($t, $test_src_exe, $test_dst_exe) = @_; - # attempt to make this test + # Attempt to make this test. my $flag = ($test_mode eq 'release')?(BoxPlatform::make_flag('RELEASE')):''; - my $make_res = system("cd test/$t && $make_command $flag"); + my $make_res; + my $test_dst_dir = "$test_mode/test/$t"; + my $start_time = time(); + + if($cmake_build) + { + appveyor_test_status($t, "Running", 0); + + # Test executables have a different name on Windows to work around + # restrictions on running different executables with the same name. + my $test_src_dir = "test/$t"; + + my @commands = ( + "cmake -E remove_directory $test_dst_dir", + "cmake -E copy_directory $test_src_dir $test_dst_dir", + "cmake -E copy $test_src_exe $test_dst_dir/$test_dst_exe", + # We could do a "make install" here, to ensure that everything + # is up to date, but it's really slow, verbose and wasteful: + # "cmake --build infrastructure/cmake/build --target install", + ); + + # Our CMake buildsystem doesn't do anything to support testextra files + # (Makfile syntax), so fake it. + if (-r "$test_src_dir/testextra") + { + open EXTRA, "$test_src_dir/testextra" + or die "$test_src_dir/testextra: $!"; + foreach my $line (<EXTRA>) + { + chomp $line; + if ($line =~ m/^mkdir (.*)/) + { + push @commands, "cmake -E make_directory $test_dst_dir/$1"; + } + elsif ($line =~ m/^rm -rf (.*)/) + { + push @commands, "cmake -E remove_directory $test_dst_dir/$1"; + } + elsif ($line =~ m/^cp (.*)\*\.\* (.*)/) + { + my ($src, $dst) = ($1, $2); + push @commands, "cmake -E copy_directory ". + "$test_dst_dir/$src $test_dst_dir/$dst"; + } + else + { + die "Unsupported command in ". + "$test_src_dir/testextra: $line"; + } + } + } + + foreach my $command (@commands) + { + $make_res = system($command); + if ($make_res != 0) + { + push @results, "$t: pre-test command failed: $command"; + appveyor_test_status($t, "NotRunnable", time() - $start_time, + "pre-test command failed: $command"); + last; + } + } + } + else + { + my $quiet = $verbose_build ? "VERBOSE=1" : ""; + $make_res = system("cd test/$t && $make_command $quiet $flag"); + $test_dst_exe = "_test$platform_exe_ext"; + } + if($make_res != 0) { push @results,"$t: make failed"; + appveyor_test_status($t, "NotRunnable", time() - $start_time, + "pre-test commands failed"); $exit_code = 2; return; } my $logfile = "test-$t.log"; + my $test_res; + + if($prepare_only) + { + appveyor_test_status($t, "Skipped", time() - $start_time, + "we are only preparing this test"); + return; + } # run it - my $test_res = system("cd $test_mode/test/$t ; ./t 2>&1 " . - "| tee ../../../$logfile"); + if($cmake_build) + { + # no tee.exe on Windows, so let's do it ourselves. + open LOG, ">$logfile" or die "$logfile: $!"; + chdir("$base_dir/$test_mode/test/$t"); + + open TEE, "$test_dst_exe |" + or die "$test_dst_dir/$test_dst_exe: $!"; + + while (my $line = <TEE>) + { + print $line; + print LOG $line; + } + close LOG; + close TEE; + chdir($base_dir); + } + else + { + chdir($base_dir); + $test_res = system("cd $test_mode/test/$t ; sh t 2>&1 " . + "| tee ../../../$logfile"); + } # open test results if(open RESULTS, $logfile) @@ -123,19 +291,38 @@ sub runtest } close RESULTS; - chomp $last; - $last =~ s/\r//; - push @results, "$t: $last"; - - if ($last ne "PASSED") - { + if(!defined $last) + { + push @results, "$t: test produced no output"; + appveyor_test_status($t, "Failed", time() - $start_time, + "test produced no output"); $exit_code = 1; } + else + { + chomp $last; + $last =~ s/\r//; + push @results, "$t: $last"; + + if ($last ne "PASSED") + { + $exit_code = 1; + appveyor_test_status($t, "Failed", time() - $start_time, + "test ended with: $last"); + } + else + { + appveyor_test_status($t, "Passed", time() - $start_time); + } + } } else { + my $cwd = getcwd(); push @results, - "$t: failed to open test log file: $logfile: $!"; + "$t: failed to open test log file: $logfile: $! (in $cwd)"; + appveyor_test_status($t, "Inconclusive", time() - $start_time, + "failed to open test log file: $logfile: $!"); } # delete test results diff --git a/test/backupdiff/testbackupdiff.cpp b/test/backupdiff/testbackupdiff.cpp index 816f50d1..ec3f24e2 100644 --- a/test/backupdiff/testbackupdiff.cpp +++ b/test/backupdiff/testbackupdiff.cpp @@ -15,6 +15,7 @@ #include "Test.h" #include "BackupClientCryptoKeys.h" #include "BackupStoreFile.h" +#include "BackupStoreFileEncodeStream.h" #include "BackupStoreFilenameClear.h" #include "FileStream.h" #include "BackupStoreFileWire.h" diff --git a/test/backupstore/Makefile.extra b/test/backupstore/Makefile.extra deleted file mode 100644 index b319736c..00000000 --- a/test/backupstore/Makefile.extra +++ /dev/null @@ -1 +0,0 @@ -link-extra: ../../lib/backupstore/HousekeepStoreAccount.o diff --git a/test/backupstore/testbackupstore.cpp b/test/backupstore/testbackupstore.cpp index 661973e0..6441d66c 100644 --- a/test/backupstore/testbackupstore.cpp +++ b/test/backupstore/testbackupstore.cpp @@ -15,17 +15,22 @@ #include "Archive.h" #include "BackupClientCryptoKeys.h" #include "BackupClientFileAttributes.h" +#include "BackupProtocol.h" #include "BackupStoreAccountDatabase.h" #include "BackupStoreAccounts.h" +#include "BackupStoreConfigVerify.h" #include "BackupStoreConstants.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" -#include "BackupStoreInfo.h" +#include "BackupStoreFile.h" #include "BackupStoreFilenameClear.h" +#include "BackupStoreFileEncodeStream.h" +#include "BackupStoreInfo.h" +#include "BackupStoreObjectMagic.h" #include "BackupStoreRefCountDatabase.h" -#include "BackupStoreFile.h" #include "BoxPortsAndFiles.h" #include "CollectInBufferStream.h" +#include "Configuration.h" #include "FileStream.h" #include "HousekeepStoreAccount.h" #include "MemBlockStream.h" @@ -37,15 +42,27 @@ #include "ServerControl.h" #include "Socket.h" #include "SocketStreamTLS.h" +#include "StoreStructure.h" +#include "StoreTestUtils.h" #include "TLSContext.h" #include "Test.h" -#include "autogen_BackupProtocol.h" +#include "ZeroStream.h" #include "MemLeakFindOn.h" - #define ENCFILE_SIZE 2765 +// Make some test attributes +#define ATTR1_SIZE 245 +#define ATTR2_SIZE 23 +#define ATTR3_SIZE 122 + +#define SHORT_TIMEOUT 5000 + +int attr1[ATTR1_SIZE]; +int attr2[ATTR2_SIZE]; +int attr3[ATTR3_SIZE]; + typedef struct { BackupStoreFilenameClear fn; @@ -94,7 +111,7 @@ typedef struct #define TEST_FILE_FOR_PATCHING_SIZE ((128*1024)+2564) #define UPLOAD_PATCH_EN 2 -uploadtest uploads[] = +uploadtest uploads[] = { {"0", BackupStoreFilenameClear(), 324, 455, 0, 0, false, false}, {"1", BackupStoreFilenameClear(), 3232432, 2674, 0, 0, true, false}, // old ver @@ -121,11 +138,42 @@ static const char *uploads_filenames[] = {"49587fds", "cvhjhj324", "sdfcscs324", // file which will be moved (as well as it's old version) #define UPLOAD_FILE_TO_MOVE 8 +#define UNLINK_IF_EXISTS(filename) \ + if (FileExists(filename)) { TEST_THAT(unlink(filename) == 0); } + +//! Simplifies calling setUp() with the current function name in each test. +#define SETUP_TEST_BACKUPSTORE() \ + SETUP(); \ + if (ServerIsAlive(bbstored_pid)) \ + TEST_THAT_OR(StopServer(), FAIL); \ + ExpectedRefCounts.resize(BACKUPSTORE_ROOT_DIRECTORY_ID + 1); \ + set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); \ + TEST_THAT_OR(create_account(10000, 20000), FAIL); + +//! Checks account for errors and shuts down daemons at end of every test. +bool teardown_test_backupstore() +{ + bool status = true; + + if (FileExists("testfiles/0_0/backup/01234567/info.rf")) + { + TEST_THAT_OR(check_reference_counts(), status = false); + TEST_THAT_OR(check_account(), status = false); + } + + return status; +} + +#define TEARDOWN_TEST_BACKUPSTORE() \ + if (ServerIsAlive(bbstored_pid)) \ + StopServer(); \ + TEST_THAT(teardown_test_backupstore()); \ + TEARDOWN(); // Nice random data for testing written files class R250 { public: - // Set up internal state table with 32-bit random numbers. + // Set up internal state table with 32-bit random numbers. // The bizarre bit-twiddling is because rand() returns 16 bits of which // the bottom bit is always zero! Hence, I use only some of the bits. // You might want to do something better than this.... @@ -142,7 +190,7 @@ public: // stir up the numbers to ensure they're random - for (int j = 0; j != stateLen * 4; ++j) + for (int j = 0; j != stateLen * 4; ++j) (void) next(); } @@ -174,7 +222,7 @@ int SkipEntries(int e, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) do { skip = false; - + if(FlagsMustBeSet != BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING) { if((ens[e].flags & FlagsMustBeSet) != FlagsMustBeSet) @@ -186,20 +234,20 @@ int SkipEntries(int e, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { skip = true; } - + if(skip) { ++e; } } while(skip && e < DIR_NUM); - + return e; } void CheckEntries(BackupStoreDirectory &rDir, int16_t FlagsMustBeSet, int16_t FlagsNotToBeSet) { int e = 0; - + BackupStoreDirectory::Iterator i(rDir); BackupStoreDirectory::Entry *en = 0; while((en = i.Next()) != 0) @@ -213,16 +261,18 @@ void CheckEntries(BackupStoreDirectory &rDir, int16_t FlagsMustBeSet, int16_t Fl TEST_THAT(en->GetName() == ens[e].fn && en->GetModificationTime() == ens[e].mod && en->GetObjectID() == ens[e].id && en->GetFlags() == ens[e].flags && en->GetSizeInBlocks() == ens[e].size); // next - ++e; + ++e; } - + // Got them all? TEST_THAT(en == 0); TEST_THAT(DIR_NUM == SkipEntries(e, FlagsMustBeSet, FlagsNotToBeSet)); } -int test1(int argc, const char *argv[]) +bool test_filename_encoding() { + SETUP_TEST_BACKUPSTORE(); + // test some basics -- encoding and decoding filenames { // Make some filenames in various ways @@ -232,11 +282,11 @@ int test1(int argc, const char *argv[]) BackupStoreFilenameClear fn3(fn1); TEST_THAT(fn1 == fn2); TEST_THAT(fn1 == fn3); - + // Check that it's been encrypted std::string name(fn2.GetEncodedFilename()); TEST_THAT(name.find("name") == name.npos); - + // Bung it in a stream, get it out in a Clear filename { CollectInBufferStream stream; @@ -247,6 +297,7 @@ int test1(int argc, const char *argv[]) TEST_THAT(fn4.GetClearFilename() == "filenameXYZ"); TEST_THAT(fn4 == fn1); } + // Bung it in a stream, get it out in a server non-Clear filename (two of them into the same var) { BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf "); @@ -260,6 +311,7 @@ int test1(int argc, const char *argv[]) fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5 == fno); } + // Same again with clear strings { BackupStoreFilenameClear fno("pinglet dksfnsf jksjdf "); @@ -273,6 +325,7 @@ int test1(int argc, const char *argv[]) fn5.ReadFromStream(stream, IOStream::TimeOutInfinite); TEST_THAT(fn5.GetClearFilename() == "pinglet dksfnsf jksjdf "); } + // Test a very big filename { const char *fnr = "01234567890123456789012345678901234567890123456789" @@ -292,19 +345,23 @@ int test1(int argc, const char *argv[]) TEST_THAT(fn9.GetClearFilename() == fnr); TEST_THAT(fn9 == fnLong); } + // Test a filename which went wrong once { BackupStoreFilenameClear dodgy("content-negotiation.html"); } } - return 0; + + TEARDOWN_TEST_BACKUPSTORE(); } -int test2(int argc, const char *argv[]) +bool test_backupstore_directory() { + SETUP_TEST_BACKUPSTORE(); + { // Now play with directories - + // Fill in... BackupStoreDirectory dir1(12, 98); for(int e = 0; e < DIR_NUM; ++e) @@ -313,27 +370,25 @@ int test2(int argc, const char *argv[]) } // Got the right number TEST_THAT(dir1.GetNumberOfEntries() == DIR_NUM); - + // Stick it into a stream and get it out again { CollectInBufferStream stream; dir1.WriteToStream(stream); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_NUM); TEST_THAT(dir2.GetObjectID() == 12); TEST_THAT(dir2.GetContainerID() == 98); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } - + // Then do selective writes and reads { CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } @@ -341,8 +396,7 @@ int test2(int argc, const char *argv[]) CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_INCLUDE_EVERYTHING, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_DIRS); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_Dir, BackupStoreDirectory::Entry::Flags_EXCLUDE_NOTHING); } @@ -350,12 +404,11 @@ int test2(int argc, const char *argv[]) CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - DIR_OLD); CheckEntries(dir2, BackupStoreDirectory::Entry::Flags_File, BackupStoreDirectory::Entry::Flags_OldVersion); } - + // Finally test deleting items { dir1.DeleteEntry(12312312321LL); @@ -364,11 +417,10 @@ int test2(int argc, const char *argv[]) CollectInBufferStream stream; dir1.WriteToStream(stream, BackupStoreDirectory::Entry::Flags_File); stream.SetForReading(); - BackupStoreDirectory dir2; - dir2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir2(stream); TEST_THAT(dir2.GetNumberOfEntries() == DIR_FILES - 1); } - + // Check attributes { int attrI[4] = {1, 2, 3, 4}; @@ -380,73 +432,70 @@ int test2(int argc, const char *argv[]) CollectInBufferStream stream; d1.WriteToStream(stream); stream.SetForReading(); - BackupStoreDirectory d2; - d2.ReadFromStream(stream, IOStream::TimeOutInfinite); + BackupStoreDirectory d2(stream); TEST_THAT(d2.GetAttributes() == attr); TEST_THAT(d2.GetAttributesModTime() == 56234987324232LL); } } - return 0; + + TEARDOWN_TEST_BACKUPSTORE(); } void write_test_file(int t) { std::string filename("testfiles/test"); filename += uploads[t].fnextra; - printf("%s\n", filename.c_str()); - + BOX_TRACE("Writing " << filename); + FileStream write(filename.c_str(), O_WRONLY | O_CREAT); - + R250 r(uploads[t].seed); - + unsigned char *data = (unsigned char*)malloc(uploads[t].size); for(int l = 0; l < uploads[t].size; ++l) { data[l] = r.next() & 0xff; } write.Write(data, uploads[t].size); - + free(data); } void test_test_file(int t, IOStream &rStream) { // Decode to a file - BackupStoreFile::DecodeFile(rStream, "testfiles/test_download", IOStream::TimeOutInfinite); - + BackupStoreFile::DecodeFile(rStream, "testfiles/test_download", SHORT_TIMEOUT); + // Compare... FileStream in("testfiles/test_download"); TEST_THAT(in.BytesLeftToRead() == uploads[t].size); - + R250 r(uploads[t].seed); - + unsigned char *data = (unsigned char*)malloc(uploads[t].size); TEST_THAT(in.ReadFullBuffer(data, uploads[t].size, 0 /* not interested in bytes read if this fails */)); - + for(int l = 0; l < uploads[t].size; ++l) { TEST_THAT(data[l] == (r.next() & 0xff)); } - + free(data); in.Close(); TEST_THAT(unlink("testfiles/test_download") == 0); } -void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID) +void assert_everything_deleted(BackupProtocolCallable &protocol, int64_t DirID) { - printf("Test for del: %llx\n", (unsigned long long)DirID); - + BOX_TRACE("Test for del: " << BOX_FORMAT_OBJECTID(DirID)); + // Command std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( DirID, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - + BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; int files = 0; @@ -457,34 +506,24 @@ void test_everything_deleted(BackupProtocolClient &protocol, int64_t DirID) { dirs++; // Recurse - test_everything_deleted(protocol, en->GetObjectID()); + assert_everything_deleted(protocol, en->GetObjectID()); } else { files++; } + // Check it's deleted - TEST_THAT(en->GetFlags() & BackupProtocolListDirectory::Flags_Deleted); + TEST_THAT(en->IsDeleted()); } - + // Check there were the right number of files and directories TEST_THAT(files == 3); TEST_THAT(dirs == 0 || dirs == 2); } -std::vector<uint32_t> ExpectedRefCounts; - -void set_refcount(int64_t ObjectID, uint32_t RefCount = 1) -{ - if ((int64_t)ExpectedRefCounts.size() <= ObjectID); - { - ExpectedRefCounts.resize(ObjectID + 1, 0); - } - ExpectedRefCounts[ObjectID] = RefCount; -} - void create_file_in_dir(std::string name, std::string source, int64_t parentId, - BackupProtocolClient &protocol, BackupStoreRefCountDatabase& rRefCount) + BackupProtocolCallable &protocol, BackupStoreRefCountDatabase* pRefCount) { BackupStoreFilenameClear name_encoded("file_One"); std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile( @@ -496,15 +535,22 @@ void create_file_in_dir(std::string name, std::string source, int64_t parentId, 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ name_encoded, - *upload)); + upload)); int64_t objectId = stored->GetObjectID(); - TEST_EQUAL(objectId, rRefCount.GetLastObjectIDUsed()); - TEST_EQUAL(1, rRefCount.GetRefCount(objectId)) + if (pRefCount) + { + TEST_EQUAL(objectId, pRefCount->GetLastObjectIDUsed()); + TEST_EQUAL(1, pRefCount->GetRefCount(objectId)) + } set_refcount(objectId, 1); } -int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, - const char *name, int depth, BackupStoreRefCountDatabase& rRefCount) +const box_time_t FAKE_MODIFICATION_TIME = 0xfeedfacedeadbeefLL; +const box_time_t FAKE_ATTR_MODIFICATION_TIME = 0xdeadbeefcafebabeLL; + +int64_t create_test_data_subdirs(BackupProtocolCallable &protocol, + int64_t indir, const char *name, int depth, + BackupStoreRefCountDatabase* pRefCount) { // Create a directory int64_t subdirid = 0; @@ -512,72 +558,80 @@ int64_t create_test_data_subdirs(BackupProtocolClient &protocol, int64_t indir, { // Create with dummy attributes int attrS = 0; - MemBlockStream attr(&attrS, sizeof(attrS)); + std::auto_ptr<IOStream> attr(new MemBlockStream(&attrS, sizeof(attrS))); std::auto_ptr<BackupProtocolSuccess> dirCreate(protocol.QueryCreateDirectory( - indir, - 9837429842987984LL, dirname, attr)); - subdirid = dirCreate->GetObjectID(); + indir, FAKE_ATTR_MODIFICATION_TIME, dirname, attr)); + subdirid = dirCreate->GetObjectID(); + } + + BOX_TRACE("Creating subdirs, depth = " << depth << ", dirid = " << + BOX_FORMAT_OBJECTID(subdirid)); + + if (pRefCount) + { + TEST_EQUAL(subdirid, pRefCount->GetLastObjectIDUsed()); + TEST_EQUAL(1, pRefCount->GetRefCount(subdirid)) } - - printf("Create subdirs, depth = %d, dirid = %llx\n", depth, - (unsigned long long)subdirid); - TEST_EQUAL(subdirid, rRefCount.GetLastObjectIDUsed()); - TEST_EQUAL(1, rRefCount.GetRefCount(subdirid)) set_refcount(subdirid, 1); - + // Put more directories in it, if we haven't gone down too far if(depth > 0) { create_test_data_subdirs(protocol, subdirid, "dir_One", - depth - 1, rRefCount); + depth - 1, pRefCount); create_test_data_subdirs(protocol, subdirid, "dir_Two", - depth - 1, rRefCount); + depth - 1, pRefCount); } - + // Stick some files in it - create_file_in_dir("file_One", "testfiles/file1", subdirid, protocol, - rRefCount); - create_file_in_dir("file_Two", "testfiles/file1", subdirid, protocol, - rRefCount); - create_file_in_dir("file_Three", "testfiles/file1", subdirid, protocol, - rRefCount); + create_file_in_dir("file_One", "testfiles/test1", subdirid, protocol, + pRefCount); + create_file_in_dir("file_Two", "testfiles/test1", subdirid, protocol, + pRefCount); + create_file_in_dir("file_Three", "testfiles/test1", subdirid, protocol, + pRefCount); return subdirid; } - -void check_dir_after_uploads(BackupProtocolClient &protocol, const StreamableMemBlock &Attributes) +void check_dir_after_uploads(BackupProtocolCallable &protocol, + const StreamableMemBlock &Attributes) { // Command std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); - TEST_THAT(dirreply->GetObjectID() == BackupProtocolListDirectory::RootDirectory); + TEST_THAT(dirreply->GetObjectID() == BACKUPSTORE_ROOT_DIRECTORY_ID); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - TEST_THAT(dir.GetNumberOfEntries() == UPLOAD_NUM + 1 /* for the first test file */); + BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); + TEST_EQUAL(UPLOAD_NUM, dir.GetNumberOfEntries()); TEST_THAT(!dir.HasAttributes()); // Check them! BackupStoreDirectory::Iterator i(dir); - // Discard first - BackupStoreDirectory::Entry *en = i.Next(); - TEST_THAT(en != 0); + BackupStoreDirectory::Entry *en; for(int t = 0; t < UPLOAD_NUM; ++t) { en = i.Next(); TEST_THAT(en != 0); - TEST_THAT(en->GetName() == uploads[t].name); - TEST_THAT(en->GetObjectID() == uploads[t].allocated_objid); - TEST_THAT(en->GetModificationTime() == uploads[t].mod_time); + if (en == 0) continue; + TEST_LINE(uploads[t].name == en->GetName(), + "uploaded file " << t << " name"); + BackupStoreFilenameClear clear(en->GetName()); + TEST_EQUAL_LINE(uploads[t].name.GetClearFilename(), + clear.GetClearFilename(), + "uploaded file " << t << " name"); + TEST_EQUAL_LINE(uploads[t].allocated_objid, en->GetObjectID(), + "uploaded file " << t << " ID"); + TEST_EQUAL_LINE(uploads[t].mod_time, en->GetModificationTime(), + "uploaded file " << t << " modtime"); int correct_flags = BackupProtocolListDirectory::Flags_File; if(uploads[t].should_be_old_version) correct_flags |= BackupProtocolListDirectory::Flags_OldVersion; if(uploads[t].delete_file) correct_flags |= BackupProtocolListDirectory::Flags_Deleted; - TEST_THAT(en->GetFlags() == correct_flags); + TEST_EQUAL_LINE(correct_flags, en->GetFlags(), + "uploaded file " << t << " flags"); if(t == UPLOAD_ATTRS_EN) { TEST_THAT(en->HasAttributes()); @@ -589,12 +643,11 @@ void check_dir_after_uploads(BackupProtocolClient &protocol, const StreamableMem // No attributes on this one TEST_THAT(!en->HasAttributes()); } - } + } en = i.Next(); TEST_THAT(en == 0); } - typedef struct { int objectsNotDel; @@ -602,7 +655,8 @@ typedef struct int old; } recursive_count_objects_results; -void recursive_count_objects_r(BackupProtocolClient &protocol, int64_t id, recursive_count_objects_results &results) +void recursive_count_objects_r(BackupProtocolCallable &protocol, int64_t id, + recursive_count_objects_results &results) { // Command std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( @@ -610,21 +664,19 @@ void recursive_count_objects_r(BackupProtocolClient &protocol, int64_t id, recur BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); // Check them! BackupStoreDirectory::Iterator i(dir); // Discard first BackupStoreDirectory::Entry *en = 0; - + while((en = i.Next()) != 0) { if((en->GetFlags() & (BackupStoreDirectory::Entry::Flags_Deleted | BackupStoreDirectory::Entry::Flags_OldVersion)) == 0) results.objectsNotDel++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Deleted) results.deleted++; if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_OldVersion) results.old++; - + if(en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) { recursive_count_objects_r(protocol, en->GetObjectID(), results); @@ -632,27 +684,14 @@ void recursive_count_objects_r(BackupProtocolClient &protocol, int64_t id, recur } } -void recursive_count_objects(const char *hostname, int64_t id, recursive_count_objects_results &results) -{ - // Context - TLSContext context; - context.Initialise(false /* client */, - "testfiles/clientCerts.pem", - "testfiles/clientPrivKey.pem", - "testfiles/clientTrustedCAs.pem"); +TLSContext context; +void recursive_count_objects(int64_t id, recursive_count_objects_results &results) +{ // Get a connection - SocketStreamTLS connReadOnly; - connReadOnly.Open(context, Socket::TypeINET, hostname, - BOX_PORT_BBSTORED_TEST); - BackupProtocolClient protocolReadOnly(connReadOnly); + BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", + "backup/01234567/", 0, false); - { - std::auto_ptr<BackupProtocolVersion> serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); - } - // Count objects recursive_count_objects_r(protocolReadOnly, id, results); @@ -673,8 +712,8 @@ bool check_block_index(const char *encoded_file, IOStream &rBlockIndex) { char buffer1[2048]; char buffer2[2048]; - int s = enc.Read(buffer1, sizeof(buffer1)); - if(rBlockIndex.Read(buffer2, s) != s) + int s = enc.Read(buffer1, sizeof(buffer1), SHORT_TIMEOUT); + if(rBlockIndex.Read(buffer2, s, SHORT_TIMEOUT) != s) { same = false; break; @@ -685,19 +724,19 @@ bool check_block_index(const char *encoded_file, IOStream &rBlockIndex) break; } } - + if(rBlockIndex.StreamDataLeft()) { same = false; - + // Absorb all this excess data so procotol is in the first state char buffer[2048]; while(rBlockIndex.StreamDataLeft()) { - rBlockIndex.Read(buffer, sizeof(buffer)); + rBlockIndex.Read(buffer, sizeof(buffer), SHORT_TIMEOUT); } } - + return same; } @@ -726,18 +765,77 @@ bool check_files_same(const char *f1, const char *f2) break; } } - + if(f2s.StreamDataLeft()) { same = false; } - + return same; } +std::auto_ptr<RaidFileRead> get_raid_file(int64_t ObjectID) +{ + std::string filename; + StoreStructure::MakeObjectFilename(ObjectID, + "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, + filename, false /* EnsureDirectoryExists */); + return RaidFileRead::Open(0, filename); +} + +int64_t create_directory(BackupProtocolCallable& protocol, + int64_t parent_dir_id = BACKUPSTORE_ROOT_DIRECTORY_ID); +int64_t create_file(BackupProtocolCallable& protocol, int64_t subdirid, + const std::string& remote_filename = ""); + +bool run_housekeeping_and_check_account(BackupProtocolLocal2& protocol) +{ + protocol.QueryFinished(); + bool check_account_status; + TEST_THAT(check_account_status = run_housekeeping_and_check_account()); + bool check_refcount_status; + TEST_THAT(check_refcount_status = check_reference_counts()); + protocol.Reopen(); + return check_account_status & check_refcount_status; +} -void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protocolReadOnly) +bool test_temporary_refcount_db_is_independent() { + SETUP_TEST_BACKUPSTORE(); + + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + std::auto_ptr<BackupStoreRefCountDatabase> temp( + BackupStoreRefCountDatabase::Create( + apAccounts->GetEntry(0x1234567))); + std::auto_ptr<BackupStoreRefCountDatabase> perm( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), + true // ReadOnly + )); + + TEST_CHECK_THROWS(temp->GetRefCount(2), + BackupStoreException, UnknownObjectRefCountRequested); + TEST_CHECK_THROWS(perm->GetRefCount(2), + BackupStoreException, UnknownObjectRefCountRequested); + temp->AddReference(2); + TEST_EQUAL(1, temp->GetRefCount(2)); + TEST_CHECK_THROWS(perm->GetRefCount(2), + BackupStoreException, UnknownObjectRefCountRequested); + temp->Discard(); + + // Need to delete perm object so it doesn't keep a filehandle open, + // preventing tearDown from rewriting the refcount DB and thus causing + // test failure. + perm.reset(); + + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_server_housekeeping() +{ + SETUP_TEST_BACKUPSTORE(); + int encfile[ENCFILE_SIZE]; { for(int l = 0; l < ENCFILE_SIZE; ++l) @@ -747,71 +845,64 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco // Write this to a file { - FileStream f("testfiles/file1", O_WRONLY | O_CREAT | O_EXCL); + FileStream f("testfiles/file1", O_WRONLY | O_CREAT); f.Write(encfile, sizeof(encfile)); } - } - // Read the root directory a few times (as it's cached, so make sure it doesn't hurt anything) - for(int l = 0; l < 3; ++l) - { - // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); - // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - TEST_THAT(dir.GetNumberOfEntries() == 0); - } + // We need complete control over housekeeping, so use a local client + // instead of a network client + daemon. - // Read the dir from the readonly connection (make sure it gets in the cache) - { - // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); - // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - TEST_THAT(dir.GetNumberOfEntries() == 0); - } + BackupProtocolLocal2 protocol(0x01234567, "test", "backup/01234567/", + 0, false); + + int root_dir_blocks = get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID)->GetDiscUsageInBlocks(); + TEST_THAT(check_num_files(0, 0, 0, 1)); + TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, + root_dir_blocks)); // Store a file -- first make the encoded file - BackupStoreFilenameClear store1name("testfiles/file1"); + BackupStoreFilenameClear store1name("file1"); { - FileStream out("testfiles/file1_upload1", O_WRONLY | O_CREAT | O_EXCL); - std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/file1", BackupProtocolListDirectory::RootDirectory, store1name)); + FileStream out("testfiles/file1_upload1", O_WRONLY | O_CREAT); + std::auto_ptr<IOStream> encoded( + BackupStoreFile::EncodeFile("testfiles/file1", + BACKUPSTORE_ROOT_DIRECTORY_ID, store1name)); encoded->CopyStreamTo(out); } -// printf("SKIPPING\n"); -// goto skip; { + // TODO FIXME move most of the code immediately below into + // test_server_commands. + + // TODO FIXME use COMMAND macro for all commands to check the returned + // object ID. + #define COMMAND(command, objectid) \ + TEST_EQUAL(objectid, protocol.command->GetObjectID()); + // Then send it - int64_t store1objid = 0; + int64_t store1objid; { - FileStream upload("testfiles/file1_upload1"); + std::auto_ptr<IOStream> upload(new FileStream("testfiles/file1_upload1")); std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ store1name, upload)); store1objid = stored->GetObjectID(); - TEST_THAT(store1objid == 2); + TEST_EQUAL_LINE(2, store1objid, "wrong ObjectID for newly " + "uploaded file"); } + + // Update expected reference count of this new object set_refcount(store1objid, 1); + // And retrieve it { // Retrieve as object - std::auto_ptr<BackupProtocolSuccess> getfile(protocol.QueryGetObject(store1objid)); - TEST_THAT(getfile->GetObjectID() == store1objid); + COMMAND(QueryGetObject(store1objid), store1objid); + // BLOCK { // Get stream @@ -821,14 +912,17 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco filestream->CopyStreamTo(f); f.SetForReading(); // Get and decode + UNLINK_IF_EXISTS("testfiles/file1_upload_retrieved"); BackupStoreFile::DecodeFile(f, "testfiles/file1_upload_retrieved", IOStream::TimeOutInfinite); } // Retrieve as file - std::auto_ptr<BackupProtocolSuccess> getobj(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, store1objid)); - TEST_THAT(getobj->GetObjectID() == store1objid); + COMMAND(QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, store1objid), store1objid); + // BLOCK { + UNLINK_IF_EXISTS("testfiles/file1_upload_retrieved_str"); + // Get stream std::auto_ptr<IOStream> filestream(protocol.ReceiveStream()); // Get and decode @@ -842,13 +936,14 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } + { FileStream in("testfiles/file1_upload_retrieved_str"); int encfile_i[ENCFILE_SIZE]; in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } - + // Retrieve the block index, by ID { std::auto_ptr<BackupProtocolSuccess> getblockindex(protocol.QueryGetBlockIndexByID(store1objid)); @@ -857,26 +952,26 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco // Check against uploaded file TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream)); } + // and again, by name { - std::auto_ptr<BackupProtocolSuccess> getblockindex(protocol.QueryGetBlockIndexByName(BackupProtocolListDirectory::RootDirectory, store1name)); + std::auto_ptr<BackupProtocolSuccess> getblockindex(protocol.QueryGetBlockIndexByName(BACKUPSTORE_ROOT_DIRECTORY_ID, store1name)); TEST_THAT(getblockindex->GetObjectID() == store1objid); std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream()); // Check against uploaded file TEST_THAT(check_block_index("testfiles/file1_upload1", *blockIndexStream)); } } + // Get the directory again, and see if the entry is in it { // Command std::auto_ptr<BackupProtocolSuccess> dirreply(protocol.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocol.ReceiveStream(), SHORT_TIMEOUT); TEST_THAT(dir.GetNumberOfEntries() == 1); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(); @@ -888,182 +983,303 @@ void test_server_1(BackupProtocolClient &protocol, BackupProtocolClient &protoco TEST_THAT(en->GetModificationTime() == 0x123456789abcdefLL); TEST_THAT(en->GetAttributesHash() == 0x7362383249872dfLL); TEST_THAT(en->GetObjectID() == store1objid); - TEST_THAT(en->GetSizeInBlocks() < ((ENCFILE_SIZE * 4 * 3) / 2 / 2048)+2); + TEST_EQUAL(6, en->GetSizeInBlocks()); TEST_THAT(en->GetFlags() == BackupStoreDirectory::Entry::Flags_File); } } + int file1_blocks = get_raid_file(store1objid)->GetDiscUsageInBlocks(); + TEST_THAT(check_num_files(1, 0, 0, 1)); + TEST_THAT(check_num_blocks(protocol, file1_blocks, 0, 0, root_dir_blocks, + file1_blocks + root_dir_blocks)); + + // Upload again, as a patch to the original file. + int64_t patch1_id = BackupStoreFile::QueryStoreFileDiff(protocol, + "testfiles/file1", // LocalFilename + BACKUPSTORE_ROOT_DIRECTORY_ID, // DirectoryObjectID + store1objid, // DiffFromFileID + 0x7362383249872dfLL, // AttributesHash + store1name // StoreFilename + ); + TEST_EQUAL_LINE(3, patch1_id, "wrong ObjectID for newly uploaded " + "patch file"); + + // We need to check the old file's size, because it's been replaced + // by a reverse diff, and patch1_id is a complete file, not a diff. + int patch1_blocks = get_raid_file(store1objid)->GetDiscUsageInBlocks(); + + // It will take extra blocks, even though there are no changes, because + // the server code is not smart enough to realise that the file + // contents are identical, so it will create an empty patch. + + TEST_THAT(check_num_files(1, 1, 0, 1)); + TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks, 0, + root_dir_blocks, file1_blocks + patch1_blocks + root_dir_blocks)); + + // Change the file and upload again, as a patch to the original file. + { + FileStream out("testfiles/file1", O_WRONLY | O_APPEND); + std::string appendix = "appendix!"; + out.Write(appendix.c_str(), appendix.size()); + out.Close(); + } + + int64_t patch2_id = BackupStoreFile::QueryStoreFileDiff(protocol, + "testfiles/file1", // LocalFilename + BACKUPSTORE_ROOT_DIRECTORY_ID, // DirectoryObjectID + patch1_id, // DiffFromFileID + 0x7362383249872dfLL, // AttributesHash + store1name // StoreFilename + ); + TEST_EQUAL_LINE(4, patch2_id, "wrong ObjectID for newly uploaded " + "patch file"); + + // How many blocks used by the new file? + // We need to check the old file's size, because it's been replaced + // by a reverse diff, and patch1_id is a complete file, not a diff. + int patch2_blocks = get_raid_file(patch1_id)->GetDiscUsageInBlocks(); + + TEST_THAT(check_num_files(1, 2, 0, 1)); + TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks + patch2_blocks, 0, + root_dir_blocks, file1_blocks + patch1_blocks + patch2_blocks + + root_dir_blocks)); + + // Housekeeping should not change anything just yet + protocol.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + + TEST_THAT(check_num_files(1, 2, 0, 1)); + TEST_THAT(check_num_blocks(protocol, file1_blocks, patch1_blocks + patch2_blocks, 0, + root_dir_blocks, file1_blocks + patch1_blocks + patch2_blocks + + root_dir_blocks)); + + // Upload not as a patch, but as a completely different file. This + // marks the previous file as old (because the filename is the same) + // but used to not adjust the number of old/deleted files properly. + int64_t replaced_id = BackupStoreFile::QueryStoreFileDiff(protocol, + "testfiles/file1", // LocalFilename + BACKUPSTORE_ROOT_DIRECTORY_ID, // DirectoryObjectID + 0, // DiffFromFileID + 0x7362383249872dfLL, // AttributesHash + store1name // StoreFilename + ); + TEST_EQUAL_LINE(5, replaced_id, "wrong ObjectID for newly uploaded " + "full file"); + + // How many blocks used by the new file? This time we need to check + // the new file, because it's not a patch. + int replaced_blocks = get_raid_file(replaced_id)->GetDiscUsageInBlocks(); + + TEST_THAT(check_num_files(1, 3, 0, 1)); + TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current + file1_blocks + patch1_blocks + patch2_blocks, // old + 0, // deleted + root_dir_blocks, // directories + file1_blocks + patch1_blocks + patch2_blocks + replaced_blocks + + root_dir_blocks)); // total + + // Housekeeping should not change anything just yet + protocol.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + + TEST_THAT(check_num_files(1, 3, 0, 1)); + TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current + file1_blocks + patch1_blocks + patch2_blocks, // old + 0, // deleted + root_dir_blocks, // directories + file1_blocks + patch1_blocks + patch2_blocks + replaced_blocks + + root_dir_blocks)); // total + + // But if we reduce the limits, then it will + protocol.QueryFinished(); + TEST_THAT(change_account_limits( + "14B", // replaced_blocks + file1_blocks + root_dir_blocks + "2000B")); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + + TEST_THAT(check_num_files(1, 1, 0, 1)); + TEST_THAT(check_num_blocks(protocol, replaced_blocks, // current + file1_blocks, // old + 0, // deleted + root_dir_blocks, // directories + file1_blocks + replaced_blocks + root_dir_blocks)); // total + + // Check that deleting files is accounted for as well + protocol.QueryDeleteFile( + BACKUPSTORE_ROOT_DIRECTORY_ID, // InDirectory + store1name); // Filename + + // The old version file is deleted as well! + TEST_THAT(check_num_files(0, 1, 2, 1)); + TEST_THAT(check_num_blocks(protocol, 0, // current + file1_blocks, // old + replaced_blocks + file1_blocks, // deleted + root_dir_blocks, // directories + file1_blocks + replaced_blocks + root_dir_blocks)); + + // Reduce limits again, check that removed files are subtracted from + // block counts. + protocol.QueryFinished(); + TEST_THAT(change_account_limits("0B", "2000B")); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + + TEST_THAT(check_num_files(0, 0, 0, 1)); + TEST_THAT(check_num_blocks(protocol, 0, 0, 0, root_dir_blocks, root_dir_blocks)); + + // Used to not consume the stream + std::auto_ptr<IOStream> upload(new ZeroStream(1000)); + TEST_COMMAND_RETURNS_ERROR(protocol, QueryStoreFile( + BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, + 0, /* use for attr hash too */ + 99999, /* diff from ID */ + uploads[0].name, + upload), + Err_DiffFromFileDoesNotExist); + + // TODO FIXME These tests should not be here, but in + // test_server_commands. But make sure you use a network protocol, + // not a local one, when you move them. + // Try using GetFile on a directory { - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolSuccess> getFile(protocol.QueryGetFile(BackupProtocolListDirectory::RootDirectory, BackupProtocolListDirectory::RootDirectory)), - ConnectionException, Conn_Protocol_UnexpectedReply); + int64_t subdirid = create_directory(protocol); + TEST_COMMAND_RETURNS_ERROR(protocol, + QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, subdirid), + Err_FileDoesNotVerify); } -} -void init_context(TLSContext& rContext) -{ - rContext.Initialise(false /* client */, - "testfiles/clientCerts.pem", - "testfiles/clientPrivKey.pem", - "testfiles/clientTrustedCAs.pem"); + // Try retrieving an object that doesn't exist. That used to return + // BackupProtocolSuccess(NoObject) for no apparent reason. + TEST_COMMAND_RETURNS_ERROR(protocol, QueryGetObject(store1objid + 1), + Err_DoesNotExist); + + // Close the protocol, so we can housekeep the account + protocol.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + + ExpectedRefCounts.resize(3); // stop test failure in teardown_test_backupstore() + TEARDOWN_TEST_BACKUPSTORE(); } -std::auto_ptr<SocketStreamTLS> open_conn(const char *hostname, - TLSContext& rContext) +int64_t create_directory(BackupProtocolCallable& protocol, int64_t parent_dir_id) { - init_context(rContext); - std::auto_ptr<SocketStreamTLS> conn(new SocketStreamTLS); - conn->Open(rContext, Socket::TypeINET, hostname, - BOX_PORT_BBSTORED_TEST); - return conn; + // Create a directory + BackupStoreFilenameClear dirname("lovely_directory"); + // Attributes + std::auto_ptr<IOStream> attr(new MemBlockStream(attr1, sizeof(attr1))); + + std::auto_ptr<BackupProtocolSuccess> dirCreate( + protocol.QueryCreateDirectory2( + parent_dir_id, FAKE_ATTR_MODIFICATION_TIME, + FAKE_MODIFICATION_TIME, dirname, attr)); + + int64_t subdirid = dirCreate->GetObjectID(); + set_refcount(subdirid, 1); + return subdirid; } -std::auto_ptr<BackupProtocolClient> test_server_login(const char *hostname, - TLSContext& rContext, std::auto_ptr<SocketStreamTLS>& rapConn) +int64_t create_file(BackupProtocolCallable& protocol, int64_t subdirid, + const std::string& remote_filename) { - rapConn = open_conn(hostname, rContext); - - // Make a protocol - std::auto_ptr<BackupProtocolClient> protocol(new - BackupProtocolClient(*rapConn)); - - // Check the version - std::auto_ptr<BackupProtocolVersion> serverVersion( - protocol->QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + // Stick a file in it + write_test_file(0); - // Login - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf( - protocol->QueryLogin(0x01234567, 0)); + BackupStoreFilenameClear remote_filename_encoded; + if (remote_filename.empty()) + { + remote_filename_encoded = uploads[0].name; + } + else + { + remote_filename_encoded = remote_filename; + } - return protocol; + std::string filename("testfiles/test0"); + int64_t modtime; + std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename, + subdirid, remote_filename_encoded, &modtime)); + + std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile( + subdirid, + modtime, + modtime, /* use for attr hash too */ + 0, /* diff from ID */ + remote_filename_encoded, + upload)); + + int64_t subdirfileid = stored->GetObjectID(); + set_refcount(subdirfileid, 1); + return subdirfileid; } -void run_housekeeping(BackupStoreAccountDatabase::Entry& rAccount) +bool assert_writable_connection_fails(BackupProtocolCallable& protocol) { - std::string rootDir = BackupStoreAccounts::GetAccountRoot(rAccount); - int discSet = rAccount.GetDiscSet(); - - // Do housekeeping on this account - HousekeepStoreAccount housekeeping(rAccount.GetID(), rootDir, - discSet, NULL); - housekeeping.DoHousekeeping(true /* keep trying forever */); + std::auto_ptr<BackupProtocolVersion> serverVersion + (protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); + TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + TEST_COMMAND_RETURNS_ERROR_OR(protocol, QueryLogin(0x01234567, 0), + Err_CannotLockStoreForWriting, return false); + protocol.QueryFinished(); + return true; } -// Run housekeeping (for which we need to disconnect ourselves) and check -// that it doesn't change the numbers of files. -// -// Also check that bbstoreaccounts doesn't change anything +int64_t assert_readonly_connection_succeeds(BackupProtocolCallable& protocol) +{ + // TODO FIXME share code with test_server_login + std::auto_ptr<BackupProtocolVersion> serverVersion + (protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); + TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + std::auto_ptr<BackupProtocolLoginConfirmed> loginConf + (protocol.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); + return loginConf->GetClientStoreMarker(); +} -void run_housekeeping_and_check_account(const char *hostname, - TLSContext& rContext, std::auto_ptr<SocketStreamTLS>& rapConn, - std::auto_ptr<BackupProtocolClient>& rapProtocol) +bool test_multiple_uploads() { - rapProtocol->QueryFinished(); - std::auto_ptr<BackupStoreAccountDatabase> apAccounts( - BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - BackupStoreAccountDatabase::Entry account = - apAccounts->GetEntry(0x1234567); - run_housekeeping(account); + SETUP_TEST_BACKUPSTORE(); + TEST_THAT_OR(StartServer(), FAIL); - TEST_THAT(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf check 01234567 fix") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + std::auto_ptr<BackupProtocolCallable> apProtocol = + connect_and_login(context); - rapProtocol = test_server_login(hostname, rContext, rapConn); -} + // TODO FIXME replace protocolReadOnly with apProtocolReadOnly. + BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", + "backup/01234567/", 0, true); // ReadOnly -int test_server(const char *hostname) -{ - TLSContext context; - std::auto_ptr<SocketStreamTLS> conn; - std::auto_ptr<BackupProtocolClient> apProtocol = - test_server_login(hostname, context, conn); - - // Make some test attributes - #define ATTR1_SIZE 245 - #define ATTR2_SIZE 23 - #define ATTR3_SIZE 122 - int attr1[ATTR1_SIZE]; - int attr2[ATTR2_SIZE]; - int attr3[ATTR3_SIZE]; + // Read the root directory a few times (as it's cached, so make sure it doesn't hurt anything) + for(int l = 0; l < 3; ++l) { - R250 r(3465657); - for(int l = 0; l < ATTR1_SIZE; ++l) {attr1[l] = r.next();} - for(int l = 0; l < ATTR2_SIZE; ++l) {attr2[l] = r.next();} - for(int l = 0; l < ATTR3_SIZE; ++l) {attr3[l] = r.next();} + // Command + apProtocol->QueryListDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */); + // Stream + BackupStoreDirectory dir(apProtocol->ReceiveStream(), + apProtocol->GetTimeout()); + TEST_THAT(dir.GetNumberOfEntries() == 0); } - // BLOCK - { - // Get it logging - FILE *protocolLog = ::fopen("testfiles/protocol.log", "w"); - TEST_THAT(protocolLog != 0); - apProtocol->SetLogToFile(protocolLog); - -#ifndef WIN32 - // Check that we can't open a new connection which requests write permissions - { - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, hostname, - BOX_PORT_BBSTORED_TEST); - BackupProtocolClient protocol(conn); - std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)), - ConnectionException, Conn_Protocol_UnexpectedReply); - protocol.QueryFinished(); - } -#endif - - // Set the client store marker - apProtocol->QuerySetClientStoreMarker(0x8732523ab23aLL); - -#ifndef WIN32 - // Open a new connection which is read only - SocketStreamTLS connReadOnly; - connReadOnly.Open(context, Socket::TypeINET, hostname, - BOX_PORT_BBSTORED_TEST); - BackupProtocolClient protocolReadOnly(connReadOnly); - - // Get it logging - FILE *protocolReadOnlyLog = ::fopen("testfiles/protocolReadOnly.log", "w"); - TEST_THAT(protocolReadOnlyLog != 0); - protocolReadOnly.SetLogToFile(protocolReadOnlyLog); - - { - std::auto_ptr<BackupProtocolVersion> serverVersion(protocolReadOnly.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocolReadOnly.QueryLogin(0x01234567, BackupProtocolLogin::Flags_ReadOnly)); - - // Check client store marker - TEST_THAT(loginConf->GetClientStoreMarker() == 0x8732523ab23aLL); - } -#else // WIN32 - #define protocolReadOnly (*apProtocol) -#endif - - test_server_1(*apProtocol, protocolReadOnly); - - #define TEST_NUM_FILES(files, old, deleted, dirs) \ - { \ - std::auto_ptr<BackupStoreInfo> apInfo = \ - BackupStoreInfo::Load(0x1234567, \ - "backup/01234567/", 0, true); \ - TEST_EQUAL_LINE(files, apInfo->GetNumFiles(), \ - "num files"); \ - TEST_EQUAL_LINE(old, apInfo->GetNumOldFiles(), \ - "old files"); \ - TEST_EQUAL_LINE(deleted, apInfo->GetNumDeletedFiles(), \ - "deleted files"); \ - TEST_EQUAL_LINE(dirs, apInfo->GetNumDirectories(), \ - "directories"); \ - } + // Read the dir from the readonly connection (make sure it gets in the cache) + // Command + protocolReadOnly.QueryListDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes */); + // Stream + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + protocolReadOnly.GetTimeout()); + TEST_THAT(dir.GetNumberOfEntries() == 0); - TEST_NUM_FILES(1, 0, 0, 1); - run_housekeeping_and_check_account(hostname, context, - conn, apProtocol); - TEST_NUM_FILES(1, 0, 0, 1); + // TODO FIXME dedent + { + TEST_THAT(check_num_files(0, 0, 0, 1)); // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); @@ -1078,16 +1294,16 @@ int test_server(const char *hostname) filename += uploads[t].fnextra; int64_t modtime = 0; - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), BackupProtocolListDirectory::RootDirectory, uploads[t].name, &modtime)); + std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[t].name, &modtime)); TEST_THAT(modtime != 0); - + std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, modtime, modtime, /* use it for attr hash too */ 0, /* diff from ID */ uploads[t].name, - *upload)); + upload)); uploads[t].allocated_objid = stored->GetObjectID(); uploads[t].mod_time = modtime; if(maxID < stored->GetObjectID()) maxID = stored->GetObjectID(); @@ -1095,35 +1311,73 @@ int test_server(const char *hostname) BOX_TRACE("wrote file " << filename << " to server " "as object " << BOX_FORMAT_OBJECTID(stored->GetObjectID())); - TEST_NUM_FILES(t + 2, 0, 0, 1); - run_housekeeping_and_check_account(hostname, context, - conn, apProtocol); - TEST_NUM_FILES(t + 2, 0, 0, 1); + // Some of the uploaded files replace old ones, increasing + // the old file count instead of the current file count. + int expected_num_old_files = 0; + if (t >= 8) expected_num_old_files++; + if (t >= 12) expected_num_old_files++; + if (t >= 13) expected_num_old_files++; + int expected_num_current_files = t + 1 - expected_num_old_files; + + TEST_THAT(check_num_files(expected_num_current_files, + expected_num_old_files, 0, 1)); + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + apProtocol = connect_and_login(context); + protocolReadOnly.Reopen(); + + TEST_THAT(check_num_files(expected_num_current_files, + expected_num_old_files, 0, 1)); } // Add some attributes onto one of them { - TEST_NUM_FILES(UPLOAD_NUM + 1, 0, 0, 1); - MemBlockStream attrnew(attr3, sizeof(attr3)); + TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 0, 1)); + std::auto_ptr<IOStream> attrnew( + new MemBlockStream(attr3, sizeof(attr3))); std::auto_ptr<BackupProtocolSuccess> set(apProtocol->QuerySetReplacementFileAttributes( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, 32498749832475LL, uploads[UPLOAD_ATTRS_EN].name, attrnew)); TEST_THAT(set->GetObjectID() == uploads[UPLOAD_ATTRS_EN].allocated_objid); - TEST_NUM_FILES(UPLOAD_NUM + 1, 0, 0, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 3, 3, 0, 1)); } - + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + apProtocol = connect_and_login(context); + protocolReadOnly.Reopen(); + // Delete one of them (will implicitly delete an old version) { std::auto_ptr<BackupProtocolSuccess> del(apProtocol->QueryDeleteFile( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[UPLOAD_DELETE_EN].name)); TEST_THAT(del->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid); - TEST_NUM_FILES(UPLOAD_NUM, 0, 1, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); } +#ifdef _MSC_VER + BOX_TRACE("1"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + apProtocol = connect_and_login(context); + protocolReadOnly.Reopen(); + +#ifdef _MSC_VER + BOX_TRACE("2"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + // Check that the block index can be obtained by name even though it's been deleted { // Fetch the raw object @@ -1131,11 +1385,11 @@ int test_server(const char *hostname) FileStream out("testfiles/downloaddelobj", O_WRONLY | O_CREAT); std::auto_ptr<BackupProtocolSuccess> getobj(apProtocol->QueryGetObject(uploads[UPLOAD_DELETE_EN].allocated_objid)); std::auto_ptr<IOStream> objstream(apProtocol->ReceiveStream()); - objstream->CopyStreamTo(out); + objstream->CopyStreamTo(out, apProtocol->GetTimeout()); } // query index and test std::auto_ptr<BackupProtocolSuccess> getblockindex(apProtocol->QueryGetBlockIndexByName( - BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_DELETE_EN].name)); + BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[UPLOAD_DELETE_EN].name)); TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_DELETE_EN].allocated_objid); std::auto_ptr<IOStream> blockIndexStream(apProtocol->ReceiveStream()); TEST_THAT(check_block_index("testfiles/downloaddelobj", *blockIndexStream)); @@ -1145,12 +1399,17 @@ int test_server(const char *hostname) for(int t = 0; t < UPLOAD_NUM; ++t) { printf("%d\n", t); - std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, uploads[t].allocated_objid)); + std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[t].allocated_objid)); TEST_THAT(getFile->GetObjectID() == uploads[t].allocated_objid); std::auto_ptr<IOStream> filestream(apProtocol->ReceiveStream()); test_test_file(t, *filestream); } +#ifdef _MSC_VER + BOX_TRACE("3"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + { StreamableMemBlock attrtest(attr3, sizeof(attr3)); @@ -1161,10 +1420,15 @@ int test_server(const char *hostname) // And on the read/write one check_dir_after_uploads(*apProtocol, attrtest); } - + // sleep to ensure that the timestamp on the file will change ::safe_sleep(1); +#ifdef _MSC_VER + BOX_TRACE("4"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + // Check diffing and rsync like stuff... // Build a modified file { @@ -1172,7 +1436,7 @@ int test_server(const char *hostname) TEST_THAT(TestGetFileSize(TEST_FILE_FOR_PATCHING) == TEST_FILE_FOR_PATCHING_SIZE); FileStream in(TEST_FILE_FOR_PATCHING); void *buf = ::malloc(TEST_FILE_FOR_PATCHING_SIZE); - FileStream out(TEST_FILE_FOR_PATCHING ".mod", O_WRONLY | O_CREAT | O_EXCL); + FileStream out(TEST_FILE_FOR_PATCHING ".mod", O_WRONLY | O_CREAT); TEST_THAT(in.Read(buf, TEST_FILE_FOR_PATCHING_PATCH_AT) == TEST_FILE_FOR_PATCHING_PATCH_AT); out.Write(buf, TEST_FILE_FOR_PATCHING_PATCH_AT); char insert[13] = "INSERTINSERT"; @@ -1182,51 +1446,66 @@ int test_server(const char *hostname) ::free(buf); } - TEST_NUM_FILES(UPLOAD_NUM, 0, 1, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); // Run housekeeping (for which we need to disconnect // ourselves) and check that it doesn't change the numbers // of files + +#ifdef _MSC_VER + BOX_TRACE("5"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); - run_housekeeping(account); +#ifdef _MSC_VER + BOX_TRACE("6"); + system("dir testfiles\\0_0\\backup\\01234567"); +#endif + TEST_EQUAL(0, run_housekeeping(account)); - // Also check that bbstoreaccounts doesn't change anything - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf check 01234567 fix") == 0); + // Also check that bbstoreaccounts doesn't change anything, + // using an external process instead of the internal one. + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf check 01234567 fix") == 0, + FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - apProtocol = test_server_login(hostname, context, conn); + apProtocol = connect_and_login(context); + protocolReadOnly.Reopen(); - TEST_NUM_FILES(UPLOAD_NUM, 0, 1, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 4, 3, 2, 1)); { // Fetch the block index for this one std::auto_ptr<BackupProtocolSuccess> getblockindex(apProtocol->QueryGetBlockIndexByName( - BackupProtocolListDirectory::RootDirectory, uploads[UPLOAD_PATCH_EN].name)); + BACKUPSTORE_ROOT_DIRECTORY_ID, uploads[UPLOAD_PATCH_EN].name)); TEST_THAT(getblockindex->GetObjectID() == uploads[UPLOAD_PATCH_EN].allocated_objid); std::auto_ptr<IOStream> blockIndexStream(apProtocol->ReceiveStream()); - + // Do the patching bool isCompletelyDifferent = false; int64_t modtime; std::auto_ptr<IOStream> patchstream( BackupStoreFile::EncodeFileDiff( - TEST_FILE_FOR_PATCHING ".mod", - BackupProtocolListDirectory::RootDirectory, - uploads[UPLOAD_PATCH_EN].name, - uploads[UPLOAD_PATCH_EN].allocated_objid, + TEST_FILE_FOR_PATCHING ".mod", + BACKUPSTORE_ROOT_DIRECTORY_ID, + uploads[UPLOAD_PATCH_EN].name, + uploads[UPLOAD_PATCH_EN].allocated_objid, *blockIndexStream, - IOStream::TimeOutInfinite, + SHORT_TIMEOUT, NULL, // pointer to DiffTimer impl &modtime, &isCompletelyDifferent)); TEST_THAT(isCompletelyDifferent == false); // Sent this to a file, so we can check the size, rather than uploading it directly { - FileStream patch(TEST_FILE_FOR_PATCHING ".patch", O_WRONLY | O_CREAT | O_EXCL); + FileStream patch(TEST_FILE_FOR_PATCHING ".patch", O_WRONLY | O_CREAT); patchstream->CopyStreamTo(patch); } // Make sure the stream is a plausible size for a patch containing only one new block @@ -1234,9 +1513,9 @@ int test_server(const char *hostname) // Upload it int64_t patchedID = 0; { - FileStream uploadpatch(TEST_FILE_FOR_PATCHING ".patch"); + std::auto_ptr<IOStream> uploadpatch(new FileStream(TEST_FILE_FOR_PATCHING ".patch")); std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile( - BackupProtocolListDirectory::RootDirectory, + BACKUPSTORE_ROOT_DIRECTORY_ID, modtime, modtime, /* use it for attr hash too */ uploads[UPLOAD_PATCH_EN].allocated_objid, /* diff from ID */ @@ -1250,71 +1529,81 @@ int test_server(const char *hostname) set_refcount(patchedID, 1); // Then download it to check it's OK - std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BackupProtocolListDirectory::RootDirectory, patchedID)); + std::auto_ptr<BackupProtocolSuccess> getFile(apProtocol->QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, patchedID)); TEST_THAT(getFile->GetObjectID() == patchedID); std::auto_ptr<IOStream> filestream(apProtocol->ReceiveStream()); - BackupStoreFile::DecodeFile(*filestream, TEST_FILE_FOR_PATCHING ".downloaded", IOStream::TimeOutInfinite); + BackupStoreFile::DecodeFile(*filestream, + TEST_FILE_FOR_PATCHING ".downloaded", SHORT_TIMEOUT); // Check it's the same TEST_THAT(check_files_same(TEST_FILE_FOR_PATCHING ".downloaded", TEST_FILE_FOR_PATCHING ".mod")); - - TEST_NUM_FILES(UPLOAD_NUM, 1, 1, 1); + TEST_THAT(check_num_files(UPLOAD_NUM - 4, 4, 2, 1)); } + } - // Create a directory - int64_t subdirid = 0; - BackupStoreFilenameClear dirname("lovely_directory"); - { - // Attributes - MemBlockStream attr(attr1, sizeof(attr1)); - std::auto_ptr<BackupProtocolSuccess> dirCreate(apProtocol->QueryCreateDirectory( - BackupProtocolListDirectory::RootDirectory, - 9837429842987984LL, dirname, attr)); - subdirid = dirCreate->GetObjectID(); - TEST_THAT(subdirid == maxID + 1); + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); - TEST_NUM_FILES(UPLOAD_NUM, 1, 1, 2); - } + TEARDOWN_TEST_BACKUPSTORE(); +} - set_refcount(subdirid, 1); +bool test_server_commands() +{ + SETUP_TEST_BACKUPSTORE(); - // Stick a file in it - int64_t subdirfileid = 0; - { - std::string filename("testfiles/test0"); - int64_t modtime; - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile(filename.c_str(), subdirid, uploads[0].name, &modtime)); + std::auto_ptr<BackupProtocolLocal2> apProtocol( + new BackupProtocolLocal2(0x01234567, "test", + "backup/01234567/", 0, false)); - std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile( - subdirid, - modtime, - modtime, /* use for attr hash too */ - 0, /* diff from ID */ - uploads[0].name, - *upload)); - subdirfileid = stored->GetObjectID(); + // Try using GetFile on an object ID that doesn't exist in the directory + { + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, + BACKUPSTORE_ROOT_DIRECTORY_ID), + Err_DoesNotExistInDirectory); + } - TEST_NUM_FILES(UPLOAD_NUM + 1, 1, 1, 2); + // BLOCK + // TODO FIXME dedent this block. + { + // Create a directory + int64_t subdirid = create_directory(*apProtocol); + TEST_THAT(check_num_files(0, 0, 0, 2)); + + // Try using GetFile on the directory + { + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryGetFile(BACKUPSTORE_ROOT_DIRECTORY_ID, + subdirid), + Err_FileDoesNotVerify); } - set_refcount(subdirfileid, 1); + // Stick a file in it + int64_t subdirfileid = create_file(*apProtocol, subdirid); + TEST_THAT(check_num_files(1, 0, 0, 2)); + + BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", + "backup/01234567/", 0, true); // read-only - printf("\n==== Checking upload using read-only connection\n"); - // Check the directories on the read only connection + BOX_TRACE("Checking root directory using read-only connection"); { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - TEST_THAT(dir.GetNumberOfEntries() == UPLOAD_NUM + 3 /* for the first test file, the patched upload, and this new dir */); + protocolReadOnly.QueryListDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes! */); // Stream + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); + + // UPLOAD_NUM test files, patch uploaded and new dir + TEST_EQUAL(1, dir.GetNumberOfEntries()); // Check the last one... BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; BackupStoreDirectory::Entry *t = 0; + BackupStoreFilenameClear dirname("lovely_directory"); + while((t = i.Next()) != 0) { if(en != 0) @@ -1325,158 +1614,214 @@ int test_server(const char *hostname) } en = t; } - // Does it look right? + + // Check that the last entry looks right + TEST_EQUAL(subdirid, en->GetObjectID()); TEST_THAT(en->GetName() == dirname); - TEST_THAT(en->GetFlags() == BackupProtocolListDirectory::Flags_Dir); - TEST_THAT(en->GetObjectID() == subdirid); - TEST_THAT(en->GetModificationTime() == 0); // dirs don't have modification times. + TEST_EQUAL(BackupProtocolListDirectory::Flags_Dir, en->GetFlags()); + int64_t actual_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); + TEST_EQUAL(actual_size, en->GetSizeInBlocks()); + TEST_EQUAL(FAKE_MODIFICATION_TIME, en->GetModificationTime()); } + BOX_TRACE("Checking subdirectory using read-only connection"); { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( + TEST_EQUAL(subdirid, + protocolReadOnly.QueryListDirectory( subdirid, BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, true /* get attributes */)); - TEST_THAT(dirreply->GetObjectID() == subdirid); - // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + true /* get attributes */)->GetObjectID()); + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); TEST_THAT(dir.GetNumberOfEntries() == 1); - // Check the last one... + // Check the (only) one... BackupStoreDirectory::Iterator i(dir); - // Discard first BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); - // Does it look right? + + // Check that it looks right + TEST_EQUAL(subdirfileid, en->GetObjectID()); TEST_THAT(en->GetName() == uploads[0].name); - TEST_THAT(en->GetFlags() == BackupProtocolListDirectory::Flags_File); - TEST_THAT(en->GetObjectID() == subdirfileid); + TEST_EQUAL(BackupProtocolListDirectory::Flags_File, en->GetFlags()); + int64_t actual_size = get_raid_file(subdirfileid)->GetDiscUsageInBlocks(); + TEST_EQUAL(actual_size, en->GetSizeInBlocks()); TEST_THAT(en->GetModificationTime() != 0); // Attributes TEST_THAT(dir.HasAttributes()); - TEST_THAT(dir.GetAttributesModTime() == 9837429842987984LL); + TEST_EQUAL(FAKE_ATTR_MODIFICATION_TIME, + dir.GetAttributesModTime()); StreamableMemBlock attr(attr1, sizeof(attr1)); TEST_THAT(dir.GetAttributes() == attr); } - printf("done.\n\n"); - // Check that we don't get attributes if we don't ask for them + BOX_TRACE("Checking that we don't get attributes if we don't ask for them"); { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - subdirid, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes! */)); + protocolReadOnly.QueryListDirectory( + subdirid, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes! */); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); TEST_THAT(!dir.HasAttributes()); } - // sleep to ensure that the timestamp on the file will change + // Sleep to ensure that the timestamp on the file will change, + // invalidating the read-only connection's cache of the + // directory, and forcing it to be reloaded. ::safe_sleep(1); // Change attributes on the directory { - MemBlockStream attrnew(attr2, sizeof(attr2)); + std::auto_ptr<IOStream> attrnew( + new MemBlockStream(attr2, sizeof(attr2))); std::auto_ptr<BackupProtocolSuccess> changereply(apProtocol->QueryChangeDirAttributes( subdirid, 329483209443598LL, attrnew)); TEST_THAT(changereply->GetObjectID() == subdirid); } + // Check the new attributes { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - subdirid, - 0, // no flags - BackupProtocolListDirectory::Flags_EXCLUDE_EVERYTHING, true /* get attributes */)); + protocolReadOnly.QueryListDirectory( + subdirid, + 0, // no flags + BackupProtocolListDirectory::Flags_EXCLUDE_EVERYTHING, + true /* get attributes */); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); TEST_THAT(dir.GetNumberOfEntries() == 0); // Attributes TEST_THAT(dir.HasAttributes()); - TEST_THAT(dir.GetAttributesModTime() == 329483209443598LL); + TEST_EQUAL(329483209443598LL, dir.GetAttributesModTime()); StreamableMemBlock attrtest(attr2, sizeof(attr2)); TEST_THAT(dir.GetAttributes() == attrtest); } - - // sleep to ensure that the timestamp on the file will change + + BackupStoreFilenameClear& oldName(uploads[0].name); + int64_t root_file_id = create_file(*apProtocol, BACKUPSTORE_ROOT_DIRECTORY_ID); + TEST_THAT(check_num_files(2, 0, 0, 2)); + + // Upload a new version of the file as well, to ensure that the + // old version is moved along with the current version. + root_file_id = BackupStoreFile::QueryStoreFileDiff(*apProtocol, + "testfiles/test0", BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, // DiffFromFileID + 0, // AttributesHash + oldName); + set_refcount(root_file_id, 1); + TEST_THAT(check_num_files(2, 1, 0, 2)); + + // Check that it's in the root directory (it won't be for long) + protocolReadOnly.QueryListDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, 0, false); + TEST_THAT(BackupStoreDirectory(protocolReadOnly.ReceiveStream()) + .FindEntryByID(root_file_id) != NULL); + + BackupStoreFilenameClear newName("moved-files"); + + // Sleep before modifying the root directory, to ensure that + // the timestamp on the file it's stored in will change when + // we modify it, invalidating the read-only connection's cache + // and forcing it to reload the root directory, next time we + // ask for its contents. ::safe_sleep(1); // Test moving a file { - BackupStoreFilenameClear newName("moved-files"); - - std::auto_ptr<BackupProtocolSuccess> rep(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, - BackupProtocolListDirectory::RootDirectory, + std::auto_ptr<BackupProtocolSuccess> rep(apProtocol->QueryMoveObject(root_file_id, + BACKUPSTORE_ROOT_DIRECTORY_ID, subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName)); - TEST_THAT(rep->GetObjectID() == uploads[UPLOAD_FILE_TO_MOVE].allocated_objid); + TEST_EQUAL(root_file_id, rep->GetObjectID()); } // Try some dodgy renames { + // File doesn't exist at all + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject(-1, + BACKUPSTORE_ROOT_DIRECTORY_ID, subdirid, + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_DoesNotExistInDirectory); BackupStoreFilenameClear newName("moved-files"); - TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject( + uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, BackupProtocolListDirectory::RootDirectory, - subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName), - ConnectionException, Conn_Protocol_UnexpectedReply); - TEST_CHECK_THROWS(apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, subdirid, - subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName), - ConnectionException, Conn_Protocol_UnexpectedReply); + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_DoesNotExistInDirectory); + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject( + uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, + subdirid, + subdirid, + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_DoesNotExistInDirectory); } - // Rename within a directory - { - BackupStoreFilenameClear newName("moved-files-x"); - apProtocol->QueryMoveObject(uploads[UPLOAD_FILE_TO_MOVE].allocated_objid, + // File exists, but not in this directory (we just moved it) + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject(root_file_id, + BACKUPSTORE_ROOT_DIRECTORY_ID, subdirid, - subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, newName); - } + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_DoesNotExistInDirectory); + + // Moving file to same directory that it's already in, + // with the same name + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryMoveObject(root_file_id, + subdirid, + subdirid, + BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName), + Err_TargetNameExists); - // Check it's all gone from the root directory... + // Rename within a directory (successfully) { - // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); - // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - // Read all entries - BackupStoreDirectory::Iterator i(dir); - BackupStoreDirectory::Entry *en = 0; - while((en = i.Next()) != 0) - { - TEST_THAT(en->GetName() != uploads[UPLOAD_FILE_TO_MOVE].name); - } + BackupStoreFilenameClear newName2("moved-files-x"); + apProtocol->QueryMoveObject(root_file_id, subdirid, + subdirid, BackupProtocolMoveObject::Flags_MoveAllWithSameName, + newName2); } + // Check it's all gone from the root directory... + protocolReadOnly.QueryListDirectory(BACKUPSTORE_ROOT_DIRECTORY_ID, + 0, 0, false); + TEST_THAT(BackupStoreDirectory(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT).FindEntryByID(root_file_id) == NULL); + // Check the old and new versions are in the other directory { + BackupStoreFilenameClear notThere("moved-files"); BackupStoreFilenameClear lookFor("moved-files-x"); // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - subdirid, - BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); + protocolReadOnly.QueryListDirectory( + subdirid, + BackupProtocolListDirectory::Flags_INCLUDE_EVERYTHING, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes */); + // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); + // Check entries BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; @@ -1484,6 +1829,10 @@ int test_server(const char *hostname) bool foundOld = false; while((en = i.Next()) != 0) { + // If we find the old name, then the rename + // operation didn't work. + TEST_THAT(en->GetName() != notThere); + if(en->GetName() == lookFor) { if(en->GetFlags() == (BackupStoreDirectory::Entry::Flags_File)) foundCurrent = true; @@ -1501,52 +1850,63 @@ int test_server(const char *hostname) int64_t subsubdirid = 0; int64_t subsubfileid = 0; { + // TODO FIXME use create_dir() and create_file() instead. BackupStoreFilenameClear nd("sub2"); // Attributes - MemBlockStream attr(attr1, sizeof(attr1)); - std::auto_ptr<BackupProtocolSuccess> dirCreate(apProtocol->QueryCreateDirectory( - subdirid, - 9837429842987984LL, nd, attr)); - subsubdirid = dirCreate->GetObjectID(); + std::auto_ptr<IOStream> attr(new MemBlockStream(attr1, + sizeof(attr1))); + subsubdirid = apProtocol->QueryCreateDirectory(subdirid, + FAKE_ATTR_MODIFICATION_TIME, nd, attr)->GetObjectID(); + + write_test_file(2); - FileStream upload("testfiles/file1_upload1"); - BackupStoreFilenameClear nf("file2"); + BackupStoreFilenameClear file2("file2"); + std::auto_ptr<IOStream> upload( + BackupStoreFile::EncodeFile("testfiles/test2", + BACKUPSTORE_ROOT_DIRECTORY_ID, file2)); std::auto_ptr<BackupProtocolSuccess> stored(apProtocol->QueryStoreFile( subsubdirid, 0x123456789abcdefLL, /* modification time */ 0x7362383249872dfLL, /* attr hash */ 0, /* diff from ID */ - nf, + file2, upload)); subsubfileid = stored->GetObjectID(); } set_refcount(subsubdirid, 1); set_refcount(subsubfileid, 1); + TEST_THAT(check_num_files(3, 1, 0, 3)); + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + apProtocol->Reopen(); + protocolReadOnly.Reopen(); // Query names -- test that invalid stuff returns not found OK { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(3248972347823478927LL, subsubdirid)); - TEST_THAT(nameRep->GetNumNameElements() == 0); + TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(subsubfileid, 2342378424LL)); - TEST_THAT(nameRep->GetNumNameElements() == 0); + TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(38947234789LL, 2342378424LL)); - TEST_THAT(nameRep->GetNumNameElements() == 0); + TEST_THAT(nameRep->GetNumNameElements() == 0); } { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, 2234342378424LL)); - TEST_THAT(nameRep->GetNumNameElements() == 0); + TEST_THAT(nameRep->GetNumNameElements() == 0); } // Query names... first, get info for the file { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(subsubfileid, subsubdirid)); std::auto_ptr<IOStream> namestream(apProtocol->ReceiveStream()); - + TEST_THAT(nameRep->GetNumNameElements() == 3); TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_File); TEST_THAT(nameRep->GetModificationTime() == 0x123456789abcdefLL); @@ -1564,7 +1924,7 @@ int test_server(const char *hostname) { std::auto_ptr<BackupProtocolObjectName> nameRep(apProtocol->QueryGetObjectName(BackupProtocolGetObjectName::ObjectID_DirectoryOnly, subsubdirid)); std::auto_ptr<IOStream> namestream(apProtocol->ReceiveStream()); - + TEST_THAT(nameRep->GetNumNameElements() == 2); TEST_THAT(nameRep->GetFlags() == BackupProtocolListDirectory::Flags_Dir); static const char *testnames[] = {"sub2","lovely_directory"}; @@ -1575,82 +1935,348 @@ int test_server(const char *hostname) TEST_THAT(fn.GetClearFilename() == testnames[l]); } } - + //} skip: - std::auto_ptr<BackupStoreRefCountDatabase> apRefCount( - BackupStoreRefCountDatabase::Load( - apAccounts->GetEntry(0x1234567), true)); - // Create some nice recursive directories - int64_t dirtodelete = create_test_data_subdirs(*apProtocol, - BackupProtocolListDirectory::RootDirectory, - "test_delete", 6 /* depth */, *apRefCount); - + TEST_THAT(check_reference_counts()); + + write_test_file(1); + int64_t dirtodelete; + + { + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + std::auto_ptr<BackupStoreRefCountDatabase> apRefCount( + BackupStoreRefCountDatabase::Load( + apAccounts->GetEntry(0x1234567), true)); + + + dirtodelete = create_test_data_subdirs(*apProtocol, + BACKUPSTORE_ROOT_DIRECTORY_ID, + "test_delete", 6 /* depth */, apRefCount.get()); + } + + TEST_THAT(check_reference_counts()); + + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(check_reference_counts()); + // And delete them + apProtocol->Reopen(); + protocolReadOnly.Reopen(); + { std::auto_ptr<BackupProtocolSuccess> dirdel(apProtocol->QueryDeleteDirectory( dirtodelete)); TEST_THAT(dirdel->GetObjectID() == dirtodelete); } + apProtocol->QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(check_reference_counts()); + protocolReadOnly.Reopen(); + // Get the root dir, checking for deleted items { // Command - std::auto_ptr<BackupProtocolSuccess> dirreply(protocolReadOnly.QueryListDirectory( - BackupProtocolListDirectory::RootDirectory, - BackupProtocolListDirectory::Flags_Dir | BackupProtocolListDirectory::Flags_Deleted, - BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, false /* no attributes */)); + protocolReadOnly.QueryListDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + BackupProtocolListDirectory::Flags_Dir | + BackupProtocolListDirectory::Flags_Deleted, + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes */); // Stream - BackupStoreDirectory dir; - std::auto_ptr<IOStream> dirstream(protocolReadOnly.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); - + BackupStoreDirectory dir(protocolReadOnly.ReceiveStream(), + SHORT_TIMEOUT); + // Check there's only that one entry TEST_THAT(dir.GetNumberOfEntries() == 1); - + BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = i.Next(); TEST_THAT(en != 0); if(en) { - TEST_THAT(en->GetObjectID() == dirtodelete); + TEST_EQUAL(dirtodelete, en->GetObjectID()); BackupStoreFilenameClear n("test_delete"); TEST_THAT(en->GetName() == n); } - + // Then... check everything's deleted - test_everything_deleted(protocolReadOnly, dirtodelete); + assert_everything_deleted(protocolReadOnly, dirtodelete); } - + + // Undelete and check that block counts are restored properly + apProtocol->Reopen(); + TEST_EQUAL(dirtodelete, + apProtocol->QueryUndeleteDirectory(dirtodelete)->GetObjectID()); + // Finish the connections -#ifndef WIN32 - protocolReadOnly.QueryFinished(); -#endif apProtocol->QueryFinished(); - - // Close logs -#ifndef WIN32 - ::fclose(protocolReadOnlyLog); -#endif - ::fclose(protocolLog); + protocolReadOnly.QueryFinished(); + + TEST_THAT(run_housekeeping_and_check_account()); + TEST_THAT(check_reference_counts()); } - - return 0; + + TEARDOWN_TEST_BACKUPSTORE(); +} + +int get_object_size(BackupProtocolCallable& protocol, int64_t ObjectID, + int64_t ContainerID) +{ + // Get the root directory cached in the read-only connection + protocol.QueryListDirectory(ContainerID, 0, // FlagsMustBeSet + BackupProtocolListDirectory::Flags_EXCLUDE_NOTHING, + false /* no attributes */); + + BackupStoreDirectory dir(protocol.ReceiveStream()); + BackupStoreDirectory::Entry *en = dir.FindEntryByID(ObjectID); + TEST_THAT_OR(en != 0, return -1); + TEST_EQUAL_OR(ObjectID, en->GetObjectID(), return -1); + return en->GetSizeInBlocks(); +} + +bool write_dir(BackupStoreDirectory& dir) +{ + std::string rfn; + StoreStructure::MakeObjectFilename(dir.GetObjectID(), + "backup/01234567/" /* mStoreRoot */, 0 /* mStoreDiscSet */, + rfn, false); // EnsureDirectoryExists + RaidFileWrite rfw(0, rfn); + rfw.Open(true); // AllowOverwrite + dir.WriteToStream(rfw); + rfw.Commit(/* ConvertToRaidNow */ true); + return true; +} + +bool test_directory_parent_entry_tracks_directory_size() +{ + SETUP_TEST_BACKUPSTORE(); + + BackupProtocolLocal2 protocol(0x01234567, "test", "backup/01234567/", + 0, false); + BackupProtocolLocal2 protocolReadOnly(0x01234567, "test", + "backup/01234567/", 0, true); // read only + + int64_t subdirid = create_directory(protocol); + + // Get the root directory cached in the read-only connection, and + // test that the initial size is correct. + int old_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); + TEST_THAT(old_size > 0); + TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Sleep to ensure that the directory file timestamp changes, so that + // the read-only connection will discard its cached copy. + safe_sleep(1); + + // Start adding files until the size on disk increases. This is + // guaranteed to happen eventually :) + int new_size = old_size; + int64_t last_added_file_id = 0; + std::string last_added_filename; + + for (int i = 0; new_size == old_size; i++) + { + std::ostringstream name; + name << "testfile_" << i; + last_added_filename = name.str(); + last_added_file_id = create_file(protocol, subdirid, name.str()); + new_size = get_raid_file(subdirid)->GetDiscUsageInBlocks(); + } + + // Check that the root directory entry has been updated + TEST_EQUAL(new_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Now delete an entry, and check that the size is reduced + protocol.QueryDeleteFile(subdirid, + BackupStoreFilenameClear(last_added_filename)); + ExpectedRefCounts[last_added_file_id] = 0; + + // Reduce the limits, to remove it permanently from the store + protocol.QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(change_account_limits("0B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account()); + set_refcount(last_added_file_id, 0); + protocol.Reopen(); + protocolReadOnly.Reopen(); + + TEST_EQUAL(old_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); + + // Check that the entry in the root directory was updated too + TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Push the limits back up + protocol.QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(change_account_limits("1000B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + protocolReadOnly.Reopen(); + + // Now modify the root directory to remove its entry for this one + BackupStoreDirectory root(*get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID), + IOStream::TimeOutInfinite); + BackupStoreDirectory::Entry *en = root.FindEntryByID(subdirid); + TEST_THAT_OR(en, return false); + BackupStoreDirectory::Entry enCopy(*en); + root.DeleteEntry(subdirid); + TEST_THAT(write_dir(root)); + + // Add a directory, this should try to push the object size back up, + // which will try to modify the subdir's entry in its parent, which + // no longer exists, which should just log an error instead of + // aborting/segfaulting. + create_directory(protocol, subdirid); + + // Repair the error ourselves, as bbstoreaccounts can't. + protocol.QueryFinished(); + enCopy.SetSizeInBlocks(get_raid_file(subdirid)->GetDiscUsageInBlocks()); + root.AddEntry(enCopy); + TEST_THAT(write_dir(root)); + + // We also have to remove the entry for lovely_directory created by + // create_directory(), because otherwise we can't create it again. + // (Perhaps it should not have been committed because we failed to + // update the parent, but currently it is.) + BackupStoreDirectory subdir(*get_raid_file(subdirid), + IOStream::TimeOutInfinite); + { + BackupStoreDirectory::Iterator i(subdir); + en = i.FindMatchingClearName( + BackupStoreFilenameClear("lovely_directory")); + } + TEST_THAT_OR(en, return false); + protocol.Reopen(); + protocol.QueryDeleteDirectory(en->GetObjectID()); + set_refcount(en->GetObjectID(), 0); + + // This should have fixed the error, so we should be able to add the + // entry now. This should push the object size back up. + int64_t dir2id = create_directory(protocol, subdirid); + TEST_EQUAL(new_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); + TEST_EQUAL(new_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Delete it again, which should reduce the object size again + protocol.QueryDeleteDirectory(dir2id); + set_refcount(dir2id, 0); + + // Reduce the limits, to remove it permanently from the store + protocol.QueryFinished(); + protocolReadOnly.QueryFinished(); + TEST_THAT(change_account_limits("0B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account()); + protocol.Reopen(); + protocolReadOnly.Reopen(); + + // Check that the entry in the root directory was updated + TEST_EQUAL(old_size, get_raid_file(subdirid)->GetDiscUsageInBlocks()); + TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Check that bbstoreaccounts check fix will detect and repair when + // a directory's parent entry has the wrong size for the directory. + + protocol.QueryFinished(); + + root.ReadFromStream(*get_raid_file(BACKUPSTORE_ROOT_DIRECTORY_ID), + IOStream::TimeOutInfinite); + en = root.FindEntryByID(subdirid); + TEST_THAT_OR(en != 0, return false); + en->SetSizeInBlocks(1234); + + // Sleep to ensure that the directory file timestamp changes, so that + // the read-only connection will discard its cached copy. + safe_sleep(1); + TEST_THAT(write_dir(root)); + + TEST_EQUAL(1234, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + + // Sleep to ensure that the directory file timestamp changes, so that + // the read-only connection will discard its cached copy. + safe_sleep(1); + + protocolReadOnly.QueryFinished(); + TEST_EQUAL(1, check_account_for_errors()); + + protocolReadOnly.Reopen(); + TEST_EQUAL(old_size, get_object_size(protocolReadOnly, subdirid, + BACKUPSTORE_ROOT_DIRECTORY_ID)); + protocolReadOnly.QueryFinished(); + + TEARDOWN_TEST_BACKUPSTORE(); } -int test3(int argc, const char *argv[]) +bool test_cannot_open_multiple_writable_connections() +{ + SETUP_TEST_BACKUPSTORE(); + + // First try a local protocol. This works even on Windows. + BackupProtocolLocal2 protocolWritable(0x01234567, "test", + "backup/01234567/", 0, false); // Not read-only + + // Set the client store marker + protocolWritable.QuerySetClientStoreMarker(0x8732523ab23aLL); + + // First try a local protocol. This works even on Windows. + { + BackupStoreContext bsContext(0x01234567, (HousekeepingInterface *)NULL, "test"); + bsContext.SetClientHasAccount("backup/01234567/", 0); + BackupProtocolLocal protocolWritable2(bsContext); + TEST_THAT(assert_writable_connection_fails(protocolWritable2)); + } + + { + BackupStoreContext bsContext(0x01234567, (HousekeepingInterface *)NULL, "test"); + bsContext.SetClientHasAccount("backup/01234567/", 0); + BackupProtocolLocal protocolReadOnly(bsContext); + TEST_EQUAL(0x8732523ab23aLL, + assert_readonly_connection_succeeds(protocolReadOnly)); + } + + // Try network connections too. + TEST_THAT_OR(StartServer(), return false); + + BackupProtocolClient protocolWritable3(open_conn("localhost", context)); + TEST_THAT(assert_writable_connection_fails(protocolWritable3)); + + // Do not dedent. Object needs to go out of scope to release lock + { + BackupProtocolClient protocolReadOnly2(open_conn("localhost", context)); + TEST_EQUAL(0x8732523ab23aLL, + assert_readonly_connection_succeeds(protocolReadOnly2)); + } + + protocolWritable.QueryFinished(); + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_encoding() { // Now test encoded files // TODO: This test needs to check failure situations as well as everything working, // but this will be saved for the full implementation. + + SETUP_TEST_BACKUPSTORE(); + int encfile[ENCFILE_SIZE]; { for(int l = 0; l < ENCFILE_SIZE; ++l) { encfile[l] = l * 173; } - + // Encode and decode a small block (shouldn't be compressed) { #define SMALL_BLOCK_SIZE 251 @@ -1658,14 +2284,14 @@ int test3(int argc, const char *argv[]) TEST_THAT(encBlockSize > SMALL_BLOCK_SIZE); BackupStoreFile::EncodingBuffer encoded; encoded.Allocate(encBlockSize / 8); // make sure reallocation happens - + // Encode! int encSize = BackupStoreFile::EncodeChunk(encfile, SMALL_BLOCK_SIZE, encoded); // Check the header says it's not been compressed TEST_THAT((encoded.mpBuffer[0] & 1) == 0); // Check the output size has been inflated (no compression) TEST_THAT(encSize > SMALL_BLOCK_SIZE); - + // Decode it int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(SMALL_BLOCK_SIZE); TEST_THAT(decBlockSize > SMALL_BLOCK_SIZE); @@ -1673,10 +2299,10 @@ int test3(int argc, const char *argv[]) int decSize = BackupStoreFile::DecodeChunk(encoded.mpBuffer, encSize, decoded, decBlockSize); TEST_THAT(decSize < decBlockSize); TEST_THAT(decSize == SMALL_BLOCK_SIZE); - + // Check it came out of the wash the same TEST_THAT(::memcmp(encfile, decoded, SMALL_BLOCK_SIZE) == 0); - + free(decoded); } @@ -1686,14 +2312,14 @@ int test3(int argc, const char *argv[]) TEST_THAT(encBlockSize > ENCFILE_SIZE); BackupStoreFile::EncodingBuffer encoded; encoded.Allocate(encBlockSize / 8); // make sure reallocation happens - + // Encode! int encSize = BackupStoreFile::EncodeChunk(encfile, ENCFILE_SIZE, encoded); // Check the header says it's compressed TEST_THAT((encoded.mpBuffer[0] & 1) == 1); // Check the output size make it likely that it's compressed (is very compressible data) TEST_THAT(encSize < ENCFILE_SIZE); - + // Decode it int decBlockSize = BackupStoreFile::OutputBufferSizeForKnownOutputSize(ENCFILE_SIZE); TEST_THAT(decBlockSize > ENCFILE_SIZE); @@ -1704,37 +2330,74 @@ int test3(int argc, const char *argv[]) // Check it came out of the wash the same TEST_THAT(::memcmp(encfile, decoded, ENCFILE_SIZE) == 0); - + free(decoded); } - + // The test block to a file { - FileStream f("testfiles/testenc1", O_WRONLY | O_CREAT | O_EXCL); + FileStream f("testfiles/testenc1", O_WRONLY | O_CREAT); f.Write(encfile, sizeof(encfile)); } - + // Encode it { - FileStream out("testfiles/testenc1_enc", O_WRONLY | O_CREAT | O_EXCL); + FileStream out("testfiles/testenc1_enc", O_WRONLY | O_CREAT); BackupStoreFilenameClear name("testfiles/testenc1"); std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testenc1", 32, name)); encoded->CopyStreamTo(out); } - + // Verify it { FileStream enc("testfiles/testenc1_enc"); TEST_THAT(BackupStoreFile::VerifyEncodedFileFormat(enc) == true); + + // And using the stream-based interface, writing different + // block sizes at a time. + CollectInBufferStream contents; + enc.Seek(0, IOStream::SeekType_Absolute); + enc.CopyStreamTo(contents); + contents.SetForReading(); + + enc.Seek(0, IOStream::SeekType_End); + size_t file_size = enc.GetPosition(); + TEST_EQUAL(file_size, contents.GetSize()); + + for(int buffer_size = 1; ; buffer_size <<= 1) + { + enc.Seek(0, IOStream::SeekType_Absolute); + CollectInBufferStream temp_copy; + BackupStoreFile::VerifyStream verifier(&temp_copy); + enc.CopyStreamTo(verifier, IOStream::TimeOutInfinite, + buffer_size); + + // The block index is only validated on Close(), which + // CopyStreamTo() doesn't do. + verifier.Close(); + + temp_copy.SetForReading(); + TEST_EQUAL(file_size, temp_copy.GetSize()); + TEST_THAT(memcmp(contents.GetBuffer(), + temp_copy.GetBuffer(), file_size) == 0); + + // Keep doubling buffer size until we've copied the + // entire encoded file in a single pass, then stop. + if(buffer_size > file_size) + { + break; + } + } } - + // Decode it { + UNLINK_IF_EXISTS("testfiles/testenc1_orig"); FileStream enc("testfiles/testenc1_enc"); BackupStoreFile::DecodeFile(enc, "testfiles/testenc1_orig", IOStream::TimeOutInfinite); } - + // Read in rebuilt original, and compare contents { TEST_THAT(TestGetFileSize("testfiles/testenc1_orig") == sizeof(encfile)); @@ -1743,7 +2406,7 @@ int test3(int argc, const char *argv[]) in.Read(encfile_i, sizeof(encfile_i)); TEST_THAT(memcmp(encfile, encfile_i, sizeof(encfile)) == 0); } - + // Check how many blocks it had, and test the stream based interface { FileStream enc("testfiles/testenc1_enc"); @@ -1756,11 +2419,11 @@ int test3(int argc, const char *argv[]) TEST_THAT(decoded->GetNumBlocks() == 3); } - + // Test that the last block in a file, if less than 256 bytes, gets put into the last block { #define FILE_SIZE_JUST_OVER ((4096*2)+58) - FileStream f("testfiles/testenc2", O_WRONLY | O_CREAT | O_EXCL); + FileStream f("testfiles/testenc2", O_WRONLY | O_CREAT); f.Write(encfile + 2, FILE_SIZE_JUST_OVER); BackupStoreFilenameClear name("testenc2"); std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testenc2", 32, name)); @@ -1772,11 +2435,11 @@ int test3(int argc, const char *argv[]) decoded->CopyStreamTo(d, IOStream::TimeOutInfinite, 879 /* buffer block size */); d.SetForReading(); TEST_THAT(d.GetSize() == FILE_SIZE_JUST_OVER); - TEST_THAT(memcmp(encfile + 2, d.GetBuffer(), FILE_SIZE_JUST_OVER) == 0); + TEST_THAT(memcmp(encfile + 2, d.GetBuffer(), FILE_SIZE_JUST_OVER) == 0); TEST_THAT(decoded->GetNumBlocks() == 2); } - + // Test that reordered streams work too { FileStream enc("testfiles/testenc1_enc"); @@ -1790,25 +2453,39 @@ int test3(int argc, const char *argv[]) TEST_THAT(decoded->GetNumBlocks() == 3); } - + } + + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_symlinks() +{ + SETUP_TEST_BACKUPSTORE(); + #ifndef WIN32 // no symlinks on Win32 - // Try out doing this on a symlink - { - TEST_THAT(::symlink("does/not/exist", "testfiles/testsymlink") == 0); - BackupStoreFilenameClear name("testsymlink"); - std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testsymlink", 32, name)); - // Can't decode it from the stream, because it's in file order, and doesn't have the - // required properties to be able to reorder it. So buffer it... - CollectInBufferStream b; - encoded->CopyStreamTo(b); - b.SetForReading(); - // Decode it - BackupStoreFile::DecodeFile(b, "testfiles/testsymlink_2", IOStream::TimeOutInfinite); - } + UNLINK_IF_EXISTS("testfiles/testsymlink"); + TEST_THAT(::symlink("does/not/exist", "testfiles/testsymlink") == 0); + BackupStoreFilenameClear name("testsymlink"); + std::auto_ptr<IOStream> encoded(BackupStoreFile::EncodeFile("testfiles/testsymlink", 32, name)); + + // Can't decode it from the stream, because it's in file order, and doesn't have the + // required properties to be able to reorder it. So buffer it... + CollectInBufferStream b; + encoded->CopyStreamTo(b); + b.SetForReading(); + + // Decode it + UNLINK_IF_EXISTS("testfiles/testsymlink_2"); + BackupStoreFile::DecodeFile(b, "testfiles/testsymlink_2", IOStream::TimeOutInfinite); #endif - } - // Store info + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_store_info() +{ + SETUP_TEST_BACKUPSTORE(); + { RaidFileWrite::CreateDirectory(0, "test-info"); BackupStoreInfo::CreateNew(76, "test-info/", 0, 3461231233455433LL, 2934852487LL); @@ -1841,6 +2518,7 @@ int test3(int argc, const char *argv[]) TEST_CHECK_THROWS(info->RemovedDeletedDirectory(9), BackupStoreException, StoreInfoDirNotInList); info->Save(); } + { std::auto_ptr<BackupStoreInfo> info(BackupStoreInfo::Load(76, "test-info/", 0, true)); TEST_THAT(info->GetBlocksUsed() == 7); @@ -1855,68 +2533,79 @@ int test3(int argc, const char *argv[]) TEST_THAT(delfiles[1] == 4); } -//printf("SKIPPINGTESTS---------\n"); -//return 0; - - // Context - TLSContext context; - context.Initialise(false /* client */, - "testfiles/clientCerts.pem", - "testfiles/clientPrivKey.pem", - "testfiles/clientTrustedCAs.pem"); + TEARDOWN_TEST_BACKUPSTORE(); +} +bool test_login_without_account() +{ // First, try logging in without an account having been created... just make sure login fails. - std::string cmd = BBSTORED " " + bbstored_args + - " testfiles/bbstored.conf"; - int pid = LaunchServer(cmd.c_str(), "testfiles/bbstored.pid"); - - TEST_THAT(pid != -1 && pid != 0); - if(pid <= 0) - { - return 1; - } - - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); + SETUP_TEST_BACKUPSTORE(); + delete_account(); + TEST_THAT_OR(StartServer(), FAIL); // BLOCK { // Open a connection to the server - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", - BOX_PORT_BBSTORED_TEST); - - // Make a protocol - BackupProtocolClient protocol(conn); + BackupProtocolClient protocol(open_conn("localhost", context)); // Check the version std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)), - ConnectionException, Conn_Protocol_UnexpectedReply); - + TEST_COMMAND_RETURNS_ERROR(protocol, QueryLogin(0x01234567, 0), + Err_BadLogin); + // Finish the connection protocol.QueryFinished(); } - // Create an account for the test client - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf create 01234567 0 " - "10000B 20000B") == 0); + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_bbstoreaccounts_create() +{ + SETUP_TEST_BACKUPSTORE(); + + // Delete the account, and create it again using bbstoreaccounts + delete_account(); + + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf -Wwarning create 01234567 0 " + "10000B 20000B") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_bbstoreaccounts_delete() +{ + SETUP_TEST_BACKUPSTORE(); + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf -Wwarning delete 01234567 yes") == 0, FAIL); + TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + + // Recreate the account so that teardown_test_backupstore() doesn't freak out + TEST_THAT(create_account(10000, 20000)); + + TEARDOWN_TEST_BACKUPSTORE(); +} + +// Test that login fails on a disabled account +bool test_login_with_disabled_account() +{ + SETUP_TEST_BACKUPSTORE(); + TEST_THAT_OR(StartServer(), FAIL); + TEST_THAT(TestDirExists("testfiles/0_0/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_1/backup/01234567")); TEST_THAT(TestDirExists("testfiles/0_2/backup/01234567")); TEST_THAT(TestGetFileSize("testfiles/accounts.txt") > 8); + // make sure something is written to it - std::auto_ptr<BackupStoreAccountDatabase> apAccounts( BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); - std::auto_ptr<BackupStoreRefCountDatabase> apReferences( BackupStoreRefCountDatabase::Load( apAccounts->GetEntry(0x1234567), true)); @@ -1924,224 +2613,193 @@ int test3(int argc, const char *argv[]) apReferences->GetLastObjectIDUsed()); TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) apReferences.reset(); - - // Test that login fails on a disabled account - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf enabled 01234567 no") == 0); + + // Test that login fails on a disabled account + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf enabled 01234567 no") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + // BLOCK { // Open a connection to the server - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", - BOX_PORT_BBSTORED_TEST); - - // Make a protocol - BackupProtocolClient protocol(conn); + BackupProtocolClient protocol(open_conn("localhost", context)); // Check the version std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); // Login - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolLoginConfirmed> - loginConf(protocol.QueryLogin(0x01234567, 0)), - ConnectionException, Conn_Protocol_UnexpectedReply); - int type, subType; - TEST_EQUAL_LINE(true, protocol.GetLastError(type, subType), - "expected a protocol error"); - TEST_EQUAL_LINE(BackupProtocolError::ErrorType, type, - "expected a BackupProtocolError"); - TEST_EQUAL_LINE(BackupProtocolError::Err_DisabledAccount, subType, - "expected an Err_DisabledAccount"); - + TEST_COMMAND_RETURNS_ERROR(protocol, QueryLogin(0x01234567, 0), + Err_DisabledAccount); + // Finish the connection protocol.QueryFinished(); } - // Re-enable the account so that subsequent logins should succeed - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf enabled 01234567 yes") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + TEARDOWN_TEST_BACKUPSTORE(); +} - // Delete the refcount database and log in again, - // check that it is recreated automatically but with - // no objects in it, to ensure seamless upgrade. - TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw")); +bool test_login_with_no_refcount_db() +{ + SETUP_TEST_BACKUPSTORE(); - std::auto_ptr<SocketStreamTLS> conn; - test_server_login("localhost", context, conn)->QueryFinished(); + // The account is already enabled, but doing it again shouldn't hurt + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf enabled 01234567 yes") == 0, FAIL); + TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + // Delete the refcount database and try to log in again. Check that + // we're locked out of the account until housekeeping has recreated + // the refcount db. + TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); + TEST_CHECK_THROWS(BackupProtocolLocal2 protocolLocal(0x01234567, + "test", "backup/01234567/", 0, false), // Not read-only + BackupStoreException, CorruptReferenceCountDatabase); + + // Run housekeeping, check that it fixes the refcount db + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); BackupStoreAccountDatabase::Entry account = apAccounts->GetEntry(0x1234567); - apReferences = BackupStoreRefCountDatabase::Load(account, true); - TEST_EQUAL(0, apReferences->GetLastObjectIDUsed()); - - TEST_THAT(ServerIsAlive(pid)); + TEST_EQUAL_LINE(1, run_housekeeping(account), + "Housekeeping should report 1 error if the refcount db is missing"); + TEST_THAT(FileExists("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); - run_housekeeping(account); + // And that we can log in afterwards + BackupProtocolLocal2(0x01234567, "test", "backup/01234567/", 0, + false).QueryFinished(); // Not read-only // Check that housekeeping fixed the ref counts - TEST_EQUAL(BACKUPSTORE_ROOT_DIRECTORY_ID, - apReferences->GetLastObjectIDUsed()); - TEST_EQUAL(1, apReferences->GetRefCount(BACKUPSTORE_ROOT_DIRECTORY_ID)) + TEST_THAT(check_reference_counts()); - TEST_THAT(ServerIsAlive(pid)); + // Start a server and try again, remotely. This is difficult to debug + // because housekeeping may fix the refcount database while we're + // stepping through. + TEST_THAT_THROWONFAIL(StartServer()); + TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); + TEST_CHECK_THROWS(connect_and_login(context), + ConnectionException, Protocol_UnexpectedReply); - set_refcount(BACKUPSTORE_ROOT_DIRECTORY_ID, 1); + TEST_THAT(ServerIsAlive(bbstored_pid)); - TEST_THAT(test_server("localhost") == 0); + // Run housekeeping, check that it fixes the refcount db + TEST_EQUAL_LINE(1, run_housekeeping(account), + "Housekeeping should report 1 error if the refcount db is missing"); + TEST_THAT(FileExists("testfiles/0_0/backup/01234567/refcount.rdb.rfw")); + TEST_THAT(check_reference_counts()); - // test that all object reference counts have the - // expected values - TEST_EQUAL(ExpectedRefCounts.size() - 1, - apReferences->GetLastObjectIDUsed()); - for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID; - i < ExpectedRefCounts.size(); i++) - { - TEST_EQUAL_LINE(ExpectedRefCounts[i], - apReferences->GetRefCount(i), - "object " << BOX_FORMAT_OBJECTID(i)); - } + // And that we can log in afterwards + connect_and_login(context)->QueryFinished(); - // Delete the refcount database again, and let - // housekeeping recreate it and fix the ref counts. - // This could also happen after upgrade, if a housekeeping - // runs before the user logs in. - apReferences.reset(); - TEST_EQUAL(0, ::unlink("testfiles/0_0/backup/01234567/refcount.db.rfw")); - run_housekeeping(account); - apReferences = BackupStoreRefCountDatabase::Load(account, true); + TEARDOWN_TEST_BACKUPSTORE(); +} - TEST_EQUAL(ExpectedRefCounts.size() - 1, - apReferences->GetLastObjectIDUsed()); - for (unsigned int i = BACKUPSTORE_ROOT_DIRECTORY_ID; - i < ExpectedRefCounts.size(); i++) - { - TEST_EQUAL_LINE(ExpectedRefCounts[i], - apReferences->GetRefCount(i), - "object " << BOX_FORMAT_OBJECTID(i)); - } - +bool test_housekeeping_deletes_files() +{ // Test the deletion of objects by the housekeeping system + + SETUP_TEST_BACKUPSTORE(); + + BackupProtocolLocal2 protocolLocal(0x01234567, "test", + "backup/01234567/", 0, false); // Not read-only + + // Create some nice recursive directories + write_test_file(1); + int64_t dirtodelete = create_test_data_subdirs(protocolLocal, + BACKUPSTORE_ROOT_DIRECTORY_ID, "test_delete", 6 /* depth */, + NULL /* pRefCount */); + + TEST_EQUAL(dirtodelete, + protocolLocal.QueryDeleteDirectory(dirtodelete)->GetObjectID()); + assert_everything_deleted(protocolLocal, dirtodelete); + protocolLocal.QueryFinished(); + // First, things as they are now. + TEST_THAT_OR(StartServer(), FAIL); recursive_count_objects_results before = {0,0,0}; + recursive_count_objects(BACKUPSTORE_ROOT_DIRECTORY_ID, before); - recursive_count_objects("localhost", BackupProtocolListDirectory::RootDirectory, before); - - TEST_THAT(before.objectsNotDel != 0); + TEST_EQUAL(0, before.objectsNotDel); TEST_THAT(before.deleted != 0); TEST_THAT(before.old != 0); // Kill it - TEST_THAT(KillServer(pid)); - ::sleep(1); - TEST_THAT(!ServerIsAlive(pid)); - - #ifdef WIN32 - TEST_THAT(unlink("testfiles/bbstored.pid") == 0); - #else - TestRemoteProcessMemLeaks("bbstored.memleaks"); - #endif - - // Set a new limit on the account -- leave the hard limit - // high to make sure the target for freeing space is the - // soft limit. - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf setlimit 01234567 " - "10B 20000B") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - - // Start things up - pid = LaunchServer(BBSTORED " testfiles/bbstored.conf", - "testfiles/bbstored.pid"); + TEST_THAT(StopServer()); - ::sleep(1); - TEST_THAT(ServerIsAlive(pid)); - - // wait for housekeeping to happen - printf("waiting for housekeeping:\n"); - for(int l = 0; l < 30; ++l) - { - ::sleep(1); - printf("."); - fflush(stdout); - } - printf("\n"); + // Reduce the store limits, so housekeeping will remove all old files. + // Leave the hard limit high, so we know that housekeeping's target + // for freeing space is the soft limit. + TEST_THAT(change_account_limits("0B", "20000B")); + TEST_THAT(run_housekeeping_and_check_account()); // Count the objects again recursive_count_objects_results after = {0,0,0}; - recursive_count_objects("localhost", - BackupProtocolListDirectory::RootDirectory, - after); - - // If these tests fail then try increasing the timeout above - TEST_THAT(after.objectsNotDel == before.objectsNotDel); - TEST_THAT(after.deleted == 0); - TEST_THAT(after.old == 0); - + recursive_count_objects(BACKUPSTORE_ROOT_DIRECTORY_ID, after); + TEST_EQUAL(before.objectsNotDel, after.objectsNotDel); + TEST_EQUAL(0, after.deleted); + TEST_EQUAL(0, after.old); + + // Adjust reference counts on deleted files, so that the final checks in + // teardown_test_backupstore() don't fail. + ExpectedRefCounts.resize(2); + + // Delete the account to stop teardown_test_backupstore from checking it. + // TODO FIXME investigate the block count mismatch that teardown_test_backupstore + // catches if we don't delete the account. + delete_account(); + + TEARDOWN_TEST_BACKUPSTORE(); +} + +bool test_account_limits_respected() +{ + SETUP_TEST_BACKUPSTORE(); + TEST_THAT_OR(StartServer(), FAIL); + // Set a really small hard limit - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + TEST_THAT_OR(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf setlimit 01234567 " - "10B 20B") == 0); + "2B 2B") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - // Try to upload a file and create a directory, and check an error is generated + // Try to upload a file and create a directory, both of which would exceed the + // current account limits, and check that each command returns an error. { - // Open a connection to the server - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", - BOX_PORT_BBSTORED_TEST); - - // Make a protocol - BackupProtocolClient protocol(conn); - - // Check the version - std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); - TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); + write_test_file(3); - // Login - std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, 0)); - - int64_t modtime = 0; - + // Open a connection to the server + std::auto_ptr<BackupProtocolCallable> apProtocol( + connect_and_login(context)); BackupStoreFilenameClear fnx("exceed-limit"); - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/test3", BackupProtocolListDirectory::RootDirectory, fnx, &modtime)); + int64_t modtime = 0; + std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("testfiles/test3", BACKUPSTORE_ROOT_DIRECTORY_ID, fnx, &modtime)); TEST_THAT(modtime != 0); - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile( - BackupProtocolListDirectory::RootDirectory, + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryStoreFile( + BACKUPSTORE_ROOT_DIRECTORY_ID, modtime, modtime, /* use it for attr hash too */ - 0, /* diff from ID */ + 0, /* diff from ID */ fnx, - *upload)), - ConnectionException, Conn_Protocol_UnexpectedReply); + upload), + Err_StorageLimitExceeded); - MemBlockStream attr(&modtime, sizeof(modtime)); + // This currently causes a fatal error on the server, which + // kills the connection. TODO FIXME return an error instead. + std::auto_ptr<IOStream> attr(new MemBlockStream(&modtime, sizeof(modtime))); BackupStoreFilenameClear fnxd("exceed-limit-dir"); - TEST_CHECK_THROWS(std::auto_ptr<BackupProtocolSuccess> dirCreate(protocol.QueryCreateDirectory( - BackupProtocolListDirectory::RootDirectory, - 9837429842987984LL, fnxd, attr)), - ConnectionException, Conn_Protocol_UnexpectedReply); - + TEST_COMMAND_RETURNS_ERROR(*apProtocol, + QueryCreateDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, + FAKE_ATTR_MODIFICATION_TIME, fnxd, attr), + Err_StorageLimitExceeded); - // Finish the connection - protocol.QueryFinished(); + // Finish the connection. + apProtocol->QueryFinished(); } - // Kill it again - TEST_THAT(KillServer(pid)); - ::sleep(1); - TEST_THAT(!ServerIsAlive(pid)); - - #ifdef WIN32 - TEST_THAT(unlink("testfiles/bbstored.pid") == 0); - #else - TestRemoteProcessMemLeaks("bbstored.memleaks"); - #endif - - return 0; + TEARDOWN_TEST_BACKUPSTORE(); } int multi_server() @@ -2149,14 +2807,14 @@ int multi_server() printf("Starting server for connection from remote machines...\n"); // Create an account for the test client - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS + TEST_THAT_OR(::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf create 01234567 0 " - "30000B 40000B") == 0); + "30000B 40000B") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); // First, try logging in without an account having been created... just make sure login fails. - int pid = LaunchServer(BBSTORED " testfiles/bbstored_multi.conf", + int pid = LaunchServer(BBSTORED " testfiles/bbstored_multi.conf", "testfiles/bbstored.pid"); TEST_THAT(pid != -1 && pid != 0); @@ -2172,43 +2830,33 @@ int multi_server() printf("Terminating server...\n"); // Kill it - TEST_THAT(KillServer(pid)); - ::sleep(1); - TEST_THAT(!ServerIsAlive(pid)); - - #ifdef WIN32 - TEST_THAT(unlink("testfiles/bbstored.pid") == 0); - #else - TestRemoteProcessMemLeaks("bbstored.memleaks"); - #endif + TEST_THAT(StopServer()); } - return 0; } -void test_open_files_with_limited_win32_permissions() +bool test_open_files_with_limited_win32_permissions() { #ifdef WIN32 // this had better work, or bbstored will die when combining diffs - char* file = "foo"; + const char* file = "foo"; - DWORD accessRights = FILE_READ_ATTRIBUTES | + DWORD accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY | FILE_READ_EA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_DATA | FILE_WRITE_EA /*| FILE_ALL_ACCESS*/; DWORD shareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; HANDLE h1 = CreateFileA(file, accessRights, shareMode, - NULL, OPEN_ALWAYS, FILE_FLAG_BACKUP_SEMANTICS, NULL); - assert(h1 != INVALID_HANDLE_VALUE); + NULL, OPEN_ALWAYS, // create file if it doesn't exist + FILE_FLAG_BACKUP_SEMANTICS, NULL); TEST_THAT(h1 != INVALID_HANDLE_VALUE); - accessRights = FILE_READ_ATTRIBUTES | + accessRights = FILE_READ_ATTRIBUTES | FILE_LIST_DIRECTORY | FILE_READ_EA; HANDLE h2 = CreateFileA(file, accessRights, shareMode, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - assert(h2 != INVALID_HANDLE_VALUE); TEST_THAT(h2 != INVALID_HANDLE_VALUE); CloseHandle(h2); @@ -2221,6 +2869,8 @@ void test_open_files_with_limited_win32_permissions() CloseHandle(h2); CloseHandle(h1); #endif + + return true; } void compare_backupstoreinfo_values_to_expected @@ -2280,20 +2930,18 @@ void compare_backupstoreinfo_values_to_expected TEST_EQUAL_LINE(expected_account_enabled, actual.IsAccountEnabled(), test_phase << " AccountEnabled"); - TEST_EQUAL_LINE(extra_data.GetSize(), actual.GetExtraData().GetSize(), + TEST_EQUAL_LINE(extra_data.GetSize(), actual.GetExtraData().GetSize(), test_phase << " extra data has wrong size"); TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), actual.GetExtraData().GetBuffer(), extra_data.GetSize()), test_phase << " extra data has wrong contents"); } -int test_read_old_backupstoreinfo_files() +bool test_read_old_backupstoreinfo_files() { + SETUP_TEST_BACKUPSTORE(); + // Create an account for the test client - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf create 01234567 0 " - "10000B 20000B") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); std::auto_ptr<BackupStoreInfo> apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), @@ -2326,7 +2974,7 @@ int test_read_old_backupstoreinfo_files() apArchive->Write((int64_t) 14); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); - + apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded from v1", info_v1, @@ -2334,10 +2982,10 @@ int test_read_old_backupstoreinfo_files() true /* enabled by default */); apInfo->SetAccountName("bonk"); - + // Save the info again apInfo->Save(/* allowOverwrite */ true); - + // Check that it was saved in the new Archive format std::auto_ptr<RaidFileRead> rfr(RaidFileRead::Open(0, info_filename, 0)); int32_t magic; @@ -2356,7 +3004,7 @@ int test_read_old_backupstoreinfo_files() /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("loaded in v1, resaved in v2", info_v1, *apInfo, "bonk", true); - + // Check that the new AccountEnabled flag is saved properly apInfo->SetAccountEnabled(false); apInfo->Save(/* allowOverwrite */ true); @@ -2375,7 +3023,7 @@ int test_read_old_backupstoreinfo_files() // Now save the info in v2 format without the AccountEnabled flag // (boxbackup 0.11 format) and check that the flag is set to true // for backwards compatibility - + rfw.reset(new RaidFileWrite(0, info_filename)); rfw->Open(/* allowOverwrite */ true); magic = htonl(INFO_MAGIC_VALUE_2); @@ -2392,7 +3040,7 @@ int test_read_old_backupstoreinfo_files() apArchive->Write(apInfo->GetBlocksInDirectories()); apArchive->Write(apInfo->GetBlocksSoftLimit()); apArchive->Write(apInfo->GetBlocksHardLimit()); - apArchive->Write(apInfo->GetNumFiles()); + apArchive->Write(apInfo->GetNumCurrentFiles()); apArchive->Write(apInfo->GetNumOldFiles()); apArchive->Write(apInfo->GetNumDeletedFiles()); apArchive->Write(apInfo->GetNumDirectories()); @@ -2401,7 +3049,7 @@ int test_read_old_backupstoreinfo_files() apArchive->Write(apInfo->GetDeletedDirectories()[1]); rfw->Commit(/* ConvertToRaidNow */ true); rfw.reset(); - + apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); compare_backupstoreinfo_values_to_expected("saved in v2 without " @@ -2411,7 +3059,7 @@ int test_read_old_backupstoreinfo_files() // Rewrite using full length, so that the first 4 bytes of extra data // doesn't get swallowed by "extra data". apInfo->Save(/* allowOverwrite */ true); - + // Append some extra data after the known account values, to simulate a // new addition to the store format. Check that this extra data is loaded // and resaved with the info file. We made the mistake of deleting it in @@ -2431,7 +3079,7 @@ int test_read_old_backupstoreinfo_files() rfw.reset(); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, /* ReadOnly */ false); - TEST_EQUAL_LINE(extra_data.GetSize(), apInfo->GetExtraData().GetSize(), + TEST_EQUAL_LINE(extra_data.GetSize(), apInfo->GetExtraData().GetSize(), "wrong amount of extra data loaded from info file"); TEST_EQUAL_LINE(0, memcmp(extra_data.GetBuffer(), apInfo->GetExtraData().GetBuffer(), extra_data.GetSize()), @@ -2441,16 +3089,16 @@ int test_read_old_backupstoreinfo_files() apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); compare_backupstoreinfo_values_to_expected("saved in future format " "with extra_data", info_v1, *apInfo, "test", true, extra_data); - + // Check that the new bbstoreaccounts command sets the flag properly - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf enabled 01234567 no") == 0); + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf enabled 01234567 no") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); TEST_EQUAL_LINE(false, apInfo->IsAccountEnabled(), "'bbstoreaccounts disabled no' should have reset AccountEnabled flag"); - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf enabled 01234567 yes") == 0); + TEST_THAT_OR(::system(BBSTOREACCOUNTS + " -c testfiles/bbstored.conf enabled 01234567 yes") == 0, FAIL); TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); apInfo = BackupStoreInfo::Load(0x1234567, "backup/01234567/", 0, true); TEST_EQUAL_LINE(true, apInfo->IsAccountEnabled(), @@ -2489,48 +3137,119 @@ int test_read_old_backupstoreinfo_files() "BackupStoreInfo::CreateForRegeneration and reloaded", info_v1, *apInfo, "spurtle", false /* AccountEnabled */, extra_data); - // Delete the account to leave the store in the same state as before + // Delete the account to stop teardown_test_backupstore checking it for errors. apInfo.reset(); - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS - " -c testfiles/bbstored.conf delete 01234567 yes") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - - return 0; + TEST_THAT(delete_account()); + + TEARDOWN_TEST_BACKUPSTORE(); +} + +// Test that attributes can be correctly read from and written to the standard +// format, for compatibility with other servers and clients. See +// http://mailman.uk.freebsd.org/pipermail../public/boxbackup/2010-November/005818.html and +// http://lists.boxbackup.org/pipermail/boxbackup/2011-February/005978.html for +// details of the problems with packed structs. +bool test_read_write_attr_streamformat() +{ + SETUP_TEST_BACKUPSTORE(); + + // Construct a minimal valid directory with 1 entry in memory using Archive, and + // try to read it back. + CollectInBufferStream buf; + Archive darc(buf, IOStream::TimeOutInfinite); + + // Write a dir_StreamFormat structure + darc.Write((int32_t)OBJECTMAGIC_DIR_MAGIC_VALUE); // mMagicValue + darc.Write((int32_t)1); // mNumEntries + darc.Write((int64_t)0x0123456789abcdef); // mObjectID + darc.Write((int64_t)0x0000000000000001); // mContainerID + darc.Write((uint64_t)0x23456789abcdef01); // mAttributesModTime + darc.Write((int32_t)BackupStoreDirectory::Option_DependencyInfoPresent); + // mOptionsPresent + // 36 bytes to here + + // Write fake attributes to make it valid. + StreamableMemBlock attr; + attr.WriteToStream(buf); + // 40 bytes to here + + // Write a single entry in an en_StreamFormat structure + darc.Write((uint64_t)0x3456789012345678); // mModificationTime + darc.Write((int64_t)0x0000000000000002); // mObjectID + darc.Write((int64_t)0x0000000000000003); // mSizeInBlocks + darc.Write((uint64_t)0x0000000000000004); // mAttributesHash + darc.WriteInt16((int16_t)0x3141); // mFlags + // 74 bytes to here + + // Then a BackupStoreFilename + BackupStoreFilename fn; + fn.SetAsClearFilename("hello"); + fn.WriteToStream(buf); + // 81 bytes to here + + // Then a StreamableMemBlock for attributes + attr.WriteToStream(buf); + // 85 bytes to here + + // Then an en_StreamFormatDepends for dependency info. + darc.Write((uint64_t)0x0000000000000005); // mDependsNewer + darc.Write((uint64_t)0x0000000000000006); // mDependsOlder + // 101 bytes to here + + // Make sure none of the fields was expanded in transit by Archive. + TEST_EQUAL(101, buf.GetSize()); + + buf.SetForReading(); + BackupStoreDirectory dir(buf); + + TEST_EQUAL(1, dir.GetNumberOfEntries()); + TEST_EQUAL(0x0123456789abcdef, dir.GetObjectID()); + TEST_EQUAL(0x0000000000000001, dir.GetContainerID()); + TEST_EQUAL(0x23456789abcdef01, dir.GetAttributesModTime()); + + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry* pen = i.Next(); + TEST_THAT_OR(pen != NULL, FAIL); + TEST_EQUAL(0x3456789012345678, pen->GetModificationTime()); + TEST_EQUAL(0x0000000000000002, pen->GetObjectID()); + TEST_EQUAL(0x0000000000000003, pen->GetSizeInBlocks()); + TEST_EQUAL(0x0000000000000004, pen->GetAttributesHash()); + TEST_EQUAL(0x0000000000000005, pen->GetDependsNewer()); + TEST_EQUAL(0x0000000000000006, pen->GetDependsOlder()); + TEST_EQUAL(0x3141, pen->GetFlags()); + + CollectInBufferStream buf2; + dir.WriteToStream(buf2); + buf2.SetForReading(); + buf.Seek(0, IOStream::SeekType_Absolute); + TEST_EQUAL(101, buf2.GetSize()); + TEST_EQUAL(buf.GetSize(), buf2.GetSize()); + + // Test that the stream written out for the Directory is exactly the same as the + // one we hand-crafted earlier. + TEST_EQUAL(0, memcmp(buf.GetBuffer(), buf2.GetBuffer(), buf.GetSize())); + + TEARDOWN_TEST_BACKUPSTORE(); } int test(int argc, const char *argv[]) { - test_open_files_with_limited_win32_permissions(); + TEST_THAT(test_open_files_with_limited_win32_permissions()); // Initialise the raid file controller RaidFileController &rcontroller = RaidFileController::GetController(); rcontroller.Initialise("testfiles/raidfile.conf"); - - int ret = test_read_old_backupstoreinfo_files(); - if (ret != 0) - { - return ret; - } - + + TEST_THAT(test_read_old_backupstoreinfo_files()); + // SSL library SSLLib::Initialise(); - - // Give a test key for the filenames -// BackupStoreFilenameClear::SetBlowfishKey(FilenameEncodingKey, sizeof(FilenameEncodingKey)); - // And set the encoding to blowfish -// BackupStoreFilenameClear::SetEncodingMethod(BackupStoreFilename::Encoding_Blowfish); - - // And for directory attributes -- need to set it, as used in file encoding -// BackupClientFileAttributes::SetBlowfishKey(AttributesEncodingKey, sizeof(AttributesEncodingKey)); - - // And finally for file encoding -// BackupStoreFile::SetBlowfishKeys(FileEncodingKey, sizeof(FileEncodingKey), FileBlockEntryEncodingKey, sizeof(FileBlockEntryEncodingKey)); // Use the setup crypto command to set up all these keys, so that the bbackupquery command can be used // for seeing what's going on. - BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); - - // encode in some filenames -- can't do static initialisation + BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); + + // encode in some filenames -- can't do static initialisation // because the key won't be set up when these are initialised { MEMLEAKFINDER_NO_LEAKS @@ -2544,36 +3263,43 @@ int test(int argc, const char *argv[]) uploads[l].name = BackupStoreFilenameClear(uploads_filenames[l]); } } - + // Trace errors out SET_DEBUG_SSLLIB_TRACE_ERRORS - if(argc == 2 && strcmp(argv[1], "server") == 0) - { - return multi_server(); - } - if(argc == 3 && strcmp(argv[1], "client") == 0) { - return test_server(argv[2]); + R250 r(3465657); + for(int l = 0; l < ATTR1_SIZE; ++l) {attr1[l] = r.next();} + for(int l = 0; l < ATTR2_SIZE; ++l) {attr2[l] = r.next();} + for(int l = 0; l < ATTR3_SIZE; ++l) {attr3[l] = r.next();} } -// large file test -/* { - int64_t modtime = 0; - std::auto_ptr<IOStream> upload(BackupStoreFile::EncodeFile("/Users/ben/temp/large.tar", - BackupProtocolListDirectory::RootDirectory, uploads[0].name, &modtime)); - TEST_THAT(modtime != 0); - FileStream write("testfiles/large.enc", O_WRONLY | O_CREAT); - upload->CopyStreamTo(write); - } -printf("SKIPPING TESTS ------------------------------------------------------\n"); -return 0;*/ - int r = 0; - r = test1(argc, argv); - if(r != 0) return r; - r = test2(argc, argv); - if(r != 0) return r; - r = test3(argc, argv); - if(r != 0) return r; - return 0; + + TEST_THAT(test_filename_encoding()); + TEST_THAT(test_temporary_refcount_db_is_independent()); + TEST_THAT(test_bbstoreaccounts_create()); + TEST_THAT(test_bbstoreaccounts_delete()); + TEST_THAT(test_backupstore_directory()); + TEST_THAT(test_directory_parent_entry_tracks_directory_size()); + TEST_THAT(test_cannot_open_multiple_writable_connections()); + TEST_THAT(test_encoding()); + TEST_THAT(test_symlinks()); + TEST_THAT(test_store_info()); + + context.Initialise(false /* client */, + "testfiles/clientCerts.pem", + "testfiles/clientPrivKey.pem", + "testfiles/clientTrustedCAs.pem"); + + TEST_THAT(test_login_without_account()); + TEST_THAT(test_login_with_disabled_account()); + TEST_THAT(test_login_with_no_refcount_db()); + TEST_THAT(test_server_housekeeping()); + TEST_THAT(test_server_commands()); + TEST_THAT(test_account_limits_respected()); + TEST_THAT(test_multiple_uploads()); + TEST_THAT(test_housekeeping_deletes_files()); + TEST_THAT(test_read_write_attr_streamformat()); + + return finish_test_suite(); } diff --git a/test/backupstore/testfiles/query.conf b/test/backupstore/testfiles/query.conf index 984ace6c..a25f148c 100644 --- a/test/backupstore/testfiles/query.conf +++ b/test/backupstore/testfiles/query.conf @@ -17,6 +17,7 @@ MinimumFileAge = 4 MaxUploadWait = 24 FileTrackingSizeThreshold = 1024 DiffingUploadSizeThreshold = 1024 +StorePort = 22011 Server { diff --git a/test/backupstorefix/testbackupstorefix.cpp b/test/backupstorefix/testbackupstorefix.cpp index bc9911f0..38492bd1 100644 --- a/test/backupstorefix/testbackupstorefix.cpp +++ b/test/backupstorefix/testbackupstorefix.cpp @@ -18,9 +18,10 @@ #include "Test.h" #include "BackupClientCryptoKeys.h" +#include "BackupProtocol.h" +#include "BackupStoreAccounts.h" #include "BackupStoreCheck.h" #include "BackupStoreConstants.h" -#include "BackupStoreContext.h" #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreFile.h" @@ -29,6 +30,7 @@ #include "BackupStoreInfo.h" #include "BufferedWriteStream.h" #include "FileStream.h" +#include "IOStreamGetLine.h" #include "RaidFileController.h" #include "RaidFileException.h" #include "RaidFileRead.h" @@ -36,6 +38,7 @@ #include "RaidFileWrite.h" #include "ServerControl.h" #include "StoreStructure.h" +#include "StoreTestUtils.h" #include "ZeroStream.h" #include "MemLeakFindOn.h" @@ -66,7 +69,7 @@ delete root, copy a file to it instead (equivalent to deleting it too) */ -std::string storeRoot("backup/01234567/"); +std::string accountRootDir("backup/01234567/"); int discSetNum = 0; std::map<std::string, int32_t> nameToID; @@ -76,33 +79,13 @@ std::map<int32_t, bool> objectIsDir; ::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf check 01234567"); \ ::system(BBSTOREACCOUNTS " -c testfiles/bbstored.conf check 01234567 fix"); -bool check_fix_internal(int expected_num_errors) -{ - BackupStoreCheck checker(storeRoot, discSetNum, - 0x01234567, true /* FixErrors */, false /* Quiet */); - checker.Check(); - if (expected_num_errors == -1) - { - TEST_THAT(checker.ErrorsFound()); - return checker.ErrorsFound(); - } - else - { - TEST_EQUAL(expected_num_errors, checker.GetNumErrorsFound()); - return checker.GetNumErrorsFound() == expected_num_errors; - } -} - -#define RUN_CHECK_INTERNAL(expected_num_errors) \ - TEST_THAT(check_fix_internal(expected_num_errors)) - // Get ID of an object given a filename int32_t getID(const char *name) { std::map<std::string, int32_t>::iterator i(nameToID.find(std::string(name))); TEST_THAT(i != nameToID.end()); if(i == nameToID.end()) return -1; - + return i->second; } @@ -110,7 +93,7 @@ int32_t getID(const char *name) std::string getObjectName(int32_t id) { std::string fn; - StoreStructure::MakeObjectFilename(id, storeRoot, discSetNum, fn, false); + StoreStructure::MakeObjectFilename(id, accountRootDir, discSetNum, fn, false); return fn; } @@ -166,11 +149,12 @@ typedef struct int flags; } dir_en_check; -void check_dir(BackupStoreDirectory &dir, dir_en_check *ck) +bool check_dir(BackupStoreDirectory &dir, dir_en_check *ck) { BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en; - + bool ok = true; + while((en = i.Next()) != 0) { BackupStoreFilenameClear clear(en->GetName()); @@ -185,9 +169,10 @@ void check_dir(BackupStoreDirectory &dir, dir_en_check *ck) TEST_THAT(en->GetFlags() == ck->flags); ++ck; } - - TEST_THAT(en == 0); - TEST_THAT(ck->name == -1); + + TEST_EQUAL_OR((void *)NULL, (void *)en, ok = false); + TEST_EQUAL_OR(ck->name, -1, ok = false); + return ok; } typedef struct @@ -199,7 +184,7 @@ void check_dir_dep(BackupStoreDirectory &dir, checkdepinfoen *ck) { BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en; - + while((en = i.Next()) != 0) { TEST_THAT(ck->id != -1); @@ -215,7 +200,7 @@ void check_dir_dep(BackupStoreDirectory &dir, checkdepinfoen *ck) "Wrong Older dependency for " << BOX_FORMAT_OBJECTID(ck->id)); ++ck; } - + TEST_THAT(en == 0); TEST_THAT(ck->id == -1); } @@ -229,19 +214,25 @@ void test_dir_fixing() 2 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2); e->SetDependsNewer(3); - + TEST_THAT(dir.CheckAndFix() == true); TEST_THAT(dir.CheckAndFix() == false); dir_en_check ck[] = { {-1, 0, 0} }; - - check_dir(dir, ck); + + TEST_THAT(check_dir(dir, ck)); } { BackupStoreDirectory dir; + /* + Entry *AddEntry(const BackupStoreFilename &rName, + box_time_t ModificationTime, int64_t ObjectID, + int64_t SizeInBlocks, int16_t Flags, + uint64_t AttributesHash); + */ dir.AddEntry(fnames[0], 12, 2 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2); dir.AddEntry(fnames[1], 12, 2 /* id */, 1, @@ -251,14 +242,23 @@ void test_dir_fixing() dir.AddEntry(fnames[0], 12, 5 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2); - + + /* + typedef struct + { + int name; + int64_t id; + int flags; + } dir_en_check; + */ + dir_en_check ck[] = { {1, 2, BackupStoreDirectory::Entry::Flags_File}, {0, 3, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion}, {0, 5, BackupStoreDirectory::Entry::Flags_File}, {-1, 0, 0} }; - + TEST_THAT(dir.CheckAndFix() == true); TEST_THAT(dir.CheckAndFix() == false); check_dir(dir, ck); @@ -270,7 +270,7 @@ void test_dir_fixing() dir.AddEntry(fnames[1], 12, 10 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_Dir | BackupStoreDirectory::Entry::Flags_OldVersion, 2); dir.AddEntry(fnames[0], 12, 3 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2); dir.AddEntry(fnames[0], 12, 5 /* id */, 1, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion, 2); - + dir_en_check ck[] = { {0, 2, BackupStoreDirectory::Entry::Flags_File | BackupStoreDirectory::Entry::Flags_OldVersion}, {1, 10, BackupStoreDirectory::Entry::Flags_Dir}, @@ -278,7 +278,7 @@ void test_dir_fixing() {0, 5, BackupStoreDirectory::Entry::Flags_File}, {-1, 0, 0} }; - + TEST_THAT(dir.CheckAndFix() == true); TEST_THAT(dir.CheckAndFix() == false); check_dir(dir, ck); @@ -309,7 +309,7 @@ void test_dir_fixing() 5 /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2); TEST_THAT(e5 != 0); e5->SetDependsOlder(4); - + // This should all be nice and valid TEST_THAT(dir.CheckAndFix() == false); static checkdepinfoen c1[] = {{2, 3, 0}, {3, 4, 2}, {4, 5, 3}, {5, 0, 4}, {-1, 0, 0}}; @@ -337,8 +337,14 @@ void test_dir_fixing() } int64_t fake_upload(BackupProtocolLocal& client, const std::string& file_path, - int64_t diff_from_id) + int64_t diff_from_id, int64_t container_id = BACKUPSTORE_ROOT_DIRECTORY_ID, + BackupStoreFilename* fn = NULL) { + if(fn == NULL) + { + fn = &fnames[0]; + } + std::auto_ptr<IOStream> upload; if(diff_from_id) { @@ -348,11 +354,13 @@ int64_t fake_upload(BackupProtocolLocal& client, const std::string& file_path, std::auto_ptr<IOStream> blockIndexStream(client.ReceiveStream()); upload = BackupStoreFile::EncodeFileDiff( file_path, - BACKUPSTORE_ROOT_DIRECTORY_ID, fnames[0], - diff_from_id, *blockIndexStream, + container_id, + *fn, + diff_from_id, + *blockIndexStream, IOStream::TimeOutInfinite, NULL, // DiffTimer implementation - 0 /* not interested in the modification time */, + 0 /* not interested in the modification time */, NULL // isCompletelyDifferent ); } @@ -360,25 +368,26 @@ int64_t fake_upload(BackupProtocolLocal& client, const std::string& file_path, { upload = BackupStoreFile::EncodeFile( file_path, - BACKUPSTORE_ROOT_DIRECTORY_ID, fnames[0], - NULL, + container_id, + *fn, + NULL, NULL, // pLogger NULL // pRunStatusProvider ); } - return client.QueryStoreFile(BACKUPSTORE_ROOT_DIRECTORY_ID, + return client.QueryStoreFile(container_id, 1, // ModificationTime 2, // AttributesHash diff_from_id, // DiffFromFileID - fnames[0], // rFilename + *fn, // rFilename upload)->GetObjectID(); } void read_bb_dir(int64_t objectId, BackupStoreDirectory& dir) { std::string fn; - StoreStructure::MakeObjectFilename(1 /* root */, storeRoot, + StoreStructure::MakeObjectFilename(1 /* root */, accountRootDir, discSetNum, fn, true /* EnsureDirectoryExists */); std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(discSetNum, @@ -397,11 +406,8 @@ void login_client_and_check_empty(BackupProtocolCallable& client) static checkdepinfoen start_deps[] = {{-1, 0, 0}}; check_dir_dep(dir, start_deps); - client.QueryVersion(BACKUP_STORE_SERVER_VERSION); - client.QueryLogin(0x01234567, 0 /* read/write */); - read_bb_dir(1 /* root */, dir); - + // Everything should be OK at the moment TEST_THAT(dir.CheckAndFix() == false); @@ -420,8 +426,8 @@ void check_root_dir_ok(dir_en_check after_entries[], { // Check the store, check that the error is detected and // repaired, by removing x1 from the directory. - RUN_CHECK_INTERNAL(0); - + TEST_EQUAL(0, check_account_for_errors()); + // Read the directory back in, check that it's empty BackupStoreDirectory dir; read_bb_dir(1 /* root */, dir); @@ -435,7 +441,7 @@ void check_and_fix_root_dir(dir_en_check after_entries[], { // Check the store, check that the error is detected and // repaired. - RUN_CHECK_INTERNAL(-1); + TEST_THAT(check_account_for_errors() > 0); check_root_dir_ok(after_entries, after_deps); } @@ -449,7 +455,7 @@ int test(int argc, const char *argv[]) } // Test the backupstore directory fixing - // FIXME reenable: test_dir_fixing(); + test_dir_fixing(); // Initialise the raidfile controller RaidFileController &rcontroller = RaidFileController::GetController(); @@ -470,20 +476,18 @@ int test(int argc, const char *argv[]) "exist is really deleted"); { - BackupStoreContext ctx(0x01234567, - *(HousekeepingInterface *)NULL, - "test" /* rConnectionDetails */); - ctx.SetClientHasAccount(storeRoot, discSetNum); - BackupProtocolLocal client(ctx); + BackupProtocolLocal2 client(0x01234567, "test", accountRootDir, + discSetNum, false); login_client_and_check_empty(client); std::string file_path = "testfiles/TestDir1/cannes/ict/metegoguered/oats"; int x1id = fake_upload(client, file_path, 0); + client.QueryFinished(); // Now break the reverse dependency by deleting x1 (the file, // not the directory entry) std::string x1FileName; - StoreStructure::MakeObjectFilename(x1id, storeRoot, discSetNum, + StoreStructure::MakeObjectFilename(x1id, accountRootDir, discSetNum, x1FileName, true /* EnsureDirectoryExists */); RaidFileWrite deleteX1(discSetNum, x1FileName); deleteX1.Delete(); @@ -497,11 +501,8 @@ int test(int argc, const char *argv[]) "exist is really deleted"); { - BackupStoreContext ctx(0x01234567, - *(HousekeepingInterface *)NULL, - "test" /* rConnectionDetails */); - ctx.SetClientHasAccount(storeRoot, discSetNum); - BackupProtocolLocal client(ctx); + BackupProtocolLocal2 client(0x01234567, "test", accountRootDir, + discSetNum, false); login_client_and_check_empty(client); std::string file_path = "testfiles/TestDir1/cannes/ict/metegoguered/oats"; @@ -514,6 +515,7 @@ int test(int argc, const char *argv[]) fs.Close(); int x1aid = fake_upload(client, file_path, x1id); + client.QueryFinished(); // Check that we've ended up with the right preconditions // for the tests below. @@ -530,7 +532,7 @@ int test(int argc, const char *argv[]) // Now break the reverse dependency by deleting x1a (the file, // not the directory entry) std::string x1aFileName; - StoreStructure::MakeObjectFilename(x1aid, storeRoot, discSetNum, + StoreStructure::MakeObjectFilename(x1aid, accountRootDir, discSetNum, x1aFileName, true /* EnsureDirectoryExists */); RaidFileWrite deleteX1a(discSetNum, x1aFileName); deleteX1a.Delete(); @@ -545,56 +547,95 @@ int test(int argc, const char *argv[]) "raidfile is corrupted doesn't crash"); // Start the bbstored server - BOX_TRACE(" === Starting bbstored server: " BBSTORED - " testfiles/bbstored.conf"); - int bbstored_pid = LaunchServer(BBSTORED " testfiles/bbstored.conf", - "testfiles/bbstored.pid"); - TEST_THAT(bbstored_pid > 0); - if (bbstored_pid <= 0) return 1; - - ::sleep(1); - TEST_THAT(ServerIsAlive(bbstored_pid)); - - std::string cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd.conf"; - int bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - TEST_THAT(bbackupd_pid > 0); - if (bbackupd_pid <= 0) return 1; - - ::safe_sleep(1); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - - // Wait 4 more seconds for the files to be old enough - // to upload - ::safe_sleep(4); - - // Upload files to create a nice store directory - ::sync_and_wait(); - - // Stop bbackupd - #ifdef WIN32 - terminate_bbackupd(bbackupd_pid); - // implicit check for memory leaks - #else - TEST_THAT(KillServer(bbackupd_pid)); - TestRemoteProcessMemLeaks("bbackupd.memleaks"); - #endif + TEST_THAT_OR(StartServer(), return 1); + + // Instead of starting a client, read the file listing file created by + // testbackupstorefix.pl and upload them in the correct order, so that the object + // IDs will not vary depending on the order in which readdir() returns entries. + { + FileStream listing("testfiles/file-listing.txt", O_RDONLY); + IOStreamGetLine getline(listing); + std::map<std::string, int64_t> dirname_to_id; + std::string line; + BackupProtocolLocal2 client(0x01234567, "test", accountRootDir, + discSetNum, false); + + for(getline.GetLine(line, true); line != ""; getline.GetLine(line, true)) + { + std::string full_path = line; + ASSERT(StartsWith("testfiles/TestDir1/", full_path)); + + bool is_dir = (full_path[full_path.size() - 1] == '/'); + if(is_dir) + { + full_path = full_path.substr(0, full_path.size() - 1); + } + + std::string::size_type last_slash = full_path.rfind('/'); + int64_t container_id; + std::string filename; + + if(full_path == "testfiles/TestDir1") + { + container_id = BACKUPSTORE_ROOT_DIRECTORY_ID; + filename = "Test1"; + } + else + { + std::string containing_dir = + full_path.substr(0, last_slash); + container_id = dirname_to_id[containing_dir]; + filename = full_path.substr(last_slash + 1); + } + + BackupStoreFilenameClear fn(filename); + if(is_dir) + { + std::auto_ptr<IOStream> attr_stream( + new CollectInBufferStream); + ((CollectInBufferStream &) + *attr_stream).SetForReading(); + + dirname_to_id[full_path] = client.QueryCreateDirectory( + container_id, 0, // AttributesModTime + fn, attr_stream)->GetObjectID(); + } + else + { + fake_upload(client, line, 0, container_id, &fn); + } + } + } + + // Check that we're starting off with the right numbers of files and blocks. + // Otherwise the test that check the counts after breaking things will fail + // because the numbers won't match. + TEST_EQUAL(0, check_account_for_errors()); + { + std::auto_ptr<BackupProtocolAccountUsage2> usage = + BackupProtocolLocal2(0x01234567, "test", + "backup/01234567/", 0, + false).QueryGetAccountUsage2(); + TEST_EQUAL(usage->GetNumCurrentFiles(), 114); + TEST_EQUAL(usage->GetNumDirectories(), 28); + TEST_EQUAL(usage->GetBlocksUsed(), 284); + TEST_EQUAL(usage->GetBlocksInCurrentFiles(), 228); + TEST_EQUAL(usage->GetBlocksInDirectories(), 56); + } BOX_INFO(" === Add a reference to a file that doesn't exist, check " "that it's removed"); { BackupStoreDirectory dir; read_bb_dir(1 /* root */, dir); - + dir.AddEntry(fnames[0], 12, 0x1234567890123456LL /* id */, 1, BackupStoreDirectory::Entry::Flags_File, 2); - + std::string fn; - StoreStructure::MakeObjectFilename(1 /* root */, storeRoot, + StoreStructure::MakeObjectFilename(1 /* root */, accountRootDir, discSetNum, fn, true /* EnsureDirectoryExists */); - std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(discSetNum, - fn)); RaidFileWrite d(discSetNum, fn); d.Open(true /* allow overwrite */); dir.WriteToStream(d); @@ -603,27 +644,21 @@ int test(int argc, const char *argv[]) read_bb_dir(1 /* root */, dir); TEST_THAT(dir.FindEntryByID(0x1234567890123456LL) != 0); - // Check it - BackupStoreCheck checker(storeRoot, discSetNum, - 0x01234567, true /* FixErrors */, false /* Quiet */); - checker.Check(); - // Should just be greater than 1 really, we don't know quite // how good the checker is (or will become) at spotting errors! // But this will help us catch changes in checker behaviour, // so it's not a bad thing to test. - TEST_EQUAL(2, checker.GetNumErrorsFound()); + TEST_EQUAL(2, check_account_for_errors()); - file = RaidFileRead::Open(discSetNum, fn); + std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(discSetNum, + fn)); dir.ReadFromStream(*file, IOStream::TimeOutInfinite); TEST_THAT(dir.FindEntryByID(0x1234567890123456LL) == 0); } - if (failures > 0) return 1; - // Generate a list of all the object IDs TEST_THAT_ABORTONFAIL(::system(BBACKUPQUERY " -Wwarning " - "-c testfiles/bbackupd.conf \"list -r\" quit " + "-c testfiles/bbackupd.conf \"list -R\" quit " "> testfiles/initial-listing.txt") == 0); // And load it in @@ -647,17 +682,17 @@ int test(int argc, const char *argv[]) ::fclose(f); } - // ------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------ BOX_INFO(" === Delete store info, add random file"); { // Delete store info - RaidFileWrite del(discSetNum, storeRoot + "info"); + RaidFileWrite del(discSetNum, accountRootDir + "info"); del.Delete(); } { // Add a spurious file RaidFileWrite random(discSetNum, - storeRoot + "randomfile"); + accountRootDir + "randomfile"); random.Open(); random.Write("test", 4); random.Commit(true); @@ -672,13 +707,28 @@ int test(int argc, const char *argv[]) // Check the random file doesn't exist { TEST_THAT(!RaidFileRead::FileExists(discSetNum, - storeRoot + "01/randomfile")); + accountRootDir + "01/randomfile")); } - // ------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------ BOX_INFO(" === Delete an entry for an object from dir, change that " "object to be a patch, check it's deleted"); { + // Temporarily stop the server, so it doesn't repair the refcount error. Except + // on win32, where hard-killing the server can leave a lockfile in place, + // breaking the rest of the test. +#ifdef WIN32 + // Wait for the server to finish housekeeping first, by getting a lock on + // the account. + std::auto_ptr<BackupStoreAccountDatabase> apAccounts( + BackupStoreAccountDatabase::Read("testfiles/accounts.txt")); + BackupStoreAccounts acc(*apAccounts); + NamedLock lock; + acc.LockAccount(0x1234567, lock); +#else + TEST_THAT(StopServer()); +#endif + // Open dir and find entry int64_t delID = getID("Test1/cannes/ict/metegoguered/oats"); { @@ -688,7 +738,7 @@ int test(int argc, const char *argv[]) dir.DeleteEntry(delID); SaveDirectory("Test1/cannes/ict/metegoguered", dir); } - + // Adjust that entry // // IMPORTANT NOTE: There's a special hack in testbackupstorefix.pl to make sure that @@ -727,41 +777,36 @@ int test(int argc, const char *argv[]) f.Commit(true /* write now! */); } -#ifndef BOX_RELEASE_BUILD - // Delete two of the three raidfiles and their parent - // directories. This used to crash bbstoreaccounts check. - // We can only do this, without destroying the entire store, - // in debug mode, where the store has a far deeper - // structure. - // This will destroy or damage objects 18-1b and 58-5b, - // some repairably. - #define RUN(x) TEST_THAT(system(x) == 0); - RUN("mv testfiles/0_0/backup/01234567/02/01/o00.rf " - "testfiles/0_0/backup/01234567/02/01/o00.rfw"); // 0x18 - RUN("mv testfiles/0_1/backup/01234567/02/01/o01.rf " - "testfiles/0_1/backup/01234567/02/01/o01.rfw"); // 0x19 - //RUN("mv testfiles/0_2/backup/01234567/02/01/o02.rf " - // "testfiles/0_0/backup/01234567/02/01/o02.rfw"); // 0x1a - RUN("mv testfiles/0_0/backup/01234567/02/01/o03.rf " - "testfiles/0_0/backup/01234567/02/01/o03.rfw"); // 0x1b - RUN("mv testfiles/0_0/backup/01234567/02/01/01/o00.rf " - "testfiles/0_0/backup/01234567/02/01/01/o00.rfw"); // 0x58 - RUN("mv testfiles/0_1/backup/01234567/02/01/01/o01.rf " - "testfiles/0_1/backup/01234567/02/01/01/o01.rfw"); // 0x59 - //RUN("mv testfiles/0_2/backup/01234567/02/01/01/o02.rf " - // "testfiles/0_0/backup/01234567/02/01/01/o02.rfw"); // 0x5a - RUN("mv testfiles/0_0/backup/01234567/02/01/01/o03.rf " - "testfiles/0_0/backup/01234567/02/01/01/o03.rfw"); // 0x5b - // RUN("rm -r testfiles/0_1/backup/01234567/02/01"); - RUN("rm -r testfiles/0_2/backup/01234567/02/01"); - #undef RUN -#endif // BOX_RELEASE_BUILD - // Fix it - RUN_CHECK_INTERNAL(3); + // ERROR: Object 0x44 is unattached. + // ERROR: BlocksUsed changed from 284 to 282 + // ERROR: BlocksInCurrentFiles changed from 228 to 226 + // ERROR: NumCurrentFiles changed from 114 to 113 + // WARNING: Reference count of object 0x44 changed from 1 to 0 +#ifdef WIN32 + lock.ReleaseLock(); +#endif + TEST_EQUAL(5, check_account_for_errors()); + { + std::auto_ptr<BackupProtocolAccountUsage2> usage = + BackupProtocolLocal2(0x01234567, "test", + "backup/01234567/", 0, + false).QueryGetAccountUsage2(); + TEST_EQUAL(usage->GetNumCurrentFiles(), 113); + TEST_EQUAL(usage->GetNumDirectories(), 28); + TEST_EQUAL(usage->GetBlocksUsed(), 282); + TEST_EQUAL(usage->GetBlocksInCurrentFiles(), 226); + TEST_EQUAL(usage->GetBlocksInDirectories(), 56); + } + + // Start the server again, so testbackupstorefix.pl can run bbackupquery which + // connects to it. Except on win32, where we didn't stop it earlier. +#ifndef WIN32 + TEST_THAT(StartServer()); +#endif // Check - TEST_THAT(::system(PERL_EXECUTABLE + TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 1") == 0); @@ -773,25 +818,9 @@ int test(int argc, const char *argv[]) // file, so checking for AsRaid excludes this possibility. RaidFileController &rcontroller(RaidFileController::GetController()); RaidFileDiscSet rdiscSet(rcontroller.GetDiscSet(discSetNum)); - TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( - rdiscSet, "backup/01234567/02/01/o00")); - TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( - rdiscSet, "backup/01234567/02/01/o01")); - TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( - rdiscSet, "backup/01234567/02/01/o02")); - TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( - rdiscSet, "backup/01234567/02/01/o03")); - TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( - rdiscSet, "backup/01234567/02/01/01/o00")); - TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( - rdiscSet, "backup/01234567/02/01/01/o01")); - TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( - rdiscSet, "backup/01234567/02/01/01/o02")); - TEST_EQUAL(RaidFileUtil::AsRaid, RaidFileUtil::RaidFileExists( - rdiscSet, "backup/01234567/02/01/01/o03")); } - - // ------------------------------------------------------------------------------------------------ + + // ------------------------------------------------------------------------------------------------ BOX_INFO(" === Delete directory, change container ID of another, " "duplicate entry in dir, spurious file size, delete file"); { @@ -825,16 +854,19 @@ int test(int argc, const char *argv[]) } SaveDirectory("Test1/cannes/ict/peep", dir); } - // Delete a directory + + // Delete a directory. The checker should be able to reconstruct it using the + // ContainerID of the contained files. DeleteObject("Test1/pass/cacted/ming"); + // Delete a file DeleteObject("Test1/cannes/ict/scely"); - // We don't know quite how good the checker is (or will become) at - // spotting errors! But asserting an exact number will help us catch + // We don't know quite how good the checker is (or will become) at + // spotting errors! But asserting an exact number will help us catch // changes in checker behaviour, so it's not a bad thing to test. - // The 11 errors are: + // The 12 errors that we currently expect are: // ERROR: Directory ID 0xb references object 0x3e which does not exist. // ERROR: Removing directory entry 0x3e from directory 0xb // ERROR: Directory ID 0xc had invalid entries, fixed @@ -846,11 +878,23 @@ int test(int argc, const char *argv[]) // ERROR: BlocksInCurrentFiles changed from 226 to 220 // ERROR: BlocksInDirectories changed from 56 to 54 // ERROR: NumFiles changed from 113 to 110 + // WARNING: Reference count of object 0x3e changed from 1 to 0 + + TEST_EQUAL(12, check_account_for_errors()); - RUN_CHECK_INTERNAL(11); + { + std::auto_ptr<BackupProtocolAccountUsage2> usage = + BackupProtocolLocal2(0x01234567, "test", + "backup/01234567/", 0, + false).QueryGetAccountUsage2(); + TEST_EQUAL(usage->GetBlocksUsed(), 278); + TEST_EQUAL(usage->GetBlocksInCurrentFiles(), 220); + TEST_EQUAL(usage->GetBlocksInDirectories(), 54); + TEST_EQUAL(usage->GetNumCurrentFiles(), 110); + } // Check everything is as it should be - TEST_THAT(::system(PERL_EXECUTABLE + TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 2") == 0); { BackupStoreDirectory dir; @@ -881,7 +925,7 @@ int test(int argc, const char *argv[]) } } - // ------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------ BOX_INFO(" === Modify the obj ID of dir, delete dir with no members, " "add extra reference to a file"); // Set bad object ID @@ -905,28 +949,32 @@ int test(int argc, const char *argv[]) dir2.AddEntry(*en); SaveDirectory("Test1/divel/torsines/cruishery", dir2); } + // Fix it RUN_CHECK + // Check everything is as it should be - TEST_THAT(::system(PERL_EXECUTABLE + TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 3") == 0); { BackupStoreDirectory dir; LoadDirectory("Test1/foreomizes/stemptinevidate/ict", dir); TEST_THAT(dir.GetObjectID() == getID("Test1/foreomizes/stemptinevidate/ict")); } - - // ------------------------------------------------------------------------------------------------ + + // ------------------------------------------------------------------------------------------------ BOX_INFO(" === Orphan files and dirs without being recoverable"); - DeleteObject("Test1/dir1"); - DeleteObject("Test1/dir1/dir2"); + DeleteObject("Test1/dir1"); + DeleteObject("Test1/dir1/dir2"); + // Fix it RUN_CHECK + // Check everything is where it is predicted to be - TEST_THAT(::system(PERL_EXECUTABLE + TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 4") == 0); - // ------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------ BOX_INFO(" === Corrupt file and dir"); // File CorruptObject("Test1/foreomizes/stemptinevidate/algoughtnerge", @@ -940,7 +988,7 @@ int test(int argc, const char *argv[]) TEST_THAT(::system(PERL_EXECUTABLE " testfiles/testbackupstorefix.pl check 5") == 0); - // ------------------------------------------------------------------------------------------------ + // ------------------------------------------------------------------------------------------------ BOX_INFO(" === Overwrite root with a file"); { std::auto_ptr<RaidFileRead> r(RaidFileRead::Open(discSetNum, getObjectName(getID("Test1/pass/shuted/brightinats/milamptimaskates")))); diff --git a/test/backupstorefix/testfiles/testbackupstorefix.pl.in b/test/backupstorefix/testfiles/testbackupstorefix.pl.in index 28dc439f..fc807155 100755 --- a/test/backupstorefix/testfiles/testbackupstorefix.pl.in +++ b/test/backupstorefix/testfiles/testbackupstorefix.pl.in @@ -2,7 +2,7 @@ use strict; my @words = split /\s+/,<<__E; -nes ment foreomizes restout offety nount stemptinevidate ristraigation algoughtnerge nont ict aduals backyalivers scely peep hyphs olworks ning dro rogarcer poducts eatinizers bank magird backs bud metegoguered con mes prisionsidenning oats nost vulgarmiscar pass red rad cacted ded oud ming red emeated compt sly thetter shuted defeve plagger wished brightinats tillishellult arreenies honing ation recyclingentivell milamptimaskates debaffessly battenteriset +nes ment foreomizes restout offety nount stemptinevidate ristraigation algoughtnerge nont ict aduals backyalivers scely peep hyphs olworks ning dro rogarcer poducts eatinizers bank magird backs bud metegoguered com mes prisionsidenning oats nost vulgarmiscar pass red rad cacted ded oud ming red emeated compt sly thetter shuted defeve plagger wished brightinats tillishellult arreenies honing ation recyclingentivell milamptimaskates debaffessly battenteriset bostopring prearnies mailatrisepatheryingic divel ing middle torsines quarcharattendlegipsied resteivate acingladdrairevents cruishery flowdemobiologgermanciolt ents subver honer paulounces relessition dunhoutpositivessiveng suers emancess cons cheating winneggs flow ditiespaynes constrannotalimentievolutal ing repowellike stucablest ablemates impsychocks sorts degruman lace scons cords unsertracturce tumottenting locapersethithend pushotty polly red rialgolfillopmeninflirer skied relocis hetterabbed undaunatermisuresocioll cont posippory fibruting cannes storm callushlike warnook imulatrougge dicreamentsvily spical fishericating roes carlylisticaller __E @@ -34,16 +34,26 @@ my @check_move = ( if($ARGV[0] eq 'init') { + open(my $fh, ">>", "testfiles/file-listing.txt") + or die "cannot open testfiles/file-listing.txt: $!"; # create the initial tree of words - make_dir('testfiles/TestDir1', 0, 4, 0); - + make_dir($fh, 'testfiles/TestDir1', 0, 4, 0); + # add some useful extra bits to it - mkdir('testfiles/TestDir1/dir-no-members', 0755); - mkdir('testfiles/TestDir1/dir1', 0755); - mkdir('testfiles/TestDir1/dir1/dir2', 0755); - mkdir('testfiles/TestDir1/dir1/dir2/dir3', 0755); - make_file('testfiles/TestDir1/dir1/dir2/file1'); - make_file('testfiles/TestDir1/dir1/dir2/dir3/file2'); + foreach my $subdir ( + 'testfiles/TestDir1/dir-no-members', + 'testfiles/TestDir1/dir1', + 'testfiles/TestDir1/dir1/dir2', + 'testfiles/TestDir1/dir1/dir2/dir3', + ) + { + mkdir($subdir, 0755); + print $fh "$subdir/\n"; + } + make_file($fh, 'testfiles/TestDir1/dir1/dir2/file1'); + make_file($fh, 'testfiles/TestDir1/dir1/dir2/dir3/file2'); + + close $fh; } elsif($ARGV[0] eq 'check') { @@ -95,7 +105,7 @@ elsif($ARGV[0] eq 'check') # read in the new listing, and compare open LISTING,"../../bin/bbackupquery/bbackupquery -Wwarning " . "-c testfiles/bbackupd.conf " . - "\"list -r\" quit |" + "\"list -R\" quit |" or die "Can't open list utility"; open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' or die "can't open copy listing file"; @@ -105,6 +115,7 @@ elsif($ARGV[0] eq 'check') print LISTING_COPY; chomp; s/\r//; s/ \[FILENAME NOT ENCRYPTED\]//; + next if /^WARNING: \*\*\*\* BackupStoreFilename encoded with Clear encoding \*\*\*\*/; if(exists $expected{$_}) { delete $expected{$_} @@ -131,7 +142,7 @@ elsif($ARGV[0] eq 'reroot') { open LISTING,"../../bin/bbackupquery/bbackupquery -Wwarning " . "-c testfiles/bbackupd.conf " . - "\"list -r\" quit |" + "\"list -R\" quit |" or die "Can't open list utility"; open LISTING_COPY,'>testfiles/listing'.$ARGV[1].'.txt' or die "can't open copy listing file"; @@ -142,6 +153,7 @@ elsif($ARGV[0] eq 'reroot') print LISTING_COPY; chomp; s/\[FILENAME NOT ENCRYPTED\]//; + next if /^WARNING: \*\*\*\* BackupStoreFilename encoded with Clear encoding \*\*\*\*/; my ($id,$type,$name) = split / /; $count++; if($name !~ /\Alost\+found0/) @@ -170,20 +182,19 @@ else sub make_dir { - my ($dir,$start,$quantity,$level) = @_; - + my ($fh,$dir,$start,$quantity,$level) = @_; return $start if $level >= 4; mkdir $dir,0755; - + print $fh "$dir/\n"; return $start if $start > $#words; - + while($start <= $#words && $quantity > 0) { my $subdirs = length($words[$start]) - 2; $subdirs = 2 if $subdirs > 2; my $entries = $subdirs + 1; - + for(0 .. ($entries - 1)) { my $w = $words[$start + $_]; @@ -196,26 +207,31 @@ sub make_dir } print FL "\n"; close FL; + + print $fh "$dir/$w\n"; } + $start += $entries; my $w = $words[$start + $_]; - $start = make_dir("$dir/$w", $start + 1, $subdirs, $level + 1); - + $start = make_dir($fh, "$dir/$w", $start + 1, $subdirs, $level + 1); + $quantity--; } - + return $start; } sub make_file { - my ($fn) = @_; - + my ($fh, $fn) = @_; + open FL,'>'.$fn or die "can't open $fn for writing"; for(0 .. 255) { print FL $fn } close FL; + + print $fh "$fn\n"; } diff --git a/test/backupstorepatch/testbackupstorepatch.cpp b/test/backupstorepatch/testbackupstorepatch.cpp index fab56523..46f278ad 100644 --- a/test/backupstorepatch/testbackupstorepatch.cpp +++ b/test/backupstorepatch/testbackupstorepatch.cpp @@ -22,6 +22,7 @@ #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreFile.h" +#include "BackupStoreFileEncodeStream.h" #include "BackupStoreFilenameClear.h" #include "BackupStoreInfo.h" #include "BoxPortsAndFiles.h" @@ -73,6 +74,7 @@ int test_file_remove_order[] = {0, 2, 3, 5, 8, 1, 4, -1}; #define NUMBER_FILES ((sizeof(test_files) / sizeof(test_files[0]))) #define FIRST_FILE_SIZE (64*1024+3) #define BUFFER_SIZE (256*1024) +#define SHORT_TIMEOUT 5000 // Chunk of memory to use for copying files, etc static void *buffer = 0; @@ -193,7 +195,7 @@ void create_test_files() FileStream out(fnt, O_WRONLY | O_CREAT); // Copy up to the change point - int b = previous.Read(buffer, test_files[f].ChangePoint, IOStream::TimeOutInfinite); + int b = previous.Read(buffer, test_files[f].ChangePoint, SHORT_TIMEOUT); out.Write(buffer, b); // Add new bytes? @@ -208,7 +210,7 @@ void create_test_files() previous.Seek(test_files[f].DeleteBytes, IOStream::SeekType_Relative); } // Copy rest of data - b = previous.Read(buffer, BUFFER_SIZE, IOStream::TimeOutInfinite); + b = previous.Read(buffer, BUFFER_SIZE, SHORT_TIMEOUT); out.Write(buffer, b); } } @@ -244,7 +246,7 @@ void test_depends_in_dirs() // Load the directory back in BackupStoreDirectory dir2; FileStream in("testfiles/dir.1"); - dir2.ReadFromStream(in, IOStream::TimeOutInfinite); + dir2.ReadFromStream(in, SHORT_TIMEOUT); // Check entries TEST_THAT(dir2.GetNumberOfEntries() == 4); for(int i = 2; i <= 5; ++i) @@ -272,7 +274,7 @@ void test_depends_in_dirs() { BackupStoreDirectory dir3; FileStream in("testfiles/dir.2"); - dir3.ReadFromStream(in, IOStream::TimeOutInfinite); + dir3.ReadFromStream(in, SHORT_TIMEOUT); dir3.Dump(0, true); for(int i = 2; i <= 5; ++i) { @@ -344,12 +346,13 @@ int test(int argc, const char *argv[]) { // Open a connection to the server - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", + SocketStreamTLS *pConn = new SocketStreamTLS; + std::auto_ptr<SocketStream> apConn(pConn); + pConn->Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED_TEST); // Make a protocol - BackupProtocolClient protocol(conn); + BackupProtocolClient protocol(apConn); // Login { @@ -370,7 +373,7 @@ int test(int argc, const char *argv[]) BackupProtocolListDirectory::RootDirectory, storeFilename)); std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile( BackupProtocolListDirectory::RootDirectory, ModificationTime, - ModificationTime, 0 /* no diff from file ID */, storeFilename, *upload)); + ModificationTime, 0 /* no diff from file ID */, storeFilename, upload)); test_files[0].IDOnServer = stored->GetObjectID(); test_files[0].IsCompletelyDifferent = true; ModificationTime += MODIFICATION_TIME_INC; @@ -409,7 +412,8 @@ int test(int argc, const char *argv[]) // Upload the patch to the store std::auto_ptr<BackupProtocolSuccess> stored(protocol.QueryStoreFile( BackupProtocolListDirectory::RootDirectory, ModificationTime, - ModificationTime, isCompletelyDifferent?(0):(diffFromID), storeFilename, *patchStream)); + ModificationTime, isCompletelyDifferent?(0):(diffFromID), + storeFilename, patchStream)); ModificationTime += MODIFICATION_TIME_INC; // Store details @@ -439,7 +443,7 @@ int test(int argc, const char *argv[]) // Stream BackupStoreDirectory dir; std::auto_ptr<IOStream> dirstream(protocol.ReceiveStream()); - dir.ReadFromStream(*dirstream, IOStream::TimeOutInfinite); + dir.ReadFromStream(*dirstream, SHORT_TIMEOUT); BackupStoreDirectory::Iterator i(dir); BackupStoreDirectory::Entry *en = 0; @@ -452,7 +456,6 @@ int test(int argc, const char *argv[]) // Finish the connection protocol.QueryFinished(); - conn.Close(); } // Fill in initial dependency information @@ -480,7 +483,7 @@ int test(int argc, const char *argv[]) BackupStoreDirectory dir; { std::auto_ptr<RaidFileRead> dirStream(RaidFileRead::Open(0, "backup/01234567/o01")); - dir.ReadFromStream(*dirStream, IOStream::TimeOutInfinite); + dir.ReadFromStream(*dirStream, SHORT_TIMEOUT); dir.Dump(0, true); // Check that dependency info is correct @@ -501,9 +504,13 @@ int test(int argc, const char *argv[]) storeRootDir, discSet, filenameOut, false /* don't bother ensuring the directory exists */); - TEST_EQUAL(RaidFileUtil::NoFile, + std::ostringstream msg; + msg << "Unreferenced object " << + test_files[f].IDOnServer << + " was not deleted by housekeeping"; + TEST_EQUAL_LINE(RaidFileUtil::NoFile, RaidFileUtil::RaidFileExists( - rfd, filenameOut)); + rfd, filenameOut), msg.str()); } else { @@ -526,10 +533,11 @@ int test(int argc, const char *argv[]) } // Open a connection to the server (need to do this each time, otherwise housekeeping won't delete files) - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", + SocketStreamTLS *pConn = new SocketStreamTLS; + std::auto_ptr<SocketStream> apConn(pConn); + pConn->Open(context, Socket::TypeINET, "localhost", BOX_PORT_BBSTORED_TEST); - BackupProtocolClient protocol(conn); + BackupProtocolClient protocol(apConn); { std::auto_ptr<BackupProtocolVersion> serverVersion(protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION)); TEST_THAT(serverVersion->GetVersion() == BACKUP_STORE_SERVER_VERSION); @@ -564,7 +572,7 @@ int test(int argc, const char *argv[]) // Get stream std::auto_ptr<IOStream> filestream(protocol.ReceiveStream()); // Get and decode - BackupStoreFile::DecodeFile(*filestream, filename_fetched, IOStream::TimeOutInfinite); + BackupStoreFile::DecodeFile(*filestream, filename_fetched, SHORT_TIMEOUT); } } // Test for identicalness @@ -575,13 +583,12 @@ int test(int argc, const char *argv[]) std::auto_ptr<BackupProtocolSuccess> getblockindex(protocol.QueryGetBlockIndexByID(test_files[f].IDOnServer)); TEST_THAT(getblockindex->GetObjectID() == test_files[f].IDOnServer); std::auto_ptr<IOStream> blockIndexStream(protocol.ReceiveStream()); - TEST_THAT(BackupStoreFile::CompareFileContentsAgainstBlockIndex(filename, *blockIndexStream, IOStream::TimeOutInfinite)); + TEST_THAT(BackupStoreFile::CompareFileContentsAgainstBlockIndex(filename, *blockIndexStream, SHORT_TIMEOUT)); } } // Close the connection protocol.QueryFinished(); - conn.Close(); // Mark one of the elements as deleted if(test_file_remove_order[deleteIndex] == -1) @@ -603,10 +610,13 @@ int test(int argc, const char *argv[]) writedir.Commit(true); } + // Get the revision number of the root directory, before housekeeping makes any changes. + int64_t first_revision = 0; + RaidFileRead::FileExists(0, "backup/01234567/o01", &first_revision); + #ifdef WIN32 - // Cannot signal bbstored to do housekeeping now, - // so just wait until we're sure it's done - wait_for_operation(12, "housekeeping to run"); + // Cannot signal bbstored to do housekeeping now, and we don't need to, as we will + // wait up to 32 seconds and detect automatically when it has finished. #else // Send the server a restart signal, so it does // housekeeping immediately, and wait for it to happen @@ -615,10 +625,8 @@ int test(int argc, const char *argv[]) ::kill(pid, SIGHUP); #endif - // Get the revision number of the info file - int64_t first_revision = 0; - RaidFileRead::FileExists(0, "backup/01234567/o01", &first_revision); - for(int l = 0; l < 32; ++l) + // Wait for changes to be written back to the root directory. + for(int secs_remaining = 32; secs_remaining >= 0; secs_remaining--) { // Sleep a while, and print a dot ::sleep(1); @@ -626,7 +634,7 @@ int test(int argc, const char *argv[]) ::fflush(stdout); // Early end? - if(l > 2) + if(!TestFileExists("testfiles/0_0/backup/01234567/write.lock")) { int64_t revid = 0; RaidFileRead::FileExists(0, "backup/01234567/o01", &revid); @@ -635,6 +643,8 @@ int test(int argc, const char *argv[]) break; } } + + TEST_LINE(secs_remaining != 0, "No changes detected to root directory after 32 seconds"); } ::printf("\n"); diff --git a/test/basicserver/Makefile.extra b/test/basicserver/Makefile.extra index e6a4675e..4b21d37d 100644 --- a/test/basicserver/Makefile.extra +++ b/test/basicserver/Makefile.extra @@ -1,12 +1,12 @@ MAKEPROTOCOL = ../../lib/server/makeprotocol.pl -GEN_CMD = $(MAKEPROTOCOL) testprotocol.txt +GEN_CMD = $(MAKEPROTOCOL) TestProtocol.txt # AUTOGEN SEEDING -autogen_TestProtocol.cpp: $(MAKEPROTOCOL) testprotocol.txt +autogen_TestProtocol.cpp: $(MAKEPROTOCOL) TestProtocol.txt $(_PERL) $(GEN_CMD) -autogen_TestProtocolServer.h: $(MAKEPROTOCOL) testprotocol.txt +autogen_TestProtocolServer.h: $(MAKEPROTOCOL) TestProtocol.txt $(_PERL) $(GEN_CMD) diff --git a/test/basicserver/TestCommands.cpp b/test/basicserver/TestCommands.cpp index 5238819f..bdbdffeb 100644 --- a/test/basicserver/TestCommands.cpp +++ b/test/basicserver/TestCommands.cpp @@ -11,6 +11,11 @@ #include "MemLeakFindOn.h" +std::auto_ptr<TestProtocolMessage> TestProtocolReplyable::HandleException(BoxException& e) const +{ + throw; +} + std::auto_ptr<TestProtocolMessage> TestProtocolHello::DoCommand(TestProtocolReplyable &rProtocol, TestContext &rContext) const { if(mNumber32 != 41 || mNumber16 != 87 || mNumber8 != 11 || mText != "pingu") @@ -48,7 +53,8 @@ public: std::auto_ptr<TestProtocolMessage> TestProtocolGetStream::DoCommand(TestProtocolReplyable &rProtocol, TestContext &rContext) const { // make a new stream object - CollectInBufferStream *pstream = mUncertainSize?(new UncertainBufferStream):(new CollectInBufferStream); + std::auto_ptr<CollectInBufferStream> apStream( + mUncertainSize?(new UncertainBufferStream):(new CollectInBufferStream)); // Data. int values[24273]; @@ -59,19 +65,21 @@ std::auto_ptr<TestProtocolMessage> TestProtocolGetStream::DoCommand(TestProtocol { values[x] = v++; } - pstream->Write(values, sizeof(values)); + apStream->Write(values, sizeof(values)); } // Finished - pstream->SetForReading(); + apStream->SetForReading(); // Get it to be sent - rProtocol.SendStreamAfterCommand(pstream); + rProtocol.SendStreamAfterCommand((std::auto_ptr<IOStream>)apStream); return std::auto_ptr<TestProtocolMessage>(new TestProtocolGetStream(mStartingValue, mUncertainSize)); } -std::auto_ptr<TestProtocolMessage> TestProtocolSendStream::DoCommand(TestProtocolReplyable &rProtocol, TestContext &rContext) const +std::auto_ptr<TestProtocolMessage> TestProtocolSendStream::DoCommand( + TestProtocolReplyable &rProtocol, TestContext &rContext, + IOStream& rDataStream) const { if(mValue != 0x73654353298ffLL) { @@ -79,15 +87,14 @@ std::auto_ptr<TestProtocolMessage> TestProtocolSendStream::DoCommand(TestProtoco } // Get a stream - std::auto_ptr<IOStream> stream(rProtocol.ReceiveStream()); - bool uncertain = (stream->BytesLeftToRead() == IOStream::SizeOfStreamUnknown); + bool uncertain = (rDataStream.BytesLeftToRead() == IOStream::SizeOfStreamUnknown); // Count how many bytes in it int bytes = 0; char buffer[125]; - while(stream->StreamDataLeft()) + while(rDataStream.StreamDataLeft()) { - bytes += stream->Read(buffer, sizeof(buffer)); + bytes += rDataStream.Read(buffer, sizeof(buffer)); } // tell the caller how many bytes there were diff --git a/test/basicserver/testprotocol.txt b/test/basicserver/TestProtocol.txt index 5bca9f49..5bca9f49 100644 --- a/test/basicserver/testprotocol.txt +++ b/test/basicserver/TestProtocol.txt diff --git a/test/basicserver/testbasicserver.cpp b/test/basicserver/testbasicserver.cpp index 976bdd92..6f2def54 100644 --- a/test/basicserver/testbasicserver.cpp +++ b/test/basicserver/testbasicserver.cpp @@ -11,7 +11,6 @@ #include "Box.h" #include <stdio.h> -#include <unistd.h> #include <time.h> #include <typeinfo> @@ -36,6 +35,10 @@ // in ms #define COMMS_READ_TIMEOUT 4 #define COMMS_SERVER_WAIT_BEFORE_REPLYING 40 +// Use a longer timeout to give Srv2TestConversations time to write 20 MB to each of +// three child processes before starting to read it back again, without the children +// timing out and aborting. +#define SHORT_TIMEOUT 30000 class basicdaemon : public Daemon { @@ -103,6 +106,12 @@ void testservers_connection(SocketStream &rStream) } if(line == "LARGEDATA") { + // This part of the test is timing-sensitive, because we write + // 20 MB to the test and then have to wait while it reads 20 MB + // from the other two children before writing anything back to us. + // We could timeout waiting for it to talk to us again. So we + // increased the SHORT_TIMEOUT from 5 seconds to 30 to allow + // more time. { // Send lots of data char data[LARGE_DATA_BLOCK_SIZE]; @@ -112,7 +121,7 @@ void testservers_connection(SocketStream &rStream) } for(int s = 0; s < (LARGE_DATA_SIZE / LARGE_DATA_BLOCK_SIZE); ++s) { - rStream.Write(data, sizeof(data)); + rStream.Write(data, sizeof(data), SHORT_TIMEOUT); } } { @@ -120,7 +129,8 @@ void testservers_connection(SocketStream &rStream) char buf[1024]; int total = 0; int r = 0; - while(total < LARGE_DATA_SIZE && (r = rStream.Read(buf, sizeof(buf))) != 0) + while(total < LARGE_DATA_SIZE && + (r = rStream.Read(buf, sizeof(buf), SHORT_TIMEOUT)) != 0) { total += r; } @@ -142,7 +152,7 @@ void testservers_connection(SocketStream &rStream) } for(int s = 0; s < (LARGE_DATA_SIZE / LARGE_DATA_BLOCK_SIZE); ++s) { - rStream.Write(data, sizeof(data)); + rStream.Write(data, sizeof(data), SHORT_TIMEOUT); } } @@ -170,7 +180,7 @@ public: testserver() {} ~testserver() {} - void Connection(SocketStream &rStream); + void Connection(std::auto_ptr<SocketStream> apStream); virtual const char *DaemonName() const { @@ -190,9 +200,9 @@ const ConfigurationVerify *testserver::GetConfigVerify() const static ConfigurationVerify verifyserver[] = { { - "Server", - 0, - verifyserverkeys, + "Server", /* mName */ + 0, /* mpSubConfigurations */ + verifyserverkeys, /* mpKeys */ ConfigTest_Exists | ConfigTest_LastEntry, 0 } @@ -200,9 +210,9 @@ const ConfigurationVerify *testserver::GetConfigVerify() const static ConfigurationVerify verify = { - "root", - verifyserver, - 0, + "root", /* mName */ + verifyserver, /* mpSubConfigurations */ + 0, /* mpKeys */ ConfigTest_Exists | ConfigTest_LastEntry, 0 }; @@ -210,9 +220,9 @@ const ConfigurationVerify *testserver::GetConfigVerify() const return &verify; } -void testserver::Connection(SocketStream &rStream) +void testserver::Connection(std::auto_ptr<SocketStream> apStream) { - testservers_connection(rStream); + testservers_connection(*apStream); } class testProtocolServer : public testserver @@ -221,7 +231,7 @@ public: testProtocolServer() {} ~testProtocolServer() {} - void Connection(SocketStream &rStream); + void Connection(std::auto_ptr<SocketStream> apStream); virtual const char *DaemonName() const { @@ -229,9 +239,9 @@ public: } }; -void testProtocolServer::Connection(SocketStream &rStream) +void testProtocolServer::Connection(std::auto_ptr<SocketStream> apStream) { - TestProtocolServer server(rStream); + TestProtocolServer server(apStream); TestContext context; server.DoServer(context); } @@ -243,7 +253,7 @@ public: testTLSserver() {} ~testTLSserver() {} - void Connection(SocketStreamTLS &rStream); + void Connection(std::auto_ptr<SocketStreamTLS> apStream); virtual const char *DaemonName() const { @@ -283,9 +293,9 @@ const ConfigurationVerify *testTLSserver::GetConfigVerify() const return &verify; } -void testTLSserver::Connection(SocketStreamTLS &rStream) +void testTLSserver::Connection(std::auto_ptr<SocketStreamTLS> apStream) { - testservers_connection(rStream); + testservers_connection(*apStream); } @@ -336,15 +346,21 @@ void Srv2TestConversations(const std::vector<IOStream *> &conns) } for(unsigned int c = 0; c < conns.size(); ++c) { - conns[c]->Write("LARGEDATA\n", 10); + conns[c]->Write("LARGEDATA\n", 10, SHORT_TIMEOUT); } + // This part of the test is timing-sensitive, because we read 20 MB from each of + // three daemon processes, then write 20 MB to each of them, then read back + // another 20 MB from each of them. Each child could timeout waiting for us to + // read from it, or write to it, while we're servicing another child. So we + // increased the SHORT_TIMEOUT from 5 seconds to 30 to allow enough time. for(unsigned int c = 0; c < conns.size(); ++c) { // Receive lots of data char buf[1024]; int total = 0; int r = 0; - while(total < LARGE_DATA_SIZE && (r = conns[c]->Read(buf, sizeof(buf))) != 0) + while(total < LARGE_DATA_SIZE && + (r = conns[c]->Read(buf, sizeof(buf), SHORT_TIMEOUT)) != 0) { total += r; } @@ -360,7 +376,7 @@ void Srv2TestConversations(const std::vector<IOStream *> &conns) } for(int s = 0; s < (LARGE_DATA_SIZE / LARGE_DATA_BLOCK_SIZE); ++s) { - conns[c]->Write(data, sizeof(data)); + conns[c]->Write(data, sizeof(data), SHORT_TIMEOUT); } } for(unsigned int c = 0; c < conns.size(); ++c) @@ -369,7 +385,8 @@ void Srv2TestConversations(const std::vector<IOStream *> &conns) char buf[1024]; int total = 0; int r = 0; - while(total < LARGE_DATA_SIZE && (r = conns[c]->Read(buf, sizeof(buf))) != 0) + while(total < LARGE_DATA_SIZE && + (r = conns[c]->Read(buf, sizeof(buf), SHORT_TIMEOUT)) != 0) { total += r; } @@ -412,7 +429,8 @@ void TestStreamReceive(TestProtocolClient &protocol, int value, bool uncertainst while(stream->StreamDataLeft()) { // Read some data - int bytes = stream->Read(((char*)values) + bytesleft, sizeof(values) - bytesleft); + int bytes = stream->Read(((char*)values) + bytesleft, + sizeof(values) - bytesleft, SHORT_TIMEOUT); bytessofar += bytes; bytes += bytesleft; int n = bytes / 4; @@ -481,7 +499,7 @@ int test(int argc, const char *argv[]) // Launch a basic server { - std::string cmd = "./test --test-daemon-args="; + std::string cmd = TEST_EXECUTABLE " --test-daemon-args="; cmd += test_args; cmd += " srv1 testfiles/srv1.conf"; int pid = LaunchServer(cmd, "testfiles/srv1.pid"); @@ -527,7 +545,7 @@ int test(int argc, const char *argv[]) // Launch a test forking server { - std::string cmd = "./test --test-daemon-args="; + std::string cmd = TEST_EXECUTABLE " --test-daemon-args="; cmd += test_args; cmd += " srv2 testfiles/srv2.conf"; int pid = LaunchServer(cmd, "testfiles/srv2.pid"); @@ -597,7 +615,7 @@ int test(int argc, const char *argv[]) // Launch a test SSL server { - std::string cmd = "./test --test-daemon-args="; + std::string cmd = TEST_EXECUTABLE " --test-daemon-args="; cmd += test_args; cmd += " srv3 testfiles/srv3.conf"; int pid = LaunchServer(cmd, "testfiles/srv3.pid"); @@ -678,7 +696,7 @@ int test(int argc, const char *argv[]) //protocolserver: // Launch a test protocol handling server { - std::string cmd = "./test --test-daemon-args="; + std::string cmd = TEST_EXECUTABLE " --test-daemon-args="; cmd += test_args; cmd += " srv4 testfiles/srv4.conf"; int pid = LaunchServer(cmd, "testfiles/srv4.pid"); @@ -691,15 +709,15 @@ int test(int argc, const char *argv[]) TEST_THAT(ServerIsAlive(pid)); // Open a connection to it - SocketStream conn; + std::auto_ptr<SocketStream> apConn(new SocketStream); #ifdef WIN32 - conn.Open(Socket::TypeINET, "localhost", 2003); + apConn->Open(Socket::TypeINET, "localhost", 2003); #else - conn.Open(Socket::TypeUNIX, "testfiles/srv4.sock"); + apConn->Open(Socket::TypeUNIX, "testfiles/srv4.sock"); #endif // Create a protocol - TestProtocolClient protocol(conn); + TestProtocolClient protocol(apConn); // Simple query { @@ -719,11 +737,13 @@ int test(int argc, const char *argv[]) // Try to send a stream { - CollectInBufferStream s; + std::auto_ptr<CollectInBufferStream> + s(new CollectInBufferStream()); char buf[1663]; - s.Write(buf, sizeof(buf)); - s.SetForReading(); - std::auto_ptr<TestProtocolGetStream> reply(protocol.QuerySendStream(0x73654353298ffLL, s)); + s->Write(buf, sizeof(buf)); + s->SetForReading(); + std::auto_ptr<TestProtocolGetStream> reply(protocol.QuerySendStream(0x73654353298ffLL, + (std::auto_ptr<IOStream>)s)); TEST_THAT(reply->GetStartingValue() == sizeof(buf)); } diff --git a/test/bbackupd/Makefile.extra b/test/bbackupd/Makefile.extra deleted file mode 100644 index 705345a9..00000000 --- a/test/bbackupd/Makefile.extra +++ /dev/null @@ -1,16 +0,0 @@ -link-extra: ../../bin/bbackupd/autogen_ClientException.o \ - ../../bin/bbackupd/BackupClientContext.o \ - ../../bin/bbackupd/BackupClientDeleteList.o \ - ../../bin/bbackupd/BackupClientDirectoryRecord.o \ - ../../bin/bbackupd/Win32BackupService.o \ - ../../bin/bbackupd/BackupClientInodeToIDMap.o \ - ../../bin/bbackupd/Win32ServiceFunctions.o \ - ../../bin/bbackupd/BackupDaemon.o \ - ../../bin/bbstored/BBStoreDHousekeeping.o \ - ../../lib/backupstore/HousekeepStoreAccount.o \ - ../../lib/backupstore/autogen_BackupProtocol.o \ - ../../lib/backupstore/BackupStoreContext.o \ - ../../lib/backupstore/BackupCommands.o \ - ../../bin/bbstored/BackupStoreDaemon.o \ - ../../bin/bbackupquery/BackupQueries.o \ - ../../bin/bbackupquery/autogen_Documentation.o diff --git a/test/bbackupd/testbbackupd.cpp b/test/bbackupd/testbbackupd.cpp index 6cd8f27d..cc602f22 100644 --- a/test/bbackupd/testbbackupd.cpp +++ b/test/bbackupd/testbbackupd.cpp @@ -21,7 +21,6 @@ #include <sys/stat.h> #include <limits.h> #include <string.h> -#include <unistd.h> #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> @@ -36,18 +35,24 @@ #include <signal.h> #endif +#ifdef WIN32 + #include <process.h> +#endif + #include <map> #ifdef HAVE_SYSCALL #include <sys/syscall.h> #endif -#include "autogen_BackupProtocol.h" #include "BackupClientCryptoKeys.h" +#include "BackupClientContext.h" #include "BackupClientFileAttributes.h" +#include "BackupClientInodeToIDMap.h" #include "BackupClientRestore.h" #include "BackupDaemon.h" #include "BackupDaemonConfigVerify.h" +#include "BackupProtocol.h" #include "BackupQueries.h" #include "BackupStoreAccounts.h" #include "BackupStoreConstants.h" @@ -56,6 +61,7 @@ #include "BackupStoreDirectory.h" #include "BackupStoreException.h" #include "BackupStoreConfigVerify.h" +#include "BackupStoreFileEncodeStream.h" #include "BoxPortsAndFiles.h" #include "BoxTime.h" #include "BoxTimeToUnix.h" @@ -67,11 +73,13 @@ #include "intercept.h" #include "IOStreamGetLine.h" #include "LocalProcessStream.h" +#include "MemBlockStream.h" #include "RaidFileController.h" #include "SSLLib.h" #include "ServerControl.h" #include "Socket.h" #include "SocketStreamTLS.h" +#include "StoreTestUtils.h" #include "TLSContext.h" #include "Test.h" #include "Timer.h" @@ -85,16 +93,16 @@ #endif // two cycles and a bit -#define TIME_TO_WAIT_FOR_BACKUP_OPERATION 12 +#define TIME_TO_WAIT_FOR_BACKUP_OPERATION 12 +#define SHORT_TIMEOUT 5000 +#define BACKUP_ERROR_DELAY_SHORTENED 10 +#define DEFAULT_BBACKUPD_CONFIG_FILE "testfiles/bbackupd.conf" void wait_for_backup_operation(const char* message) { wait_for_operation(TIME_TO_WAIT_FOR_BACKUP_OPERATION, message); } -int bbstored_pid = 0; -int bbackupd_pid = 0; - #ifdef HAVE_SYS_XATTR_H bool readxattr_into_map(const char *filename, std::map<std::string,std::string> &rOutput) { @@ -103,6 +111,16 @@ bool readxattr_into_map(const char *filename, std::map<std::string,std::string> ssize_t xattrNamesBufferSize = llistxattr(filename, NULL, 0); if(xattrNamesBufferSize < 0) { +#if HAVE_DECL_ENOTSUP + if(errno == ENOTSUP) + { + // Pretend that it worked, leaving an empty map, so + // that the rest of the attribute comparison will + // proceed as normal. + return true; + } +#endif // HAVE_DECL_ENOTSUP + return false; } else if(xattrNamesBufferSize > 0) @@ -306,8 +324,187 @@ bool attrmatch(const char *f1, const char *f2) return (s1.st_mode == s2.st_mode && s1.st_uid == s2.st_uid && s1.st_gid == s2.st_gid); } -int test_basics() +bool unpack_files(const std::string& archive_file, + const std::string& destination_dir = "testfiles", + const std::string& tar_options = "") +{ + BOX_INFO("Unpacking test fixture archive into " << destination_dir + << ": " << archive_file); + +#ifdef _MSC_VER // No tar, use 7zip. + // 7za only extracts the tgz file to a tar file, which we have to extract in a + // separate step. + std::string cmd = std::string("7za x testfiles/") + archive_file + ".tgz -aos " + "-otestfiles >nul:"; + TEST_LINE_OR(::system(cmd.c_str()) == 0, cmd, return false); + + cmd = std::string("7za x testfiles/") + archive_file + ".tar -aos " + "-o" + destination_dir + " -x!.\\TestDir1\\symlink? -x!.\\test2 >nul:"; +#elif defined WIN32 // Cygwin + MinGW, we can use real tar. + std::string cmd("tar xz"); + cmd += tar_options + " -f testfiles/" + archive_file + ".tgz " + + "-C " + destination_dir; +#else // Unixish, but Solaris tar doesn't like decompressing gzip files. + std::string cmd("gzip -d < testfiles/"); + cmd += archive_file + ".tgz | ( cd " + destination_dir + " && tar xf" + + tar_options + " -)"; +#endif + + TEST_LINE_OR(::system(cmd.c_str()) == 0, cmd, return false); + return true; +} + +Daemon* spDaemon = NULL; + +bool configure_bbackupd(BackupDaemon& bbackupd, const std::string& config_file) +{ + // Stop bbackupd initialisation from changing the console logging level + // and the program name tag. + Logger& console(Logging::GetConsole()); + Logger::LevelGuard undo_log_level_change(console, console.GetLevel()); + Logging::Tagger undo_program_name_change; + + std::vector<std::string> args; + size_t last_arg_start = 0; + for (size_t pos = 0; pos <= bbackupd_args.size(); pos++) + { + char c; + + if (pos == bbackupd_args.size()) + { + c = ' '; // finish last argument + } + else + { + c = bbackupd_args[pos]; + } + + if (c == ' ') + { + if (last_arg_start < pos) + { + std::string last_arg = + bbackupd_args.substr(last_arg_start, + pos - last_arg_start); + args.push_back(last_arg); + } + + last_arg_start = pos + 1; + } + } + + MemoryBlockGuard<const char **> argv_buffer(sizeof(const char*) * (args.size() + 1)); + const char **argv = argv_buffer; + argv_buffer[0] = "bbackupd"; + for (size_t i = 0; i < args.size(); i++) + { + argv_buffer[i + 1] = args[i].c_str(); + } + + TEST_EQUAL_LINE(0, bbackupd.ProcessOptions(args.size() + 1, argv), + "processing command-line options"); + + bbackupd.Configure(config_file); + bbackupd.InitCrypto(); + + return true; +} + +bool kill_running_daemons() +{ + bool success = true; + + if(FileExists("testfiles/bbstored.pid")) + { + TEST_THAT_OR(KillServer("testfiles/bbstored.pid", true), success = false); + } + + if(FileExists("testfiles/bbackupd.pid")) + { + TEST_THAT_OR(KillServer("testfiles/bbackupd.pid", true), success = false); + } + + return success; +} + +bool setup_test_bbackupd(BackupDaemon& bbackupd, bool do_unpack_files = true, + bool do_start_bbstored = true) +{ + Timers::Cleanup(false); // don't throw exception if not initialised + Timers::Init(); + + if (do_start_bbstored) + { + TEST_THAT_OR(StartServer(), FAIL); + } + + if (do_unpack_files) + { + TEST_THAT_OR(unpack_files("test_base"), FAIL); + // Older versions of GNU tar fail to set the timestamps on + // symlinks, which makes them appear too recent to be backed + // up immediately, causing test_bbackupd_uploads_files() for + // example to fail. So restore the timestamps manually. + // http://lists.gnu.org/archive/html/bug-tar/2009-08/msg00007.html + // http://git.savannah.gnu.org/cgit/tar.git/plain/NEWS?id=release_1_24 + #ifdef HAVE_UTIMENSAT + const struct timespec times[2] = { + {1065707200, 0}, + {1065707200, 0}, + }; + const char * filenames[] = { + "testfiles/TestDir1/symlink1", + "testfiles/TestDir1/symlink2", + "testfiles/TestDir1/symlink3", + NULL, + }; + for (int i = 0; filenames[i] != NULL; i++) + { + TEST_THAT_OR(utimensat(AT_FDCWD, filenames[i], + times, AT_SYMLINK_NOFOLLOW) == 0, + BOX_LOG_SYS_ERROR("Failed to change " + "timestamp on symlink: " << + filenames[i])); + } + #endif + } + + TEST_THAT_OR(configure_bbackupd(bbackupd, "testfiles/bbackupd.conf"), + FAIL); + spDaemon = &bbackupd; + return true; +} + +//! Simplifies calling setUp() with the current function name in each test. +#define SETUP_TEST_BBACKUPD() \ + SETUP(); \ + TEST_THAT(bbackupd_pid == 0 || StopClient()); \ + TEST_THAT(bbstored_pid == 0 || StopServer()); \ + TEST_THAT(kill_running_daemons()); \ + TEST_THAT(create_account(10000, 20000)); + +#define SETUP_WITHOUT_FILES() \ + SETUP_TEST_BBACKUPD(); \ + BackupDaemon bbackupd; \ + TEST_THAT_OR(setup_test_bbackupd(bbackupd, false), FAIL); \ + TEST_THAT_OR(::mkdir("testfiles/TestDir1", 0755) == 0, FAIL); + +#define SETUP_WITH_BBSTORED() \ + SETUP_TEST_BBACKUPD(); \ + BackupDaemon bbackupd; \ + TEST_THAT_OR(setup_test_bbackupd(bbackupd), FAIL); + +#define TEARDOWN_TEST_BBACKUPD() \ + TEST_THAT(bbackupd_pid == 0 || StopClient()); \ + TEST_THAT(bbstored_pid == 0 || StopServer()); \ + TEST_THAT(kill_running_daemons()); \ + TEARDOWN(); + +bool test_basics() { + SETUP_TEST_BBACKUPD(); + TEST_THAT_OR(unpack_files("test_base"), FAIL); + // Read attributes from files BackupClientFileAttributes t1; t1.ReadAttributes("testfiles/test1"); @@ -324,7 +521,7 @@ int test_basics() BackupClientFileAttributes t3; { - Logging::Guard guard(Log::ERROR); + Logger::LevelGuard(Logging::GetConsole(), Log::ERROR); TEST_CHECK_THROWS(t3.ReadAttributes("doesn't exist"), CommonException, OSFileError); } @@ -338,6 +535,8 @@ int test_basics() // Apply attributes to these new files t1.WriteAttributes("testfiles/test1_n"); #ifdef WIN32 + // We can't apply symlink attributes on Win32, so use a normal file's + // attributes instead. t1.WriteAttributes("testfiles/test2_n"); #else t2.WriteAttributes("testfiles/test2_n"); @@ -345,7 +544,7 @@ int test_basics() #ifndef WIN32 { - Logging::Guard guard(Log::ERROR); + Logger::LevelGuard(Logging::GetConsole(), Log::ERROR); TEST_CHECK_THROWS(t1.WriteAttributes("testfiles/test1_nXX"), CommonException, OSFileError); TEST_CHECK_THROWS(t3.WriteAttributes("doesn't exist"), @@ -423,55 +622,10 @@ int test_basics() finish_with_write_xattr_test(); #endif // HAVE_SYS_XATTR_H - return 0; -} - -int test_setupaccount() -{ - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " - "testfiles/bbstored.conf create 01234567 0 1000B 2000B") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - return 0; -} - -int test_run_bbstored() -{ - std::string cmd = BBSTORED " " + bbstored_args + - " testfiles/bbstored.conf"; - bbstored_pid = LaunchServer(cmd, "testfiles/bbstored.pid"); - - TEST_THAT(bbstored_pid != -1 && bbstored_pid != 0); - - if(bbstored_pid > 0) - { - ::safe_sleep(1); - TEST_THAT(ServerIsAlive(bbstored_pid)); - return 0; // success - } - - return 1; -} - -int test_kill_bbstored(bool wait_for_process = false) -{ - TEST_THAT(KillServer(bbstored_pid, wait_for_process)); - ::safe_sleep(1); - TEST_THAT(!ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbstored_pid)) - { - bbstored_pid = 0; - } - - #ifdef WIN32 - TEST_THAT(unlink("testfiles/bbstored.pid") == 0); - #else - TestRemoteProcessMemLeaks("bbstored.memleaks"); - #endif - - return 0; + TEARDOWN_TEST_BBACKUPD(); } -int64_t GetDirID(BackupProtocolClient &protocol, const char *name, int64_t InDirectory) +int64_t GetDirID(BackupProtocolCallable &protocol, const char *name, int64_t InDirectory) { protocol.QueryListDirectory( InDirectory, @@ -513,10 +667,11 @@ void do_interrupted_restore(const TLSContext &context, int64_t restoredirid) // child process { // connect and log in - SocketStreamTLS conn; - conn.Open(context, Socket::TypeINET, "localhost", - 22011); - BackupProtocolClient protocol(conn); + SocketStreamTLS* pConn = new SocketStreamTLS; + std::auto_ptr<SocketStream> apConn(pConn); + pConn->Open(context, Socket::TypeINET, "localhost", 22011); + BackupProtocolClient protocol(apConn); + protocol.QueryVersion(BACKUP_STORE_SERVER_VERSION); std::auto_ptr<BackupProtocolLoginConfirmed> loginConf(protocol.QueryLogin(0x01234567, @@ -579,7 +734,7 @@ void do_interrupted_restore(const TLSContext &context, int64_t restoredirid) #endif // !WIN32 #ifdef WIN32 -bool set_file_time(const char* filename, FILETIME creationTime, +bool set_file_time(const char* filename, FILETIME creationTime, FILETIME lastModTime, FILETIME lastAccessTime) { HANDLE handle = openfile(filename, O_RDWR, 0); @@ -595,7 +750,7 @@ bool set_file_time(const char* filename, FILETIME creationTime, } #endif -void intercept_setup_delay(const char *filename, unsigned int delay_after, +void intercept_setup_delay(const char *filename, unsigned int delay_after, int delay_ms, int syscall_to_delay); bool intercept_triggered(); @@ -603,7 +758,7 @@ int64_t SearchDir(BackupStoreDirectory& rDir, const std::string& rChildName) { BackupStoreDirectory::Iterator i(rDir); - BackupStoreFilenameClear child(rChildName.c_str()); + BackupStoreFilenameClear child(rChildName); BackupStoreDirectory::Entry *en = i.FindMatchingClearName(child); if (en == 0) return 0; int64_t id = en->GetObjectID(); @@ -612,50 +767,18 @@ int64_t SearchDir(BackupStoreDirectory& rDir, return id; } -SocketStreamTLS sSocket; - -std::auto_ptr<BackupProtocolClient> Connect(TLSContext& rContext) -{ - sSocket.Open(rContext, Socket::TypeINET, - "localhost", 22011); - std::auto_ptr<BackupProtocolClient> connection; - connection.reset(new BackupProtocolClient(sSocket)); - connection->Handshake(); - std::auto_ptr<BackupProtocolVersion> - serverVersion(connection->QueryVersion( - BACKUP_STORE_SERVER_VERSION)); - if(serverVersion->GetVersion() != - BACKUP_STORE_SERVER_VERSION) - { - THROW_EXCEPTION(BackupStoreException, - WrongServerVersion); - } - return connection; -} - -std::auto_ptr<BackupProtocolClient> ConnectAndLogin(TLSContext& rContext, - int flags) -{ - std::auto_ptr<BackupProtocolClient> connection(Connect(rContext)); - connection->QueryLogin(0x01234567, flags); - return connection; -} - std::auto_ptr<BackupStoreDirectory> ReadDirectory ( - BackupProtocolClient& rClient, + BackupProtocolCallable& rClient, int64_t id = BackupProtocolListDirectory::RootDirectory ) { std::auto_ptr<BackupProtocolSuccess> dirreply( rClient.QueryListDirectory(id, false, 0, false)); - std::auto_ptr<IOStream> dirstream(rClient.ReceiveStream()); - std::auto_ptr<BackupStoreDirectory> apDir(new BackupStoreDirectory()); - apDir->ReadFromStream(*dirstream, rClient.GetTimeout()); + std::auto_ptr<BackupStoreDirectory> apDir( + new BackupStoreDirectory(rClient.ReceiveStream(), SHORT_TIMEOUT)); return apDir; } - -Daemon* spDaemon = NULL; int start_internal_daemon() { @@ -737,6 +860,8 @@ static int readdir_stop_time = 0; static char stat_hook_filename[512]; // First test hook, during the directory scanning stage, returns empty. +// (Where is this stage? I can't find it, so I switched from using +// readdir_test_hook_1 to readdir_test_hook_2 in intercept tests.) // This will not match the directory on the store, so a sync will start. // We set up the next intercept for the same directory by passing NULL. @@ -756,8 +881,9 @@ extern "C" struct dirent *readdir_test_hook_1(DIR *dir) return NULL; } -// Second test hook, during the directory sync stage, keeps returning -// new filenames until the timer expires, then disables the intercept. +// Second test hook, called by BackupClientDirectoryRecord::SyncDirectory, +// keeps returning new filenames until the timer expires, then disables the +// intercept. extern "C" struct dirent *readdir_test_hook_2(DIR *dir) { @@ -809,7 +935,10 @@ extern "C" struct dirent *readdir_test_hook_2(DIR *dir) #endif // sleep a bit to reduce the number of dirents returned - ::safe_sleep(1); + if (time_now < readdir_stop_time) + { + ::safe_sleep(1); + } return &readdir_test_dirent; } @@ -839,497 +968,665 @@ int lstat_test_post_hook(int old_ret, const char *file_name, struct stat *buf) return old_ret; } -bool test_entry_deleted(BackupStoreDirectory& rDir, +bool test_entry_deleted(BackupStoreDirectory& rDir, const std::string& rName) { BackupStoreDirectory::Iterator i(rDir); BackupStoreDirectory::Entry *en = i.FindMatchingClearName( BackupStoreFilenameClear(rName)); - TEST_THAT(en != 0); - if (en == 0) return false; + TEST_THAT_OR(en != 0, return false); int16_t flags = en->GetFlags(); - TEST_THAT(flags && BackupStoreDirectory::Entry::Flags_Deleted); + TEST_LINE(flags && BackupStoreDirectory::Entry::Flags_Deleted, + rName + " should have been deleted"); return flags && BackupStoreDirectory::Entry::Flags_Deleted; } -bool compare_all(BackupQueries::ReturnCode::Type expected_status, - std::string config_file = "testfiles/bbackupd.conf") +bool compare(BackupQueries::ReturnCode::Type expected_status, + const std::string& bbackupquery_options = "", + const std::string& compare_options = "-acQ") { std::string cmd = BBACKUPQUERY; cmd += " "; cmd += (expected_status == BackupQueries::ReturnCode::Compare_Same) - ? "-Werror" : "-Wwarning"; - cmd += " -c "; - cmd += config_file; - cmd += " \"compare -acQ\" quit"; + ? "-Wwarning" : "-Werror"; + cmd += " -c testfiles/bbackupd.conf "; + cmd += " " + bbackupquery_options; + cmd += " \"compare " + compare_options + "\" quit"; int returnValue = ::system(cmd.c_str()); - int expected_system_result = (int) expected_status; - #ifndef WIN32 - expected_system_result <<= 8; - #endif +#ifndef WIN32 + expected_system_result <<= 8; +#endif TEST_EQUAL_LINE(expected_system_result, returnValue, "compare return value"); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); return (returnValue == expected_system_result); } -#define TEST_COMPARE(...) \ - TEST_THAT(compare_all(BackupQueries::ReturnCode::__VA_ARGS__)); +bool compare_local(BackupQueries::ReturnCode::Type expected_status, + BackupProtocolCallable& client, + const std::string& compare_options = "acQ") +{ + std::auto_ptr<Configuration> config = + load_config_file(DEFAULT_BBACKUPD_CONFIG_FILE, BackupDaemonConfigVerify); + TEST_THAT_OR(config.get(), return false); + BackupQueries bbackupquery(client, *config, false); + + std::vector<std::string> args; + bool opts[256] = {}; + for (std::string::const_iterator i = compare_options.begin(); + i != compare_options.end(); i++) + { + opts[(unsigned char)*i] = true; + } + bbackupquery.CommandCompare(args, opts); + TEST_EQUAL_OR(expected_status, bbackupquery.GetReturnCode(), + return false); + return true; +} -int test_bbackupd() +bool bbackupquery(const std::string& arguments, + const std::string& memleaks_file = "bbackupquery.memleaks") { - // First, wait for a normal period to make sure the last changes - // attributes are within a normal backup timeframe. - // wait_for_backup_operation(); + std::string cmd = BBACKUPQUERY; + cmd += " -c testfiles/bbackupd.conf " + arguments + " quit"; - // Connection gubbins - TLSContext context; - context.Initialise(false /* client */, - "testfiles/clientCerts.pem", - "testfiles/clientPrivKey.pem", - "testfiles/clientTrustedCAs.pem"); + int returnValue = ::system(cmd.c_str()); - printf("\n==== Testing that ReadDirectory on nonexistent directory " - "does not crash\n"); - { - std::auto_ptr<BackupProtocolClient> client = ConnectAndLogin( - context, 0 /* read-write */); - - { - Logging::Guard guard(Log::ERROR); - TEST_CHECK_THROWS(ReadDirectory(*client, 0x12345678), - ConnectionException, - Conn_Protocol_UnexpectedReply); - } +#ifndef WIN32 + returnValue >>= 8; +#endif - client->QueryFinished(); - sSocket.Close(); - } + TestRemoteProcessMemLeaks(memleaks_file.c_str()); + TEST_EQUAL(returnValue, BackupQueries::ReturnCode::Command_OK); + return (returnValue == BackupQueries::ReturnCode::Command_OK); +} - // unpack the files for the initial test - TEST_THAT(::system("rm -rf testfiles/TestDir1") == 0); - TEST_THAT(::mkdir("testfiles/TestDir1", 0777) == 0); +bool restore(const std::string& location, const std::string& dest_dir) +{ + std::string cmd = "\"restore " + location + " " + dest_dir + "\""; + TEST_THAT_OR(bbackupquery(cmd), FAIL); + return true; +} - #ifdef WIN32 - TEST_THAT(::system("tar xzvf testfiles/spacetest1.tgz " - "-C testfiles/TestDir1") == 0); - #else - TEST_THAT(::system("gzip -d < testfiles/spacetest1.tgz " - "| ( cd testfiles/TestDir1 && tar xf - )") == 0); - #endif +bool touch_and_wait(const std::string& filename) +{ + int fd = open(filename.c_str(), O_CREAT | O_WRONLY, 0755); + TEST_THAT(fd > 0); + if (fd <= 0) return false; -#ifdef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE - printf("\n==== Skipping intercept-based KeepAlive tests " - "on this platform.\n"); -#else - printf("\n==== Testing SSL KeepAlive messages\n"); + // write again, to update the file's timestamp + int write_result = write(fd, "z", 1); + TEST_EQUAL_LINE(1, write_result, "Buffer write"); + if (write_result != 1) return false; - { - #ifdef WIN32 - #error TODO: implement threads on Win32, or this test \ - will not finish properly - #endif + TEST_THAT(close(fd) == 0); - // bbackupd daemon will try to initialise timers itself - Timers::Cleanup(); - - // something to diff against (empty file doesn't work) - int fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); - TEST_THAT(fd > 0); + // wait long enough to put file into sync window + wait_for_operation(5, "locally modified file to " + "mature for sync"); + return true; +} - char buffer[10000]; - memset(buffer, 0, sizeof(buffer)); +TLSContext sTlsContext; - TEST_EQUAL_LINE(sizeof(buffer), - write(fd, buffer, sizeof(buffer)), - "Buffer write"); - TEST_THAT(close(fd) == 0); - - int pid = start_internal_daemon(); - wait_for_backup_operation("internal daemon to run a sync"); - TEST_THAT(stop_internal_daemon(pid)); - - // two-second delay on the first read() of f1 - // should mean that a single keepalive is sent, - // and diff does not abort. - intercept_setup_delay("testfiles/TestDir1/spacetest/f1", - 0, 2000, SYS_read, 1); - TEST_THAT(unlink("testfiles/bbackupd.log") == 0); - - pid = start_internal_daemon(); - intercept_clear_setup(); - - fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); - TEST_THAT(fd > 0); - // write again, to update the file's timestamp - TEST_EQUAL_LINE(sizeof(buffer), - write(fd, buffer, sizeof(buffer)), - "Buffer write"); - TEST_THAT(close(fd) == 0); +#define TEST_COMPARE(...) \ + TEST_THAT(compare(BackupQueries::ReturnCode::__VA_ARGS__)); +#define TEST_COMPARE_LOCAL(...) \ + TEST_THAT(compare_local(BackupQueries::ReturnCode::__VA_ARGS__)); - wait_for_backup_operation("internal daemon to sync " - "spacetest/f1"); - // can't test whether intercept was triggered, because - // it's in a different process. - // TEST_THAT(intercept_triggered()); - TEST_THAT(stop_internal_daemon(pid)); +bool search_for_file(const std::string& filename) +{ + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, BackupProtocolLogin::Flags_ReadOnly); - // check that keepalive was written to logs, and - // diff was not aborted, i.e. upload was a diff - FileStream fs("testfiles/bbackupd.log", O_RDONLY); - IOStreamGetLine reader(fs); - bool found1 = false; + std::auto_ptr<BackupStoreDirectory> dir = ReadDirectory(*client); + int64_t testDirId = SearchDir(*dir, filename); + client->QueryFinished(); - while (!reader.IsEOF()) - { - std::string line; - TEST_THAT(reader.GetLine(line)); - if (line == "Send GetBlockIndexByName(0x3,\"f1\")") - { - found1 = true; - break; - } - } + return (testDirId != 0); +} - TEST_THAT(found1); - if (found1) - { - std::string line; - TEST_THAT(reader.GetLine(line)); - std::string comp = "Receive Success(0x"; - TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), - line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receiving stream, size 124", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Send GetIsAlive()", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive IsAlive()", line); +class MockClientContext : public BackupClientContext +{ +public: + BackupProtocolCallable& mrClient; + int mNumKeepAlivesPolled; + int mKeepAliveTime; + + MockClientContext + ( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode, + BackupProtocolCallable& rClient + ) + : BackupClientContext(rResolver, rTLSContext, + rHostname, Port, AccountNumber, ExtendedLogging, + ExtendedLogToFile, ExtendedLogFile, + rProgressNotifier, TcpNiceMode), + mrClient(rClient), + mNumKeepAlivesPolled(0), + mKeepAliveTime(-1) + { } + + BackupProtocolCallable &GetConnection() + { + return mrClient; + } - TEST_THAT(reader.GetLine(line)); - comp = "Send StoreFile(0x3,"; - TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), - line); - comp = ",\"f1\")"; - std::string sub = line.substr(line.size() - comp.size()); - TEST_EQUAL_LINE(comp, sub, line); - std::string comp2 = ",0x0,"; - sub = line.substr(line.size() - comp.size() - - comp2.size() + 1, comp2.size()); - TEST_LINE(comp2 != sub, line); - } + virtual BackupProtocolCallable* GetOpenConnection() const + { + return &mrClient; + } - Timers::Init(); - - // stop early to make debugging easier - if (failures) return 1; - - // four-second delay on first read() of f1 - // should mean that no keepalives were sent, - // because diff was immediately aborted - // before any matching blocks could be found. - intercept_setup_delay("testfiles/TestDir1/spacetest/f1", - 0, 4000, SYS_read, 1); - - { - BackupDaemon bbackupd; - bbackupd.Configure("testfiles/bbackupd.conf"); - bbackupd.InitCrypto(); - - fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); - TEST_THAT(fd > 0); - // write again, to update the file's timestamp - TEST_EQUAL_LINE(1, write(fd, "z", 1), "Buffer write"); - TEST_THAT(close(fd) == 0); + void SetKeepAliveTime(int iSeconds) + { + mKeepAliveTime = iSeconds; + BackupClientContext::SetKeepAliveTime(iSeconds); + } - // wait long enough to put file into sync window - wait_for_operation(5, "locally modified file to " - "mature for sync"); + virtual void DoKeepAlive() + { + mNumKeepAlivesPolled++; + BackupClientContext::DoKeepAlive(); + } +}; - bbackupd.RunSyncNow(); - TEST_THAT(intercept_triggered()); - intercept_clear_setup(); - Timers::Cleanup(); - } +class MockBackupDaemon : public BackupDaemon +{ + BackupProtocolCallable& mrClient; + +public: + MockBackupDaemon(BackupProtocolCallable &rClient) + : mrClient(rClient) + { } + + std::auto_ptr<BackupClientContext> GetNewContext + ( + LocationResolver &rResolver, + TLSContext &rTLSContext, + const std::string &rHostname, + int32_t Port, + uint32_t AccountNumber, + bool ExtendedLogging, + bool ExtendedLogToFile, + std::string ExtendedLogFile, + ProgressNotifier &rProgressNotifier, + bool TcpNiceMode + ) + { + std::auto_ptr<BackupClientContext> context( + new MockClientContext(rResolver, + rTLSContext, rHostname, Port, + AccountNumber, ExtendedLogging, + ExtendedLogToFile, ExtendedLogFile, + rProgressNotifier, TcpNiceMode, mrClient)); + return context; + } +}; - // check that the diff was aborted, i.e. upload was not a diff - found1 = false; +bool test_readdirectory_on_nonexistent_dir() +{ + SETUP_WITH_BBSTORED(); - while (!reader.IsEOF()) + { + std::auto_ptr<BackupProtocolCallable> client = connect_and_login( + sTlsContext, 0 /* read-write */); + { - std::string line; - TEST_THAT(reader.GetLine(line)); - if (line == "Send GetBlockIndexByName(0x3,\"f1\")") - { - found1 = true; - break; - } + Logger::LevelGuard(Logging::GetConsole(), Log::ERROR); + TEST_CHECK_THROWS(ReadDirectory(*client, 0x12345678), + ConnectionException, + Protocol_UnexpectedReply); + TEST_PROTOCOL_ERROR_OR(*client, Err_DoesNotExist,); } - TEST_THAT(found1); - if (found1) - { - std::string line; - TEST_THAT(reader.GetLine(line)); - std::string comp = "Receive Success(0x"; - TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), - line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receiving stream, size 124", line); + client->QueryFinished(); + } - // delaying for 4 seconds in one step means that - // the diff timer and the keepalive timer will - // both expire, and the diff timer is honoured first, - // so there will be no keepalives. + TEARDOWN_TEST_BBACKUPD(); +} - TEST_THAT(reader.GetLine(line)); - comp = "Send StoreFile(0x3,"; - TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), - line); - comp = ",0x0,\"f1\")"; - std::string sub = line.substr(line.size() - comp.size()); - TEST_EQUAL_LINE(comp, sub, line); - } +bool test_bbackupquery_parser_escape_slashes() +{ + SETUP_WITH_BBSTORED(); + + BackupProtocolLocal2 connection(0x01234567, "test", + "backup/01234567/", 0, false); + + BackupClientFileAttributes attr; + attr.ReadAttributes("testfiles/TestDir1", + false /* put mod times in the attributes, please */); + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); + BackupStoreFilenameClear dirname("foo"); + int64_t foo_id = connection.QueryCreateDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory + 0, // attrModTime, + dirname, // dirname, + attrStream)->GetObjectID(); + + attrStream.reset(new MemBlockStream(attr)); + dirname = BackupStoreFilenameClear("/bar"); + int64_t bar_id = connection.QueryCreateDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory + 0, // attrModTime, + dirname, // dirname, + attrStream)->GetObjectID(); + + std::auto_ptr<Configuration> config = + load_config_file(DEFAULT_BBACKUPD_CONFIG_FILE, BackupDaemonConfigVerify); + TEST_THAT_OR(config.get(), return false); + BackupQueries query(connection, *config, false); // read-only + + TEST_EQUAL(foo_id, query.FindDirectoryObjectID("foo")); + TEST_EQUAL(foo_id, query.FindDirectoryObjectID("/foo")); + TEST_EQUAL(0, query.FindDirectoryObjectID("\\/foo")); + TEST_EQUAL(0, query.FindDirectoryObjectID("/bar")); + TEST_EQUAL(bar_id, query.FindDirectoryObjectID("\\/bar")); + connection.QueryFinished(); + + TEARDOWN_TEST_BBACKUPD(); +} - if (failures > 0) +bool test_getobject_on_nonexistent_file() +{ + SETUP_WITH_BBSTORED(); + + { + std::auto_ptr<Configuration> config = + load_config_file(DEFAULT_BBACKUPD_CONFIG_FILE, BackupDaemonConfigVerify); + TEST_THAT_OR(config.get(), return false); + + std::auto_ptr<BackupProtocolCallable> connection = + connect_and_login(sTlsContext, 0 /* read-write */); + BackupQueries query(*connection, *config, false); // read-only + std::vector<std::string> args; + args.push_back("2"); // object ID + args.push_back("testfiles/2.obj"); // output file + bool opts[256]; + + Capture capture; + Logging::TempLoggerGuard guard(&capture); + query.CommandGetObject(args, opts); + std::vector<Capture::Message> messages = capture.GetMessages(); + TEST_THAT(!messages.empty()); + if (!messages.empty()) { - // stop early to make debugging easier - Timers::Init(); - return 1; + std::string last_message = messages.back().message; + TEST_EQUAL("Object ID 0x2 does not exist on store.", + last_message); } - - intercept_setup_delay("testfiles/TestDir1/spacetest/f1", - 0, 1000, SYS_read, 3); - pid = start_internal_daemon(); - intercept_clear_setup(); + } - fd = open("testfiles/TestDir1/spacetest/f1", O_WRONLY); - TEST_THAT(fd > 0); - // write again, to update the file's timestamp - TEST_EQUAL_LINE(sizeof(buffer), - write(fd, buffer, sizeof(buffer)), - "Buffer write"); - TEST_THAT(close(fd) == 0); + TEARDOWN_TEST_BBACKUPD(); +} + +// ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0)) in +// BackupStoreFileEncodeStream::Recipe::Recipe once failed, apparently because +// a zero byte file had a block index but no entries in it. But this test +// doesn't reproduce the error, so it's not enabled for now. - wait_for_backup_operation("internal daemon to sync " - "spacetest/f1 again"); - // can't test whether intercept was triggered, because - // it's in a different process. - // TEST_THAT(intercept_triggered()); - TEST_THAT(stop_internal_daemon(pid)); +bool test_replace_zero_byte_file_with_nonzero_byte_file() +{ + SETUP_TEST_BBACKUPD(); - // check that the diff was aborted, i.e. upload was not a diff - found1 = false; + TEST_THAT_OR(mkdir("testfiles/TestDir1", 0755) == 0, FAIL); + FileStream emptyFile("testfiles/TestDir1/f2", + O_WRONLY | O_CREAT | O_EXCL, 0755); + wait_for_operation(5, "f2 to be old enough"); - while (!reader.IsEOF()) + BackupProtocolLocal2 client(0x01234567, "test", + "backup/01234567/", 0, false); + MockBackupDaemon bbackupd(client); + TEST_THAT(configure_bbackupd(bbackupd, "testfiles/bbackupd.conf")); + bbackupd.RunSyncNow(); + TEST_COMPARE_LOCAL(Compare_Same, client); + + MemBlockStream stream("Hello world"); + stream.CopyStreamTo(emptyFile); + emptyFile.Close(); + wait_for_operation(5, "f2 to be old enough"); + + bbackupd.RunSyncNow(); + TEST_COMPARE_LOCAL(Compare_Same, client); + + TEARDOWN_TEST_BBACKUPD(); +} + +// This caused the issue reported by Brendon Baumgartner and described in my +// email to the Box Backup list on Mon, 21 Apr 2014 at 18:44:38. If the +// directory disappears then we used to try to send an empty attributes block +// to the server, which is illegal. +bool test_backup_disappearing_directory() +{ + SETUP_WITH_BBSTORED(); + + class BackupClientDirectoryRecordHooked : public BackupClientDirectoryRecord + { + public: + BackupClientDirectoryRecordHooked(int64_t ObjectID, + const std::string &rSubDirName) + : BackupClientDirectoryRecord(ObjectID, rSubDirName), + mDeletedOnce(false) + { } + bool mDeletedOnce; + bool UpdateItems(SyncParams &rParams, const std::string &rLocalPath, + const std::string &rRemotePath, + const Location& rBackupLocation, + BackupStoreDirectory *pDirOnStore, + std::vector<BackupStoreDirectory::Entry *> &rEntriesLeftOver, + std::vector<std::string> &rFiles, + const std::vector<std::string> &rDirs) { - std::string line; - TEST_THAT(reader.GetLine(line)); - if (line == "Send GetBlockIndexByName(0x3,\"f1\")") + if(!mDeletedOnce) { - found1 = true; - break; + TEST_THAT(::rmdir("testfiles/TestDir1/dir23") == 0); + mDeletedOnce = true; } + + return BackupClientDirectoryRecord::UpdateItems(rParams, + rLocalPath, rRemotePath, rBackupLocation, + pDirOnStore, rEntriesLeftOver, rFiles, rDirs); } + }; - TEST_THAT(found1); - if (found1) - { - std::string line; - TEST_THAT(reader.GetLine(line)); - std::string comp = "Receive Success(0x"; - TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), - line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receiving stream, size 124", line); + BackupClientContext clientContext + ( + bbackupd, // rLocationResolver + sTlsContext, + "localhost", + BOX_PORT_BBSTORED_TEST, + 0x01234567, + false, // ExtendedLogging + false, // ExtendedLogFile + "", // extendedLogFile + bbackupd, // rProgressNotifier + false // TcpNice + ); + + BackupClientInodeToIDMap oldMap, newMap; + oldMap.OpenEmpty(); + newMap.Open("testfiles/test_map.db", false, true); + clientContext.SetIDMaps(&oldMap, &newMap); + + BackupClientDirectoryRecord::SyncParams params( + bbackupd, // rRunStatusProvider, + bbackupd, // rSysadminNotifier, + bbackupd, // rProgressNotifier, + clientContext, + &bbackupd); + params.mSyncPeriodEnd = GetCurrentBoxTime(); + + BackupClientFileAttributes attr; + attr.ReadAttributes("testfiles/TestDir1", + false /* put mod times in the attributes, please */); + std::auto_ptr<IOStream> attrStream(new MemBlockStream(attr)); + BackupStoreFilenameClear dirname("Test1"); + std::auto_ptr<BackupProtocolSuccess> + dirCreate(clientContext.GetConnection().QueryCreateDirectory( + BACKUPSTORE_ROOT_DIRECTORY_ID, // containing directory + 0, // attrModTime, + dirname, // dirname, + attrStream)); + clientContext.CloseAnyOpenConnection(); + + // Object ID for later creation + int64_t oid = dirCreate->GetObjectID(); + BackupClientDirectoryRecordHooked record(oid, "Test1"); + + TEST_COMPARE(Compare_Different); + + Location fakeLocation; + record.SyncDirectory(params, + BACKUPSTORE_ROOT_DIRECTORY_ID, + "testfiles/TestDir1", // locationPath, + "/whee", // remotePath + fakeLocation); + clientContext.CloseAnyOpenConnection(); + TEST_COMPARE(Compare_Same); - // delaying for 3 seconds in steps of 1 second - // means that the keepalive timer will expire 3 times, - // and on the 3rd time the diff timer will expire too. - // The diff timer is honoured first, so there will be - // only two keepalives. - - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Send GetIsAlive()", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive IsAlive()", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Send GetIsAlive()", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive IsAlive()", line); + // Run another backup, check that we haven't got an inconsistent + // state that causes a crash. + record.SyncDirectory(params, + BACKUPSTORE_ROOT_DIRECTORY_ID, + "testfiles/TestDir1", // locationPath, + "/whee", // remotePath + fakeLocation); + clientContext.CloseAnyOpenConnection(); + TEST_COMPARE(Compare_Same); - // but two matching blocks should have been found - // already, so the upload should be a diff. + // Now recreate it and run another backup, check that we haven't got + // an inconsistent state that causes a crash or prevents us from + // creating the directory if it appears later. + TEST_THAT(::mkdir("testfiles/TestDir1/dir23", 0755) == 0); + TEST_COMPARE(Compare_Different); + + record.SyncDirectory(params, + BACKUPSTORE_ROOT_DIRECTORY_ID, + "testfiles/TestDir1", // locationPath, + "/whee", // remotePath + fakeLocation); + clientContext.CloseAnyOpenConnection(); + TEST_COMPARE(Compare_Same); - TEST_THAT(reader.GetLine(line)); - comp = "Send StoreFile(0x3,"; - TEST_EQUAL_LINE(comp, line.substr(0, comp.size()), - line); - comp = ",\"f1\")"; - std::string sub = line.substr(line.size() - comp.size()); - TEST_EQUAL_LINE(comp, sub, line); - std::string comp2 = ",0x0,"; - sub = line.substr(line.size() - comp.size() - - comp2.size() + 1, comp2.size()); - TEST_LINE(comp2 != sub, line); - } + TEARDOWN_TEST_BBACKUPD(); +} - // Check that no read error has been reported yet - TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); +class KeepAliveBackupProtocolLocal : public BackupProtocolLocal2 +{ +public: + int mNumKeepAlivesSent; + int mNumKeepAlivesReceived; + +public: + KeepAliveBackupProtocolLocal(int32_t AccountNumber, + const std::string& ConnectionDetails, + const std::string& AccountRootDir, int DiscSetNumber, + bool ReadOnly) + : BackupProtocolLocal2(AccountNumber, ConnectionDetails, AccountRootDir, + DiscSetNumber, ReadOnly), + mNumKeepAlivesSent(0), + mNumKeepAlivesReceived(0) + { } + + std::auto_ptr<BackupProtocolIsAlive> Query(const BackupProtocolGetIsAlive &rQuery) + { + mNumKeepAlivesSent++; + std::auto_ptr<BackupProtocolIsAlive> response = + BackupProtocolLocal::Query(rQuery); + mNumKeepAlivesReceived++; + return response; + } +}; - if (failures > 0) - { - // stop early to make debugging easier - Timers::Init(); - return 1; - } +bool test_ssl_keepalives() +{ + SETUP_TEST_BBACKUPD(); - intercept_setup_readdir_hook("testfiles/TestDir1/spacetest/d1", - readdir_test_hook_1); - - // time for at least two keepalives - readdir_stop_time = time(NULL) + 12 + 2; + KeepAliveBackupProtocolLocal connection(0x01234567, "test", "backup/01234567/", + 0, false); + MockBackupDaemon bbackupd(connection); + TEST_THAT_OR(setup_test_bbackupd(bbackupd), FAIL); - pid = start_internal_daemon(); - intercept_clear_setup(); + // Test that sending a keepalive actually works, when the timeout has expired, + // but doesn't send anything at the beginning: + { + MockClientContext context( + bbackupd, // rResolver + sTlsContext, // rTLSContext + "localhost", // rHostname + BOX_PORT_BBSTORED_TEST, + 0x01234567, // AccountNumber + false, // ExtendedLogging + false, // ExtendedLogToFile + "", // ExtendedLogFile + bbackupd, // rProgressNotifier + false, // TcpNiceMode + connection); // rClient - std::string touchfile = - "testfiles/TestDir1/spacetest/d1/touch-me"; + // Set the timeout to 1 second + context.SetKeepAliveTime(1); + + // Check that DoKeepAlive() does nothing right now + context.DoKeepAlive(); + TEST_EQUAL(0, connection.mNumKeepAlivesSent); + + // Sleep until just before the timer expires, check that DoKeepAlive() + // still does nothing. + ShortSleep(MilliSecondsToBoxTime(900), true); + context.DoKeepAlive(); + TEST_EQUAL(0, connection.mNumKeepAlivesSent); + + // Sleep until just after the timer expires, check that DoKeepAlive() + // sends a GetIsAlive message now. + ShortSleep(MilliSecondsToBoxTime(200), true); + context.DoKeepAlive(); + TEST_EQUAL(1, connection.mNumKeepAlivesSent); + TEST_EQUAL(1, connection.mNumKeepAlivesReceived); + TEST_EQUAL(3, context.mNumKeepAlivesPolled); + } - fd = open(touchfile.c_str(), O_CREAT | O_WRONLY, 0700); - TEST_THAT(fd > 0); - // write again, to update the file's timestamp - TEST_EQUAL_LINE(sizeof(buffer), - write(fd, buffer, sizeof(buffer)), - "Buffer write"); - TEST_THAT(close(fd) == 0); + // Do the initial backup. There are no existing files to diff, so the only + // keepalives polled should be the ones for each directory entry while reading + // directories, and the one in UpdateItems(), which is also once per item (file + // or directory). test_base.tgz has 16 directory entries, so we expect 2 * 16 = 32 + // keepalives in total. Except on Windows where there are no symlinks, and when + // compiled with MSVC we exclude them from the tar extract operation as 7za + // complains about them, so there should be 3 files less, and thus only 26 + // keepalives. +#ifdef _MSC_VER + #define NUM_KEEPALIVES_BASE 26 +#else + #define NUM_KEEPALIVES_BASE 32 +#endif - wait_for_backup_operation("internal daemon to scan " - "spacetest/d1"); - // can't test whether intercept was triggered, because - // it's in a different process. - // TEST_THAT(intercept_triggered()); - TEST_THAT(stop_internal_daemon(pid)); + std::auto_ptr<BackupClientContext> apContext = bbackupd.RunSyncNow(); + MockClientContext* pContext = (MockClientContext *)(apContext.get()); + TEST_EQUAL(NUM_KEEPALIVES_BASE, pContext->mNumKeepAlivesPolled); + TEST_EQUAL(1, pContext->mKeepAliveTime); + + // Calculate the number of blocks that will be in ./TestDir1/x1/dsfdsfs98.fd, + // which is 4269 bytes long. + int64_t NumBlocks; + int32_t BlockSize, LastBlockSize; + BackupStoreFileEncodeStream::CalculateBlockSizes(4269, NumBlocks, BlockSize, LastBlockSize); + TEST_EQUAL(4096, BlockSize); + TEST_EQUAL(173, LastBlockSize); + TEST_EQUAL(2, NumBlocks); + + // Now modify the file and run another backup. It's the only file that should be + // diffed, and DoKeepAlive() should be called for each block size in the original + // file, times the number of times that block size fits into the new file, + // i.e. 1 + (4269 / 256) = 18 times (plus the same 32 while scanning, as above). - // check that keepalives were sent during the dir search - found1 = false; + { + int fd = open("testfiles/TestDir1/x1/dsfdsfs98.fd", O_WRONLY); + TEST_THAT_OR(fd > 0, FAIL); - // skip to next login - while (!reader.IsEOF()) - { - std::string line; - TEST_THAT(reader.GetLine(line)); - if (line == "Send ListDirectory(0x3,0xffff,0xc,true)") - { - found1 = true; - break; - } - } + char buffer[4000]; + memset(buffer, 0, sizeof(buffer)); - TEST_THAT(found1); - if (found1) - { - found1 = false; + TEST_EQUAL_LINE(sizeof(buffer), + write(fd, buffer, sizeof(buffer)), + "Buffer write"); + TEST_THAT(close(fd) == 0); - while (!reader.IsEOF()) - { - std::string line; - TEST_THAT(reader.GetLine(line)); - if (line == "Send ListDirectory(0x3,0xffffffff,0xc,true)") - { - found1 = true; - break; - } - } - } + wait_for_operation(5, "modified file to be old enough"); + } - if (found1) - { - std::string line; - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive Success(0x3)", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receiving stream, size 425", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Send GetIsAlive()", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive IsAlive()", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Send GetIsAlive()", line); - TEST_THAT(reader.GetLine(line)); - TEST_EQUAL("Receive IsAlive()", line); - } + apContext = bbackupd.RunSyncNow(); + pContext = (MockClientContext *)(apContext.get()); + TEST_EQUAL(NUM_KEEPALIVES_BASE + (4269/4096) + (4269/173), + pContext->mNumKeepAlivesPolled); + TEARDOWN_TEST_BBACKUPD(); +} - if (failures > 0) - { - // stop early to make debugging easier - Timers::Init(); - return 1; - } +bool test_backup_hardlinked_files() +{ + SETUP_WITH_BBSTORED(); - TEST_THAT(unlink(touchfile.c_str()) == 0); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); - // restore timers for rest of tests - Timers::Init(); - } -#endif // PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE + // Create some hard links. First in the same directory: + TEST_THAT(link("testfiles/TestDir1/x1/dsfdsfs98.fd", + "testfiles/TestDir1/x1/hardlink1") == 0); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); - // Check that no read error has been reported yet - TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + // Now in a different directory + TEST_THAT(mkdir("testfiles/TestDir1/x2", 0755) == 0); + TEST_THAT(link("testfiles/TestDir1/x1/dsfdsfs98.fd", + "testfiles/TestDir1/x2/hardlink2") == 0); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); - std::string cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd.conf"; + // Now delete one of them + TEST_THAT(unlink("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); - ::safe_sleep(1); + // And another. + TEST_THAT(unlink("testfiles/TestDir1/x1/hardlink1") == 0); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + TEARDOWN_TEST_BBACKUPD(); +} - if(bbackupd_pid > 0) - { - printf("\n==== Testing that backup pauses when " - "store is full\n"); +bool test_backup_pauses_when_store_is_full() +{ + SETUP_WITHOUT_FILES(); + unpack_files("spacetest1", "testfiles/TestDir1"); + TEST_THAT_OR(StartClient(), FAIL); + // TODO FIXME dedent + { // wait for files to be uploaded - BOX_TRACE("Waiting for all outstanding files to be uploaded") + BOX_TRACE("Waiting for all outstanding files to be uploaded...") wait_for_sync_end(); - BOX_TRACE("done.") + BOX_TRACE("Done. Comparing to check that it worked...") + TEST_COMPARE(Compare_Same); + + // BLOCK + { + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, 0 /* read-write */); + TEST_THAT(check_num_files(5, 0, 0, 9)); + TEST_THAT(check_num_blocks(*client, 10, 0, 0, 18, 28)); + client->QueryFinished(); + } - // Set limit to something very small - // 26 blocks will be used at this point. - // (12 files + location * 2 for raidfile) - // 20 is what we'll need in a minute + // Set limit to something very small. + // 28 blocks are used at this point. // set soft limit to 0 to ensure that all deleted files // are deleted immediately by housekeeping - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " - "testfiles/bbstored.conf setlimit 01234567 0B 20B") - == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); + TEST_THAT(change_account_limits("0B", "20B")); // Unpack some more files - #ifdef WIN32 - TEST_THAT(::system("tar xzvf testfiles/spacetest2.tgz " - "-C testfiles/TestDir1") == 0); - #else - TEST_THAT(::system("gzip -d < testfiles/spacetest2.tgz " - "| ( cd testfiles/TestDir1 && tar xf - )") == 0); - #endif + unpack_files("spacetest2", "testfiles/TestDir1"); // Delete a file and a directory TEST_THAT(::unlink("testfiles/TestDir1/spacetest/f1") == 0); +#ifdef WIN32 + TEST_THAT(::system("rd /s/q testfiles\\TestDir1\\spacetest\\d7") == 0); +#else TEST_THAT(::system("rm -rf testfiles/TestDir1/spacetest/d7") == 0); +#endif - // The following files should be on the server: + // The following files should be in the backup directory: // 00000001 -d---- 00002 (root) // 00000002 -d---- 00002 Test1 // 00000003 -d---- 00002 Test1/spacetest @@ -1339,22 +1636,26 @@ int test_bbackupd() // 00000007 f----- 00002 Test1/spacetest/d1/f3 // 00000008 f----- 00002 Test1/spacetest/d1/f4 // 00000009 -d---- 00002 Test1/spacetest/d2 + // 00000009 -d---- 00002 Test1/spacetest/d6 // 0000000a -d---- 00002 Test1/spacetest/d3 // 0000000b -d---- 00002 Test1/spacetest/d3/d4 // 0000000c f----- 00002 Test1/spacetest/d3/d4/f5 // 0000000d -d---- 00002 Test1/spacetest/d6 + // 0000000d -d---- 00002 Test1/spacetest/d6/d8 + // 0000000d -d---- 00002 Test1/spacetest/d6/d8/f7 // 0000000e -dX--- 00002 Test1/spacetest/d7 - // This is 28 blocks total, of which 2 in deleted files + // + // root + location + spacetest1 + spacetest2 = 17 files + // = 34 blocks with raidfile. Of which 2 in deleted files // and 18 in directories. Note that f1 and d7 may or may // not be deleted yet. // - // spacetest1 + spacetest2 = 16 files = 32 blocks with raidfile - // minus one file and one dir is 28 blocks - // - // d2/f6, d6/d8 and d6/d8/f7 are new - // even if the client marks f1 and d7 as deleted, and - // housekeeping deleted them, the backup cannot complete - // if the limit is 20 blocks. + // The files and dirs from spacetest1 are already on the server + // (28 blocks). If we set the limit to 20 then the client will + // notice as soon as it tries to create the new files and dirs + // from spacetest2. It should still delete f1 and d7, but that + // won't bring it back under the hard limit, so no files from + // spacetest2 should be uploaded. BOX_TRACE("Waiting for sync for bbackupd to notice that the " "store is full"); @@ -1363,82 +1664,144 @@ int test_bbackupd() BOX_TRACE("Compare to check that there are differences"); TEST_COMPARE(Compare_Different); - BOX_TRACE("Compare finished."); // Check that the notify script was run TEST_THAT(TestFileExists("testfiles/notifyran.store-full.1")); // But only once! TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2")); - - // Kill the daemon - terminate_bbackupd(bbackupd_pid); - wait_for_operation(5, "housekeeping to remove the " - "deleted files"); - - // This removes f1 and d7, which were previously marked - // as deleted, so total usage drops by 4 blocks to 24. + // We can't guarantee to get in before housekeeping runs, so it's safer + // (more reliable) to wait for it to finish. But we also need to stop the + // client first, otherwise it might be connected when housekeeping tries + // to run, and that would prevent it from running, causing test to fail. + TEST_THAT_OR(StopClient(), FAIL); + wait_for_operation(5, "housekeeping to run"); // BLOCK { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, 0 /* read-write */); - - std::auto_ptr<BackupProtocolAccountUsage> usage( - client->QueryGetAccountUsage()); - TEST_EQUAL_LINE(24, usage->GetBlocksUsed(), - "blocks used"); - TEST_EQUAL_LINE(0, usage->GetBlocksInDeletedFiles(), - "deleted blocks"); - TEST_EQUAL_LINE(16, usage->GetBlocksInDirectories(), - "directory blocks"); + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, + BackupProtocolLogin::Flags_ReadOnly); + std::auto_ptr<BackupStoreDirectory> root_dir = + ReadDirectory(*client, BACKUPSTORE_ROOT_DIRECTORY_ID); + int64_t test_dir_id = SearchDir(*root_dir, "Test1"); + TEST_THAT_OR(test_dir_id, FAIL); + std::auto_ptr<BackupStoreDirectory> test_dir = + ReadDirectory(*client, test_dir_id); + + int64_t spacetest_dir_id = SearchDir(*test_dir, "spacetest"); + TEST_THAT_OR(spacetest_dir_id, FAIL); + std::auto_ptr<BackupStoreDirectory> spacetest_dir = + ReadDirectory(*client, spacetest_dir_id); + + int64_t d2_id = SearchDir(*spacetest_dir, "d2"); + int64_t d6_id = SearchDir(*spacetest_dir, "d6"); + TEST_THAT_OR(d2_id != 0, FAIL); + TEST_THAT_OR(d6_id != 0, FAIL); + std::auto_ptr<BackupStoreDirectory> d2_dir = + ReadDirectory(*client, d2_id); + std::auto_ptr<BackupStoreDirectory> d6_dir = + ReadDirectory(*client, d6_id); + + // None of the new files should have been uploaded + TEST_EQUAL(SearchDir(*d2_dir, "f6"), 0); + TEST_EQUAL(SearchDir(*d6_dir, "d8"), 0); + + // But f1 and d7 should have been deleted. + TEST_EQUAL(SearchDir(*spacetest_dir, "f1"), 0); + TEST_EQUAL(SearchDir(*spacetest_dir, "d7"), 0); + + TEST_THAT(check_num_files(4, 0, 0, 8)); + TEST_THAT(check_num_blocks(*client, 8, 0, 0, 16, 24)); client->QueryFinished(); - sSocket.Close(); } + } - if (failures) return 1; + // Increase the limit again, check that all files are backed up on the + // next run. + TEST_THAT(change_account_limits("0B", "34B")); + TEST_THAT_OR(StartClient(), FAIL); + wait_for_sync_end(); + TEST_COMPARE(Compare_Same); - // ensure time is different to refresh the cache - ::safe_sleep(1); + TEARDOWN_TEST_BBACKUPD(); +} - BOX_TRACE("Restart bbackupd with more exclusions"); - // Start again with a new config that excludes d3 and f2, - // and hence also d3/d4 and d3/d4/f5. bbackupd should mark - // them as deleted and housekeeping should clean up, - // making space to upload the new files. - // total required: (13-2-4+3)*2 = 20 blocks - /* - cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd-exclude.conf"; - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - */ - - BackupDaemon bbackupd; +bool test_bbackupd_exclusions() +{ + SETUP_WITHOUT_FILES(); + + TEST_THAT(unpack_files("spacetest1", "testfiles/TestDir1")); + // Delete a file and a directory + TEST_THAT(::unlink("testfiles/TestDir1/spacetest/f1") == 0); +#ifdef WIN32 + TEST_THAT(::system("rd /s/q testfiles\\TestDir1\\spacetest\\d7") == 0); +#else + TEST_THAT(::system("rm -rf testfiles/TestDir1/spacetest/d7") == 0); +#endif + + // We need to be OVER the limit, i.e. >24 blocks, or + // BackupClientContext will mark us over limit immediately on + // connection. + TEST_THAT(change_account_limits("0B", "25B")); + + // Initial run to get the files backed up + { + bbackupd.RunSyncNow(); + TEST_THAT(!bbackupd.StorageLimitExceeded()); + + // BLOCK { - const char* argv[] = { "bbackupd", - bbackupd_args.c_str() }; - TEST_EQUAL_LINE(0, bbackupd.ProcessOptions(2, argv), - "processing command-line options"); + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, 0 /* read-write */); + TEST_THAT(check_num_files(4, 0, 0, 8)); + TEST_THAT(check_num_blocks(*client, 8, 0, 0, 16, 24)); + client->QueryFinished(); } + } - bbackupd.Configure("testfiles/bbackupd-exclude.conf"); - bbackupd.InitCrypto(); - BOX_TRACE("done."); + // Create a directory and then try to run a backup. This should try + // to create the directory on the server, fail, and catch the error. + // The directory that we create, spacetest/d6/d8, is included in + // spacetest2.tgz, so we can ignore this for counting files after we + // unpack spacetest2.tgz. + TEST_THAT(::mkdir("testfiles/TestDir1/spacetest/d6/d8", 0755) == 0); + bbackupd.RunSyncNow(); + TEST_THAT(bbackupd.StorageLimitExceeded()); - // Should be marked as deleted by this run - // wait_for_sync_end(); + // BLOCK + { + TEST_THAT(unpack_files("spacetest2", "testfiles/TestDir1")); + bbackupd.RunSyncNow(); + TEST_THAT(bbackupd.StorageLimitExceeded()); + + // BLOCK { - // Logging::Guard guard(Log::ERROR); - bbackupd.RunSyncNow(); + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, 0 /* read-write */); + TEST_THAT(check_num_files(4, 0, 0, 8)); + TEST_THAT(check_num_blocks(*client, 8, 0, 0, 16, 24)); + client->QueryFinished(); } + } + // TODO FIXME dedent + { + // Start again with a new config that excludes d3 and f2, + // and hence also d3/d4 and d3/d4/f5. bbackupd should mark + // them as deleted and housekeeping should later clean up, + // making space to upload the new files. + // total required: (13-2-4+3)*2 = 20 blocks + + TEST_THAT(configure_bbackupd(bbackupd, "testfiles/bbackupd-exclude.conf")); + // Should be marked as deleted by this run. Hold onto the + // BackupClientContext to stop housekeeping from running. + std::auto_ptr<BackupClientContext> apClientContext = + bbackupd.RunSyncNow(); + // Housekeeping has not yet deleted the files, so there's not + // enough space to upload the new ones. TEST_THAT(bbackupd.StorageLimitExceeded()); // Check that the notify script was run @@ -1446,31 +1809,35 @@ int test_bbackupd() // But only twice! // TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.3")); - // All these should be marked as deleted but hopefully - // not removed by housekeeping yet: - // f1 deleted + // All these should be marked as deleted but not removed by + // housekeeping yet: // f2 excluded - // d1 excluded (why?) - // d1/f3 excluded (why?) // d3 excluded // d3/d4 excluded // d3/d4/f5 excluded - // d7 deleted // Careful with timing here, these files will be removed by - // housekeeping the next time it runs. On Win32, housekeeping - // runs immediately after disconnect, but only if enough time - // has elapsed since the last housekeeping. Since the - // backup run closely follows the last one, housekeeping - // should not run afterwards. On other platforms, we want to - // get in immediately after the backup and hope that - // housekeeping doesn't beat us to it. - - BOX_TRACE("Find out whether bbackupd marked files as deleted"); + // housekeeping the next time it runs. We hold onto the client + // context (and hence an open connection) to stop it from + // running for now. + + // But we can't do that on Windows, because bbstored only + // support one simultaneous connection. So we have to hope that + // housekeeping has run recently enough that it doesn't need to + // run again when we disconnect. + + BOX_INFO("Finding out whether bbackupd marked files as deleted"); + + // TODO FIXME dedent { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, 0 /* read-write */); +#ifdef WIN32 + apClientContext.reset(); +#endif + + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, + BackupProtocolLogin::Flags_ReadOnly); - std::auto_ptr<BackupStoreDirectory> rootDir = + std::auto_ptr<BackupStoreDirectory> rootDir = ReadDirectory(*client); int64_t testDirId = SearchDir(*rootDir, "Test1"); @@ -1479,7 +1846,7 @@ int test_bbackupd() std::auto_ptr<BackupStoreDirectory> Test1_dir = ReadDirectory(*client, testDirId); - int64_t spacetestDirId = SearchDir(*Test1_dir, + int64_t spacetestDirId = SearchDir(*Test1_dir, "spacetest"); TEST_THAT(spacetestDirId != 0); @@ -1500,49 +1867,41 @@ int test_bbackupd() TEST_THAT(test_entry_deleted(*spacetest_dir, "d3")); int64_t d3_id = SearchDir(*spacetest_dir, "d3"); - TEST_THAT(d3_id != 0); + TEST_THAT_OR(d3_id != 0, FAIL); std::auto_ptr<BackupStoreDirectory> d3_dir = ReadDirectory(*client, d3_id); TEST_THAT(test_entry_deleted(*d3_dir, "d4")); int64_t d4_id = SearchDir(*d3_dir, "d4"); - TEST_THAT(d4_id != 0); + TEST_THAT_OR(d4_id != 0, FAIL); std::auto_ptr<BackupStoreDirectory> d4_dir = ReadDirectory(*client, d4_id); TEST_THAT(test_entry_deleted(*d4_dir, "f5")); - std::auto_ptr<BackupProtocolAccountUsage> usage( - client->QueryGetAccountUsage()); - TEST_EQUAL_LINE(24, usage->GetBlocksUsed(), - "blocks used"); - TEST_EQUAL_LINE(4, usage->GetBlocksInDeletedFiles(), - "deleted blocks"); - TEST_EQUAL_LINE(16, usage->GetBlocksInDirectories(), - "directory blocks"); // d1/f3 and d1/f4 are the only two files on the // server which are not deleted, they use 2 blocks // each, the rest is directories and 2 deleted files - // (f1 and d3/d4/f5) + // (f2 and d3/d4/f5) + TEST_THAT(check_num_files(2, 0, 2, 8)); + TEST_THAT(check_num_blocks(*client, 4, 0, 4, 16, 24)); // Log out. client->QueryFinished(); - sSocket.Close(); } - BOX_TRACE("done."); - - if (failures) return 1; - wait_for_operation(5, "housekeeping to remove the " - "deleted files"); + // Release our BackupClientContext and open connection, and + // force housekeeping to run now. + apClientContext.reset(); + TEST_THAT(run_housekeeping_and_check_account()); - BOX_TRACE("Check that the files were removed"); + BOX_INFO("Checking that the files were removed"); { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, 0 /* read-write */); - - std::auto_ptr<BackupStoreDirectory> rootDir = + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, 0 /* read-write */); + + std::auto_ptr<BackupStoreDirectory> rootDir = ReadDirectory(*client); int64_t testDirId = SearchDir(*rootDir, "Test1"); @@ -1551,7 +1910,7 @@ int test_bbackupd() std::auto_ptr<BackupStoreDirectory> Test1_dir = ReadDirectory(*client, testDirId); - int64_t spacetestDirId = SearchDir(*Test1_dir, + int64_t spacetestDirId = SearchDir(*Test1_dir, "spacetest"); TEST_THAT(spacetestDirId != 0); @@ -1563,25 +1922,16 @@ int test_bbackupd() TEST_THAT(SearchDir(*spacetest_dir, "d3") == 0); TEST_THAT(SearchDir(*spacetest_dir, "d7") == 0); - std::auto_ptr<BackupProtocolAccountUsage> usage( - client->QueryGetAccountUsage()); - TEST_EQUAL_LINE(16, usage->GetBlocksUsed(), - "blocks used"); - TEST_EQUAL_LINE(0, usage->GetBlocksInDeletedFiles(), - "deleted blocks"); - TEST_EQUAL_LINE(12, usage->GetBlocksInDirectories(), - "directory blocks"); - // d1/f3 and d1/f4 are the only two files on the - // server, they use 2 blocks each, the rest is - // directories. + // f2, d3, d3/d4 and d3/d4/f5 have been removed. + // The files were counted as deleted files before, the + // deleted directories just as directories. + TEST_THAT(check_num_files(2, 0, 0, 6)); + TEST_THAT(check_num_blocks(*client, 4, 0, 0, 12, 16)); // Log out. client->QueryFinished(); - sSocket.Close(); } - if (failures) return 1; - // Need 22 blocks free to upload everything TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " "testfiles/bbstored.conf setlimit 01234567 0B 22B") @@ -1595,111 +1945,82 @@ int test_bbackupd() // Check that the contents of the store are the same // as the contents of the disc - TEST_COMPARE(Compare_Same, "testfiles/bbackupd-exclude.conf"); + TEST_COMPARE(Compare_Same, "-c testfiles/bbackupd-exclude.conf"); BOX_TRACE("done."); // BLOCK { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, 0 /* read-write */); - - std::auto_ptr<BackupProtocolAccountUsage> usage( - client->QueryGetAccountUsage()); - TEST_EQUAL_LINE(22, usage->GetBlocksUsed(), - "blocks used"); - TEST_EQUAL_LINE(0, usage->GetBlocksInDeletedFiles(), - "deleted blocks"); - TEST_EQUAL_LINE(14, usage->GetBlocksInDirectories(), - "directory blocks"); + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, 0 /* read-write */); + + TEST_THAT(check_num_files(4, 0, 0, 7)); + TEST_THAT(check_num_blocks(*client, 8, 0, 0, 14, 22)); + // d2/f6, d6/d8 and d6/d8/f7 are new // i.e. 2 new files, 1 new directory client->QueryFinished(); - sSocket.Close(); } + } - if (failures) return 1; - - // Put the limit back - TEST_THAT_ABORTONFAIL(::system(BBSTOREACCOUNTS " -c " - "testfiles/bbstored.conf setlimit 01234567 " - "1000B 2000B") == 0); - TestRemoteProcessMemLeaks("bbstoreaccounts.memleaks"); - - // Start again with the old config - BOX_TRACE("Restart bbackupd with original configuration"); - // terminate_bbackupd(); - cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd.conf"; - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); - ::safe_sleep(1); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - BOX_TRACE("done."); + TEARDOWN_TEST_BBACKUPD(); +} - // unpack the initial files again - #ifdef WIN32 - TEST_THAT(::system("tar xzvf testfiles/test_base.tgz " - "-C testfiles") == 0); - #else - TEST_THAT(::system("gzip -d < testfiles/test_base.tgz " - "| ( cd testfiles && tar xf - )") == 0); - #endif +bool test_bbackupd_uploads_files() +{ + SETUP_WITH_BBSTORED(); - wait_for_backup_operation("bbackupd to upload more files"); - - // Check that the contents of the store are the same - // as the contents of the disc - // (-a = all, -c = give result in return code) - BOX_TRACE("Check that all files were uploaded successfully"); + // TODO FIXME dedent + { + // The files were all unpacked with timestamps in the past, + // so no delay should be needed to make them eligible to be + // backed up. + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - BOX_TRACE("done."); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; } // Check that no read error has been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); - #ifndef WIN32 // requires fork - printf("\n==== Testing that bbackupd responds correctly to " - "connection failure\n"); + TEARDOWN_TEST_BBACKUPD(); +} - { - // Kill the daemons - terminate_bbackupd(bbackupd_pid); - test_kill_bbstored(); +bool test_bbackupd_responds_to_connection_failure() +{ + SETUP_TEST_BBACKUPD(); + TEST_THAT_OR(unpack_files("test_base"), FAIL); +#ifdef WIN32 + BOX_NOTICE("skipping test on this platform"); // requires fork +#else // !WIN32 + // TODO FIXME dedent + { // create a new file to force an upload const char* new_file = "testfiles/TestDir1/force-upload-2"; - int fd = open(new_file, - O_CREAT | O_EXCL | O_WRONLY, 0700); - if (fd <= 0) - { - perror(new_file); - } - TEST_THAT(fd > 0); - + int fd = open(new_file, O_CREAT | O_EXCL | O_WRONLY, 0700); + TEST_THAT_OR(fd >= 0, + BOX_LOG_SYS_ERROR(BOX_FILE_MESSAGE(new_file, + "failed to create new file")); + FAIL); + const char* control_string = "whee!\n"; TEST_THAT(write(fd, control_string, strlen(control_string)) == (int)strlen(control_string)); close(fd); - // sleep to make it old enough to upload - safe_sleep(4); + wait_for_operation(5, "new file to be old enough"); + // Start a bbstored with a test hook that makes it terminate + // on the first StoreFile command, breaking the connection to + // bbackupd. class MyHook : public BackupStoreContext::TestHook { + public: + int trigger_count; + MyHook() : trigger_count(0) { } + virtual std::auto_ptr<BackupProtocolMessage> StartCommand( const BackupProtocolMessage& rCommand) { @@ -1707,104 +2028,69 @@ int test_bbackupd() BackupProtocolStoreFile::TypeID) { // terminate badly - THROW_EXCEPTION(CommonException, - Internal); + trigger_count++; + THROW_EXCEPTION(ConnectionException, + TLSReadFailed); } return std::auto_ptr<BackupProtocolMessage>(); } }; - MyHook hook; - bbstored_pid = fork(); - - if (bbstored_pid < 0) + class MockBackupProtocolLocal : public BackupProtocolLocal2 { - BOX_LOG_SYS_ERROR("failed to fork()"); - return 1; - } - - if (bbstored_pid == 0) - { - // in fork child - TEST_THAT(setsid() != -1); - - if (!Logging::IsEnabled(Log::TRACE)) - { - Logging::SetGlobalLevel(Log::NOTHING); - } - - // BackupStoreDaemon must be destroyed before exit(), - // to avoid memory leaks being reported. + public: + MyHook hook; + MockBackupProtocolLocal(int32_t AccountNumber, + const std::string& ConnectionDetails, + const std::string& AccountRootDir, int DiscSetNumber, + bool ReadOnly) + : BackupProtocolLocal2(AccountNumber, ConnectionDetails, + AccountRootDir, DiscSetNumber, ReadOnly) { - BackupStoreDaemon bbstored; - bbstored.SetTestHook(hook); - bbstored.SetRunInForeground(true); - bbstored.Main("testfiles/bbstored.conf"); + GetContext().SetTestHook(hook); } + virtual ~MockBackupProtocolLocal() { } + }; - Timers::Cleanup(); // avoid memory leaks - exit(0); - } - - // in fork parent - bbstored_pid = WaitForServerStartup("testfiles/bbstored.pid", - bbstored_pid); + MockBackupProtocolLocal client(0x01234567, "test", + "backup/01234567/", 0, false); + MockBackupDaemon bbackupd(client); + TEST_THAT_OR(setup_test_bbackupd(bbackupd, false, false), FAIL); TEST_THAT(::system("rm -f testfiles/notifyran.store-full.*") == 0); - - // Ignore SIGPIPE so that when the connection is broken, - // the daemon doesn't terminate. - ::signal(SIGPIPE, SIG_IGN); + std::auto_ptr<BackupClientContext> apClientContext; { - Log::Level newLevel = Logging::GetGlobalLevel(); + Console& console(Logging::GetConsole()); + Logger::LevelGuard guard(console); - if (!Logging::IsEnabled(Log::TRACE)) + if (console.GetLevel() < Log::TRACE) { - newLevel = Log::NOTHING; + console.Filter(Log::NOTHING); } - Logging::Guard guard(newLevel); - - BackupDaemon bbackupd; - bbackupd.Configure("testfiles/bbackupd.conf"); - bbackupd.InitCrypto(); - bbackupd.RunSyncNowWithExceptionHandling(); + apClientContext = bbackupd.RunSyncNowWithExceptionHandling(); } - ::signal(SIGPIPE, SIG_DFL); - + // Should only have been triggered once + TEST_EQUAL(1, client.hook.trigger_count); TEST_THAT(TestFileExists("testfiles/notifyran.backup-error.1")); TEST_THAT(!TestFileExists("testfiles/notifyran.backup-error.2")); TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.1")); - - test_kill_bbstored(true); - - if (failures > 0) - { - // stop early to make debugging easier - return 1; - } - - TEST_THAT(test_run_bbstored() == 0); - - cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd.conf"; - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); - ::safe_sleep(1); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; } - #endif // !WIN32 +#endif // !WIN32 - #ifndef WIN32 - printf("\n==== Testing that absolute symlinks are not followed " - "during restore\n"); + TEARDOWN_TEST_BBACKUPD(); +} +bool test_absolute_symlinks_not_followed_during_restore() +{ + SETUP_WITH_BBSTORED(); + +#ifdef WIN32 + BOX_NOTICE("skipping test on this platform"); // requires symlinks +#else + // TODO FIXME dedent { #define SYM_DIR "testfiles" DIRECTORY_SEPARATOR "TestDir1" \ DIRECTORY_SEPARATOR "symlink_test" @@ -1825,26 +2111,24 @@ int test_bbackupd() char buf[PATH_MAX]; TEST_THAT(getcwd(buf, sizeof(buf)) == buf); std::string path = buf; - path += DIRECTORY_SEPARATOR SYM_DIR + // testfiles/TestDir1/symlink_test/a/subdir -> + // testfiles/TestDir1/symlink_test/b/link + path += DIRECTORY_SEPARATOR SYM_DIR DIRECTORY_SEPARATOR "a" DIRECTORY_SEPARATOR "subdir"; - TEST_THAT(symlink(path.c_str(), SYM_DIR + TEST_THAT(symlink(path.c_str(), SYM_DIR DIRECTORY_SEPARATOR "b" DIRECTORY_SEPARATOR "link") == 0); // also test symlink-to-self loop does not break restore TEST_THAT(symlink("self", SYM_DIR "/self") == 0); - wait_for_operation(4, "symlinks to be old enough"); - sync_and_wait(); + wait_for_operation(5, "symlinks to be old enough"); + bbackupd.RunSyncNow(); // Check that the backup was successful, i.e. no differences TEST_COMPARE(Compare_Same); - // now stop bbackupd and update the test file, - // make the original directory unreadable - terminate_bbackupd(bbackupd_pid); - fp = fopen(SYM_DIR DIRECTORY_SEPARATOR "a" DIRECTORY_SEPARATOR "subdir" DIRECTORY_SEPARATOR "content", "w"); @@ -1876,89 +2160,23 @@ int test_bbackupd() TEST_EQUAL("after", line); #undef SYM_DIR + } +#endif - /* - // This is not worth testing or fixing. - // - #ifndef PLATFORM_CLIB_FNS_INTERCEPTION_IMPOSSIBLE - printf("\n==== Testing that symlinks to other filesystems " - "can be backed up as roots\n"); - - intercept_setup_lstat_post_hook(lstat_test_post_hook); - TEST_THAT(symlink("TestDir1", "testfiles/symlink-to-TestDir1") - == 0); - - struct stat stat_st, lstat_st; - TEST_THAT(stat("testfiles/symlink-to-TestDir1", &stat_st) == 0); - TEST_THAT(lstat("testfiles/symlink-to-TestDir1", &lstat_st) == 0); - TEST_EQUAL_LINE((stat_st.st_dev ^ 0xFFFF), lstat_st.st_dev, - "stat vs lstat"); - - BackupDaemon bbackupd; - bbackupd.Configure("testfiles/bbackupd-symlink.conf"); - bbackupd.InitCrypto(); - bbackupd.RunSyncNow(); - intercept_clear_setup(); - - compareReturnValue = ::system(BBACKUPQUERY " " - "-c testfiles/bbackupd.conf " - "-l testfiles/query0a.log " - "-Wwarning \"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - - // and again using the symlink during compare - compareReturnValue = ::system(BBACKUPQUERY " " - "-c testfiles/bbackupd-symlink.conf " - "-l testfiles/query0a.log " - "-Wwarning \"compare -acQ\" quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - #endif - */ - - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); - ::safe_sleep(1); + TEARDOWN_TEST_BBACKUPD(); +} - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - } - #endif // !WIN32 +// Testing that nonexistent locations are backed up if they are created later +bool test_initially_missing_locations_are_not_forgotten() +{ + SETUP_WITH_BBSTORED(); - printf("\n==== Testing that nonexistent locations are backed up " - "if they are created later\n"); - // ensure that the directory does not exist at the start - TEST_THAT(::system("rm -rf testfiles/TestDir2") == 0); + TEST_THAT(!FileExists("testfiles/TestDir2")); + TEST_THAT(configure_bbackupd(bbackupd, "testfiles/bbackupd-temploc.conf")); // BLOCK { - // Kill the daemon - terminate_bbackupd(bbackupd_pid); - - // Delete any old result marker files - TEST_RETURN(0, system("rm -f testfiles/notifyran.*")); - - // Start it with a config that has a temporary location - // whose path does not exist yet - std::string cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd-temploc.conf"; - - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - TEST_THAT(!TestFileExists("testfiles/notifyran.backup-start.1")); TEST_THAT(!TestFileExists("testfiles/notifyran.backup-start.2")); TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); @@ -1967,7 +2185,7 @@ int test_bbackupd() TEST_THAT(!TestFileExists("testfiles/notifyran.backup-finish.1")); TEST_THAT(!TestFileExists("testfiles/notifyran.backup-finish.2")); - sync_and_wait(); + bbackupd.RunSyncNowWithExceptionHandling(); TEST_COMPARE(Compare_Same); TEST_THAT( TestFileExists("testfiles/notifyran.backup-start.1")); @@ -1979,31 +2197,15 @@ int test_bbackupd() TEST_THAT(!TestFileExists("testfiles/notifyran.backup-finish.2")); // Did it actually get created? Should not have been! - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, - BackupProtocolLogin::Flags_ReadOnly); - - std::auto_ptr<BackupStoreDirectory> dir = - ReadDirectory(*client); - int64_t testDirId = SearchDir(*dir, "Test2"); - TEST_THAT(testDirId == 0); - client->QueryFinished(); - sSocket.Close(); + TEST_THAT_OR(!search_for_file("Test2"), FAIL); } // create the location directory and unpack some files into it TEST_THAT(::mkdir("testfiles/TestDir2", 0777) == 0); - - #ifdef WIN32 - TEST_THAT(::system("tar xzvf testfiles/spacetest1.tgz " - "-C testfiles/TestDir2") == 0); - #else - TEST_THAT(::system("gzip -d < testfiles/spacetest1.tgz " - "| ( cd testfiles/TestDir2 && tar xf - )") == 0); - #endif + TEST_THAT(unpack_files("spacetest1", "testfiles/TestDir2")); // check that the files are backed up now - sync_and_wait(); + bbackupd.RunSyncNowWithExceptionHandling(); TEST_COMPARE(Compare_Same); TEST_THAT( TestFileExists("testfiles/notifyran.backup-start.2")); @@ -2016,187 +2218,128 @@ int test_bbackupd() TEST_THAT(!TestFileExists("testfiles/notifyran.backup-finish.3")); // BLOCK - { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, - BackupProtocolLogin::Flags_ReadOnly); - - std::auto_ptr<BackupStoreDirectory> dir = - ReadDirectory(*client, - BackupProtocolListDirectory::RootDirectory); - int64_t testDirId = SearchDir(*dir, "Test2"); - TEST_THAT(testDirId != 0); + TEST_THAT_OR(search_for_file("Test2"), FAIL); + TEARDOWN_TEST_BBACKUPD(); +} - client->QueryFinished(); - sSocket.Close(); - } +bool test_redundant_locations_deleted_on_time() +{ + SETUP_WITH_BBSTORED(); - printf("\n==== Testing that redundant locations are deleted on time\n"); + // create the location directory and unpack some files into it + TEST_THAT(::mkdir("testfiles/TestDir2", 0777) == 0); + TEST_THAT(unpack_files("spacetest1", "testfiles/TestDir2")); - // BLOCK + // Use a daemon with the TestDir2 location configured to back it up + // to the server. { - // Kill the daemon - terminate_bbackupd(bbackupd_pid); - - // Start it again with the normal config (no Test2) - cmd = BBACKUPD " " + bbackupd_args + " testfiles/bbackupd.conf"; - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); - - ::safe_sleep(1); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - // Test2 should be deleted after 10 seconds (4 runs) - wait_for_sync_end(); - wait_for_sync_end(); - wait_for_sync_end(); - - // not yet! should still be there - - { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, - BackupProtocolLogin::Flags_ReadOnly); - - std::auto_ptr<BackupStoreDirectory> dir = - ReadDirectory(*client); - int64_t testDirId = SearchDir(*dir, "Test2"); - TEST_THAT(testDirId != 0); + TEST_THAT(configure_bbackupd(bbackupd, "testfiles/bbackupd-temploc.conf")); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); + } - client->QueryFinished(); - sSocket.Close(); - } + // Now use a daemon with no temporary location, which should delete + // it after 10 seconds + { + TEST_THAT(configure_bbackupd(bbackupd, "testfiles/bbackupd.conf")); - wait_for_sync_end(); + // Initial run to start the countdown to destruction + bbackupd.RunSyncNow(); - // NOW it should be gone + // Not deleted yet! + TEST_THAT(search_for_file("Test2")); - { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, - BackupProtocolLogin::Flags_ReadOnly); - - std::auto_ptr<BackupStoreDirectory> root_dir = - ReadDirectory(*client); + wait_for_operation(9, "just before Test2 should be deleted"); + bbackupd.RunSyncNow(); + TEST_THAT(search_for_file("Test2")); - TEST_THAT(test_entry_deleted(*root_dir, "Test2")); + // Now wait until after it should be deleted + wait_for_operation(2, "just after Test2 should be deleted"); + bbackupd.RunSyncNow(); - client->QueryFinished(); - sSocket.Close(); - } + TEST_THAT(search_for_file("Test2")); + std::auto_ptr<BackupProtocolCallable> client = connect_and_login( + sTlsContext, 0 /* read-write */); + std::auto_ptr<BackupStoreDirectory> root_dir = + ReadDirectory(*client, BACKUPSTORE_ROOT_DIRECTORY_ID); + TEST_THAT(test_entry_deleted(*root_dir, "Test2")); } - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - if(bbackupd_pid > 0) - { - // Delete any old result marker files - TEST_RETURN(0, system("rm -f testfiles/notifyran.*")); - - printf("\n==== Check that read-only directories and " - "their contents can be restored.\n"); + TEARDOWN_TEST_BBACKUPD(); +} - int compareReturnValue; +// Check that read-only directories and their contents can be restored. +bool test_read_only_dirs_can_be_restored() +{ + SETUP_WITH_BBSTORED(); + // TODO FIXME dedent + { { #ifdef WIN32 - TEST_THAT(::system("chmod 0555 testfiles/" - "TestDir1/x1") == 0); + TEST_THAT(::system("attrib +r testfiles\\TestDir1\\x1") + == 0); #else TEST_THAT(chmod("testfiles/TestDir1/x1", 0555) == 0); #endif - wait_for_sync_end(); // too new - wait_for_sync_end(); // should be backed up now - - compareReturnValue = ::system(BBACKUPQUERY " " - "-Wwarning " - "-c testfiles/bbackupd.conf " - "\"compare -cEQ Test1 testfiles/TestDir1\" " - "quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same, "", "-cEQ Test1 testfiles/TestDir1"); // check that we can restore it - compareReturnValue = ::system(BBACKUPQUERY " " - "-Wwarning " - "-c testfiles/bbackupd.conf " - "\"restore Test1 testfiles/restore1\" " - "quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - - // check that it restored properly - compareReturnValue = ::system(BBACKUPQUERY " " - "-Wwarning " - "-c testfiles/bbackupd.conf " - "\"compare -cEQ Test1 testfiles/restore1\" " - "quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(restore("Test1", "testfiles/restore1")); + TEST_COMPARE(Compare_Same, "", "-cEQ Test1 testfiles/restore1"); // Try a restore with just the remote directory name, // check that it uses the same name in the local // directory. TEST_THAT(::mkdir("testfiles/restore-test", 0700) == 0); - - compareReturnValue = ::system(BBACKUPQUERY " " - "-Wwarning " - "-c testfiles/bbackupd.conf " - "\"lcd testfiles/restore-test\" " - "\"restore Test1\" " - "quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); - TestRemoteProcessMemLeaks("testfiles/restore-test/" - "bbackupquery.memleaks"); - - // check that it restored properly - compareReturnValue = ::system(BBACKUPQUERY " " - "-Wwarning " - "-c testfiles/bbackupd.conf " - "\"compare -cEQ Test1 testfiles/restore-test/Test1\" " - "quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_THAT(bbackupquery("\"lcd testfiles/restore-test\" " + "\"restore Test1\"")); + TEST_COMPARE(Compare_Same, "", "-cEQ Test1 " + "testfiles/restore-test/Test1"); // put the permissions back to sensible values #ifdef WIN32 - TEST_THAT(::system("chmod 0755 testfiles/" - "TestDir1/x1") == 0); - TEST_THAT(::system("chmod 0755 testfiles/" - "restore1/x1") == 0); + TEST_THAT(::system("attrib -r testfiles\\TestDir1\\x1") + == 0); + TEST_THAT(::system("attrib -r testfiles\\restore1\\x1") + == 0); + TEST_THAT(::system("attrib -r testfiles\\restore-test\\" + "Test1\\x1") == 0); #else TEST_THAT(chmod("testfiles/TestDir1/x1", 0755) == 0); TEST_THAT(chmod("testfiles/restore1/x1", 0755) == 0); + TEST_THAT(chmod("testfiles/restore-test/Test1/x1", + 0755) == 0); #endif } + } -#ifdef WIN32 - printf("\n==== Check that filenames in UTF-8 " - "can be backed up\n"); + TEARDOWN_TEST_BBACKUPD(); +} +// Check that filenames in UTF-8 can be backed up +bool test_unicode_filenames_can_be_backed_up() +{ + SETUP_WITH_BBSTORED(); + +#ifndef WIN32 + BOX_NOTICE("skipping test on this platform"); + // requires ConvertConsoleToUtf8() +#else + // TODO FIXME dedent + { // We have no guarantee that a random Unicode string can be // represented in the user's character set, so we go the // other way, taking three random characters from the - // character set and converting them to Unicode. + // character set and converting them to Unicode. Unless the + // console codepage is CP_UTF8, in which case our random + // characters are not valid, so we use the UTF8 version + // of them instead. // // We hope that these characters are valid in most // character sets, but they probably are not in multibyte @@ -2216,7 +2359,8 @@ int test_bbackupd() // accented characters, which are supported in code page // 850. Depending on your locale, YYMV (your yak may vomit). - std::string foreignCharsNative("\x91\x9b\x86"); + std::string foreignCharsNative = (GetConsoleCP() == CP_UTF8) + ? "\xc3\xa6\xc3\xb8\xc3\xa5" : "\x91\x9b\x86"; std::string foreignCharsUnicode; TEST_THAT(ConvertConsoleToUtf8(foreignCharsNative.c_str(), foreignCharsUnicode)); @@ -2260,46 +2404,13 @@ int test_bbackupd() consoleFileName)); // test that bbackupd will let us lcd into the local - // directory using a relative path - std::string command = BBACKUPQUERY " " - "-Wwarning " - "-c testfiles/bbackupd.conf " - "\"lcd testfiles/TestDir1/" + systemDirName + "\" " - "quit"; - compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); - - // and back out again - command = BBACKUPQUERY " " - "-Wwarning " - "-c testfiles/bbackupd.conf " - "\"lcd testfiles/TestDir1/" + systemDirName + "\" " - "\"lcd ..\" quit"; - compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); + // directory using a relative path, and back out + TEST_THAT(bbackupquery("\"lcd testfiles/TestDir1/" + + systemDirName + "\" \"lcd ..\"")); // and using an absolute path - command = BBACKUPQUERY " " - "-Wwarning " - "-c testfiles/bbackupd.conf " - "\"lcd " + cwd + "/testfiles/TestDir1/" + - systemDirName + "\" quit"; - compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); - - // and back out again - command = BBACKUPQUERY " " - "-Wwarning " - "-c testfiles/bbackupd.conf " - "\"lcd " + cwd + "/testfiles/TestDir1/" + - systemDirName + "\" " - "\"lcd ..\" quit"; - compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); + TEST_THAT(bbackupquery("\"lcd " + cwd + "/testfiles/" + + "TestDir1/" + systemDirName + "\" \"lcd ..\"")); { FileStream fs(filepath.c_str(), O_CREAT | O_RDWR); @@ -2309,17 +2420,22 @@ int test_bbackupd() TEST_EQUAL_LINE(12, fs.GetPosition(), "FileStream position"); fs.Close(); + + // Set modtime back in time to allow immediate backup + struct timeval times[2] = {}; + times[1].tv_sec = 1000000000; + TEST_THAT(emu_utimes(filepath.c_str(), times) == 0); } - wait_for_backup_operation("upload of file with unicode name"); + bbackupd.RunSyncNow(); // Compare to check that the file was uploaded TEST_COMPARE(Compare_Same); // Check that we can find it in directory listing { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, 0); + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, 0); std::auto_ptr<BackupStoreDirectory> dir = ReadDirectory( *client); @@ -2335,11 +2451,10 @@ int test_bbackupd() TEST_THAT(SearchDir(*dir, filename.c_str()) != 0); // Log out client->QueryFinished(); - sSocket.Close(); } // Check that bbackupquery shows the dir in console encoding - command = BBACKUPQUERY " -Wwarning " + std::string command = BBACKUPQUERY " -Wwarning " "-c testfiles/bbackupd.conf " "-q \"list Test1\" quit"; pid_t bbackupquery_pid; @@ -2392,35 +2507,17 @@ int test_bbackupd() // Check that bbackupquery can compare the dir when given // on the command line in system encoding. - command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-Wwarning \"compare -cEQ Test1/" + systemDirName + - " testfiles/TestDir1/" + systemDirName + "\" quit"; - - compareReturnValue = ::system(command.c_str()); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); + TEST_COMPARE(Compare_Same, "", "-cEQ Test1/" + systemDirName + + " testfiles/TestDir1/" + systemDirName); // Check that bbackupquery can restore the dir when given // on the command line in system encoding. - command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-Wwarning \"restore Test1/" + systemDirName + - " testfiles/restore-" + systemDirName + "\" quit"; - - compareReturnValue = ::system(command.c_str()); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); + TEST_THAT(restore("Test1/" + systemDirName, + "testfiles/restore-" + systemDirName)); // Compare to make sure it was restored properly. - command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-Wwarning \"compare -cEQ Test1/" + systemDirName + - " testfiles/restore-" + systemDirName + "\" quit"; - - compareReturnValue = ::system(command.c_str()); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); + TEST_COMPARE(Compare_Same, "", "-cEQ Test1/" + systemDirName + + " testfiles/restore-" + systemDirName); std::string fileToUnlink = "testfiles/restore-" + dirname + "/" + filename; @@ -2428,80 +2525,47 @@ int test_bbackupd() // Check that bbackupquery can get the file when given // on the command line in system encoding. - command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-Wwarning \"get Test1/" + systemDirName + "/" + + TEST_THAT(bbackupquery("\"get Test1/" + systemDirName + "/" + systemFileName + " " + "testfiles/restore-" + - systemDirName + "/" + systemFileName + "\" quit"; - - compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + systemDirName + "/" + systemFileName + "\"")); // And after changing directory to a relative path - command = BBACKUPQUERY " -c testfiles/bbackupd.conf " - "-Wwarning " + TEST_THAT(bbackupquery( "\"lcd testfiles\" " "\"cd Test1/" + systemDirName + "\" " + - "\"get " + systemFileName + "\" quit"; - - compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); - TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks"); + "\"get " + systemFileName + "\"")); // cannot overwrite a file that exists, so delete it std::string tmp = "testfiles/" + filename; TEST_THAT(::unlink(tmp.c_str()) == 0); // And after changing directory to an absolute path - command = BBACKUPQUERY " -c testfiles/bbackupd.conf -Wwarning " + TEST_THAT(bbackupquery( "\"lcd " + cwd + "/testfiles\" " "\"cd Test1/" + systemDirName + "\" " + - "\"get " + systemFileName + "\" quit"; - - compareReturnValue = ::system(command.c_str()); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Command_OK); - TestRemoteProcessMemLeaks("testfiles/bbackupquery.memleaks"); - - // Compare to make sure it was restored properly. - // The Get command does not restore attributes, so - // we must compare without them (-A) to succeed. - command = BBACKUPQUERY " " - "-c testfiles/bbackupd.conf " - "-Wwarning \"compare -cAEQ Test1/" + systemDirName + - " testfiles/restore-" + systemDirName + "\" quit"; + "\"get " + systemFileName + "\"")); - compareReturnValue = ::system(command.c_str()); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - - // Compare without attributes. This should fail. - command = BBACKUPQUERY " " - "-c testfiles/bbackupd.conf " - "-Werror \"compare -cEQ Test1/" + systemDirName + - " testfiles/restore-" + systemDirName + "\" quit"; - compareReturnValue = ::system(command.c_str()); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Different); -#endif // WIN32 + // Compare to make sure it was restored properly. The Get + // command does restore attributes, so we don't need to + // specify the -A option for this to succeed. + TEST_COMPARE(Compare_Same, "", "-cEQ Test1/" + systemDirName + + " testfiles/restore-" + systemDirName); // Check that no read error has been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + } +#endif // WIN32 + + TEARDOWN_TEST_BBACKUPD(); +} - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - printf("\n==== Check that SyncAllowScript is executed and can " - "pause backup\n"); - fflush(stdout); +bool test_sync_allow_script_can_pause_backup() +{ + SETUP_WITH_BBSTORED(); + TEST_THAT(StartClient()); + // TODO FIXME dedent + { { wait_for_sync_end(); // we now have 3 seconds before bbackupd @@ -2572,7 +2636,6 @@ int test_bbackupd() TEST_THAT(unlink(sync_control_file) == 0); wait_for_sync_start(); long end_time = time(NULL); - long wait_time = end_time - start_time + 2; // should be about 10 seconds @@ -2589,36 +2652,38 @@ int test_bbackupd() // check that backup has run (compare succeeds) TEST_COMPARE(Compare_Same); - - if (failures) return 1; } // Check that no read error has been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + } - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + TEARDOWN_TEST_BBACKUPD(); +} - printf("\n==== Delete file and update another, " - "create symlink.\n"); - +// Delete file and update another, create symlink. +bool test_delete_update_and_symlink_files() +{ + SETUP_WITH_BBSTORED(); + + bbackupd.RunSyncNow(); + + // TODO FIXME dedent + { // Delete a file TEST_THAT(::unlink("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); #ifndef WIN32 // New symlink - TEST_THAT(::symlink("does-not-exist", + TEST_THAT(::symlink("does-not-exist", "testfiles/TestDir1/symlink-to-dir") == 0); #endif // Update a file (will be uploaded as a diff) { - // Check that the file is over the diffing + // Check that the file is over the diffing // threshold in the bbackupd.conf file - TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") + TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") > 1024); // Add a bit to the end @@ -2626,39 +2691,37 @@ int test_bbackupd() TEST_THAT(f != 0); ::fprintf(f, "EXTRA STUFF"); ::fclose(f); - TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") + TEST_THAT(TestGetFileSize("testfiles/TestDir1/f45.df") > 1024); } // wait long enough for new files to be old enough to backup wait_for_operation(5, "new files to be old enough"); - // wait for backup daemon to do it's stuff - sync_and_wait(); + bbackupd.RunSyncNow(); // compare to make sure that it worked TEST_COMPARE(Compare_Same); // Try a quick compare, just for fun - compareReturnValue = ::system(BBACKUPQUERY " " - "-c testfiles/bbackupd.conf " - "-l testfiles/query2q.log " - "-Wwarning \"compare -acqQ\" quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - // Check that store errors are reported neatly - printf("\n==== Create store error\n"); - TEST_THAT(system("rm -f testfiles/notifyran.backup-error.*") - == 0); + TEST_COMPARE(Compare_Same, "", "-acqQ"); + } + + TEARDOWN_TEST_BBACKUPD(); +} + +// Check that store errors are reported neatly. This test uses an independent +// daemon to check the daemon's backup loop delay, so it's easier to debug +// with the command: ./t -VTttest -e test_store_error_reporting +// --bbackupd-args=-kTtbbackupd +bool test_store_error_reporting() +{ + SETUP_WITH_BBSTORED(); + TEST_THAT(StartClient()); + wait_for_sync_end(); + // TODO FIXME dedent + { // Break the store. We need a write lock on the account // while we do this, otherwise housekeeping might be running // and might rewrite the info files when it finishes, @@ -2669,10 +2732,7 @@ int test_bbackupd() ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); TEST_EQUAL_LINE(0, errs.size(), "Loading configuration file " "reported errors: " << errs); - TEST_THAT(config.get() != 0); - // Initialise the raid file controller - RaidFileController &rcontroller(RaidFileController::GetController()); - rcontroller.Initialise(config->GetKeyValue("RaidFileConf").c_str()); + TEST_THAT_OR(config.get(), return false); std::auto_ptr<BackupStoreAccountDatabase> db( BackupStoreAccountDatabase::Read( config->GetKeyValue("AccountDatabase"))); @@ -2709,18 +2769,8 @@ int test_bbackupd() // Now kill bbackupd and start one that's running in // snapshot mode, check that it automatically syncs after // an error, without waiting for another sync command. - terminate_bbackupd(bbackupd_pid); - std::string cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd-snapshot.conf"; - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); - ::safe_sleep(1); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - + TEST_THAT(StopClient()); + TEST_THAT(StartClient("testfiles/bbackupd-snapshot.conf")); sync_and_wait(); // Check that the error was reported once more @@ -2755,10 +2805,12 @@ int test_bbackupd() TEST_THAT(close(fd1) == 0); } - // bbackupd should pause for about 90 seconds from - // store_fixed_time, so check that it hasn't run after - // 85 seconds after store_fixed_time - wait_for_operation(85 - time(NULL) + store_fixed_time, + // bbackupd should pause for BACKUP_ERROR_RETRY_SECONDS (plus + // a random delay of up to mUpdateStoreInterval/64 or 0.05 + // extra seconds) from store_fixed_time, so check that it + // hasn't run just before this time + wait_for_operation(BACKUP_ERROR_DELAY_SHORTENED + + (store_fixed_time - time(NULL)) - 1, "just before bbackupd recovers"); TEST_THAT(!TestFileExists("testfiles/" "notifyran.backup-start.wait-snapshot.1")); @@ -2766,8 +2818,8 @@ int test_bbackupd() // Should not have backed up, should still get errors TEST_COMPARE(Compare_Different); - // wait another 10 seconds, bbackup should have run - wait_for_operation(10, "bbackupd to recover"); + // wait another 2 seconds, bbackup should have run + wait_for_operation(2, "bbackupd to recover"); TEST_THAT(TestFileExists("testfiles/" "notifyran.backup-start.wait-snapshot.1")); @@ -2777,7 +2829,7 @@ int test_bbackupd() TEST_THAT(::unlink("testfiles/notifyscript.tag") == 0); // Stop the snapshot bbackupd - terminate_bbackupd(bbackupd_pid); + TEST_THAT(StopClient()); // Break the store again TEST_THAT(::rename("testfiles/0_0/backup/01234567/info.rf", @@ -2796,18 +2848,8 @@ int test_bbackupd() TEST_THAT(close(fd1) == 0); } - // Restart the old bbackupd, in automatic mode - cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd.conf"; - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); - ::safe_sleep(1); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - + // Restart bbackupd in automatic mode + TEST_THAT_OR(StartClient(), FAIL); sync_and_wait(); // Fix the store again @@ -2838,10 +2880,12 @@ int test_bbackupd() TEST_THAT(close(fd1) == 0); } - // bbackupd should pause for about 90 seconds from - // store_fixed_time, so check that it hasn't run after - // 85 seconds from store_fixed_time - wait_for_operation(85 - time(NULL) + store_fixed_time, + // bbackupd should pause for BACKUP_ERROR_RETRY_SECONDS (plus + // a random delay of up to mUpdateStoreInterval/64 or 0.05 + // extra seconds) from store_fixed_time, so check that it + // hasn't run just before this time + wait_for_operation(BACKUP_ERROR_DELAY_SHORTENED + + (store_fixed_time - time(NULL)) - 1, "just before bbackupd recovers"); TEST_THAT(!TestFileExists("testfiles/" "notifyran.backup-start.wait-automatic.1")); @@ -2849,8 +2893,8 @@ int test_bbackupd() // Should not have backed up, should still get errors TEST_COMPARE(Compare_Different); - // wait another 10 seconds, bbackup should have run - wait_for_operation(10, "bbackupd to recover"); + // wait another 2 seconds, bbackup should have run + wait_for_operation(2, "bbackupd to recover"); TEST_THAT(TestFileExists("testfiles/" "notifyran.backup-start.wait-automatic.1")); @@ -2858,28 +2902,39 @@ int test_bbackupd() TEST_COMPARE(Compare_Same); TEST_THAT(::unlink("testfiles/notifyscript.tag") == 0); + } + + TEARDOWN_TEST_BBACKUPD(); +} - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; +bool test_change_file_to_symlink_and_back() +{ + SETUP_WITH_BBSTORED(); + + #ifndef WIN32 + // New symlink + TEST_THAT(::symlink("does-not-exist", + "testfiles/TestDir1/symlink-to-dir") == 0); + #endif - // Bad case: delete a file/symlink, replace it with a directory - printf("\n==== Replace symlink with directory, " - "add new directory\n"); + bbackupd.RunSyncNow(); + + // TODO FIXME dedent + { + // Bad case: delete a file/symlink, replace it with a directory. + // Replace symlink with directory, add new directory. #ifndef WIN32 TEST_THAT(::unlink("testfiles/TestDir1/symlink-to-dir") == 0); #endif - TEST_THAT(::mkdir("testfiles/TestDir1/symlink-to-dir", 0755) + TEST_THAT(::mkdir("testfiles/TestDir1/symlink-to-dir", 0755) == 0); - TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", 0755) + TEST_THAT(::mkdir("testfiles/TestDir1/x1/dir-to-file", 0755) == 0); - // NOTE: create a file within the directory to + // NOTE: create a file within the directory to // avoid deletion by the housekeeping process later #ifndef WIN32 @@ -2888,17 +2943,11 @@ int test_bbackupd() == 0); #endif - wait_for_backup_operation("bbackupd to sync the changes"); + wait_for_operation(5, "files to be old enough"); + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - // And the inverse, replace a directory with a file/symlink - printf("\n==== Replace directory with symlink\n"); #ifndef WIN32 TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file" @@ -2912,19 +2961,12 @@ int test_bbackupd() "testfiles/TestDir1/x1/dir-to-file") == 0); #endif - wait_for_backup_operation("bbackupd to sync the changes"); - + wait_for_operation(5, "files to be old enough"); + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - // And then, put it back to how it was before. - printf("\n==== Replace symlink with directory " - "(which was a symlink)\n"); + BOX_INFO("Replace symlink with directory (which was a symlink)"); #ifndef WIN32 TEST_THAT(::unlink("testfiles/TestDir1/x1" @@ -2940,21 +2982,14 @@ int test_bbackupd() == 0); #endif - wait_for_backup_operation("bbackupd to sync the changes"); - + wait_for_operation(5, "files to be old enough"); + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; // And finally, put it back to how it was before // it was put back to how it was before // This gets lots of nasty things in the store with // directories over other old directories. - printf("\n==== Put it all back to how it was\n"); #ifndef WIN32 TEST_THAT(::unlink("testfiles/TestDir1/x1/dir-to-file" @@ -2968,19 +3003,23 @@ int test_bbackupd() "testfiles/TestDir1/x1/dir-to-file") == 0); #endif - wait_for_backup_operation("bbackupd to sync the changes"); - + wait_for_operation(5, "files to be old enough"); + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); + } + + TEARDOWN_TEST_BBACKUPD(); +} - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; +bool test_file_rename_tracking() +{ + SETUP_WITH_BBSTORED(); + bbackupd.RunSyncNow(); - // rename an untracked file over an - // existing untracked file - printf("\n==== Rename over existing untracked file\n"); + // TODO FIXME dedent + { + // rename an untracked file over an existing untracked file + BOX_INFO("Rename over existing untracked file"); int fd1 = open("testfiles/TestDir1/untracked-1", O_CREAT | O_EXCL | O_WRONLY, 0700); int fd2 = open("testfiles/TestDir1/untracked-2", @@ -2997,9 +3036,7 @@ int test_bbackupd() // back up both files wait_for_operation(5, "untracked files to be old enough"); - wait_for_backup_operation("bbackupd to sync the " - "untracked files"); - + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); #ifdef WIN32 @@ -3012,21 +3049,13 @@ int test_bbackupd() TEST_THAT(!TestFileExists("testfiles/TestDir1/untracked-1")); TEST_THAT( TestFileExists("testfiles/TestDir1/untracked-2")); - wait_for_backup_operation("bbackupd to sync the untracked " - "files again"); - + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - // case which went wrong: rename a tracked file over an // existing tracked file - printf("\n==== Rename over existing tracked file\n"); - fd1 = open("testfiles/TestDir1/tracked-1", + BOX_INFO("Rename over existing tracked file"); + fd1 = open("testfiles/TestDir1/tracked-1", O_CREAT | O_EXCL | O_WRONLY, 0700); fd2 = open("testfiles/TestDir1/tracked-2", O_CREAT | O_EXCL | O_WRONLY, 0700); @@ -3045,11 +3074,7 @@ int test_bbackupd() // wait for them to be old enough to back up wait_for_operation(5, "tracked files to be old enough"); - - // back up both files - sync_and_wait(); - - // compare to make sure that it worked + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); #ifdef WIN32 @@ -3057,91 +3082,71 @@ int test_bbackupd() == 0); #endif - TEST_THAT(::rename("testfiles/TestDir1/tracked-1", + TEST_THAT(::rename("testfiles/TestDir1/tracked-1", "testfiles/TestDir1/tracked-2") == 0); TEST_THAT(!TestFileExists("testfiles/TestDir1/tracked-1")); TEST_THAT( TestFileExists("testfiles/TestDir1/tracked-2")); - wait_for_backup_operation("bbackupd to sync the tracked " - "files again"); - + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - // case which went wrong: rename a tracked file + // case which went wrong: rename a tracked file // over a deleted file - printf("\n==== Rename an existing file over a deleted file\n"); - TEST_THAT(!TestFileExists("testfiles/TestDir1/x1/dsfdsfs98.fd")); - TEST_THAT(::rename("testfiles/TestDir1/df9834.dsf", + BOX_INFO("Rename an existing file over a deleted file"); + TEST_THAT(::unlink("testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); + TEST_THAT(::rename("testfiles/TestDir1/df9834.dsf", "testfiles/TestDir1/x1/dsfdsfs98.fd") == 0); - - wait_for_backup_operation("bbackupd to sync"); + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - + // Check that no read error has been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + } - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + TEARDOWN_TEST_BBACKUPD(); +} - printf("\n==== Add files with old times, update " - "attributes of one to latest time\n"); +// Files that suddenly appear, with timestamps before the last sync window, +// and files whose size or timestamp change, should still be uploaded, even +// though they look old. +bool test_upload_very_old_files() +{ + SETUP_WITH_BBSTORED(); + bbackupd.RunSyncNow(); - // Move that file back - TEST_THAT(::rename("testfiles/TestDir1/x1/dsfdsfs98.fd", - "testfiles/TestDir1/df9834.dsf") == 0); - + // TODO FIXME dedent + { // Add some more files - // Because the 'm' option is not used, these files will + // Because the 'm' option is not used, these files will // look very old to the daemon. // Lucky it'll upload them then! - #ifdef WIN32 - TEST_THAT(::system("tar xzvf testfiles/test2.tgz " - "-C testfiles") == 0); - #else - TEST_THAT(::system("gzip -d < testfiles/test2.tgz " - "| ( cd testfiles && tar xf - )") == 0); + TEST_THAT(unpack_files("test2")); + + #ifndef WIN32 ::chmod("testfiles/TestDir1/sub23/dhsfdss/blf.h", 0415); #endif // Wait and test - wait_for_backup_operation("bbackupd to sync old files"); - + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - // Check that no read error has been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); // Check that modifying files with old timestamps // still get added - printf("\n==== Modify existing file, but change timestamp " - "to rather old\n"); - wait_for_sync_end(); + BOX_INFO("Modify existing file, but change timestamp to rather old"); // Then modify an existing file { // in the archive, it's read only #ifdef WIN32 - TEST_THAT(::system("chmod 0777 testfiles" - "/TestDir1/sub23/rand.h") == 0); + TEST_THAT(::system("attrib -r " + "testfiles\\TestDir\\sub23\\rand.h") == 0); #else - TEST_THAT(chmod("testfiles/TestDir1/sub23" - "/rand.h", 0777) == 0); + TEST_THAT(chmod("testfiles/TestDir1/sub23/rand.h", + 0777) == 0); #endif FILE *f = fopen("testfiles/TestDir1/sub23/rand.h", @@ -3170,69 +3175,60 @@ int test_bbackupd() } // Wait and test - wait_for_sync_end(); // files too new - wait_for_sync_end(); // should (not) be backed up this time - - TEST_COMPARE(Compare_Same); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); // files too new? // Check that no read error has been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); + } + + TEARDOWN_TEST_BBACKUPD(); +} + +bool test_excluded_files_are_not_backed_up() +{ + // SETUP_WITH_BBSTORED(); + SETUP_TEST_BBACKUPD(); + + BackupProtocolLocal2 client(0x01234567, "test", "backup/01234567/", + 0, false); + MockBackupDaemon bbackupd(client); + + TEST_THAT_OR(setup_test_bbackupd(bbackupd, + true, // do_unpack_files + false // do_start_bbstored + ), FAIL); + // TODO FIXME dedent + { // Add some files and directories which are marked as excluded - printf("\n==== Add files and dirs for exclusion test\n"); - #ifdef WIN32 - TEST_THAT(::system("tar xzvf testfiles/testexclude.tgz " - "-C testfiles") == 0); - #else - TEST_THAT(::system("gzip -d < " - "testfiles/testexclude.tgz " - "| ( cd testfiles && tar xf - )") == 0); - #endif + TEST_THAT(unpack_files("testexclude")); + bbackupd.RunSyncNow(); - // Wait and test - wait_for_sync_end(); - wait_for_sync_end(); - // compare with exclusions, should not find differences - TEST_COMPARE(Compare_Same); + // TEST_COMPARE(Compare_Same); + TEST_COMPARE_LOCAL(Compare_Same, client); // compare without exclusions, should find differences - compareReturnValue = ::system(BBACKUPQUERY " " - "-c testfiles/bbackupd.conf " - "-l testfiles/query3n.log " - "-Werror \"compare -acEQ\" quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Different); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + // TEST_COMPARE(Compare_Different, "", "-acEQ"); + TEST_COMPARE_LOCAL(Compare_Different, client, "acEQ"); // check that the excluded files did not make it // into the store, and the included files did - printf("\n==== Check that exclude/alwaysinclude commands " - "actually work\n"); - { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, + /* + std::auto_ptr<BackupProtocolCallable> pClient = + connect_and_login(context, BackupProtocolLogin::Flags_ReadOnly); + */ + BackupProtocolCallable* pClient = &client; - std::auto_ptr<BackupStoreDirectory> dir = - ReadDirectory(*client); + std::auto_ptr<BackupStoreDirectory> dir = + ReadDirectory(*pClient); int64_t testDirId = SearchDir(*dir, "Test1"); TEST_THAT(testDirId != 0); - dir = ReadDirectory(*client, testDirId); + dir = ReadDirectory(*pClient, testDirId); TEST_THAT(!SearchDir(*dir, "excluded_1")); TEST_THAT(!SearchDir(*dir, "excluded_2")); @@ -3247,32 +3243,44 @@ int test_bbackupd() int64_t sub23id = SearchDir(*dir, "sub23"); TEST_THAT(sub23id != 0); - dir = ReadDirectory(*client, sub23id); + dir = ReadDirectory(*pClient, sub23id); TEST_THAT(!SearchDir(*dir, "xx_not_this_dir_22")); TEST_THAT(!SearchDir(*dir, "somefile.excludethis")); - client->QueryFinished(); - sSocket.Close(); + + // client->QueryFinished(); } + } - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + TEARDOWN_TEST_BBACKUPD(); +} -#ifndef WIN32 +bool test_read_error_reporting() +{ + SETUP_WITH_BBSTORED(); + +#ifdef WIN32 + BOX_NOTICE("skipping test on this platform"); +#else + if(::getuid() == 0) + { + BOX_NOTICE("skipping test because we're running as root"); // These tests only work as non-root users. - if(::getuid() != 0) + } + else + { + // TODO FIXME detent { // Check that the error has not been reported yet TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.1")); // Check that read errors are reported neatly - printf("\n==== Add unreadable files\n"); - + BOX_INFO("Add unreadable files"); + { // Dir and file which can't be read + TEST_THAT(::mkdir("testfiles/TestDir1/sub23", + 0755) == 0); TEST_THAT(::mkdir("testfiles/TestDir1/sub23" "/read-fail-test-dir", 0000) == 0); int fd = ::open("testfiles/TestDir1" @@ -3282,9 +3290,8 @@ int test_bbackupd() ::close(fd); } - // Wait and test... - wait_for_backup_operation("bbackupd to try to sync " - "unreadable file"); + // Wait and test... with sysadmin notification + bbackupd.RunSyncNowWithExceptionHandling(); // should fail with an error due to unreadable file TEST_COMPARE(Compare_Error); @@ -3302,17 +3309,19 @@ int test_bbackupd() TEST_THAT(::chmod("testfiles/TestDir1" "/read-fail-test-file", 0770) == 0); } + } #endif - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + TEARDOWN_TEST_BBACKUPD(); +} - printf("\n==== Continuously update file, " - "check isn't uploaded\n"); - +bool test_continuously_updated_file() +{ + SETUP_WITH_BBSTORED(); + TEST_THAT(StartClient()); + + // TODO FIXME dedent + { // Make sure everything happens at the same point in the // sync cycle: wait until exactly the start of a sync wait_for_sync_start(); @@ -3321,7 +3330,8 @@ int test_bbackupd() ::safe_sleep(1); { - // Open a file, then save something to it every second + BOX_INFO("Open a file, then save something to it " + "every second for 12 seconds"); for(int l = 0; l < 12; ++l) { FILE *f = ::fopen("testfiles/TestDir1/continousupdate", "w+"); @@ -3329,23 +3339,18 @@ int test_bbackupd() fprintf(f, "Loop iteration %d\n", l); fflush(f); fclose(f); - - printf("."); - fflush(stdout); safe_sleep(1); } - printf("\n"); - fflush(stdout); // Check there's a difference - compareReturnValue = ::system("perl testfiles/" + int compareReturnValue = ::system("perl testfiles/" "extcheck1.pl"); TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - printf("\n==== Keep on continuously updating file, " - "check it is uploaded eventually\n"); + BOX_INFO("Keep on continuously updating file for " + "28 seconds, check it is uploaded eventually"); for(int l = 0; l < 28; ++l) { @@ -3355,13 +3360,8 @@ int test_bbackupd() fprintf(f, "Loop 2 iteration %d\n", l); fflush(f); fclose(f); - - printf("."); - fflush(stdout); safe_sleep(1); } - printf("\n"); - fflush(stdout); compareReturnValue = ::system("perl testfiles/" "extcheck2.pl"); @@ -3369,42 +3369,62 @@ int test_bbackupd() TEST_RETURN(compareReturnValue, 1); TestRemoteProcessMemLeaks("bbackupquery.memleaks"); } - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + } - printf("\n==== Delete directory, change attributes\n"); - + TEARDOWN_TEST_BBACKUPD(); +} + +bool test_delete_dir_change_attribute() +{ + SETUP_WITH_BBSTORED(); + bbackupd.RunSyncNow(); + + // TODO FIXME dedent + { // Delete a directory - TEST_THAT(::system("rm -rf testfiles/TestDir1/x1") == 0); - // Change attributes on an original file. - ::chmod("testfiles/TestDir1/df9834.dsf", 0423); - - // Wait and test - wait_for_backup_operation("bbackupd to sync deletion " - "of directory"); +#ifdef WIN32 + TEST_THAT(::system("rd /s/q testfiles\\TestDir1\\x1") == 0); +#else + TEST_THAT(::system("rm -r testfiles/TestDir1/x1") == 0); +#endif + // Change attributes on an existing file. +#ifdef WIN32 + TEST_EQUAL(0, system("attrib +r testfiles\\TestDir1\\df9834.dsf")); +#else + TEST_THAT(::chmod("testfiles/TestDir1/df9834.dsf", 0423) == 0); +#endif + TEST_COMPARE(Compare_Different); + + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - - printf("\n==== Restore files and directories\n"); + } + + TEARDOWN_TEST_BBACKUPD(); +} + +bool test_restore_files_and_directories() +{ + SETUP_WITH_BBSTORED(); + bbackupd.RunSyncNow(); + + // TODO FIXME dedent + { int64_t deldirid = 0; int64_t restoredirid = 0; { // connect and log in - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, BackupProtocolLogin::Flags_ReadOnly); // Find the ID of the Test1 directory - restoredirid = GetDirID(*client, "Test1", + restoredirid = GetDirID(*client, "Test1", BackupProtocolListDirectory::RootDirectory); - TEST_THAT(restoredirid != 0); + TEST_THAT_OR(restoredirid != 0, FAIL); // Test the restoration - TEST_THAT(BackupClientRestore(*client, restoredirid, + TEST_THAT(BackupClientRestore(*client, restoredirid, "Test1" /* remote */, "testfiles/restore-Test1" /* local */, true /* print progress dots */, @@ -3418,24 +3438,24 @@ int test_bbackupd() // to the server, so we'll compare later. // Make sure you can't restore a restored directory - TEST_THAT(BackupClientRestore(*client, restoredirid, - "Test1", "testfiles/restore-Test1", + TEST_THAT(BackupClientRestore(*client, restoredirid, + "Test1", "testfiles/restore-Test1", true /* print progress dots */, false /* restore deleted */, false /* undelete after */, false /* resume */, false /* keep going */) == Restore_TargetExists); - + // Find ID of the deleted directory deldirid = GetDirID(*client, "x1", restoredirid); TEST_THAT(deldirid != 0); - // Just check it doesn't bomb out -- will check this + // Just check it doesn't bomb out -- will check this // properly later (when bbackupd is stopped) - TEST_THAT(BackupClientRestore(*client, deldirid, + TEST_THAT(BackupClientRestore(*client, deldirid, "Test1", "testfiles/restore-Test1-x1", - true /* print progress dots */, + true /* print progress dots */, true /* restore deleted */, false /* undelete after */, false /* resume */, @@ -3448,7 +3468,8 @@ int test_bbackupd() fflush(stdout); { - Logging::Guard guard(Log::FATAL); + Logger::LevelGuard(Logging::GetConsole(), + Log::FATAL); TEST_THAT(BackupClientRestore(*client, restoredirid, "Test1", "testfiles/no-such-path/subdir", @@ -3462,35 +3483,44 @@ int test_bbackupd() // Log out client->QueryFinished(); - sSocket.Close(); } // Compare the restored files TEST_COMPARE(Compare_Same); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + } -#ifdef WIN32 + TEARDOWN_TEST_BBACKUPD(); +} + +bool test_compare_detects_attribute_changes() +{ + SETUP_WITH_BBSTORED(); + +#ifndef WIN32 + BOX_NOTICE("skipping test on this platform"); + // requires openfile(), GetFileTime() and attrib.exe +#else + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); + + // TODO FIXME dedent + { // make one of the files read-only, expect a compare failure - compareReturnValue = ::system("attrib +r " - "testfiles\\restore-Test1\\f1.dat"); - TEST_RETURN(compareReturnValue, 0); + int exit_status = ::system("attrib +r " + "testfiles\\TestDir1\\f1.dat"); + TEST_RETURN(exit_status, 0); TEST_COMPARE(Compare_Different); // set it back, expect no failures - compareReturnValue = ::system("attrib -r " - "testfiles\\restore-Test1\\f1.dat"); - TEST_RETURN(compareReturnValue, 0); + exit_status = ::system("attrib -r " + "testfiles\\TestDir1\\f1.dat"); + TEST_RETURN(exit_status, 0); TEST_COMPARE(Compare_Same); // change the timestamp on a file, expect a compare failure - char* testfile = "testfiles\\restore-Test1\\f1.dat"; + const char* testfile = "testfiles\\TestDir1\\f1.dat"; HANDLE handle = openfile(testfile, O_RDWR, 0); TEST_THAT(handle != INVALID_HANDLE_VALUE); @@ -3524,90 +3554,95 @@ int test_bbackupd() TEST_THAT(set_file_time(testfile, creationTime, lastModTime, lastAccessTime)); TEST_COMPARE(Compare_Same); + } #endif // WIN32 - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + TEARDOWN_TEST_BBACKUPD(); +} - printf("\n==== Add files with current time\n"); - - // Add some more files and modify others - // Use the m flag this time so they have a recent modification time - #ifdef WIN32 - TEST_THAT(::system("tar xzvmf testfiles/test3.tgz " - "-C testfiles") == 0); - #else - TEST_THAT(::system("gzip -d < testfiles/test3.tgz " - "| ( cd testfiles && tar xmf - )") == 0); - #endif +bool test_sync_new_files() +{ + SETUP_WITH_BBSTORED(); + bbackupd.RunSyncNow(); + + // TODO FIXME dedent + { + // Add some more files and modify others. Use the m flag this + // time so they have a recent modification time. + TEST_THAT(unpack_files("test3", "testfiles", "m")); + + // OpenBSD's tar interprets the "-m" option quite differently: + // it sets the time to epoch zero (1 Jan 1970) instead of the + // current time, which doesn't help us. So reset the timestamp + // on a file by touching it, so it won't be backed up. + { +#ifndef WIN32 + TEST_THAT(chmod("testfiles/TestDir1/chsh", 0755) == 0); +#endif + FileStream fs("testfiles/TestDir1/chsh", O_WRONLY); + fs.Write("a", 1); + } - // Wait and test - wait_for_backup_operation("bbackupd to sync new files"); + // At least one file is too new to be backed up on the first run. + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Different); + wait_for_operation(5, "newly added files to be old enough"); + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - // Rename directory - printf("\n==== Rename directory\n"); + } + + TEARDOWN_TEST_BBACKUPD(); +} + +bool test_rename_operations() +{ + SETUP_WITH_BBSTORED(); + + TEST_THAT(unpack_files("test2")); + TEST_THAT(unpack_files("test3")); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); + + // TODO FIXME dedent + { + BOX_INFO("Rename directory"); TEST_THAT(rename("testfiles/TestDir1/sub23/dhsfdss", "testfiles/TestDir1/renamed-dir") == 0); - wait_for_backup_operation("bbackupd to sync renamed directory"); - + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); // and again, but with quick flag - compareReturnValue = ::system(BBACKUPQUERY " " - "-c testfiles/bbackupd.conf " - "-l testfiles/query6q.log " - "-Wwarning \"compare -acqQ\" quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_COMPARE(Compare_Same, "", "-acqQ"); // Rename some files -- one under the threshold, others above - printf("\n==== Rename files\n"); - TEST_THAT(rename("testfiles/TestDir1/continousupdate", - "testfiles/TestDir1/continousupdate-ren") == 0); TEST_THAT(rename("testfiles/TestDir1/df324", "testfiles/TestDir1/df324-ren") == 0); TEST_THAT(rename("testfiles/TestDir1/sub23/find2perl", "testfiles/TestDir1/find2perl-ren") == 0); - wait_for_backup_operation("bbackupd to sync renamed files"); - + bbackupd.RunSyncNow(); TEST_COMPARE(Compare_Same); + } - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - // Check that modifying files with madly in the future - // timestamps still get added - printf("\n==== Create a file with timestamp way ahead " - "in the future\n"); - - // Time critical, so sync - wait_for_sync_start(); + TEARDOWN_TEST_BBACKUPD(); +} - // Then wait a second, to make sure the scan is complete - ::safe_sleep(1); +// Check that modifying files with madly in the future timestamps still get added +bool test_sync_files_with_timestamps_in_future() +{ + SETUP_WITH_BBSTORED(); + bbackupd.RunSyncNow(); - // Then modify an existing file + // TODO FIXME dedent + { { + TEST_THAT(::mkdir("testfiles/TestDir1/sub23", + 0755) == 0); FILE *f = fopen("testfiles/TestDir1/sub23/" "in-the-future", "w"); - TEST_THAT(f != 0); + TEST_THAT_OR(f != 0, FAIL); fprintf(f, "Back to the future!\n"); fclose(f); // and then move the time forwards! @@ -3621,17 +3656,41 @@ int test_bbackupd() } // Wait and test + bbackupd.RunSyncNow(); wait_for_backup_operation("bbackup to sync future file"); TEST_COMPARE(Compare_Same); + } + + TEARDOWN_TEST_BBACKUPD(); +} - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; +// Check change of store marker pauses daemon +bool test_changing_client_store_marker_pauses_daemon() +{ + SETUP_WITH_BBSTORED(); + TEST_THAT(StartClient()); + + // Wait for the client to upload all current files. We also time + // approximately how long a sync takes. + box_time_t sync_start_time = GetCurrentBoxTime(); + sync_and_wait(); + box_time_t sync_time = GetCurrentBoxTime() - sync_start_time; + BOX_INFO("Sync takes " << BOX_FORMAT_MICROSECONDS(sync_time)); - printf("\n==== Change client store marker\n"); + // Time how long a compare takes. On NetBSD it's 3 seconds, and that + // interferes with test timing unless we account for it. + box_time_t compare_start_time = GetCurrentBoxTime(); + // There should be no differences right now (yet). + TEST_COMPARE(Compare_Same); + box_time_t compare_time = GetCurrentBoxTime() - compare_start_time; + BOX_INFO("Compare takes " << BOX_FORMAT_MICROSECONDS(compare_time)); + // Wait for the end of another sync, to give us ~3 seconds to change + // the client store marker. + wait_for_sync_end(); + + // TODO FIXME dedent + { // Then... connect to the server, and change the // client store marker. See what that does! { @@ -3641,8 +3700,8 @@ int test_bbackupd() { try { - std::auto_ptr<BackupProtocolClient> - protocol = Connect(context); + std::auto_ptr<BackupProtocolCallable> + protocol = connect_to_bbstored(sTlsContext); // Make sure the marker isn't zero, // because that's the default, and // it should have changed @@ -3650,6 +3709,9 @@ int test_bbackupd() TEST_THAT(loginConf->GetClientStoreMarker() != 0); // Change it to something else + BOX_INFO("Changing client store marker " + "from " << loginConf->GetClientStoreMarker() << + " to 12"); protocol->QuerySetClientStoreMarker(12); // Success! @@ -3657,7 +3719,13 @@ int test_bbackupd() // Log out protocol->QueryFinished(); - sSocket.Close(); + } + catch(BoxException &e) + { + BOX_INFO("Failed to connect to bbstored, " + << tries << " retries remaining: " + << e.what()); + tries--; } catch(...) { @@ -3666,15 +3734,7 @@ int test_bbackupd() } TEST_THAT(done); } - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - printf("\n==== Check change of store marker pauses daemon\n"); - // Make a change to a file, to detect whether or not // it's hanging around waiting to retry. { @@ -3684,51 +3744,80 @@ int test_bbackupd() ::fclose(f); } - // Wait a little bit longer than usual - wait_for_operation((TIME_TO_WAIT_FOR_BACKUP_OPERATION * - 3) / 2, "bbackupd to detect changed store marker"); + // Wait for bbackupd to detect the problem. + wait_for_sync_end(); - // Test that there *are* differences + // Test that there *are* differences still, i.e. that bbackupd + // didn't successfully run a backup during that time. + BOX_INFO("Compare starting, expecting differences"); TEST_COMPARE(Compare_Different); - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + BOX_TRACE("Compare finished, expected differences"); + + // Wait out the expected delay in bbackupd. This is quite + // time-sensitive, so we use sub-second precision. + box_time_t wait = + SecondsToBoxTime(BACKUP_ERROR_DELAY_SHORTENED - 1) - + compare_time * 2; + BOX_INFO("Waiting for " << BOX_FORMAT_MICROSECONDS(wait) << " " + "until just before bbackupd recovers"); + ShortSleep(wait, true); + + // bbackupd should not have recovered yet, so there should + // still be differences. + BOX_INFO("Compare starting, expecting differences"); + TEST_COMPARE(Compare_Different); + BOX_TRACE("Compare finished, expected differences"); + + // Now wait for it to recover and finish a sync, and check that + // the differences are gone (successful backup). Wait until ~2 + // seconds after we expect the sync to have finished, to reduce + // the risk of random failure on AppVeyor when heavily loaded. + wait = sync_time + SecondsToBoxTime(6); + BOX_INFO("Waiting for " << BOX_FORMAT_MICROSECONDS(wait) << + " until just after bbackupd recovers and finishes a sync"); + ShortSleep(wait, true); + + BOX_INFO("Compare starting, expecting no differences"); + TEST_COMPARE(Compare_Same); + BOX_TRACE("Compare finished, expected no differences"); + } - wait_for_operation(100, "bbackupd to recover"); + TEARDOWN_TEST_BBACKUPD(); +} - // Then check it has backed up successfully. - TEST_COMPARE(Compare_Same); +bool test_interrupted_restore_can_be_recovered() +{ + SETUP_WITH_BBSTORED(); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; +#ifdef WIN32 + BOX_NOTICE("skipping test on this platform"); +#else + bbackupd.RunSyncNow(); -#ifndef WIN32 - printf("\n==== Interrupted restore\n"); + // TODO FIXME dedent + { { - do_interrupted_restore(context, restoredirid); + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, + BackupProtocolLogin::Flags_ReadOnly); + + // Find the ID of the Test1 directory + int64_t restoredirid = GetDirID(*client, "Test1", + BackupProtocolListDirectory::RootDirectory); + TEST_THAT_OR(restoredirid != 0, FAIL); + + do_interrupted_restore(sTlsContext, restoredirid); int64_t resumesize = 0; TEST_THAT(FileExists("testfiles/" - "restore-interrupt.boxbackupresume", + "restore-interrupt.boxbackupresume", &resumesize)); // make sure it has recorded something to resume - TEST_THAT(resumesize > 16); - - printf("\n==== Resume restore\n"); - - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, - BackupProtocolLogin::Flags_ReadOnly); + TEST_THAT(resumesize > 16); // Check that the restore fn returns resume possible, // rather than doing anything - TEST_THAT(BackupClientRestore(*client, restoredirid, - "Test1", "testfiles/restore-interrupt", + TEST_THAT(BackupClientRestore(*client, restoredirid, + "Test1", "testfiles/restore-interrupt", true /* print progress dots */, false /* restore deleted */, false /* undelete after */, @@ -3737,8 +3826,8 @@ int test_bbackupd() == Restore_ResumePossible); // Then resume it - TEST_THAT(BackupClientRestore(*client, restoredirid, - "Test1", "testfiles/restore-interrupt", + TEST_THAT(BackupClientRestore(*client, restoredirid, + "Test1", "testfiles/restore-interrupt", true /* print progress dots */, false /* restore deleted */, false /* undelete after */, @@ -3747,28 +3836,73 @@ int test_bbackupd() == Restore_Complete); client->QueryFinished(); - sSocket.Close(); + client.reset(); // Then check it has restored the correct stuff TEST_COMPARE(Compare_Same); } + } #endif // !WIN32 - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + TEARDOWN_TEST_BBACKUPD(); +} + +bool assert_x1_deleted_or_not(bool expected_deleted) +{ + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, 0 /* read-write */); + + std::auto_ptr<BackupStoreDirectory> dir = ReadDirectory(*client); + int64_t testDirId = SearchDir(*dir, "Test1"); + TEST_THAT_OR(testDirId != 0, return false); + + dir = ReadDirectory(*client, testDirId); + BackupStoreDirectory::Iterator i(*dir); + BackupStoreFilenameClear child("x1"); + BackupStoreDirectory::Entry *en = i.FindMatchingClearName(child); + TEST_THAT_OR(en != 0, return false); + TEST_EQUAL_OR(expected_deleted, en->IsDeleted(), return false); + + return true; +} + +bool test_restore_deleted_files() +{ + SETUP_WITH_BBSTORED(); + + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); - printf("\n==== Check restore deleted files\n"); + TEST_THAT(::unlink("testfiles/TestDir1/f1.dat") == 0); +#ifdef WIN32 + TEST_THAT(::system("rd /s/q testfiles\\TestDir1\\x1") == 0); +#else + TEST_THAT(::system("rm -r testfiles/TestDir1/x1") == 0); +#endif + TEST_COMPARE(Compare_Different); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); + TEST_THAT(assert_x1_deleted_or_not(true)); + + // TODO FIXME dedent + { { - std::auto_ptr<BackupProtocolClient> client = - ConnectAndLogin(context, 0 /* read-write */); + std::auto_ptr<BackupProtocolCallable> client = + connect_and_login(sTlsContext, 0 /* read-write */); + + // Find the ID of the Test1 directory + int64_t restoredirid = GetDirID(*client, "Test1", + BackupProtocolListDirectory::RootDirectory); + TEST_THAT_OR(restoredirid != 0, FAIL); + + // Find ID of the deleted directory + int64_t deldirid = GetDirID(*client, "x1", restoredirid); + TEST_THAT_OR(deldirid != 0, FAIL); // Do restore and undelete - TEST_THAT(BackupClientRestore(*client, deldirid, - "Test1", "testfiles/restore-Test1-x1-2", + TEST_THAT(BackupClientRestore(*client, deldirid, + "Test1", "testfiles/restore-Test1-x1-2", true /* print progress dots */, true /* deleted files */, true /* undelete after */, @@ -3777,181 +3911,118 @@ int test_bbackupd() == Restore_Complete); client->QueryFinished(); - sSocket.Close(); + client.reset(); // Do a compare with the now undeleted files - compareReturnValue = ::system(BBACKUPQUERY " " - "-c testfiles/bbackupd.conf " - "-l testfiles/query11.log " - "-Wwarning " - "\"compare -cEQ Test1/x1 " - "testfiles/restore-Test1-x1-2\" quit"); - TEST_RETURN(compareReturnValue, - BackupQueries::ReturnCode::Compare_Same); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); + TEST_COMPARE(Compare_Same, "", "-cEQ Test1/x1 " + "testfiles/restore-Test1-x1-2"); } // Final check on notifications TEST_THAT(!TestFileExists("testfiles/notifyran.store-full.2")); TEST_THAT(!TestFileExists("testfiles/notifyran.read-error.2")); + } - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + // should have been undeleted by restore + TEST_THAT(assert_x1_deleted_or_not(false)); -#ifdef WIN32 - printf("\n==== Testing locked file behaviour:\n"); + TEARDOWN_TEST_BBACKUPD(); +} + +bool test_locked_file_behaviour() +{ + SETUP_WITH_BBSTORED(); +#ifndef WIN32 + // There are no tests for mandatory locks on non-Windows platforms yet. + BOX_NOTICE("skipping test on this platform"); +#else + // TODO FIXME dedent + { // Test that locked files cannot be backed up, // and the appropriate error is reported. - // Wait for the sync to finish, so that we have time to work - wait_for_sync_end(); - // Now we have about three seconds to work - handle = openfile("testfiles/TestDir1/lockedfile", - O_CREAT | O_EXCL | O_LOCK, 0); - TEST_THAT(handle != INVALID_HANDLE_VALUE); + HANDLE handle = openfile("testfiles/TestDir1/f1.dat", + BOX_OPEN_LOCK, 0); + TEST_THAT_OR(handle != INVALID_HANDLE_VALUE, FAIL); - if (handle != 0) - { - // first sync will ignore the file, it's too new - wait_for_sync_end(); - TEST_THAT(!TestFileExists("testfiles/" - "notifyran.read-error.1")); - } - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - if (handle != 0) { // this sync should try to back up the file, // and fail, because it's locked - wait_for_sync_end(); + bbackupd.RunSyncNowWithExceptionHandling(); TEST_THAT(TestFileExists("testfiles/" "notifyran.read-error.1")); TEST_THAT(!TestFileExists("testfiles/" "notifyran.read-error.2")); } - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - if (handle != 0) { // now close the file and check that it is // backed up on the next run. CloseHandle(handle); - wait_for_sync_end(); + bbackupd.RunSyncNow(); // still no read errors? TEST_THAT(!TestFileExists("testfiles/" "notifyran.read-error.2")); + TEST_COMPARE(Compare_Same); } - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - if (handle != 0) - { - // compare, and check that it works - // reports the correct error message (and finishes) - TEST_THAT(compare_all(false)); - } - - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; - - if (handle != 0) { // open the file again, compare and check that compare // reports the correct error message (and finishes) - handle = openfile("testfiles/TestDir1/lockedfile", - O_LOCK, 0); - TEST_THAT(handle != INVALID_HANDLE_VALUE); + handle = openfile("testfiles/TestDir1/f1.dat", + BOX_OPEN_LOCK, 0); + TEST_THAT_OR(handle != INVALID_HANDLE_VALUE, FAIL); TEST_COMPARE(Compare_Error); // close the file again, check that compare // works again CloseHandle(handle); + TEST_COMPARE(Compare_Same); } + } +#endif // WIN32 - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + TEARDOWN_TEST_BBACKUPD(); +} - if (handle != 0) - { - TEST_COMPARE(Compare_Same); - } -#endif +bool test_backup_many_files() +{ + SETUP_WITH_BBSTORED(); - // Kill the daemon - terminate_bbackupd(bbackupd_pid); - - // Start it again - cmd = BBACKUPD " " + bbackupd_args + - " testfiles/bbackupd.conf"; - bbackupd_pid = LaunchServer(cmd, "testfiles/bbackupd.pid"); + unpack_files("test2"); + unpack_files("test3"); + unpack_files("testexclude"); + unpack_files("spacetest1", "testfiles/TestDir1"); + unpack_files("spacetest2", "testfiles/TestDir1"); - TEST_THAT(bbackupd_pid != -1 && bbackupd_pid != 0); + bbackupd.RunSyncNow(); + TEST_COMPARE(Compare_Same); - TEST_THAT(ServerIsAlive(bbackupd_pid)); - TEST_THAT(ServerIsAlive(bbstored_pid)); - if (!ServerIsAlive(bbackupd_pid)) return 1; - if (!ServerIsAlive(bbstored_pid)) return 1; - if (failures) return 1; + TEARDOWN_TEST_BBACKUPD(); +} - if(bbackupd_pid != -1 && bbackupd_pid != 0) - { - // Wait and compare (a little bit longer than usual) - wait_for_operation( - (TIME_TO_WAIT_FOR_BACKUP_OPERATION*3) / 2, - "bbackupd to sync everything"); - TEST_COMPARE(Compare_Same); +bool test_parse_incomplete_command() +{ + SETUP_TEST_BBACKUPD(); - // Kill it again - terminate_bbackupd(bbackupd_pid); - } + { + // This is not a complete command, it should not parse! + BackupQueries::ParsedCommand cmd("-od", true); + TEST_THAT(cmd.mFailed); + TEST_EQUAL((void *)NULL, cmd.pSpec); + TEST_EQUAL(0, cmd.mCompleteArgCount); } - /* - // List the files on the server - why? - ::system(BBACKUPQUERY " -q -c testfiles/bbackupd.conf " - "-l testfiles/queryLIST.log \"list -rotdh\" quit"); - TestRemoteProcessMemLeaks("bbackupquery.memleaks"); - */ - - #ifndef WIN32 - if(::getuid() == 0) - { - ::printf("WARNING: This test was run as root. " - "Some tests have been omitted.\n"); - } - #endif - - return 0; + TEARDOWN_TEST_BBACKUPD(); } -int test(int argc, const char *argv[]) +bool test_parse_syncallowscript_output() { + SETUP_TEST_BBACKUPD(); + { BackupDaemon daemon; @@ -3965,48 +4036,83 @@ int test(int argc, const char *argv[]) TEST_EQUAL(0, daemon.GetMaxBandwidthFromSyncAllowScript()); } + TEARDOWN_TEST_BBACKUPD(); +} + +int test(int argc, const char *argv[]) +{ // SSL library SSLLib::Initialise(); // Keys for subsystems BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); - // Initial files - #ifdef WIN32 - TEST_THAT(::system("tar xzvf testfiles/test_base.tgz " - "-C testfiles") == 0); - #else - TEST_THAT(::system("gzip -d < testfiles/test_base.tgz " - "| ( cd testfiles && tar xf - )") == 0); - #endif + { + std::string errs; + std::auto_ptr<Configuration> config( + Configuration::LoadAndVerify + ("testfiles/bbstored.conf", &BackupConfigFileVerify, errs)); + TEST_EQUAL_LINE(0, errs.size(), "Loading configuration file " + "reported errors: " << errs); + TEST_THAT_OR(config.get(), return 1); + // Initialise the raid file controller + RaidFileController &rcontroller(RaidFileController::GetController()); + rcontroller.Initialise(config->GetKeyValue("RaidFileConf").c_str()); + } - // Do the tests + sTlsContext.Initialise(false /* client */, + "testfiles/clientCerts.pem", + "testfiles/clientPrivKey.pem", + "testfiles/clientTrustedCAs.pem"); - int r = test_basics(); - if(r != 0) return r; - - r = test_setupaccount(); - if(r != 0) return r; + TEST_THAT(test_basics()); + TEST_THAT(test_readdirectory_on_nonexistent_dir()); + TEST_THAT(test_bbackupquery_parser_escape_slashes()); + TEST_THAT(test_getobject_on_nonexistent_file()); + // TEST_THAT(test_replace_zero_byte_file_with_nonzero_byte_file()); + TEST_THAT(test_backup_disappearing_directory()); + TEST_THAT(test_ssl_keepalives()); + TEST_THAT(test_backup_hardlinked_files()); + TEST_THAT(test_backup_pauses_when_store_is_full()); + TEST_THAT(test_bbackupd_exclusions()); + TEST_THAT(test_bbackupd_uploads_files()); + TEST_THAT(test_bbackupd_responds_to_connection_failure()); + TEST_THAT(test_absolute_symlinks_not_followed_during_restore()); + TEST_THAT(test_initially_missing_locations_are_not_forgotten()); + TEST_THAT(test_redundant_locations_deleted_on_time()); + TEST_THAT(test_read_only_dirs_can_be_restored()); + TEST_THAT(test_unicode_filenames_can_be_backed_up()); + TEST_THAT(test_sync_allow_script_can_pause_backup()); + TEST_THAT(test_delete_update_and_symlink_files()); + TEST_THAT(test_store_error_reporting()); + TEST_THAT(test_change_file_to_symlink_and_back()); + TEST_THAT(test_file_rename_tracking()); + TEST_THAT(test_upload_very_old_files()); + TEST_THAT(test_excluded_files_are_not_backed_up()); + TEST_THAT(test_read_error_reporting()); + TEST_THAT(test_continuously_updated_file()); + TEST_THAT(test_delete_dir_change_attribute()); + TEST_THAT(test_restore_files_and_directories()); + TEST_THAT(test_compare_detects_attribute_changes()); + TEST_THAT(test_sync_new_files()); + TEST_THAT(test_rename_operations()); + TEST_THAT(test_sync_files_with_timestamps_in_future()); + TEST_THAT(test_changing_client_store_marker_pauses_daemon()); + TEST_THAT(test_interrupted_restore_can_be_recovered()); + TEST_THAT(test_restore_deleted_files()); + TEST_THAT(test_locked_file_behaviour()); + TEST_THAT(test_backup_many_files()); + TEST_THAT(test_parse_incomplete_command()); + TEST_THAT(test_parse_syncallowscript_output()); + + TEST_THAT(kill_running_daemons()); - r = test_run_bbstored(); - TEST_THAT(r == 0); - if(r != 0) return r; - - r = test_bbackupd(); - if(r != 0) +#ifndef WIN32 + if(::getuid() == 0) { - if (bbackupd_pid) - { - KillServer(bbackupd_pid); - } - if (bbstored_pid) - { - KillServer(bbstored_pid); - } - return r; + BOX_WARNING("This test was run as root. Some tests have been omitted."); } - - test_kill_bbstored(); +#endif - return 0; + return finish_test_suite(); } diff --git a/test/bbackupd/testfiles/bbackupd-snapshot.conf.in b/test/bbackupd/testfiles/bbackupd-snapshot.conf.in index d245d077..73b50c6e 100644 --- a/test/bbackupd/testfiles/bbackupd-snapshot.conf.in +++ b/test/bbackupd/testfiles/bbackupd-snapshot.conf.in @@ -13,6 +13,7 @@ AccountNumber = 0x01234567 AutomaticBackup = no UpdateStoreInterval = 0 +BackupErrorDelay = 10 MinimumFileAge = 4 MaxUploadWait = 24 DeleteRedundantLocationsAfter = 10 diff --git a/test/bbackupd/testfiles/bbackupd.conf.in b/test/bbackupd/testfiles/bbackupd.conf.in index 712b58b2..f0080c4a 100644 --- a/test/bbackupd/testfiles/bbackupd.conf.in +++ b/test/bbackupd/testfiles/bbackupd.conf.in @@ -12,6 +12,7 @@ StorePort = 22011 AccountNumber = 0x01234567 UpdateStoreInterval = 3 +BackupErrorDelay = 10 MinimumFileAge = 4 MaxUploadWait = 24 DeleteRedundantLocationsAfter = 10 diff --git a/test/bbackupd/testfiles/extcheck1.pl.in b/test/bbackupd/testfiles/extcheck1.pl.in index 5b70c677..c80ddc66 100755 --- a/test/bbackupd/testfiles/extcheck1.pl.in +++ b/test/bbackupd/testfiles/extcheck1.pl.in @@ -1,9 +1,12 @@ #!@PERL@ use strict; +use File::Spec; + my $flags = $ARGV[0] or ""; +my $bbackupquery = File::Spec->catfile('..', '..', 'bin', 'bbackupquery', 'bbackupquery'); -unless(open IN,"../../bin/bbackupquery/bbackupquery -Wwarning " . +unless(open IN, "$bbackupquery -Wwarning " . "-c testfiles/bbackupd.conf " . "-l testfiles/query4.log " . "\"compare -ac$flags\" quit 2>&1 |") diff --git a/test/bbackupd/testfiles/extcheck2.pl.in b/test/bbackupd/testfiles/extcheck2.pl.in index 3671ad93..02c258f8 100755 --- a/test/bbackupd/testfiles/extcheck2.pl.in +++ b/test/bbackupd/testfiles/extcheck2.pl.in @@ -1,9 +1,12 @@ #!@PERL@ use strict; +use File::Spec; + my $flags = $ARGV[0] or ""; +my $bbackupquery = File::Spec->catfile('..', '..', 'bin', 'bbackupquery', 'bbackupquery'); -unless(open IN,"../../bin/bbackupquery/bbackupquery -Wwarning " . +unless(open IN, "$bbackupquery -Wwarning " . "-c testfiles/bbackupd.conf " . "-l testfiles/query4.log " . "\"compare -ac$flags\" quit 2>&1 |") diff --git a/test/common/testcommon.cpp b/test/common/testcommon.cpp index f47a0dba..cba40fe7 100644 --- a/test/common/testcommon.cpp +++ b/test/common/testcommon.cpp @@ -160,8 +160,9 @@ class TestLogger : public Logger bool IsTriggered() { return mTriggered; } void Reset() { mTriggered = false; } - virtual bool Log(Log::Level level, const std::string& rFile, - int line, std::string& rMessage) + virtual bool Log(Log::Level level, const std::string& file, int line, + const std::string& function, const Log::Category& category, + const std::string& message) { if (level == mTargetLevel) { @@ -261,6 +262,18 @@ int test(int argc, const char *argv[]) TEST_THAT(!TestFileExists(tempfile.c_str())); } + // Test that named locks work as expected + { + NamedLock lock1; + TEST_THAT(lock1.TryAndGetLock("testfiles/locktest")); + // With a lock held, we should not be able to acquire another. + TEST_THAT(!NamedLock().TryAndGetLock("testfiles/locktest")); + } + { + // But with the lock released, we should be able to. + TEST_THAT(NamedLock().TryAndGetLock("testfiles/locktest")); + } + // Test that memory leak detection doesn't crash { char *test = new char[1024]; @@ -273,21 +286,21 @@ int test(int argc, const char *argv[]) { Timers::Cleanup(); - TEST_THAT(memleakfinder_numleaks() == 0); + TEST_EQUAL(0, memleakfinder_numleaks()); void *block = ::malloc(12); - TEST_THAT(memleakfinder_numleaks() == 1); + TEST_EQUAL(1, memleakfinder_numleaks()); void *b2 = ::realloc(block, 128*1024); - TEST_THAT(memleakfinder_numleaks() == 1); + TEST_EQUAL(1, memleakfinder_numleaks()); ::free(b2); - TEST_THAT(memleakfinder_numleaks() == 0); + TEST_EQUAL(0, memleakfinder_numleaks()); char *test = new char[1024]; - TEST_THAT(memleakfinder_numleaks() == 1); + TEST_EQUAL(1, memleakfinder_numleaks()); MemBlockStream *s = new MemBlockStream(test,12); - TEST_THAT(memleakfinder_numleaks() == 2); + TEST_EQUAL(3, memleakfinder_numleaks()); delete s; - TEST_THAT(memleakfinder_numleaks() == 1); + TEST_EQUAL(1, memleakfinder_numleaks()); delete [] test; - TEST_THAT(memleakfinder_numleaks() == 0); + TEST_EQUAL(0, memleakfinder_numleaks()); Timers::Init(); } @@ -299,16 +312,19 @@ int test(int argc, const char *argv[]) // Check that using timer methods without initialisation // throws an assertion failure. Can only do this in debug mode #ifndef BOX_RELEASE_BUILD - TEST_CHECK_THROWS(Timers::Add(*(Timer*)NULL), - CommonException, AssertFailed); - TEST_CHECK_THROWS(Timers::Remove(*(Timer*)NULL), - CommonException, AssertFailed); - #endif - - // TEST_CHECK_THROWS(Timers::Signal(), CommonException, AssertFailed); - #ifndef BOX_RELEASE_BUILD + { TEST_CHECK_THROWS(Timers::Cleanup(), CommonException, AssertFailed); + + Timer tim(0, "tim"); + TEST_CHECK_THROWS(Timers::Add(tim), CommonException, AssertFailed); + Timers::Remove(tim); + + TEST_CHECK_THROWS(Timer t1(900, "t1"), CommonException, + AssertFailed); + + // TEST_CHECK_THROWS(Timers::Signal(), CommonException, AssertFailed); + } #endif // Check that we can initialise the timers @@ -331,47 +347,60 @@ int test(int argc, const char *argv[]) Timers::Init(); - Timer t0(0, "t0"); // should never expire - Timer t1(1000, "t1"); - Timer t2(2000, "t2"); - Timer t3(3000, "t3"); - - TEST_THAT(!t0.HasExpired()); - TEST_THAT(!t1.HasExpired()); - TEST_THAT(!t2.HasExpired()); - TEST_THAT(!t3.HasExpired()); - - safe_sleep(1); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(t1.HasExpired()); - TEST_THAT(!t2.HasExpired()); - TEST_THAT(!t3.HasExpired()); - - safe_sleep(1); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(t1.HasExpired()); - TEST_THAT(t2.HasExpired()); - TEST_THAT(!t3.HasExpired()); - - // Try both ways of resetting an existing timer. - t1 = Timer(1000, "t1a"); - t2.Reset(2000); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(!t1.HasExpired()); - TEST_THAT(!t2.HasExpired()); - TEST_THAT(!t3.HasExpired()); - - safe_sleep(1); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(t1.HasExpired()); - TEST_THAT(!t2.HasExpired()); - TEST_THAT(t3.HasExpired()); - - safe_sleep(1); - TEST_THAT(!t0.HasExpired()); - TEST_THAT(t1.HasExpired()); - TEST_THAT(t2.HasExpired()); - TEST_THAT(t3.HasExpired()); + // Ideally timers would be perfectly accurate and we could sleep for 1.0 seconds, but + // on OSX in particular they could fire 50-100 ms late (I've seen 4 ms in practice) + // and we don't want the tests to fail because of this, because we don't really need + // that kind of precision in practice. So we reduce the timer intervals by 100ms to + // be safe. + + { + Logger::LevelGuard temporary_verbosity(Logging::GetConsole(), Log::TRACE); + Console::SettingsGuard save_old_settings; + Console::SetShowTime(true); + Console::SetShowTimeMicros(true); + + Timer t0(0, "t0"); // should never expire + Timer t1(900, "t1"); + Timer t2(1900, "t2"); + Timer t3(2900, "t3"); + + TEST_THAT(!t0.HasExpired()); + TEST_THAT(!t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + safe_sleep(1); + + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + + // Try both ways of resetting an existing timer. + t1 = Timer(900, "t1a"); + t2.Reset(1900); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(!t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(!t3.HasExpired()); + + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(!t2.HasExpired()); + TEST_THAT(t3.HasExpired()); + + safe_sleep(1); + TEST_THAT(!t0.HasExpired()); + TEST_THAT(t1.HasExpired()); + TEST_THAT(t2.HasExpired()); + TEST_THAT(t3.HasExpired()); + } // Leave timers initialised for rest of test. // Test main() will cleanup after test finishes. @@ -407,20 +436,18 @@ int test(int argc, const char *argv[]) DIRECTORY_SEPARATOR "fdgetlinetest.txt"); FdGetLine getline(file); - int l = 0; - while(testfilelines[l] != 0) - { - TEST_THAT(!getline.IsEOF()); - std::string line = getline.GetLine(true); - printf("expected |%s| got |%s|\n", testfilelines[l], - line.c_str()); - TEST_LINE(strcmp(testfilelines[l], line.c_str()) == 0, - line); - l++; - } - TEST_THAT(getline.IsEOF()); - TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); + int l = 0; + while(testfilelines[l] != 0) + { + TEST_THAT(!getline.IsEOF()); + std::string line = getline.GetLine(true); + TEST_EQUAL(testfilelines[l], line); + l++; + } + TEST_THAT(getline.IsEOF()); + TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); } + // and again without pre-processing { FileHandleGuard<O_RDONLY> file("testfiles" @@ -430,7 +457,7 @@ int test(int argc, const char *argv[]) TEST_THAT_ABORTONFAIL(file2 != 0); FdGetLine getline(file); char ll[512]; - + while(!feof(file2)) { fgets(ll, sizeof(ll), file2); @@ -440,40 +467,38 @@ int test(int argc, const char *argv[]) e--; } ll[e] = '\0'; - - TEST_THAT(!getline.IsEOF()); - std::string line = getline.GetLine(false); - //printf("expected |%s| got |%s|\n", ll, line.c_str()); - TEST_THAT(strcmp(ll, line.c_str()) == 0); + + TEST_THAT(!getline.IsEOF()); + std::string line = getline.GetLine(false); + TEST_EQUAL(ll, line); } - TEST_THAT(getline.IsEOF()); - TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); + TEST_THAT(getline.IsEOF()); + TEST_CHECK_THROWS(getline.GetLine(true), CommonException, GetLineEOF); fclose(file2); } - + // Then the IOStream version of get line, seeing as we're here... { FileStream file("testfiles" DIRECTORY_SEPARATOR "fdgetlinetest.txt", O_RDONLY); IOStreamGetLine getline(file); - int l = 0; - while(testfilelines[l] != 0) - { - TEST_THAT(!getline.IsEOF()); - std::string line; - while(!getline.GetLine(line, true)) - ; - printf("expected |%s| got |%s|\n", testfilelines[l], - line.c_str()); - TEST_LINE(strcmp(testfilelines[l], line.c_str()) == 0, - line); - l++; - } - TEST_THAT(getline.IsEOF()); - std::string dummy; - TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); + int l = 0; + while(testfilelines[l] != 0) + { + TEST_THAT(!getline.IsEOF()); + std::string line; + while(!getline.GetLine(line, true)) + { + // skip line + } + TEST_EQUAL(testfilelines[l], line); + l++; + } + TEST_THAT(getline.IsEOF()); + std::string dummy; + TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); } // and again without pre-processing { @@ -495,21 +520,20 @@ int test(int argc, const char *argv[]) e--; } ll[e] = '\0'; - - TEST_THAT(!getline.IsEOF()); - std::string line; - while(!getline.GetLine(line, false)) - ; - //printf("expected |%s| got |%s|\n", ll, line.c_str()); - TEST_THAT(strcmp(ll, line.c_str()) == 0); + + TEST_THAT(!getline.IsEOF()); + std::string line; + while(!getline.GetLine(line, false)) + ; + TEST_EQUAL(ll, line); } - TEST_THAT(getline.IsEOF()); - std::string dummy; - TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); + TEST_THAT(getline.IsEOF()); + std::string dummy; + TEST_CHECK_THROWS(getline.GetLine(dummy, true), CommonException, GetLineEOF); fclose(file2); } - + // Doesn't exist { std::string errMsg; @@ -569,50 +593,69 @@ int test(int argc, const char *argv[]) TEST_THAT(sub1_3.GetKeyValue("terrible") == "absolutely"); } - static const char *file[] = + static const char *file[][2] = { - "testfiles" DIRECTORY_SEPARATOR "config2.txt", + {"testfiles" DIRECTORY_SEPARATOR "config2.txt", + "<root>.TOPlevel (key) is missing."}, // Value missing from root - "testfiles" DIRECTORY_SEPARATOR "config3.txt", + {"testfiles" DIRECTORY_SEPARATOR "config3.txt", + "Unexpected start block in test1"}, // Unexpected { - "testfiles" DIRECTORY_SEPARATOR "config4.txt", + {"testfiles" DIRECTORY_SEPARATOR "config4.txt", + "Root level has close block -- forgot to terminate subblock?"}, // Missing } - "testfiles" DIRECTORY_SEPARATOR "config5.txt", + {"testfiles" DIRECTORY_SEPARATOR "config5.txt", + "Block subconfig2 wasn't started correctly (no '{' on line of it's own)\n" + "Root level has close block -- forgot to terminate subblock?"}, // { expected, but wasn't there - "testfiles" DIRECTORY_SEPARATOR "config6.txt", + {"testfiles" DIRECTORY_SEPARATOR "config6.txt", + "test1.subconfig2.bing (key) multi value not allowed (duplicated key?)."}, // Duplicate key - "testfiles" DIRECTORY_SEPARATOR "config7.txt", + {"testfiles" DIRECTORY_SEPARATOR "config7.txt", + "Invalid configuration key: = invalid thing here!"}, // Invalid key (no name) - "testfiles" DIRECTORY_SEPARATOR "config8.txt", + {"testfiles" DIRECTORY_SEPARATOR "config8.txt", + "File ended without terminating all subblocks"}, // Not all sub blocks terminated - "testfiles" DIRECTORY_SEPARATOR "config9.txt", + {"testfiles" DIRECTORY_SEPARATOR "config9.txt", + "test1.subconfig3.carrots (key) is not a valid integer."}, // Not valid integer - "testfiles" DIRECTORY_SEPARATOR "config9b.txt", + {"testfiles" DIRECTORY_SEPARATOR "config9b.txt", + "test1.subconfig2.carrots (key) is not a valid integer."}, // Not valid integer - "testfiles" DIRECTORY_SEPARATOR "config9c.txt", + {"testfiles" DIRECTORY_SEPARATOR "config9c.txt", + "test1.subconfig2.carrots (key) is not a valid integer."}, // Not valid integer - "testfiles" DIRECTORY_SEPARATOR "config9d.txt", + {"testfiles" DIRECTORY_SEPARATOR "config9d.txt", + "test1.subconfig3.carrots (key) is not a valid integer."}, // Not valid integer - "testfiles" DIRECTORY_SEPARATOR "config10.txt", + {"testfiles" DIRECTORY_SEPARATOR "config10.txt", + "test1.subconfig.carrots (key) is missing."}, // Missing key (in subblock) - "testfiles" DIRECTORY_SEPARATOR "config11.txt", + {"testfiles" DIRECTORY_SEPARATOR "config11.txt", + "test1.subconfig3.NOTEXPECTED (key) is not a known key. Check spelling and placement."}, // Unknown key - "testfiles" DIRECTORY_SEPARATOR "config12.txt", + {"testfiles" DIRECTORY_SEPARATOR "config12.txt", + "<root>.test1.otherthing (block) is missing."}, // Missing block - "testfiles" DIRECTORY_SEPARATOR "config13.txt", + {"testfiles" DIRECTORY_SEPARATOR "config13.txt", + "<root>.test1.* (block) is missing (a block must be present).\n" + "<root>.test1.otherthing (block) is missing."}, // Subconfig (wildcarded) should exist, but missing (ie nothing present) - "testfiles" DIRECTORY_SEPARATOR "config16.txt", + {"testfiles" DIRECTORY_SEPARATOR "config16.txt", + "<root>.BoolTrue1 (key) is not a valid boolean value."}, // bad boolean value - 0 + {NULL, NULL}, }; - for(int l = 0; file[l] != 0; ++l) + for(int l = 0; file[l][0] != 0; ++l) { + HideCategoryGuard hide(ConfigurationVerify::VERIFY_ERROR); std::string errMsg; - std::auto_ptr<Configuration> pconfig(Configuration::LoadAndVerify(file[l], &verify, errMsg)); + std::auto_ptr<Configuration> pconfig(Configuration::LoadAndVerify(file[l][0], &verify, errMsg)); TEST_THAT(pconfig.get() == 0); - TEST_THAT(!errMsg.empty()); - printf("(%s) Error msg is:\n------\n%s------\n", file[l], errMsg.c_str()); + errMsg = errMsg.substr(0, errMsg.size() > 0 ? errMsg.size() - 1 : 0); + TEST_EQUAL_LINE(file[l][1], errMsg, file[l][0]); } // Check that multivalues happen as expected @@ -764,10 +807,13 @@ int test(int argc, const char *argv[]) // Add regex entries #ifdef HAVE_REGEX_SUPPORT + { + HideCategoryGuard hide(ConfigurationVerify::VERIFY_ERROR); elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")); elist.AddRegexEntries(std::string("")); TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[:not_valid")), CommonException, BadRegularExpression); TEST_THAT(elist.SizeOfRegexList() == 3); + } #else TEST_CHECK_THROWS(elist.AddRegexEntries(std::string("[a-d]+\\.reg$" "\x01" "EXCLUDE" "\x01" "^exclude$")), CommonException, RegexNotSupportedOnThisPlatform); TEST_THAT(elist.SizeOfRegexList() == 0); diff --git a/test/compress/testcompress.cpp b/test/compress/testcompress.cpp index 4a522d31..76512e6a 100644 --- a/test/compress/testcompress.cpp +++ b/test/compress/testcompress.cpp @@ -47,9 +47,10 @@ public: return 0; } - void Write(const void *pBuffer, int NBytes) + void Write(const void *pBuffer, int NBytes, + int Timeout = IOStream::TimeOutInfinite) { - buffers[(currentBuffer + 1) & 1].Write(pBuffer, NBytes); + buffers[(currentBuffer + 1) & 1].Write(pBuffer, NBytes, Timeout); } bool StreamDataLeft() { diff --git a/test/crypto/testcrypto.cpp b/test/crypto/testcrypto.cpp index 6d90e5e7..32d2efb8 100644 --- a/test/crypto/testcrypto.cpp +++ b/test/crypto/testcrypto.cpp @@ -10,7 +10,6 @@ #include "Box.h" #include <string.h> -#include <strings.h> #include <openssl/rand.h> #include "Test.h" @@ -267,7 +266,7 @@ int test(int argc, const char *argv[]) // Check rolling checksums uint8_t *checkdata_blk = (uint8_t *)malloc(CHECKSUM_DATA_SIZE); uint8_t *checkdata = checkdata_blk; - RAND_pseudo_bytes(checkdata, CHECKSUM_DATA_SIZE); + RAND_bytes(checkdata, CHECKSUM_DATA_SIZE); for(int size = CHECKSUM_BLOCK_SIZE_BASE; size <= CHECKSUM_BLOCK_SIZE_LAST; ++size) { // Test skip-roll code diff --git a/test/httpserver/testfiles/httpserver.conf b/test/httpserver/testfiles/httpserver.conf index 1a1c4644..d8c72a3f 100644 --- a/test/httpserver/testfiles/httpserver.conf +++ b/test/httpserver/testfiles/httpserver.conf @@ -1,5 +1,6 @@ - -AddressPrefix = http://localhost:1080 +# Use 127.0.0.1 instead of localhost to force use of IPv4, as that is what the server +# binds to. Windows tends to use IPv6 instead if possible, breaking the test. +AddressPrefix = http://127.0.0.1:1080 Server { diff --git a/test/httpserver/testfiles/testrequests.pl b/test/httpserver/testfiles/testrequests.pl index 85380ee0..fd4274b4 100755 --- a/test/httpserver/testfiles/testrequests.pl +++ b/test/httpserver/testfiles/testrequests.pl @@ -2,7 +2,9 @@ use strict; use LWP::UserAgent; -my $url_base = 'http://localhost:1080'; +# Use 127.0.0.1 instead of localhost to force use of IPv4, as that is what the server +# binds to. Windows tends to use IPv6 instead if possible, breaking the test. +my $url_base = 'http://127.0.0.1:1080'; my $ua = LWP::UserAgent->new(env_proxy => 0, keep_alive => 1, timeout => 30); @@ -47,7 +49,8 @@ if($response3->code() != 200) print "Redirected GET request...\n"; my $response4 = $ua->get("$url_base/redirect?key=value"); -exit 4 unless $response4->is_success(); +die "GET ".$response4->request()->url()." failed: ".$response4->content() + unless $response4->is_success(); my $content4 = $response4->content(); @@ -110,13 +113,11 @@ sub check_url my ($c,$url) = @_; unless($c =~ m~URI:</b> (.+?)</p>~) { - print "URI not found\n"; - exit(1); + die "URI not found in response: '$c'\n"; } if($url ne $1) { - print "Wrong URI in content\n"; - exit(1); + die "Wrong URI in content: expected '$url' but found '$1'\n"; } } diff --git a/test/httpserver/testhttpserver.cpp b/test/httpserver/testhttpserver.cpp index 160cb32f..469fa383 100644 --- a/test/httpserver/testhttpserver.cpp +++ b/test/httpserver/testhttpserver.cpp @@ -34,6 +34,8 @@ #include "MemLeakFindOn.h" +#define SHORT_TIMEOUT 5000 + class TestWebServer : public HTTPServer { public: @@ -134,25 +136,27 @@ int test(int argc, const char *argv[]) TestWebServer server; return server.Main("doesnotexist", argc - 1, argv + 1); } - + if(argc >= 2 && ::strcmp(argv[1], "s3server") == 0) { // Run a server S3Simulator server; return server.Main("doesnotexist", argc - 1, argv + 1); } - + +#ifndef WIN32 + TEST_THAT(system("rm -rf *.memleaks") == 0); +#endif + // Start the server - int pid = LaunchServer("./test server testfiles/httpserver.conf", "testfiles/httpserver.pid"); - TEST_THAT(pid != -1 && pid != 0); - if(pid <= 0) - { - return 0; - } + int pid = StartDaemon(0, TEST_EXECUTABLE " server testfiles/httpserver.conf", + "testfiles/httpserver.pid"); + TEST_THAT_OR(pid > 0, return 1); // Run the request script TEST_THAT(::system("perl testfiles/testrequests.pl") == 0); +#ifdef ENABLE_KEEPALIVE_SUPPORT // incomplete, need chunked encoding support #ifndef WIN32 signal(SIGPIPE, SIG_IGN); #endif @@ -182,8 +186,8 @@ int test(int argc, const char *argv[]) { sleep(1); // need time for our process to realise // that the peer has died, otherwise no SIGPIPE :( - TEST_CHECK_THROWS(request.Send(sock, - IOStream::TimeOutInfinite), + TEST_CHECK_THROWS( + request.Send(sock, SHORT_TIMEOUT), ConnectionException, SocketWriteError); sock.Close(); sock.Open(Socket::TypeINET, "localhost", 1080); @@ -191,12 +195,12 @@ int test(int argc, const char *argv[]) } else { - request.Send(sock, IOStream::TimeOutInfinite); + request.Send(sock, SHORT_TIMEOUT); } HTTPResponse response; - response.Receive(sock); - + response.Receive(sock, SHORT_TIMEOUT); + TEST_THAT(response.GetResponseCode() == HTTPResponse::Code_OK); TEST_THAT(response.GetContentType() == "text/html"); @@ -234,16 +238,25 @@ int test(int argc, const char *argv[]) TEST_EQUAL("</body>", line); TEST_THAT(getline.GetLine(line)); TEST_EQUAL("</html>", line); + + if(!response.IsKeepAlive()) + { + BOX_TRACE("Server will close the connection, closing our end too."); + sock.Close(); + sock.Open(Socket::TypeINET, "localhost", 1080); + } + else + { + BOX_TRACE("Server will keep the connection open for more requests."); + } } - - // Kill it - TEST_THAT(KillServer(pid)); - #ifdef WIN32 - TEST_THAT(unlink("testfiles/httpserver.pid") == 0); - #else - TestRemoteProcessMemLeaks("generic-httpserver.memleaks"); - #endif + sock.Close(); +#endif // ENABLE_KEEPALIVE_SUPPORT + + // Kill it + TEST_THAT(StopDaemon(pid, "testfiles/httpserver.pid", + "generic-httpserver.memleaks", true)); // correct, official signature should succeed, with lower-case header { @@ -253,16 +266,16 @@ int test(int argc, const char *argv[]) request.AddHeader("date", "Tue, 27 Mar 2007 19:36:42 +0000"); request.AddHeader("authorization", "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA="); - + S3Simulator simulator; simulator.Configure("testfiles/s3simulator.conf"); - + CollectInBufferStream response_buffer; HTTPResponse response(&response_buffer); - + simulator.Handle(request, response); TEST_EQUAL(200, response.GetResponseCode()); - + std::string response_data((const char *)response.GetBuffer(), response.GetSize()); TEST_EQUAL("omgpuppies!\n", response_data); @@ -276,16 +289,16 @@ int test(int argc, const char *argv[]) request.AddHeader("date", "Tue, 27 Mar 2007 19:36:42 +0000"); request.AddHeader("authorization", "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbB="); - + S3Simulator simulator; simulator.Configure("testfiles/s3simulator.conf"); - + CollectInBufferStream response_buffer; HTTPResponse response(&response_buffer); - + simulator.Handle(request, response); TEST_EQUAL(401, response.GetResponseCode()); - + std::string response_data((const char *)response.GetBuffer(), response.GetSize()); TEST_EQUAL("<html><head>" @@ -297,7 +310,7 @@ int test(int argc, const char *argv[]) "</html>\n", response_data); } - // S3Client tests + // S3Client tests with S3Simulator in-process server for debugging { S3Simulator simulator; simulator.Configure("testfiles/s3simulator.conf"); @@ -332,26 +345,27 @@ int test(int argc, const char *argv[]) } { - HTTPRequest request(HTTPRequest::Method_PUT, - "/newfile"); + HTTPRequest request(HTTPRequest::Method_PUT, "/newfile"); request.SetHostName("quotes.s3.amazonaws.com"); request.AddHeader("date", "Wed, 01 Mar 2006 12:00:00 GMT"); - request.AddHeader("authorization", "AWS 0PN5J17HBGZHT7JJ3X82:XtMYZf0hdOo4TdPYQknZk0Lz7rw="); + request.AddHeader("authorization", + "AWS 0PN5J17HBGZHT7JJ3X82:XtMYZf0hdOo4TdPYQknZk0Lz7rw="); request.AddHeader("Content-Type", "text/plain"); - + FileStream fs("testfiles/testrequests.pl"); fs.CopyStreamTo(request); request.SetForReading(); CollectInBufferStream response_buffer; HTTPResponse response(&response_buffer); - + S3Simulator simulator; simulator.Configure("testfiles/s3simulator.conf"); simulator.Handle(request, response); - + TEST_EQUAL(200, response.GetResponseCode()); - TEST_EQUAL("LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7", response.GetHeaderValue("x-amz-id-2")); + TEST_EQUAL("LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7", + response.GetHeaderValue("x-amz-id-2")); TEST_EQUAL("F2A8CCCA26B4B26D", response.GetHeaderValue("x-amz-request-id")); TEST_EQUAL("Wed, 01 Mar 2006 12:00:00 GMT", response.GetHeaderValue("Date")); TEST_EQUAL("Sun, 1 Jan 2006 12:00:00 GMT", response.GetHeaderValue("Last-Modified")); @@ -367,27 +381,23 @@ int test(int argc, const char *argv[]) } // Start the S3Simulator server - pid = LaunchServer("./test s3server testfiles/s3simulator.conf", + pid = StartDaemon(0, TEST_EXECUTABLE " s3server testfiles/s3simulator.conf", "testfiles/s3simulator.pid"); - TEST_THAT(pid != -1 && pid != 0); - if(pid <= 0) - { - return 0; - } - - sock.Close(); - sock.Open(Socket::TypeINET, "localhost", 1080); + TEST_THAT_OR(pid > 0, return 1); { + SocketStream sock; + sock.Open(Socket::TypeINET, "localhost", 1080); + HTTPRequest request(HTTPRequest::Method_GET, "/nonexist"); request.SetHostName("quotes.s3.amazonaws.com"); request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:0cSX/YPdtXua1aFFpYmH1tc0ajA="); request.SetClientKeepAliveRequested(true); - request.Send(sock, IOStream::TimeOutInfinite); + request.Send(sock, SHORT_TIMEOUT); HTTPResponse response; - response.Receive(sock); + response.Receive(sock, SHORT_TIMEOUT); std::string value; TEST_EQUAL(404, response.GetResponseCode()); } @@ -396,6 +406,9 @@ int test(int argc, const char *argv[]) // Make file inaccessible, should cause server to return a 403 error, // unless of course the test is run as root :) { + SocketStream sock; + sock.Open(Socket::TypeINET, "localhost", 1080); + TEST_THAT(chmod("testfiles/testrequests.pl", 0) == 0); HTTPRequest request(HTTPRequest::Method_GET, "/testrequests.pl"); @@ -403,10 +416,10 @@ int test(int argc, const char *argv[]) request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:qc1e8u8TVl2BpIxwZwsursIb8U8="); request.SetClientKeepAliveRequested(true); - request.Send(sock, IOStream::TimeOutInfinite); + request.Send(sock, SHORT_TIMEOUT); HTTPResponse response; - response.Receive(sock); + response.Receive(sock, SHORT_TIMEOUT); std::string value; TEST_EQUAL(403, response.GetResponseCode()); TEST_THAT(chmod("testfiles/testrequests.pl", 0755) == 0); @@ -414,16 +427,19 @@ int test(int argc, const char *argv[]) #endif { + SocketStream sock; + sock.Open(Socket::TypeINET, "localhost", 1080); + HTTPRequest request(HTTPRequest::Method_GET, "/testrequests.pl"); request.SetHostName("quotes.s3.amazonaws.com"); request.AddHeader("Date", "Wed, 01 Mar 2006 12:00:00 GMT"); request.AddHeader("Authorization", "AWS 0PN5J17HBGZHT7JJ3X82:qc1e8u8TVl2BpIxwZwsursIb8U8="); request.SetClientKeepAliveRequested(true); - request.Send(sock, IOStream::TimeOutInfinite); + request.Send(sock, SHORT_TIMEOUT); HTTPResponse response; - response.Receive(sock); + response.Receive(sock, SHORT_TIMEOUT); std::string value; TEST_EQUAL(200, response.GetResponseCode()); TEST_EQUAL("qBmKRcEWBBhH6XAqsKU/eg24V3jf/kWKN9dJip1L/FpbYr9FDy7wWFurfdQOEMcY", response.GetHeaderValue("x-amz-id-2")); @@ -439,6 +455,9 @@ int test(int argc, const char *argv[]) } { + SocketStream sock; + sock.Open(Socket::TypeINET, "localhost", 1080); + HTTPRequest request(HTTPRequest::Method_PUT, "/newfile"); request.SetHostName("quotes.s3.amazonaws.com"); @@ -447,9 +466,7 @@ int test(int argc, const char *argv[]) request.AddHeader("Content-Type", "text/plain"); FileStream fs("testfiles/testrequests.pl"); HTTPResponse response; - request.SendWithStream(sock, - IOStream::TimeOutInfinite /* or 10000 milliseconds */, - &fs, response); + request.SendWithStream(sock, SHORT_TIMEOUT, &fs, response); std::string value; TEST_EQUAL(200, response.GetResponseCode()); TEST_EQUAL("LriYPLdmOdAiIfgSm/F1YsViT1LW94/xUQxMsF7xiEb1a0wiIOIxl+zbwZ163pt7", response.GetHeaderValue("x-amz-id-2")); @@ -466,14 +483,10 @@ int test(int argc, const char *argv[]) TEST_THAT(f1.CompareWith(f2)); } - // Kill it - TEST_THAT(KillServer(pid)); - #ifdef WIN32 - TEST_THAT(unlink("testfiles/s3simulator.pid") == 0); - #else - TestRemoteProcessMemLeaks("generic-httpserver.memleaks"); - #endif + // Kill it + TEST_THAT(StopDaemon(pid, "testfiles/s3simulator.pid", + "s3simulator.memleaks", true)); return 0; } diff --git a/test/raidfile/testraidfile.cpp b/test/raidfile/testraidfile.cpp index 160de5c9..d771f23d 100644 --- a/test/raidfile/testraidfile.cpp +++ b/test/raidfile/testraidfile.cpp @@ -10,7 +10,6 @@ #include "Box.h" #include <fcntl.h> -#include <unistd.h> #include <stdio.h> #include <errno.h> @@ -26,6 +25,7 @@ #include "RaidFileException.h" #include "RaidFileRead.h" #include "Guards.h" +#include "intercept.h" #include "MemLeakFindOn.h" @@ -38,14 +38,6 @@ #define TRF_CAN_INTERCEPT #endif - -#ifdef TRF_CAN_INTERCEPT -// function in intercept.cpp for setting up errors -void intercept_setup_error(const char *filename, unsigned int errorafter, int errortoreturn, int syscalltoerror); -bool intercept_triggered(); -void intercept_clear_setup(); -#endif - // Nice random data for testing written files class R250 { public: @@ -216,6 +208,12 @@ void testReadingFileContents(int set, const char *filename, void *data, int data // Be nasty, and create some errors for the RAID stuff to recover from... if(TestRAIDProperties) { + HideCategoryGuard hide(RaidFileRead::OPEN_IN_RECOVERY); + hide.Add(RaidFileRead::IO_ERROR); + hide.Add(RaidFileRead::RECOVERING_IO_ERROR); + HideSpecificExceptionGuard hex(RaidFileException::ExceptionType, + RaidFileException::ErrorOpeningFileForRead); + char stripe1fn[256], stripe1fnRename[256]; sprintf(stripe1fn, "testfiles" DIRECTORY_SEPARATOR "%d_%d" DIRECTORY_SEPARATOR "%s.rf", set, startDisc, filename); @@ -273,7 +271,6 @@ void testReadingFileContents(int set, const char *filename, void *data, int data DIRECTORY_SEPARATOR ".raidfile-unreadable" DIRECTORY_SEPARATOR "%s", set, (startDisc + 1) % RAID_NUMBER_DISCS, mungefilename); - #ifdef TRF_CAN_INTERCEPT // Test I/O errors on opening diff --git a/test/s3store/testextra b/test/s3store/testextra new file mode 100644 index 00000000..798c8c67 --- /dev/null +++ b/test/s3store/testextra @@ -0,0 +1,4 @@ +mkdir testfiles/0_0 +mkdir testfiles/0_1 +mkdir testfiles/0_2 +mkdir testfiles/bbackupd-data diff --git a/test/s3store/testfiles/bbackupd.conf b/test/s3store/testfiles/bbackupd.conf new file mode 100644 index 00000000..77640e5e --- /dev/null +++ b/test/s3store/testfiles/bbackupd.conf @@ -0,0 +1,61 @@ + +CertificateFile = testfiles/clientCerts.pem +PrivateKeyFile = testfiles/clientPrivKey.pem +TrustedCAsFile = testfiles/clientTrustedCAs.pem + +KeysFile = testfiles/bbackupd.keys + +DataDirectory = testfiles/bbackupd-data + +S3Store +{ + HostName = localhost + Port = 22080 + BasePath = /subdir/ + AccessKey = 0PN5J17HBGZHT7JJ3X82 + SecretKey = uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o +} + +UpdateStoreInterval = 3 +BackupErrorDelay = 10 +MinimumFileAge = 4 +MaxUploadWait = 24 +DeleteRedundantLocationsAfter = 10 + +FileTrackingSizeThreshold = 1024 +DiffingUploadSizeThreshold = 1024 + +MaximumDiffingTime = 3 +KeepAliveTime = 1 + +ExtendedLogging = no +ExtendedLogFile = testfiles/bbackupd.log + +CommandSocket = testfiles/bbackupd.sock + +NotifyScript = /usr/bin/perl testfiles/notifyscript.pl +SyncAllowScript = /usr/bin/perl testfiles/syncallowscript.pl + +Server +{ + PidFile = testfiles/bbackupd.pid +} + +BackupLocations +{ + Test1 + { + Path = testfiles/TestDir1 + + ExcludeFile = testfiles/TestDir1/excluded_1 + ExcludeFile = testfiles/TestDir1/excluded_2 + ExcludeFilesRegex = \.excludethis$ + ExcludeFilesRegex = EXCLUDE + AlwaysIncludeFile = testfiles/TestDir1/dont.excludethis + ExcludeDir = testfiles/TestDir1/exclude_dir + ExcludeDir = testfiles/TestDir1/exclude_dir_2 + ExcludeDirsRegex = not_this_dir + AlwaysIncludeDirsRegex = ALWAYSINCLUDE + } +} + diff --git a/test/s3store/testfiles/bbackupd.keys b/test/s3store/testfiles/bbackupd.keys Binary files differnew file mode 100644 index 00000000..d9135b97 --- /dev/null +++ b/test/s3store/testfiles/bbackupd.keys diff --git a/test/s3store/testfiles/clientTrustedCAs.pem b/test/s3store/testfiles/clientTrustedCAs.pem new file mode 100644 index 00000000..2a065879 --- /dev/null +++ b/test/s3store/testfiles/clientTrustedCAs.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBjDCB9gIBADANBgkqhkiG9w0BAQUFADAPMQ0wCwYDVQQDEwRST09UMB4XDTAz +MTAwNzA4NTkzMloXDTMxMDIyMjA4NTkzMlowDzENMAsGA1UEAxMEUk9PVDCBnzAN +BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtZypR/5m5fuNMPNrSLdzwmXKqhdVZj/e +cZHUZvVuXQZboosAznDrbh8HgpuTw5vaZDEz8VfPwgIaROZDT3ztFIedLapJ7Ot9 +I4JNqSv/y3V9MKb7trTSPVvyYLqk9isLmw8wmEidJiLbWbIc2cHFXDvWNqTr2jF6 +u4Q8DvdVfAECAwEAATANBgkqhkiG9w0BAQUFAAOBgQAL1lyJ/5y44yjk2BK+tnrZ +hbK7Ghtqrq/uZ8RQq5sAme919TnPijh2tRBqSaUaD2K+Sgo3RNgUGbKhfHRU1pfM +USllHskTKiJu74ix/T3UOnjpQ946OLSl5zNsOdOgbjBDnozfPSrKeEGN0huBbmmt +SlL3iQzVXlF6NAhkzS54fQ== +-----END CERTIFICATE----- diff --git a/test/s3store/testfiles/s3simulator.conf b/test/s3store/testfiles/s3simulator.conf new file mode 100644 index 00000000..c9895e9f --- /dev/null +++ b/test/s3store/testfiles/s3simulator.conf @@ -0,0 +1,10 @@ +AccessKey = 0PN5J17HBGZHT7JJ3X82 +SecretKey = uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o +StoreDirectory = testfiles/store +AddressPrefix = http://localhost:22080 + +Server +{ + PidFile = testfiles/s3simulator.pid + ListenAddresses = inet:localhost:22080 +} diff --git a/test/s3store/testfiles/serverCerts.pem b/test/s3store/testfiles/serverCerts.pem new file mode 100644 index 00000000..92467618 --- /dev/null +++ b/test/s3store/testfiles/serverCerts.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBlzCCAQACAQQwDQYJKoZIhvcNAQEFBQAwDzENMAsGA1UEAxMEUk9PVDAeFw0w +MzEwMDcwOTAwMTFaFw0zMTAyMjIwOTAwMTFaMBkxFzAVBgNVBAMTDlNUT1JFLTAw +MDAwMDA4MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDNj1fGSCaSl/1w1lRV +I8qE6BqjvT6R0XXGdIV+dk/mHmE3NOCPcBq/gxZOYevp+QnwMc+nUSS7Px/n+q92 +cl3a8ttInfZjLqg9o/wpd6dBfH4gLTG4bEujhMt1x4bEUJk/uWfnk5FhsJXDBrlH +RJZNiS9Asme+5Zvjfz3Phy0YWwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBABhmdun/ +myn3l4SbH+PxSUaW/mSvBubFhbbl9wolwhzvGCrtY968jn464JUP1UwUnnvePUU2 +SSVPZOVCvobCfM6s20aOdlKvnn+7GZkjoFONuCw3O+1hIFTSyXFcJWBaYLuczVk1 +HfdIKKcVZ1CpAfnMhMxuu+nA7fjor4p1/K0t +-----END CERTIFICATE----- diff --git a/test/s3store/testfiles/serverPrivKey.pem b/test/s3store/testfiles/serverPrivKey.pem new file mode 100644 index 00000000..fd87607d --- /dev/null +++ b/test/s3store/testfiles/serverPrivKey.pem @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDNj1fGSCaSl/1w1lRVI8qE6BqjvT6R0XXGdIV+dk/mHmE3NOCP +cBq/gxZOYevp+QnwMc+nUSS7Px/n+q92cl3a8ttInfZjLqg9o/wpd6dBfH4gLTG4 +bEujhMt1x4bEUJk/uWfnk5FhsJXDBrlHRJZNiS9Asme+5Zvjfz3Phy0YWwIDAQAB +AoGBAI88mjo1noM528Wb4+nr5bvVDHMadJYhccMXAMqNYMGGW9GfS/dHc6wNiSaX +P0+rVIyF+R+rAEBmDTKV0Vxk9xZQuAaDKjLluDkxSxSR869D2YOWYUfvjDo3OFlT +LMZf0eE7u/3Pm0MtxPctXszqvNnmb+IvPXzttGRgUfU5G+tJAkEA+IphkGMI4A3l +4KfxotZZU+HiJbRDFpm81RzCc2709KCMkXMEz/+xkvnqlo28jqOf7PRBeq/ecsZN +8BGvtyoqVQJBANO6uj6sPI66GaRqxV83VyUUdMmL9uFOccIMqW5q0rx5UDi0mG7t +Pjjz+ul1D247+dvVxnEBeW4C85TSNbbKR+8CQQChpV7PCZo8Hs3jz1bZEZAHfmIX +I6Z+jH7EHHBbo06ty72g263FmgdkECcCxCxemQzqj/IGWVvUSiVmfhpKhqIBAkAl +XbjswpzVW4aW+7jlevDIPHn379mcHan54x4rvHKAjLBZsZWNThVDG9vWQ7B7dd48 +q9efrfDuN1shko+kOMLFAkAGIc5w0bJNC4eu91Wr6AFgTm2DntyVQ9keVhYbrwrE +xY37dgVhAWVeLDOk6eVOVSYqEI1okXPVqvfOIoRJUYkn +-----END RSA PRIVATE KEY----- diff --git a/test/s3store/testfiles/serverReq.pem b/test/s3store/testfiles/serverReq.pem new file mode 100644 index 00000000..7475d406 --- /dev/null +++ b/test/s3store/testfiles/serverReq.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIBWDCBwgIBADAZMRcwFQYDVQQDEw5TVE9SRS0wMDAwMDAwODCBnzANBgkqhkiG +9w0BAQEFAAOBjQAwgYkCgYEAzY9Xxkgmkpf9cNZUVSPKhOgao70+kdF1xnSFfnZP +5h5hNzTgj3Aav4MWTmHr6fkJ8DHPp1Ekuz8f5/qvdnJd2vLbSJ32Yy6oPaP8KXen +QXx+IC0xuGxLo4TLdceGxFCZP7ln55ORYbCVwwa5R0SWTYkvQLJnvuWb4389z4ct +GFsCAwEAAaAAMA0GCSqGSIb3DQEBBQUAA4GBAIdlFo8gbik1K/+4Ra87cQDZzn0L +wE9bZrxRMPXqGjCQ8HBCfvQMFa1Oc6fEczCJ/nmmd76j0HIXW7uYOELIT8L/Zvf5 +jw/z9/OvEOQal7H2JN2d6W4ZmYpQko5+e/bJmlrOxyBpcXk34BvyQen9pTmI6J4E +pkBN/5XUUvVJSM67 +-----END CERTIFICATE REQUEST----- diff --git a/test/s3store/testfiles/store/subdir/dirs/create-me.txt b/test/s3store/testfiles/store/subdir/dirs/create-me.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/s3store/testfiles/store/subdir/dirs/create-me.txt diff --git a/test/s3store/tests3store.cpp b/test/s3store/tests3store.cpp new file mode 100644 index 00000000..50bd2bfd --- /dev/null +++ b/test/s3store/tests3store.cpp @@ -0,0 +1,128 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: tests3store.cpp +// Purpose: Test Amazon S3 storage VFS API and utilities +// Created: 2015/06/28 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#ifndef WIN32 +# include <csignal> +#endif + +#include "BackupAccountControl.h" +#include "BackupClientCryptoKeys.h" +#include "BackupDaemonConfigVerify.h" +#include "BackupStoreDirectory.h" +#include "BackupStoreInfo.h" +#include "Configuration.h" +#include "RaidFileController.h" +#include "ServerControl.h" +#include "SSLLib.h" +#include "Test.h" +#include "Utils.h" + +#include "MemLeakFindOn.h" + +#define DEFAULT_BBACKUPD_CONFIG_FILE "testfiles/bbackupd.conf" + +int s3simulator_pid = 0; + +bool StartSimulator() +{ + s3simulator_pid = StartDaemon(s3simulator_pid, + "../../bin/s3simulator/s3simulator " + bbstored_args + + " testfiles/s3simulator.conf", "testfiles/s3simulator.pid"); + return s3simulator_pid != 0; +} + +bool StopSimulator() +{ + bool result = StopDaemon(s3simulator_pid, "testfiles/s3simulator.pid", + "s3simulator.memleaks", true); + s3simulator_pid = 0; + return result; +} + +bool kill_running_daemons() +{ + if(FileExists("testfiles/s3simulator.pid")) + { + return KillServer("testfiles/s3simulator.pid", true); + } + else + { + return true; + } +} + +//! Simplifies calling setUp() with the current function name in each test. +#define SETUP_TEST_S3SIMULATOR() \ + SETUP(); \ + TEST_THAT(kill_running_daemons()); \ + TEST_THAT(StartSimulator()); \ + +#define TEARDOWN_TEST_S3SIMULATOR() \ + TEST_THAT(s3simulator_pid == 0 || StopSimulator()); \ + TEST_THAT(kill_running_daemons()); \ + TEARDOWN(); + +bool test_create_account_with_account_control() +{ + SETUP_TEST_S3SIMULATOR(); + + std::auto_ptr<Configuration> config = load_config_file(DEFAULT_BBACKUPD_CONFIG_FILE, + BackupDaemonConfigVerify); + S3BackupAccountControl control(*config); + control.CreateAccount("test", 1000, 2000); + + FileStream fs("testfiles/store/subdir/" S3_INFO_FILE_NAME); + std::auto_ptr<BackupStoreInfo> info = BackupStoreInfo::Load(fs, fs.GetFileName(), + true); // ReadOnly + TEST_EQUAL(0, info->GetAccountID()); + TEST_EQUAL(1, info->GetLastObjectIDUsed()); + TEST_EQUAL(1, info->GetBlocksUsed()); + TEST_EQUAL(0, info->GetBlocksInCurrentFiles()); + TEST_EQUAL(0, info->GetBlocksInOldFiles()); + TEST_EQUAL(0, info->GetBlocksInDeletedFiles()); + TEST_EQUAL(1, info->GetBlocksInDirectories()); + TEST_EQUAL(0, info->GetDeletedDirectories().size()); + TEST_EQUAL(1000, info->GetBlocksSoftLimit()); + TEST_EQUAL(2000, info->GetBlocksHardLimit()); + TEST_EQUAL(0, info->GetNumCurrentFiles()); + TEST_EQUAL(0, info->GetNumOldFiles()); + TEST_EQUAL(0, info->GetNumDeletedFiles()); + TEST_EQUAL(1, info->GetNumDirectories()); + TEST_EQUAL(true, info->IsAccountEnabled()); + TEST_EQUAL(true, info->IsReadOnly()); + TEST_EQUAL(0, info->GetClientStoreMarker()); + TEST_EQUAL("test", info->GetAccountName()); + + FileStream root_stream("testfiles/store/subdir/dirs/0x1.dir"); + BackupStoreDirectory root_dir(root_stream); + TEST_EQUAL(0, root_dir.GetNumberOfEntries()); + + TEARDOWN_TEST_S3SIMULATOR(); +} + +int test(int argc, const char *argv[]) +{ + // SSL library + SSLLib::Initialise(); + + // Use the setup crypto command to set up all these keys, so that the bbackupquery command can be used + // for seeing what's going on. + BackupClientCryptoKeys_Setup("testfiles/bbackupd.keys"); + +#ifndef WIN32 + signal(SIGPIPE, SIG_IGN); +#endif + + TEST_THAT(test_create_account_with_account_control()); + + return finish_test_suite(); +} + diff --git a/win32.bat b/win32.bat deleted file mode 100644 index 5df3d709..00000000 --- a/win32.bat +++ /dev/null @@ -1,39 +0,0 @@ -@echo off
-
-echo quick and dirty to get up and running by generating the required files
-echo using Cygwin and Perl
-
-copy .\infrastructure\BoxPlatform.pm.in .\infrastructure\BoxPlatform.pm
-copy .\lib\common\BoxPortsAndFiles.h.in .\lib\common\BoxPortsAndFiles.h
-copy .\lib\common\BoxConfig-MSVC.h .\lib\common\BoxConfig.h
-
-cd .\bin\bbackupquery\ & perl ./../../bin/bbackupquery/makedocumentation.pl.in
-cd ..\..\
-
-cd .\lib\backupstore & perl ./../../lib/common/makeexception.pl.in BackupStoreException.txt & perl ./../../lib/server/makeprotocol.pl.in backupprotocol.txt
-cd ..\..\
-
-cd .\lib\compress & perl ./../../lib/common/makeexception.pl.in CompressException.txt
-cd ..\..\
-
-cd .\lib\common & perl ./../../lib/common/makeexception.pl.in CommonException.txt & perl ./../../lib/common/makeexception.pl.in ConversionException.txt
-cd ..\..\
-
-cd .\lib\raidfile & perl ./../../lib/common/makeexception.pl.in RaidFileException.txt
-cd ..\..\
-
-cd .\bin\bbackupd & perl ./../../lib/common/makeexception.pl.in ClientException.txt
-cd ..\..\
-
-cd .\lib\crypto & perl ./../../lib/common/makeexception.pl.in CipherException.txt
-cd ..\..\
-
-echo server parts - which appears as though some of the clients rely on
-
-cd .\lib\server & perl ./../../lib/common/makeexception.pl.in ServerException.txt & perl ./../../lib/common/makeexception.pl.in ConnectionException.txt
-cd ..\..\
-
-perl -pe 's/@PERL@/perl/' ./test/bbackupd/testfiles/bbackupd.conf.in > .\test\bbackupd\testfiles\bbackupd.conf
-
-echo Generating InstallJammer configuration file
-perl infrastructure/msvc/fake-config.sub.pl ./contrib/windows/installer/boxbackup.mpi.in > ./contrib/windows/installer/boxbackup.mpi
|
