From 99f8ce096bc5569adbfea1911dbcda24c28d8d8b Mon Sep 17 00:00:00 2001 From: Ben Summers Date: Fri, 14 Oct 2005 08:50:54 +0000 Subject: Box Backup 0.09 with a few tweeks --- bin/bbackupquery/BackupQueries.cpp | 1700 +++++++++++++++++++++++++++++++++ bin/bbackupquery/BackupQueries.h | 101 ++ bin/bbackupquery/Makefile.extra | 6 + bin/bbackupquery/bbackupquery.cpp | 243 +++++ bin/bbackupquery/documentation.txt | 165 ++++ bin/bbackupquery/makedocumentation.pl | 75 ++ 6 files changed, 2290 insertions(+) create mode 100755 bin/bbackupquery/BackupQueries.cpp create mode 100755 bin/bbackupquery/BackupQueries.h create mode 100755 bin/bbackupquery/Makefile.extra create mode 100755 bin/bbackupquery/bbackupquery.cpp create mode 100755 bin/bbackupquery/documentation.txt create mode 100755 bin/bbackupquery/makedocumentation.pl (limited to 'bin/bbackupquery') diff --git a/bin/bbackupquery/BackupQueries.cpp b/bin/bbackupquery/BackupQueries.cpp new file mode 100755 index 00000000..0d08f1eb --- /dev/null +++ b/bin/bbackupquery/BackupQueries.cpp @@ -0,0 +1,1700 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupQueries.cpp +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "BackupQueries.h" +#include "Utils.h" +#include "Configuration.h" +#include "autogen_BackupProtocolClient.h" +#include "BackupStoreFilenameClear.h" +#include "BackupStoreDirectory.h" +#include "IOStream.h" +#include "BoxTimeToText.h" +#include "FileStream.h" +#include "BackupStoreFile.h" +#include "TemporaryDirectory.h" +#include "FileModificationTime.h" +#include "BackupClientFileAttributes.h" +#include "CommonException.h" +#include "BackupClientRestore.h" +#include "BackupStoreException.h" +#include "ExcludeList.h" +#include "BackupClientMakeExcludeList.h" + +#include "MemLeakFindOn.h" + +#define COMPARE_RETURN_SAME 1 +#define COMPARE_RETURN_DIFFERENT 2 +#define COMPARE_RETURN_ERROR 3 + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::BackupQueries() +// Purpose: Constructor +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +BackupQueries::BackupQueries(BackupProtocolClient &rConnection, const Configuration &rConfiguration) + : mrConnection(rConnection), + mrConfiguration(rConfiguration), + mQuitNow(false), + mRunningAsRoot(false), + mWarnedAboutOwnerAttributes(false), + mReturnCode(0) // default return code +{ + mRunningAsRoot = (::geteuid() == 0); +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::~BackupQueries() +// Purpose: Destructor +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +BackupQueries::~BackupQueries() +{ +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::DoCommand(const char *) +// Purpose: Perform a command +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::DoCommand(const char *Command) +{ + // is the command a shell command? + if(Command[0] == 's' && Command[1] == 'h' && Command[2] == ' ' && Command[3] != '\0') + { + // Yes, run shell command + ::system(Command + 3); + return; + } + + // split command into components + std::vector cmdElements; + std::string options; + { + const char *c = Command; + bool inQuoted = false; + bool inOptions = false; + + std::string s; + while(*c != 0) + { + // Terminating char? + if(*c == ((inQuoted)?'"':' ')) + { + if(!s.empty()) cmdElements.push_back(s); + s.resize(0); + inQuoted = false; + inOptions = false; + } + else + { + // No. Start of quoted parameter? + if(s.empty() && *c == '"') + { + inQuoted = true; + } + // Start of options? + else if(s.empty() && *c == '-') + { + inOptions = true; + } + else + { + if(inOptions) + { + // Option char + options += *c; + } + else + { + // Normal string char + s += *c; + } + } + } + + ++c; + } + if(!s.empty()) cmdElements.push_back(s); + } + + // Check... + if(cmdElements.size() < 1) + { + // blank command + return; + } + + // Data about commands + static const char *commandNames[] = {"quit", "exit", "list", "pwd", "cd", "lcd", "sh", "getobject", "get", "compare", "restore", "help", "usage", "undelete", 0}; + static const char *validOptions[] = {"", "", "rodIFtsh", "", "od", "", "", "", "i", "alcqE", "dri", "", "", "", 0}; + #define COMMAND_Quit 0 + #define COMMAND_Exit 1 + #define COMMAND_List 2 + #define COMMAND_pwd 3 + #define COMMAND_cd 4 + #define COMMAND_lcd 5 + #define COMMAND_sh 6 + #define COMMAND_GetObject 7 + #define COMMAND_Get 8 + #define COMMAND_Compare 9 + #define COMMAND_Restore 10 + #define COMMAND_Help 11 + #define COMMAND_Usage 12 + #define COMMAND_Undelete 13 + static const char *alias[] = {"ls", 0}; + static const int aliasIs[] = {COMMAND_List, 0}; + + // Work out which command it is... + int cmd = 0; + while(commandNames[cmd] != 0 && ::strcmp(cmdElements[0].c_str(), commandNames[cmd]) != 0) + { + cmd++; + } + if(commandNames[cmd] == 0) + { + // Check for aliases + int a; + for(a = 0; alias[a] != 0; ++a) + { + if(::strcmp(cmdElements[0].c_str(), alias[a]) == 0) + { + // Found an alias + cmd = aliasIs[a]; + break; + } + } + + // No such command + if(alias[a] == 0) + { + printf("Unrecognised command: %s\n", Command); + return; + } + } + + // Arguments + std::vector args(cmdElements.begin() + 1, cmdElements.end()); + + // Set up options + bool opts[256]; + for(int o = 0; o < 256; ++o) opts[o] = false; + // BLOCK + { + // options + const char *c = options.c_str(); + while(*c != 0) + { + // Valid option? + if(::strchr(validOptions[cmd], *c) == NULL) + { + printf("Invalid option '%c' for command %s\n", *c, commandNames[cmd]); + return; + } + opts[(int)*c] = true; + ++c; + } + } + + if(cmd != COMMAND_Quit && cmd != COMMAND_Exit) + { + // If not a quit command, set the return code to zero + SetReturnCode(0); + } + + // Handle command + switch(cmd) + { + case COMMAND_Quit: + case COMMAND_Exit: + mQuitNow = true; + break; + + case COMMAND_List: + CommandList(args, opts); + break; + + case COMMAND_pwd: + { + // Simple implementation, so do it here + std::string dir(GetCurrentDirectoryName()); + printf("%s (%08llx)\n", dir.c_str(), GetCurrentDirectoryID()); + } + break; + + case COMMAND_cd: + CommandChangeDir(args, opts); + break; + + case COMMAND_lcd: + CommandChangeLocalDir(args); + break; + + case COMMAND_sh: + printf("The command to run must be specified as an argument.\n"); + break; + + case COMMAND_GetObject: + CommandGetObject(args, opts); + break; + + case COMMAND_Get: + CommandGet(args, opts); + break; + + case COMMAND_Compare: + CommandCompare(args, opts); + break; + + case COMMAND_Restore: + CommandRestore(args, opts); + break; + + case COMMAND_Usage: + CommandUsage(); + break; + + case COMMAND_Help: + CommandHelp(args); + break; + + case COMMAND_Undelete: + CommandUndelete(args, opts); + break; + + default: + break; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandList(const std::vector &, const bool *) +// Purpose: List directories (optionally recursive) +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandList(const std::vector &args, const bool *opts) +{ + #define LIST_OPTION_RECURSIVE 'r' + #define LIST_OPTION_ALLOWOLD 'o' + #define LIST_OPTION_ALLOWDELETED 'd' + #define LIST_OPTION_NOOBJECTID 'I' + #define LIST_OPTION_NOFLAGS 'F' + #define LIST_OPTION_TIMES 't' + #define LIST_OPTION_SIZEINBLOCKS 's' + #define LIST_OPTION_DISPLAY_HASH 'h' + + // default to using the current directory + int64_t rootDir = GetCurrentDirectoryID(); + + // name of base directory + std::string listRoot; // blank + + // Got a directory in the arguments? + if(args.size() > 0) + { + // Attempt to find the directory + rootDir = FindDirectoryObjectID(args[0], opts[LIST_OPTION_ALLOWOLD], opts[LIST_OPTION_ALLOWDELETED]); + if(rootDir == 0) + { + printf("Directory %s not found on store\n", args[0].c_str()); + return; + } + } + + // List it + List(rootDir, listRoot, opts, true /* first level to list */); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandList2(int64_t, const std::string &, const bool *) +// Purpose: Do the actual listing of directories and files +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::List(int64_t DirID, const std::string &rListRoot, const bool *opts, bool FirstLevel) +{ + // Generate exclude flags + int16_t excludeFlags = BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING; + if(!opts[LIST_OPTION_ALLOWOLD]) excludeFlags |= BackupProtocolClientListDirectory::Flags_OldVersion; + if(!opts[LIST_OPTION_ALLOWDELETED]) excludeFlags |= BackupProtocolClientListDirectory::Flags_Deleted; + + // Do communication + mrConnection.QueryListDirectory( + DirID, + BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, // both files and directories + excludeFlags, + true /* want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Then... display everything + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *en = 0; + while((en = i.Next()) != 0) + { + // Display this entry + BackupStoreFilenameClear clear(en->GetName()); + std::string line; + + // Object ID? + if(!opts[LIST_OPTION_NOOBJECTID]) + { + // add object ID to line + char oid[32]; + sprintf(oid, "%08llx ", en->GetObjectID()); + line += oid; + } + + // Flags? + if(!opts[LIST_OPTION_NOFLAGS]) + { + static const char *flags = BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES; + char displayflags[16]; + // make sure f is big enough + ASSERT(sizeof(displayflags) >= sizeof(BACKUPSTOREDIRECTORY_ENTRY_FLAGS_DISPLAY_NAMES) + 3); + // Insert flags + char *f = displayflags; + const char *t = flags; + int16_t en_flags = en->GetFlags(); + while(*t != 0) + { + *f = ((en_flags&1) == 0)?'-':*t; + en_flags >>= 1; + f++; + t++; + } + // attributes flags + *(f++) = (en->HasAttributes())?'a':'-'; + // terminate + *(f++) = ' '; + *(f++) = '\0'; + line += displayflags; + if(en_flags != 0) + { + line += "[ERROR: Entry has additional flags set] "; + } + } + + if(opts[LIST_OPTION_TIMES]) + { + // Show times... + line += BoxTimeToISO8601String(en->GetModificationTime()); + line += ' '; + } + + if(opts[LIST_OPTION_DISPLAY_HASH]) + { + char hash[64]; + ::sprintf(hash, "%016llx ", en->GetAttributesHash()); + line += hash; + } + + if(opts[LIST_OPTION_SIZEINBLOCKS]) + { + char num[32]; + sprintf(num, "%05lld ", en->GetSizeInBlocks()); + line += num; + } + + // add name + if(!FirstLevel) + { + line += rListRoot; + line += '/'; + } + line += clear.GetClearFilename().c_str(); + + if(!en->GetName().IsEncrypted()) + { + line += "[FILENAME NOT ENCRYPTED]"; + } + + // print line + printf("%s\n", line.c_str()); + + // Directory? + if((en->GetFlags() & BackupStoreDirectory::Entry::Flags_Dir) != 0) + { + // Recurse? + if(opts[LIST_OPTION_RECURSIVE]) + { + std::string subroot(rListRoot); + if(!FirstLevel) subroot += '/'; + subroot += clear.GetClearFilename(); + List(en->GetObjectID(), subroot, opts, false /* not the first level to list */); + } + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::FindDirectoryObjectID(const std::string &) +// Purpose: Find the object ID of a directory on the store, or return 0 for not found. +// If pStack != 0, the object is set to the stack of directories. +// Will start from the current directory stack. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::FindDirectoryObjectID(const std::string &rDirName, bool AllowOldVersion, + bool AllowDeletedDirs, std::vector > *pStack) +{ + // Split up string into elements + std::vector dirElements; + SplitString(rDirName, DIRECTORY_SEPARATOR_ASCHAR, dirElements); + + // Start from current stack, or root, whichever is required + std::vector > stack; + int64_t dirID = BackupProtocolClientListDirectory::RootDirectory; + if(rDirName.size() > 0 && rDirName[0] == '/') + { + // Root, do nothing + } + else + { + // Copy existing stack + stack = mDirStack; + if(stack.size() > 0) + { + dirID = stack[stack.size() - 1].second; + } + } + + // Generate exclude flags + int16_t excludeFlags = BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING; + if(!AllowOldVersion) excludeFlags |= BackupProtocolClientListDirectory::Flags_OldVersion; + if(!AllowDeletedDirs) excludeFlags |= BackupProtocolClientListDirectory::Flags_Deleted; + + // Read directories + for(unsigned int e = 0; e < dirElements.size(); ++e) + { + if(dirElements[e].size() > 0) + { + if(dirElements[e] == ".") + { + // Ignore. + } + else if(dirElements[e] == "..") + { + // Up one! + if(stack.size() > 0) + { + // Remove top element + stack.pop_back(); + + // New dir ID + dirID = (stack.size() > 0)?(stack[stack.size() - 1].second):BackupProtocolClientListDirectory::RootDirectory; + } + else + { + // At root anyway + dirID = BackupProtocolClientListDirectory::RootDirectory; + } + } + else + { + // Not blank element. Read current directory. + std::auto_ptr dirreply(mrConnection.QueryListDirectory( + dirID, + BackupProtocolClientListDirectory::Flags_Dir, // just directories + excludeFlags, + true /* want attributes */)); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Then... find the directory within it + BackupStoreDirectory::Iterator i(dir); + BackupStoreFilenameClear dirname(dirElements[e]); + BackupStoreDirectory::Entry *en = i.FindMatchingClearName(dirname); + if(en == 0) + { + // Not found + return 0; + } + + // Object ID for next round of searching + dirID = en->GetObjectID(); + + // Push onto stack + stack.push_back(std::pair(dirElements[e], dirID)); + } + } + } + + // If required, copy the new stack to the caller + if(pStack) + { + *pStack = stack; + } + + return dirID; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::GetCurrentDirectoryID() +// Purpose: Returns the ID of the current directory +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +int64_t BackupQueries::GetCurrentDirectoryID() +{ + // Special case for root + if(mDirStack.size() == 0) + { + return BackupProtocolClientListDirectory::RootDirectory; + } + + // Otherwise, get from the last entry on the stack + return mDirStack[mDirStack.size() - 1].second; +} + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::GetCurrentDirectoryName() +// Purpose: Gets the name of the current directory +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +std::string BackupQueries::GetCurrentDirectoryName() +{ + // Special case for root + if(mDirStack.size() == 0) + { + return std::string("/"); + } + + // Build path + std::string r; + for(unsigned int l = 0; l < mDirStack.size(); ++l) + { + r += "/"; + r += mDirStack[l].first; + } + + return r; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandChangeDir(const std::vector &) +// Purpose: Change directory command +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandChangeDir(const std::vector &args, const bool *opts) +{ + if(args.size() != 1 || args[0].size() == 0) + { + printf("Incorrect usage.\ncd [-o] [-d] \n"); + return; + } + + std::vector > newStack; + int64_t id = FindDirectoryObjectID(args[0], opts['o'], opts['d'], &newStack); + + if(id == 0) + { + printf("Directory '%s' not found\n", args[0].c_str()); + return; + } + + // Store new stack + mDirStack = newStack; +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandChangeLocalDir(const std::vector &) +// Purpose: Change local directory command +// Created: 2003/10/11 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandChangeLocalDir(const std::vector &args) +{ + if(args.size() != 1 || args[0].size() == 0) + { + printf("Incorrect usage.\nlcd \n"); + return; + } + + // Try changing directory + if(::chdir(args[0].c_str()) != 0) + { + printf((errno == ENOENT || errno == ENOTDIR)?"Directory '%s' does not exist\n":"Error changing dir to '%s'\n", + args[0].c_str()); + return; + } + + // Report current dir + char wd[PATH_MAX]; + if(::getcwd(wd, PATH_MAX) == 0) + { + printf("Error getting current directory\n"); + return; + } + + printf("Local current directory is now '%s'\n", wd); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandGetObject(const std::vector &, const bool *) +// Purpose: Gets an object without any translation. +// Created: 2003/10/11 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandGetObject(const std::vector &args, const bool *opts) +{ + // Check args + if(args.size() != 2) + { + printf("Incorrect usage.\ngetobject \n"); + return; + } + + int64_t id = ::strtoll(args[0].c_str(), 0, 16); + if(id == LLONG_MIN || id == LLONG_MAX || id == 0) + { + printf("Not a valid object ID (specified in hex)\n"); + return; + } + + // Does file exist? + struct stat st; + if(::stat(args[1].c_str(), &st) == 0 || errno != ENOENT) + { + printf("The local file %s already exists\n", args[1].c_str()); + return; + } + + // Open file + FileStream out(args[1].c_str(), O_WRONLY | O_CREAT | O_EXCL); + + // Request that object + try + { + // Request object + std::auto_ptr getobj(mrConnection.QueryGetObject(id)); + if(getobj->GetObjectID() != BackupProtocolClientGetObject::NoObject) + { + // Stream that object out to the file + std::auto_ptr objectStream(mrConnection.ReceiveStream()); + objectStream->CopyStreamTo(out); + + printf("Object ID %08llx fetched successfully.\n", id); + } + else + { + printf("Object does not exist on store.\n"); + ::unlink(args[1].c_str()); + } + } + catch(...) + { + ::unlink(args[1].c_str()); + printf("Error occured fetching object.\n"); + } +} + + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandGet(const std::vector &, const bool *) +// Purpose: Command to get a file from the store +// Created: 2003/10/12 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandGet(const std::vector &args, const bool *opts) +{ + // At least one argument? + // Check args + if(args.size() < 1 || (opts['i'] && args.size() != 2) || args.size() > 2) + { + printf("Incorrect usage.\ngetobject \n or get -i \n"); + return; + } + + // Find object ID somehow + int64_t id; + std::string localName; + // BLOCK + { + // Need to look it up in the current directory + mrConnection.QueryListDirectory( + GetCurrentDirectoryID(), + BackupProtocolClientListDirectory::Flags_File, // just files + (opts['i'])?(BackupProtocolClientListDirectory::Flags_EXCLUDE_NOTHING):(BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted), // only current versions + false /* don't want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + if(opts['i']) + { + // Specified as ID. + id = ::strtoll(args[0].c_str(), 0, 16); + if(id == LLONG_MIN || id == LLONG_MAX || id == 0) + { + printf("Not a valid object ID (specified in hex)\n"); + return; + } + + // Check that the item is actually in the directory + if(dir.FindEntryByID(id) == 0) + { + printf("ID '%08llx' not found in current directory on store.\n(You can only download objects by ID from the current directory.)\n", id); + return; + } + + // Must have a local name in the arguments (check at beginning of function ensures this) + localName = args[1]; + } + else + { + // Specified by name, find the object in the directory to get the ID + BackupStoreDirectory::Iterator i(dir); + BackupStoreFilenameClear fn(args[0]); + BackupStoreDirectory::Entry *en = i.FindMatchingClearName(fn); + + if(en == 0) + { + printf("Filename '%s' not found in current directory on store.\n(Subdirectories in path not searched.)\n", args[0].c_str()); + return; + } + + id = en->GetObjectID(); + + // Local name is the last argument, which is either the looked up filename, or + // a filename specified by the user. + localName = args[args.size() - 1]; + } + } + + // Does local file already exist? (don't want to overwrite) + struct stat st; + if(::stat(localName.c_str(), &st) == 0 || errno != ENOENT) + { + printf("The local file %s already exists, will not overwrite it.\n", localName.c_str()); + return; + } + + // Request it from the store + try + { + // Request object + mrConnection.QueryGetFile(GetCurrentDirectoryID(), id); + + // Stream containing encoded file + std::auto_ptr objectStream(mrConnection.ReceiveStream()); + + // Decode it + BackupStoreFile::DecodeFile(*objectStream, localName.c_str(), mrConnection.GetTimeout()); + + // Done. + printf("Object ID %08llx fetched sucessfully.\n", id); + } + catch(...) + { + ::unlink(args[1].c_str()); + printf("Error occured fetching file.\n"); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareParams::CompareParams() +// Purpose: Constructor +// Created: 29/1/04 +// +// -------------------------------------------------------------------------- +BackupQueries::CompareParams::CompareParams() + : mQuickCompare(false), + mIgnoreExcludes(false), + mDifferences(0), + mDifferencesExplainedByModTime(0), + mExcludedDirs(0), + mExcludedFiles(0), + mpExcludeFiles(0), + mpExcludeDirs(0), + mLatestFileUploadTime(0) +{ +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareParams::~CompareParams() +// Purpose: Destructor +// Created: 29/1/04 +// +// -------------------------------------------------------------------------- +BackupQueries::CompareParams::~CompareParams() +{ + DeleteExcludeLists(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareParams::DeleteExcludeLists() +// Purpose: Delete the include lists contained +// Created: 29/1/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CompareParams::DeleteExcludeLists() +{ + if(mpExcludeFiles != 0) + { + delete mpExcludeFiles; + mpExcludeFiles = 0; + } + if(mpExcludeDirs != 0) + { + delete mpExcludeDirs; + mpExcludeDirs = 0; + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandCompare(const std::vector &, const bool *) +// Purpose: Command to compare data on the store with local data +// Created: 2003/10/12 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandCompare(const std::vector &args, const bool *opts) +{ + // Parameters, including count of differences + BackupQueries::CompareParams params; + params.mQuickCompare = opts['q']; + params.mIgnoreExcludes = opts['E']; + + // Try and work out the time before which all files should be on the server + { + std::string syncTimeFilename(mrConfiguration.GetKeyValue("DataDirectory") + DIRECTORY_SEPARATOR_ASCHAR); + syncTimeFilename += "last_sync_start"; + // Stat it to get file time + struct stat st; + if(::stat(syncTimeFilename.c_str(), &st) == 0) + { + // Files modified after this time shouldn't be on the server, so report errors slightly differently + params.mLatestFileUploadTime = FileModificationTime(st) + - SecondsToBoxTime((uint32_t)mrConfiguration.GetKeyValueInt("MinimumFileAge")); + } + else + { + printf("Warning: couldn't determine the time of the last syncronisation -- checks not performed.\n"); + } + } + + // Quick compare? + if(params.mQuickCompare) + { + printf("WARNING: Quick compare used -- file attributes are not checked.\n"); + } + + if(!opts['l'] && opts['a'] && args.size() == 0) + { + // Compare all locations + const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations")); + for(std::list >::const_iterator i = locations.mSubConfigurations.begin(); + i != locations.mSubConfigurations.end(); ++i) + { + CompareLocation(i->first, params); + } + } + else if(opts['l'] && !opts['a'] && args.size() == 1) + { + // Compare one location + CompareLocation(args[0], params); + } + else if(!opts['l'] && !opts['a'] && args.size() == 2) + { + // Compare directory to directory + + // Can't be bothered to do all the hard work to work out which location it's on, and hence which exclude list + if(!params.mIgnoreExcludes) + { + printf("Cannot use excludes on directory to directory comparison -- use -E flag to specify ignored excludes\n"); + return; + } + else + { + // Do compare + Compare(args[0], args[1], params); + } + } + else + { + printf("Incorrect usage.\ncompare -a\n or compare -l \n or compare \n"); + return; + } + + printf("\n[ %d (of %d) differences probably due to file modifications after the last upload ]\nDifferences: %d (%d dirs excluded, %d files excluded)\n", + params.mDifferencesExplainedByModTime, params.mDifferences, params.mDifferences, params.mExcludedDirs, params.mExcludedFiles); + + // Set return code? + if(opts['c']) + { + SetReturnCode((params.mDifferences == 0)?COMPARE_RETURN_SAME:COMPARE_RETURN_DIFFERENT); + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CompareLocation(const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a location +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::CompareLocation(const std::string &rLocation, BackupQueries::CompareParams &rParams) +{ + // Find the location's sub configuration + const Configuration &locations(mrConfiguration.GetSubConfiguration("BackupLocations")); + if(!locations.SubConfigurationExists(rLocation.c_str())) + { + printf("Location %s does not exist.\n", rLocation.c_str()); + return; + } + const Configuration &loc(locations.GetSubConfiguration(rLocation.c_str())); + + try + { + // Generate the exclude lists + if(!rParams.mIgnoreExcludes) + { + rParams.mpExcludeFiles = BackupClientMakeExcludeList_Files(loc); + rParams.mpExcludeDirs = BackupClientMakeExcludeList_Dirs(loc); + } + + // Then get it compared + Compare(std::string("/") + rLocation, loc.GetKeyValue("Path"), rParams); + } + catch(...) + { + // Clean up + rParams.DeleteExcludeLists(); + throw; + } + + // Delete exclude lists + rParams.DeleteExcludeLists(); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::Compare(const std::string &, const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a store directory against a local directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::Compare(const std::string &rStoreDir, const std::string &rLocalDir, BackupQueries::CompareParams &rParams) +{ + // Get the directory ID of the directory -- only use current data + int64_t dirID = FindDirectoryObjectID(rStoreDir); + + // Found? + if(dirID == 0) + { + printf("Local directory '%s' exists, but server directory '%s' does not exist\n", rLocalDir.c_str(), rStoreDir.c_str()); + rParams.mDifferences ++; + return; + } + + // Go! + Compare(dirID, rStoreDir, rLocalDir, rParams); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::Compare(int64_t, const std::string &, BackupQueries::CompareParams &) +// Purpose: Compare a store directory against a local directory +// Created: 2003/10/13 +// +// -------------------------------------------------------------------------- +void BackupQueries::Compare(int64_t DirID, const std::string &rStoreDir, const std::string &rLocalDir, BackupQueries::CompareParams &rParams) +{ + // Get info on the local directory + struct stat st; + if(::lstat(rLocalDir.c_str(), &st) != 0) + { + // What kind of error? + if(errno == ENOTDIR) + { + printf("Local object '%s' is a file, server object '%s' is a directory\n", rLocalDir.c_str(), rStoreDir.c_str()); + rParams.mDifferences ++; + } + else if(errno == ENOENT) + { + printf("Local directory '%s' does not exist (compared to server directory '%s')\n", rLocalDir.c_str(), rStoreDir.c_str()); + } + else + { + printf("ERROR: stat on local dir '%s'\n", rLocalDir.c_str()); + } + return; + } + + // Get the directory listing from the store + mrConnection.QueryListDirectory( + DirID, + BackupProtocolClientListDirectory::Flags_INCLUDE_EVERYTHING, // get everything + BackupProtocolClientListDirectory::Flags_OldVersion | BackupProtocolClientListDirectory::Flags_Deleted, // except for old versions and deleted files + true /* want attributes */); + + // Retrieve the directory from the stream following + BackupStoreDirectory dir; + std::auto_ptr dirstream(mrConnection.ReceiveStream()); + dir.ReadFromStream(*dirstream, mrConnection.GetTimeout()); + + // Test out the attributes + if(!dir.HasAttributes()) + { + printf("Store directory '%s' doesn't have attributes.\n", rStoreDir.c_str()); + } + else + { + // Fetch the attributes + const StreamableMemBlock &storeAttr(dir.GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + + // Get attributes of local directory + BackupClientFileAttributes localAttr; + localAttr.ReadAttributes(rLocalDir.c_str(), true /* directories have zero mod times */); + + if(!(attr.Compare(localAttr, true, true /* ignore modification times */))) + { + printf("Local directory '%s' has different attributes to store directory '%s'.\n", + rLocalDir.c_str(), rStoreDir.c_str()); + rParams.mDifferences ++; + } + } + + // Open the local directory + DIR *dirhandle = ::opendir(rLocalDir.c_str()); + if(dirhandle == 0) + { + printf("ERROR: opendir on local dir '%s'\n", rLocalDir.c_str()); + return; + } + try + { + // Read the files and directories into sets + std::set localFiles; + std::set localDirs; + struct dirent *localDirEn = 0; + while((localDirEn = readdir(dirhandle)) != 0) + { + // Not . and ..! + if(localDirEn->d_name[0] == '.' && + (localDirEn->d_name[1] == '\0' || (localDirEn->d_name[1] == '.' && localDirEn->d_name[2] == '\0'))) + { + // ignore, it's . or .. + continue; + } + +#ifdef PLATFORM_dirent_BROKEN_d_type + std::string fn(rLocalDir); + fn += '/'; + fn += localDirEn->d_name; + struct stat st; + if(::lstat(fn.c_str(), &st) != 0) + { + THROW_EXCEPTION(CommonException, OSFileError) + } + + // Entry -- file or dir? + if(S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) + { + // File or symbolic link + localFiles.insert(std::string(localDirEn->d_name)); + } + else if(S_ISDIR(st.st_mode)) + { + // Directory + localDirs.insert(std::string(localDirEn->d_name)); + } +#else + // Entry -- file or dir? + if(localDirEn->d_type == DT_REG || localDirEn->d_type == DT_LNK) + { + // File or symbolic link + localFiles.insert(std::string(localDirEn->d_name)); + } + else if(localDirEn->d_type == DT_DIR) + { + // Directory + localDirs.insert(std::string(localDirEn->d_name)); + } +#endif // PLATFORM_dirent_BROKEN_d_type + } + // Close directory + if(::closedir(dirhandle) != 0) + { + printf("ERROR: closedir on local dir '%s'\n", rLocalDir.c_str()); + } + dirhandle = 0; + + // Do the same for the store directories + std::set > storeFiles; + std::set > storeDirs; + + BackupStoreDirectory::Iterator i(dir); + BackupStoreDirectory::Entry *storeDirEn = 0; + while((storeDirEn = i.Next()) != 0) + { + // Decrypt filename + BackupStoreFilenameClear name(storeDirEn->GetName()); + + // What is it? + if((storeDirEn->GetFlags() & BackupStoreDirectory::Entry::Flags_File) == BackupStoreDirectory::Entry::Flags_File) + { + // File + storeFiles.insert(std::pair(name.GetClearFilename(), storeDirEn)); + } + else + { + // Dir + storeDirs.insert(std::pair(name.GetClearFilename(), storeDirEn)); + } + } + + // Now compare files. + for(std::set >::const_iterator i = storeFiles.begin(); i != storeFiles.end(); ++i) + { + // Does the file exist locally? + std::set::const_iterator local(localFiles.find(i->first)); + if(local == localFiles.end()) + { + // Not found -- report + printf("Local file '%s/%s' does not exist, but store file '%s/%s' does.\n", + rLocalDir.c_str(), i->first.c_str(), rStoreDir.c_str(), i->first.c_str()); + rParams.mDifferences ++; + } + else + { + try + { + // make local name of file for comparison + std::string localName(rLocalDir + DIRECTORY_SEPARATOR + i->first); + + // Files the same flag? + bool equal = true; + + // File modified after last sync flag + bool modifiedAfterLastSync = false; + + if(rParams.mQuickCompare) + { + // Compare file -- fetch it + mrConnection.QueryGetBlockIndexByID(i->second->GetObjectID()); + + // Stream containing block index + std::auto_ptr blockIndexStream(mrConnection.ReceiveStream()); + + // Compare + equal = BackupStoreFile::CompareFileContentsAgainstBlockIndex(localName.c_str(), *blockIndexStream, mrConnection.GetTimeout()); + } + else + { + // Compare file -- fetch it + mrConnection.QueryGetFile(DirID, i->second->GetObjectID()); + + // Stream containing encoded file + std::auto_ptr objectStream(mrConnection.ReceiveStream()); + + // Decode it + std::auto_ptr fileOnServerStream; + // Got additional attibutes? + if(i->second->HasAttributes()) + { + // Use these attributes + const StreamableMemBlock &storeAttr(i->second->GetAttributes()); + BackupClientFileAttributes attr(storeAttr); + fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout(), &attr).release()); + } + else + { + // Use attributes stored in file + fileOnServerStream.reset(BackupStoreFile::DecodeFileStream(*objectStream, mrConnection.GetTimeout()).release()); + } + + // Should always be something in the auto_ptr, it's how the interface is defined. But be paranoid. + if(!fileOnServerStream.get()) + { + THROW_EXCEPTION(BackupStoreException, Internal) + } + + // Compare attributes + BackupClientFileAttributes localAttr; + box_time_t fileModTime = 0; + localAttr.ReadAttributes(localName.c_str(), false /* don't zero mod times */, &fileModTime); + modifiedAfterLastSync = (fileModTime > rParams.mLatestFileUploadTime); + if(!localAttr.Compare(fileOnServerStream->GetAttributes(), + true /* ignore attr mod time */, + fileOnServerStream->IsSymLink() /* ignore modification time if it's a symlink */)) + { + printf("Local file '%s/%s' has different attributes to store file '%s/%s'.\n", + rLocalDir.c_str(), i->first.c_str(), rStoreDir.c_str(), i->first.c_str()); + rParams.mDifferences ++; + if(modifiedAfterLastSync) + { + rParams.mDifferencesExplainedByModTime ++; + printf("(the file above was modified after the last sync time -- might be reason for difference)\n"); + } + else if(i->second->HasAttributes()) + { + printf("(the file above has had new attributes applied)\n"); + } + } + + // Compare contents, if it's a regular file not a link + // Remember, we MUST read the entire stream from the server. + if(!fileOnServerStream->IsSymLink()) + { + // Open the local file + FileStream l(localName.c_str()); + + // Size + IOStream::pos_type fileSizeLocal = l.BytesLeftToRead(); + IOStream::pos_type fileSizeServer = 0; + + // Test the contents + char buf1[2048]; + char buf2[2048]; + while(fileOnServerStream->StreamDataLeft() && l.StreamDataLeft()) + { + int size = fileOnServerStream->Read(buf1, sizeof(buf1), mrConnection.GetTimeout()); + fileSizeServer += size; + + if(l.Read(buf2, size) != size + || ::memcmp(buf1, buf2, size) != 0) + { + equal = false; + break; + } + } + + // Check read all the data from the server and file -- can't be equal if local and remote aren't the same length + // Can't use StreamDataLeft() test on file, because if it's the same size, it won't know + // it's EOF yet. + if(fileOnServerStream->StreamDataLeft() || fileSizeServer != fileSizeLocal) + { + equal = false; + } + + // Must always read the entire decoded string, if it's not a symlink + if(fileOnServerStream->StreamDataLeft()) + { + // Absorb all the data remaining + char buffer[2048]; + while(fileOnServerStream->StreamDataLeft()) + { + fileOnServerStream->Read(buffer, sizeof(buffer), mrConnection.GetTimeout()); + } + } + } + } + + // Report if not equal. + if(!equal) + { + printf("Local file '%s/%s' has different contents to store file '%s/%s'.\n", + rLocalDir.c_str(), i->first.c_str(), rStoreDir.c_str(), i->first.c_str()); + rParams.mDifferences ++; + if(modifiedAfterLastSync) + { + rParams.mDifferencesExplainedByModTime ++; + printf("(the file above was modified after the last sync time -- might be reason for difference)\n"); + } + else if(i->second->HasAttributes()) + { + printf("(the file above has had new attributes applied)\n"); + } + } + } + catch(BoxException &e) + { + printf("ERROR: (%d/%d) during file fetch and comparsion for '%s/%s'\n", + e.GetType(), + e.GetSubType(), + rStoreDir.c_str(), i->first.c_str()); + } + catch(...) + { + printf("ERROR: (unknown) during file fetch and comparsion for '%s/%s'\n", rStoreDir.c_str(), i->first.c_str()); + } + + // Remove from set so that we know it's been compared + localFiles.erase(local); + } + } + + // Report any files which exist on the locally, but not on the store + for(std::set::const_iterator i = localFiles.begin(); i != localFiles.end(); ++i) + { + std::string localName(rLocalDir + DIRECTORY_SEPARATOR + *i); + // Should this be ignored (ie is excluded)? + if(rParams.mpExcludeFiles == 0 || !(rParams.mpExcludeFiles->IsExcluded(localName))) + { + printf("Local file '%s/%s' exists, but store file '%s/%s' does not exist.\n", + rLocalDir.c_str(), (*i).c_str(), rStoreDir.c_str(), (*i).c_str()); + rParams.mDifferences ++; + + // Check the file modification time + { + struct stat st; + if(::stat(localName.c_str(), &st) == 0) + { + if(FileModificationTime(st) > rParams.mLatestFileUploadTime) + { + rParams.mDifferencesExplainedByModTime ++; + printf("(the file above was modified after the last sync time -- might be reason for difference)\n"); + } + } + } + } + else + { + rParams.mExcludedFiles ++; + } + } + + // Finished with the files, clear the sets to reduce memory usage slightly + localFiles.clear(); + storeFiles.clear(); + + // Now do the directories, recusively to check subdirectories + for(std::set >::const_iterator i = storeDirs.begin(); i != storeDirs.end(); ++i) + { + // Does the directory exist locally? + std::set::const_iterator local(localDirs.find(i->first)); + if(local == localDirs.end()) + { + // Not found -- report + printf("Local directory '%s/%s' does not exist, but store directory '%s/%s' does.\n", + rLocalDir.c_str(), i->first.c_str(), rStoreDir.c_str(), i->first.c_str()); + rParams.mDifferences ++; + } + else + { + // Compare directory + Compare(i->second->GetObjectID(), rStoreDir + "/" + i->first, rLocalDir + DIRECTORY_SEPARATOR + i->first, rParams); + + // Remove from set so that we know it's been compared + localDirs.erase(local); + } + } + + // Report any files which exist on the locally, but not on the store + for(std::set::const_iterator i = localDirs.begin(); i != localDirs.end(); ++i) + { + std::string localName(rLocalDir + DIRECTORY_SEPARATOR + *i); + // Should this be ignored (ie is excluded)? + if(rParams.mpExcludeDirs == 0 || !(rParams.mpExcludeDirs->IsExcluded(localName))) + { + printf("Local directory '%s/%s' exists, but store directory '%s/%s' does not exist.\n", + rLocalDir.c_str(), (*i).c_str(), rStoreDir.c_str(), (*i).c_str()); + rParams.mDifferences ++; + } + else + { + rParams.mExcludedDirs ++; + } + } + + } + catch(...) + { + if(dirhandle != 0) + { + ::closedir(dirhandle); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandRestore(const std::vector &, const bool *) +// Purpose: Restore a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandRestore(const std::vector &args, const bool *opts) +{ + // Check arguments + if(args.size() != 2) + { + printf("Incorrect usage.\nrestore [-d] [-r] [-i] \n"); + return; + } + + // Restoring deleted things? + bool restoreDeleted = opts['d']; + + // Get directory ID + int64_t dirID = 0; + if(opts['i']) + { + // Specified as ID. + dirID = ::strtoll(args[0].c_str(), 0, 16); + if(dirID == LLONG_MIN || dirID == LLONG_MAX || dirID == 0) + { + printf("Not a valid object ID (specified in hex)\n"); + return; + } + } + else + { + // Look up directory ID + dirID = FindDirectoryObjectID(args[0], false /* no old versions */, restoreDeleted /* find deleted dirs */); + } + + // Allowable? + if(dirID == 0) + { + printf("Directory %s not found on server\n", args[0].c_str()); + return; + } + if(dirID == BackupProtocolClientListDirectory::RootDirectory) + { + printf("Cannot restore the root directory -- restore locations individually.\n"); + return; + } + + // Go and restore... + switch(BackupClientRestore(mrConnection, dirID, args[1].c_str(), true /* print progress dots */, restoreDeleted, + false /* don't undelete after restore! */, opts['r'] /* resume? */)) + { + case Restore_Complete: + printf("Restore complete\n"); + break; + + case Restore_ResumePossible: + printf("Resume possible -- repeat command with -r flag to resume\n"); + break; + + case Restore_TargetExists: + printf("The target directory exists. You cannot restore over an existing directory.\n"); + break; + + default: + printf("ERROR: Unknown restore result.\n"); + break; + } +} + + + +// These are autogenerated by a script. +extern char *help_commands[]; +extern char *help_text[]; + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandHelp(const std::vector &args) +// Purpose: Display help on commands +// Created: 15/2/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandHelp(const std::vector &args) +{ + if(args.size() == 0) + { + // Display a list of all commands + printf("Available commands are:\n"); + for(int c = 0; help_commands[c] != 0; ++c) + { + printf(" %s\n", help_commands[c]); + } + printf("Type \"help \" for more information on a command.\n\n"); + } + else + { + // Display help on a particular command + int c; + for(c = 0; help_commands[c] != 0; ++c) + { + if(::strcmp(help_commands[c], args[0].c_str()) == 0) + { + // Found the command, print help + printf("\n%s\n", help_text[c]); + break; + } + } + if(help_commands[c] == 0) + { + printf("No help found for command '%s'\n", args[0].c_str()); + } + } +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUsage() +// Purpose: Display storage space used on server +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUsage() +{ + // Request full details from the server + std::auto_ptr usage(mrConnection.QueryGetAccountUsage()); + + // Display each entry in turn + int64_t hardLimit = usage->GetBlocksHardLimit(); + int32_t blockSize = usage->GetBlockSize(); + CommandUsageDisplayEntry("Used", usage->GetBlocksUsed(), hardLimit, blockSize); + CommandUsageDisplayEntry("Old files", usage->GetBlocksInOldFiles(), hardLimit, blockSize); + CommandUsageDisplayEntry("Deleted files", usage->GetBlocksInDeletedFiles(), hardLimit, blockSize); + CommandUsageDisplayEntry("Directories", usage->GetBlocksInDirectories(), hardLimit, blockSize); + CommandUsageDisplayEntry("Soft limit", usage->GetBlocksSoftLimit(), hardLimit, blockSize); + CommandUsageDisplayEntry("Hard limit", hardLimit, hardLimit, blockSize); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUsageDisplayEntry(const char *, int64_t, int64_t, int32_t) +// Purpose: Display an entry in the usage table +// Created: 19/4/04 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUsageDisplayEntry(const char *Name, int64_t Size, int64_t HardLimit, int32_t BlockSize) +{ + // Calculate size in Mb + double mb = (((double)Size) * ((double)BlockSize)) / ((double)(1024*1024)); + int64_t percent = (Size * 100) / HardLimit; + + // Bar graph + char bar[41]; + unsigned int b = (int)((Size * (sizeof(bar)-1)) / HardLimit); + if(b > sizeof(bar)-1) {b = sizeof(bar)-1;} + for(unsigned int l = 0; l < b; l++) + { + bar[l] = '*'; + } + bar[b] = '\0'; + + // Print the entryj + ::printf("%14s %10.1fMb %3d%% %s\n", Name, mb, (int32_t)percent, bar); +} + + +// -------------------------------------------------------------------------- +// +// Function +// Name: BackupQueries::CommandUndelete(const std::vector &, const bool *) +// Purpose: Undelete a directory +// Created: 23/11/03 +// +// -------------------------------------------------------------------------- +void BackupQueries::CommandUndelete(const std::vector &args, const bool *opts) +{ + // Check arguments + if(args.size() != 1) + { + printf("Incorrect usage.\nundelete \n"); + return; + } + + // Get directory ID + int64_t dirID = FindDirectoryObjectID(args[0], false /* no old versions */, true /* find deleted dirs */); + + // Allowable? + if(dirID == 0) + { + printf("Directory %s not found on server\n", args[0].c_str()); + return; + } + if(dirID == BackupProtocolClientListDirectory::RootDirectory) + { + printf("Cannot restore the root directory -- restore locations individually.\n"); + return; + } + + // Undelete + mrConnection.QueryUndeleteDirectory(dirID); +} + + + + + diff --git a/bin/bbackupquery/BackupQueries.h b/bin/bbackupquery/BackupQueries.h new file mode 100755 index 00000000..e84de6ab --- /dev/null +++ b/bin/bbackupquery/BackupQueries.h @@ -0,0 +1,101 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: BackupQueries.h +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#ifndef BACKUPQUERIES__H +#define BACKUPQUERIES__H + +#include +#include + +#include "BoxTime.h" + +class BackupProtocolClient; +class Configuration; +class ExcludeList; + +// -------------------------------------------------------------------------- +// +// Class +// Name: BackupQueries +// Purpose: Perform various queries on the backup store server. +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- +class BackupQueries +{ +public: + BackupQueries(BackupProtocolClient &rConnection, const Configuration &rConfiguration); + ~BackupQueries(); +private: + BackupQueries(const BackupQueries &); +public: + + void DoCommand(const char *Command); + + // Ready to stop? + bool Stop() {return mQuitNow;} + + // Return code? + int GetReturnCode() {return mReturnCode;} + +private: + // Commands + void CommandList(const std::vector &args, const bool *opts); + void CommandChangeDir(const std::vector &args, const bool *opts); + void CommandChangeLocalDir(const std::vector &args); + void CommandGetObject(const std::vector &args, const bool *opts); + void CommandGet(const std::vector &args, const bool *opts); + void CommandCompare(const std::vector &args, const bool *opts); + void CommandRestore(const std::vector &args, const bool *opts); + void CommandUndelete(const std::vector &args, const bool *opts); + void CommandUsage(); + void CommandUsageDisplayEntry(const char *Name, int64_t Size, int64_t HardLimit, int32_t BlockSize); + void CommandHelp(const std::vector &args); + + // Implementations + void List(int64_t DirID, const std::string &rListRoot, const bool *opts, bool FirstLevel); + class CompareParams + { + public: + CompareParams(); + ~CompareParams(); + void DeleteExcludeLists(); + bool mQuickCompare; + bool mIgnoreExcludes; + int mDifferences; + int mDifferencesExplainedByModTime; + int mExcludedDirs; + int mExcludedFiles; + const ExcludeList *mpExcludeFiles; + const ExcludeList *mpExcludeDirs; + box_time_t mLatestFileUploadTime; + }; + void CompareLocation(const std::string &rLocation, CompareParams &rParams); + void Compare(const std::string &rStoreDir, const std::string &rLocalDir, CompareParams &rParams); + void Compare(int64_t DirID, const std::string &rStoreDir, const std::string &rLocalDir, CompareParams &rParams); + + // Utility functions + int64_t FindDirectoryObjectID(const std::string &rDirName, bool AllowOldVersion = false, + bool AllowDeletedDirs = false, std::vector > *pStack = 0); + int64_t GetCurrentDirectoryID(); + std::string GetCurrentDirectoryName(); + void SetReturnCode(int code) {mReturnCode = code;} + +private: + BackupProtocolClient &mrConnection; + const Configuration &mrConfiguration; + bool mQuitNow; + std::vector > mDirStack; + bool mRunningAsRoot; + bool mWarnedAboutOwnerAttributes; + int mReturnCode; +}; + +#endif // BACKUPQUERIES__H + diff --git a/bin/bbackupquery/Makefile.extra b/bin/bbackupquery/Makefile.extra new file mode 100755 index 00000000..633ec0fc --- /dev/null +++ b/bin/bbackupquery/Makefile.extra @@ -0,0 +1,6 @@ + +# AUTOGEN SEEDING +autogen_Documentation.cpp: makedocumentation.pl documentation.txt + perl makedocumentation.pl + + diff --git a/bin/bbackupquery/bbackupquery.cpp b/bin/bbackupquery/bbackupquery.cpp new file mode 100755 index 00000000..aea0faa8 --- /dev/null +++ b/bin/bbackupquery/bbackupquery.cpp @@ -0,0 +1,243 @@ +// -------------------------------------------------------------------------- +// +// File +// Name: bbackupquery.cpp +// Purpose: Backup query utility +// Created: 2003/10/10 +// +// -------------------------------------------------------------------------- + +#include "Box.h" + +#include +#include +#include +#ifndef PLATFORM_READLINE_NOT_SUPPORTED + #ifdef PLATFORM_LINUX + #include "../../local/_linux_readline.h" + #else + #include + #include + #endif +#endif + +#include "MainHelper.h" +#include "BoxPortsAndFiles.h" +#include "BackupDaemonConfigVerify.h" +#include "SocketStreamTLS.h" +#include "Socket.h" +#include "TLSContext.h" +#include "SSLLib.h" +#include "BackupStoreConstants.h" +#include "BackupStoreException.h" +#include "autogen_BackupProtocolClient.h" +#include "BackupQueries.h" +#include "FdGetLine.h" +#include "BackupClientCryptoKeys.h" +#include "BannerText.h" + +#include "MemLeakFindOn.h" + +void PrintUsageAndExit() +{ + printf("Usage: bbackupquery [-q] [-c config_file] [-l log_file] [commands]\nAs many commands as you require.\n" \ + "If commands are multiple words, remember to enclose the command in quotes.\n" \ + "Remember to use quit command if you don't want to drop into interactive mode.\n"); + exit(1); +} + +int main(int argc, const char *argv[]) +{ + MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT("bbackupquery.memleaks", "bbackupquery") + + // Really don't want trace statements happening, even in debug mode + #ifndef NDEBUG + BoxDebugTraceOn = false; + #endif + + int returnCode = 0; + + MAINHELPER_START + + FILE *logFile = 0; + + // Filename for configuraiton file? + const char *configFilename = BOX_FILE_BBACKUPD_DEFAULT_CONFIG; + + // Flags + bool quiet = false; + bool readWrite = false; + + // See if there's another entry on the command line + int c; + while((c = getopt(argc, (char * const *)argv, "qwc:l:")) != -1) + { + switch(c) + { + case 'q': + // Quiet mode + quiet = true; + break; + + case 'w': + // Read/write mode + readWrite = true; + break; + + case 'c': + // store argument + configFilename = optarg; + break; + + case 'l': + // open log file + logFile = ::fopen(optarg, "w"); + if(logFile == 0) + { + printf("Can't open log file '%s'\n", optarg); + } + break; + + case '?': + default: + PrintUsageAndExit(); + } + } + // Adjust arguments + argc -= optind; + argv += optind; + + // Print banner? + if(!quiet) + { + const char *banner = BANNER_TEXT("Backup Query Tool"); + printf(banner); + } + + // Read in the configuration file + if(!quiet) printf("Using configuration file %s\n", configFilename); + std::string errs; + std::auto_ptr config(Configuration::LoadAndVerify(configFilename, &BackupDaemonConfigVerify, errs)); + if(config.get() == 0 || !errs.empty()) + { + printf("Invalid configuration file:\n%s", errs.c_str()); + return 1; + } + // Easier coding + const Configuration &conf(*config); + + // Setup and connect + // 1. TLS context + SSLLib::Initialise(); + // Read in the certificates creating a TLS context + TLSContext tlsContext; + std::string certFile(conf.GetKeyValue("CertificateFile")); + std::string keyFile(conf.GetKeyValue("PrivateKeyFile")); + std::string caFile(conf.GetKeyValue("TrustedCAsFile")); + tlsContext.Initialise(false /* as client */, certFile.c_str(), keyFile.c_str(), caFile.c_str()); + + // Initialise keys + BackupClientCryptoKeys_Setup(conf.GetKeyValue("KeysFile").c_str()); + + // 2. Connect to server + if(!quiet) printf("Connecting to store...\n"); + SocketStreamTLS socket; + socket.Open(tlsContext, Socket::TypeINET, conf.GetKeyValue("StoreHostname").c_str(), BOX_PORT_BBSTORED); + + // 3. Make a protocol, and handshake + if(!quiet) printf("Handshake with store...\n"); + BackupProtocolClient connection(socket); + connection.Handshake(); + + // logging? + if(logFile != 0) + { + connection.SetLogToFile(logFile); + } + + // 4. Log in to server + if(!quiet) printf("Login to store...\n"); + // Check the version of the server + { + std::auto_ptr serverVersion(connection.QueryVersion(BACKUP_STORE_SERVER_VERSION)); + if(serverVersion->GetVersion() != BACKUP_STORE_SERVER_VERSION) + { + THROW_EXCEPTION(BackupStoreException, WrongServerVersion) + } + } + // Login -- if this fails, the Protocol will exception + connection.QueryLogin(conf.GetKeyValueInt("AccountNumber"), + (readWrite)?0:(BackupProtocolClientLogin::Flags_ReadOnly)); + + // 5. Tell user. + if(!quiet) printf("Login complete.\n\nType \"help\" for a list of commands.\n\n"); + + // Set up a context for our work + BackupQueries context(connection, conf); + + // Start running commands... first from the command line + { + int c = 0; + while(c < argc && !context.Stop()) + { + context.DoCommand(argv[c++]); + } + } + + // Get commands from input +#ifndef PLATFORM_READLINE_NOT_SUPPORTED + using_history(); + char *last_cmd = 0; + while(!context.Stop()) + { + char *command = readline("query > "); + if(command == NULL) + { + // Ctrl-D pressed -- terminate now + break; + } + context.DoCommand(command); + if(last_cmd != 0 && ::strcmp(last_cmd, command) == 0) + { + free(command); + } + else + { + add_history(command); + last_cmd = command; + } + } +#else + // Version for platforms which don't have readline by default + FdGetLine getLine(fileno(stdin)); + while(!context.Stop()) + { + printf("query > "); + fflush(stdout); + std::string command(getLine.GetLine()); + context.DoCommand(command.c_str()); + } +#endif + + // Done... stop nicely + if(!quiet) printf("Logging off...\n"); + connection.QueryFinished(); + if(!quiet) printf("Session finished.\n"); + + // Return code + returnCode = context.GetReturnCode(); + + // Close log file? + if(logFile) + { + ::fclose(logFile); + } + + // Let everything be cleaned up on exit. + + MAINHELPER_END + + exit(returnCode); + return returnCode; +} + diff --git a/bin/bbackupquery/documentation.txt b/bin/bbackupquery/documentation.txt new file mode 100755 index 00000000..429caabe --- /dev/null +++ b/bin/bbackupquery/documentation.txt @@ -0,0 +1,165 @@ + +bbackupquery utility -- examine store, compare files, restore, etc. + +This file has markers for automatic help generation script -- '>' marks a start of a command/help topic, +and '<' marks the end of a section. + +Command line: +============= + +> bbackupquery [-q] [-c configfile] [commands ...] + + -q -- quiet, no information prompts + -c -- specify another bbackupd configuation file + +The commands following the options are executed, then (if there was no quit +command) an interactive mode is entered. + +If a command contains a space, enclose it in quotes. Example + + bbackupquery "list testdir1" quit + +to list the contents of testdir1, and then exit without interactive mode. +< + +Commands: +========= + +All directory names relative to a "current" directory, or from root if they +start with '/'. The initial directory is always the root directory. + + +> list [options] [directory-name] + + List contents of current directory, or specified directory. + + -r -- recursively list all files + -d -- list deleted files/directories + -o -- list old versions of files/directories + -I -- don't display object ID + -F -- don't display flags + -t -- show file modification time + (and attr mod time if has the object has attributes, ~ separated) + -s -- show file size in blocks used on server + (only very approximate indication of size locally) + +ls can be used as an alias. +< + +> ls + + Alias for 'list'. Type 'help list' for options. +< + +> cd [options] + + Change directory + + -d -- consider deleted directories for traversal + -o -- consider old versions of directories for traversal + (this option should never be useful in a correctly formed store) +< + +> pwd + + Print current directory, always root relative. +< + +> lcd + + Change local directory. + + Type "sh ls" to list the contents. +< + +> sh + + All of the parameters after the "sh" are run as a shell command. + + For example, to list the contents of the location directory, type "sh ls" +< + +> get [] +get -i + + Gets a file from the store. Object is specified as the filename within + the current directory, and local filename is optional. Ignores old and + deleted files when searching the directory for the file to retrieve. + + To get an old or deleted file, use the -i option and select the object + as a hex object ID (first column in listing). The local filename must + be specified. +< + +> compare -a +compare -l +compare + + Compares the (current) data on the store with the data on the disc. + All the data will be downloaded -- this is potentially a very long + operation. + + -a -- compare all locations + -l -- compare one backup location as specified in the configuration file. + -c -- set return code + -q -- quick compare. Only checks file contents against checksums, + doesn't do a full download + -E -- ignore exclusion settings + + Comparing with the root directory is an error, use -a option instead. + + If -c is set, then the return code (if quit is the next command) will be + 1 Comparison was exact + 2 Differences were found + 3 An error occured + This can be used for automated tests. +< + +> restore [-d] [-r] [-i] + + Restores a directory to the local disc. The local directory specified + must not exist (unless a previous restore is being restarted). + + The root cannot be restored -- restore locations individually. + + -d -- restore a deleted directory. + -r -- resume an interrupted restoration + -i -- directory name is actually an ID + + If a restore operation is interrupted for any reason, it can be restarted + using the -r switch. Restore progress information is saved in a file at + regular intervals during the restore operation to allow restarts. +< + +> getobject + + Gets the object specified by the object id (in hex) and stores the raw + contents in the local file specified. + + This is only useful for debugging as it does not decode files from the + stored format, which is encrypted and compressed. +< + +> usage + + Show space used on the server for this account. + + Used: Total amount of space used on the server. + Old files: Space used by old files + Deleted files: Space used by deleted files + Directories: Space used by the directory structure. + + When Used exceeds the soft limit, the server will start to remove old and + deleted files until the usage drops below the soft limit. + + After a while, you would expect to see the usage stay at just below the + soft limit. You only need more space if the space used by old and deleted + files is near zero. +< + +> quit + + End session and exit. +< + + diff --git a/bin/bbackupquery/makedocumentation.pl b/bin/bbackupquery/makedocumentation.pl new file mode 100755 index 00000000..a3632848 --- /dev/null +++ b/bin/bbackupquery/makedocumentation.pl @@ -0,0 +1,75 @@ +#!/usr/bin/perl +use strict; + +print "Creating built-in documentation for bbackupquery...\n"; + +open DOC,"documentation.txt" or die "Can't open documentation.txt file"; +my $section; +my %help; +my @in_order; + +while() +{ + if(m/\A>\s+(\w+)/) + { + $section = $1; + m/\A>\s+(.+)\Z/; + $help{$section} = $1."\n"; + push @in_order,$section; + } + elsif(m/\Aautogen_Documentation.cpp" or die "Can't open output file for writing"; + +print OUT <<__E; +// +// Automatically generated file, do not edit. +// + +#include "Box.h" + +#include "MemLeakFindOn.h" + +char *help_commands[] = +{ +__E + +for(@in_order) +{ + print OUT qq:\t"$_",\n:; +} + +print OUT <<__E; + 0 +}; + +char *help_text[] = +{ +__E + +for(@in_order) +{ + my $t = $help{$_}; + $t =~ s/\t/ /g; + $t =~ s/\n/\\n/g; + $t =~ s/"/\\"/g; + print OUT qq:\t"$t",\n:; +} + +print OUT <<__E; + 0 +}; + +__E + +close OUT; -- cgit v1.2.3